From baaca9d99612bec403c287861c90fc5a55c75e81 Mon Sep 17 00:00:00 2001 From: snermolaev Date: Thu, 7 Aug 2025 06:19:54 +0300 Subject: [PATCH 01/52] additional flags for ANTL4 generated cpp files commit_hash:fbafc1ab387d054144fd85a59c33eed9fcc0a4d1 --- build/ymake.core.conf | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/build/ymake.core.conf b/build/ymake.core.conf index bbcab1dab858..0bf0d751a943 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -3448,12 +3448,14 @@ when ($CLANG == "yes") { _LANG_CFLAGS_SWG=-Wno-deprecated-declarations $_DISABLE_CLANG_COVERAGE_CFLAGS _LANG_CFLAGS_LEX=-Wno-unused-variable $_DISABLE_CLANG_COVERAGE_CFLAGS _LANG_CFLAGS_BISON=-Wno-unused-but-set-variable -Wno-deprecated-copy $_DISABLE_CLANG_COVERAGE_CFLAGS + _LANG_CFLAGS_ANTLR=-Wno-unused-variable $_DISABLE_CLANG_COVERAGE_CFLAGS } otherwise { _LANG_CFLAGS_RL=-Wno-implicit-fallthrough _LANG_CFLAGS_SWG=-Wno-deprecated-declarations _LANG_CFLAGS_LEX=-Wno-unused-variable _LANG_CFLAGS_BISON=-Wno-unused-but-set-variable -Wno-deprecated-copy + _LANG_CFLAGS_ANTLR=-Wno-unused-variable } # tag:src-processing @@ -3468,7 +3470,8 @@ when ($CLANG == "yes" || $CLANG_CL == "yes" || $GCC == "yes") { ${pre=$_LANG_CFLAGS_RL;clear;ext=.rl6;noext;input:SRC} \ ${pre=$_LANG_CFLAGS_RL;clear;ext=.pyx;noext;input:SRC} \ ${pre=$_LANG_CFLAGS_SWG;clear;ext=.swg;noext;input:SRC} \ - ${pre=$_LANG_CFLAGS_LEX;clear;ext=.l;noext;input:SRC} + ${pre=$_LANG_CFLAGS_LEX;clear;ext=.l;noext;input:SRC} \ + ${pre=$_LANG_CFLAGS_ANTLR;clear;ext=.g4;noext;input:SRC} } # Allows to add single compilation unit name to the node's 'kv' section. @@ -5163,7 +5166,9 @@ _ANTLR4_VISITOR__ANTLR4_EMPTY=-no-visitor ### ### Macro to invoke ANTLR4 generator for separate lexer and parser grammars (Cpp) macro RUN_ANTLR4_CPP_SPLIT(LEXER, PARSER, OUTPUT_INCLUDES[], LISTENER?"PARSER":"_ANTLR4_EMPTY", VISITOR?"PARSER":"_ANTLR4_EMPTY", _ANTLR4_EMPTY="", Args...) { - RUN_ANTLR4(${LEXER} ${PARSER} -Dlanguage=Cpp -o ${BINDIR} ${_ANTLR4_VISITOR_$VISITOR} ${_ANTLR4_LISTENER_$LISTENER} ${Args} CWD ${BINDIR} IN ${LEXER} ${PARSER} OUT ${suf=.cpp;noext:LEXER} ${suf=.h;noext:LEXER} ${suf=.cpp;noext:PARSER} ${suf=.h;noext:PARSER} ${suf=Listener.h;noext:$LISTENER} ${suf=BaseListener.h;noext:$LISTENER} ${suf=Visitor.h;noext:$VISITOR} ${suf=BaseVisitor.h;noext:$VISITOR} OUTPUT_INCLUDES ${ARCADIA_ROOT}/contrib/libs/antlr4_cpp_runtime/src/antlr4-runtime.h ${OUTPUT_INCLUDES}) + RUN_ANTLR4(${LEXER} ${PARSER} -Dlanguage=Cpp -o ${BINDIR} ${_ANTLR4_VISITOR_$VISITOR} ${_ANTLR4_LISTENER_$LISTENER} ${Args} CWD ${BINDIR} IN ${LEXER} ${PARSER} OUT_NOAUTO ${suf=.cpp;noext:LEXER} ${suf=.h;noext:LEXER} ${suf=.cpp;noext:PARSER} ${suf=.h;noext:PARSER} ${suf=Listener.h;noext:$LISTENER} ${suf=BaseListener.h;noext:$LISTENER} ${suf=Visitor.h;noext:$VISITOR} ${suf=BaseVisitor.h;noext:$VISITOR} OUTPUT_INCLUDES ${ARCADIA_ROOT}/contrib/libs/antlr4_cpp_runtime/src/antlr4-runtime.h ${OUTPUT_INCLUDES}) + COPY_FILE(AUTO ${suf=.cpp;noext:LEXER} ${suf=.g4.cpp;noext:LEXER} OUTPUT_INCLUDES ${suf=.h;noext:LEXER} ${OUTPUT_INCLUDES}) + COPY_FILE(AUTO ${suf=.cpp;noext:PARSER} ${suf=.g4.cpp;noext:PARSER} OUTPUT_INCLUDES ${suf=.h;noext:PARSER} ${OUTPUT_INCLUDES}) PEERDIR(contrib/libs/antlr4_cpp_runtime) } @@ -5171,7 +5176,9 @@ macro RUN_ANTLR4_CPP_SPLIT(LEXER, PARSER, OUTPUT_INCLUDES[], LISTENER?"PARSER":" ### ### Macro to invoke ANTLR4 generator for combined lexer+parser grammars (Cpp) macro RUN_ANTLR4_CPP(GRAMMAR, OUTPUT_INCLUDES[], LISTENER?"GRAMMAR":"_ANTLR4_EMPTY", VISITOR?"GRAMMAR":"_ANTLR4_EMPTY", _ANTLR4_EMPTY="", Args...) { - RUN_ANTLR4(${GRAMMAR} -Dlanguage=Cpp -o ${BINDIR} ${_ANTLR4_VISITOR_$VISITOR} ${_ANTLR4_LISTENER_$LISTENER} ${Args} CWD ${BINDIR} IN ${GRAMMAR} OUT ${suf=Lexer.cpp;noext:GRAMMAR} ${suf=Lexer.h;noext:GRAMMAR} ${suf=Parser.cpp;noext:GRAMMAR} ${suf=Parser.h;noext:GRAMMAR} ${suf=Listener.h;noext:$LISTENER} ${suf=BaseListener.h;noext:$LISTENER} ${suf=Visitor.h;noext:$VISITOR} ${suf=BaseVisitor.h;noext:$VISITOR} OUTPUT_INCLUDES ${ARCADIA_ROOT}/contrib/libs/antlr4_cpp_runtime/src/antlr4-runtime.h ${OUTPUT_INCLUDES}) + RUN_ANTLR4(${GRAMMAR} -Dlanguage=Cpp -o ${BINDIR} ${_ANTLR4_VISITOR_$VISITOR} ${_ANTLR4_LISTENER_$LISTENER} ${Args} CWD ${BINDIR} IN ${GRAMMAR} OUT_NOAUTO ${suf=Lexer.cpp;noext:GRAMMAR} ${suf=Lexer.h;noext:GRAMMAR} ${suf=Parser.cpp;noext:GRAMMAR} ${suf=Parser.h;noext:GRAMMAR} ${suf=Listener.h;noext:$LISTENER} ${suf=BaseListener.h;noext:$LISTENER} ${suf=Visitor.h;noext:$VISITOR} ${suf=BaseVisitor.h;noext:$VISITOR} OUTPUT_INCLUDES ${ARCADIA_ROOT}/contrib/libs/antlr4_cpp_runtime/src/antlr4-runtime.h ${OUTPUT_INCLUDES}) + COPY_FILE(AUTO ${suf=Lexer.cpp;noext:GRAMMAR} ${suf=Lexer.g4.cpp;noext:GRAMMAR} OUTPUT_INCLUDES ${suf=Lexer.h;noext:GRAMMAR} ${OUTPUT_INCLUDES}) + COPY_FILE(AUTO ${suf=Parser.cpp;noext:GRAMMAR} ${suf=Parser.g4.cpp;noext:GRAMMAR} OUTPUT_INCLUDES ${suf=Parser.h;noext:GRAMMAR} ${OUTPUT_INCLUDES}) PEERDIR(contrib/libs/antlr4_cpp_runtime) } From acd802e58c091a779e6daecd282ff987db0a88db Mon Sep 17 00:00:00 2001 From: alevitskii Date: Thu, 7 Aug 2025 08:51:44 +0300 Subject: [PATCH 02/52] Disable clang coverage with PGO Disable clang coverage with PGO commit_hash:f548e82a0717ddb3032ec3f00b22e88d4937bc24 --- build/ymake.core.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/ymake.core.conf b/build/ymake.core.conf index 0bf0d751a943..50f21b64029f 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -707,7 +707,7 @@ module _BASE_UNIT: _BARE_UNIT { when ($CLANG) { when ($PGO_ADD == "yes") { CFLAGS+=-fprofile-instr-generate - NO_PGO_CFLAGS=-fno-profile-instr-generate + NO_PGO_CFLAGS=$_DISABLE_CLANG_COVERAGE_CFLAGS LDFLAGS+=-fprofile-instr-generate } when ($PGO_PATH) { From a073958afdf2ff4c71f34b21363d2f271e92dd1f Mon Sep 17 00:00:00 2001 From: robot-piglet Date: Thu, 7 Aug 2025 10:49:35 +0300 Subject: [PATCH 03/52] Intermediate changes commit_hash:f979a996722800ee3aee873edd83adb67cddd1ed --- contrib/libs/cxxsupp/openmp/.yandex_meta/override.nix | 4 ++-- contrib/libs/cxxsupp/openmp/ya.make | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/libs/cxxsupp/openmp/.yandex_meta/override.nix b/contrib/libs/cxxsupp/openmp/.yandex_meta/override.nix index 071076d48b16..89f2b9beb227 100644 --- a/contrib/libs/cxxsupp/openmp/.yandex_meta/override.nix +++ b/contrib/libs/cxxsupp/openmp/.yandex_meta/override.nix @@ -1,12 +1,12 @@ pkgs: attrs: with pkgs; with attrs; rec { pname = "openmp"; - version = "20.1.7"; + version = "20.1.8"; src = fetchFromGitHub { owner = "llvm"; repo = "llvm-project"; rev = "llvmorg-${version}"; - hash = "sha256-OSd26CLKziKo/eM/5rhtcWd0AxdtJk0ELA5YIxqINKs="; + hash = "sha256-ysyB/EYxi2qE9fD5x/F2zI4vjn8UDoo1Z9ukiIrjFGw="; }; buildInputs = [ pkgs.python3 ]; diff --git a/contrib/libs/cxxsupp/openmp/ya.make b/contrib/libs/cxxsupp/openmp/ya.make index 8c6f30350ca6..8c1840b045b2 100644 --- a/contrib/libs/cxxsupp/openmp/ya.make +++ b/contrib/libs/cxxsupp/openmp/ya.make @@ -12,9 +12,9 @@ LICENSE( LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(20.1.7) +VERSION(20.1.8) -ORIGINAL_SOURCE(https://github.com/llvm/llvm-project/archive/llvmorg-20.1.7.tar.gz) +ORIGINAL_SOURCE(https://github.com/llvm/llvm-project/archive/llvmorg-20.1.8.tar.gz) ADDINCL( GLOBAL contrib/libs/cxxsupp/openmp From 9765bb0d8b628b0dc07af0fe3dda757a900b669c Mon Sep 17 00:00:00 2001 From: swarmer Date: Thu, 7 Aug 2025 10:49:36 +0300 Subject: [PATCH 04/52] TPortManager shouldn't crash even if it doesn't have enough permissions to operate on lock files created by other users Create the `testing_port_locks` directory with RWX permission for all, allowing other users to create lock files within it. Don't abort if current user doesn't have enough permission to delete a lock file created by another user. KIKIMR-23792 commit_hash:d887ce073d42d93c997a0f9a0c16416860395377 --- library/cpp/testing/common/network.cpp | 16 +++- library/cpp/testing/common/ut/network_ut.cpp | 85 +++++++++++++++----- 2 files changed, 79 insertions(+), 22 deletions(-) diff --git a/library/cpp/testing/common/network.cpp b/library/cpp/testing/common/network.cpp index 9f018f69b72a..dbc80715fde2 100644 --- a/library/cpp/testing/common/network.cpp +++ b/library/cpp/testing/common/network.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -15,6 +16,7 @@ #include #include #include +#include #ifdef _darwin_ #include @@ -29,7 +31,7 @@ namespace { } \ } while (false) - class TPortGuard : public NTesting::IPort { + class TPortGuard : public NTesting::IPort, TNonCopyable { public: TPortGuard(ui16 port, THolder lock) : Lock_(std::move(lock)) @@ -38,7 +40,16 @@ namespace { } ~TPortGuard() override { - Y_VERIFY_SYSERROR(NFs::Remove(Lock_->GetName())); + try { + if (!NFs::Remove(Lock_->GetName())) { + // Deletion may fail if the current user does not have + // write permission to the PORT_SYNC_PATH directory. + // At least let's release the lock file. + Lock_->Release(); + } + } catch (const TSystemError& e) { + Y_ABORT("Failed to unlock port %d, error=%d", Port_, e.Status()); + } } ui16 Get() override { @@ -107,6 +118,7 @@ namespace { } Y_ABORT_UNLESS(SyncDir_.IsDefined()); NFs::MakeDirectoryRecursive(SyncDir_); + Chmod(SyncDir_.c_str(), NFs::FP_COMMON_FILE); // override umask Ranges_ = GetPortRanges(); TotalCount_ = 0; diff --git a/library/cpp/testing/common/ut/network_ut.cpp b/library/cpp/testing/common/ut/network_ut.cpp index 2016e26b0923..22cedf87ae91 100644 --- a/library/cpp/testing/common/ut/network_ut.cpp +++ b/library/cpp/testing/common/ut/network_ut.cpp @@ -1,20 +1,38 @@ +#include #include #include +#include #include #include #include #include +#include #include +#include #include +#include -#include +class NetworkTestBase: public ::testing::Test { +protected: + void SetUp() override { + TmpDir.ConstructInPlace(); + } + + void TearDown() override { + TmpDir.Clear(); + } + +public: + TMaybe TmpDir; +}; -static TTempDir TmpDir; +class NetworkTest: public NetworkTestBase {}; +class FreePortTest: public NetworkTestBase {}; -TEST(NetworkTest, FreePort) { - NTesting::TScopedEnvironment envGuard("PORT_SYNC_PATH", TmpDir.Name()); +TEST_F(NetworkTest, FreePort) { + NTesting::TScopedEnvironment envGuard("PORT_SYNC_PATH", TmpDir->Name()); NTesting::InitPortManagerFromEnv(); TVector ports(Reserve(100)); @@ -24,7 +42,7 @@ TEST(NetworkTest, FreePort) { THashSet uniqPorts; for (auto& port : ports) { - const TString guardPath = TmpDir.Path() / ToString(static_cast(port)); + const TString guardPath = TmpDir->Path() / ToString(static_cast(port)); EXPECT_TRUE(NFs::Exists(guardPath)); EXPECT_TRUE(uniqPorts.emplace(port).second); @@ -35,16 +53,16 @@ TEST(NetworkTest, FreePort) { } ports.clear(); for (ui16 port : uniqPorts) { - const TString guardPath = TmpDir.Path() / ToString(port); + const TString guardPath = TmpDir->Path() / ToString(port); EXPECT_FALSE(NFs::Exists(guardPath)); } } -TEST(NetworkTest, FreePortWithinRanges) { +TEST_F(NetworkTest, FreePortWithinRanges) { NTesting::TScopedEnvironment envGuard{{ - {"PORT_SYNC_PATH", TmpDir.Name()}, - {"VALID_PORT_RANGE", "3456:7654"}, - }}; + {"PORT_SYNC_PATH", TmpDir->Name()}, + {"VALID_PORT_RANGE", "3456:7654"}, + }}; NTesting::InitPortManagerFromEnv(); for (size_t i = 0; i < 100; ++i) { @@ -55,11 +73,11 @@ TEST(NetworkTest, FreePortWithinRanges) { } } -TEST(NetworkTest, GetPortRandom) { +TEST_F(NetworkTest, GetPortRandom) { NTesting::TScopedEnvironment envGuard{{ - {"PORT_SYNC_PATH", TmpDir.Name()}, - {"NO_RANDOM_PORTS", ""}, - }}; + {"PORT_SYNC_PATH", TmpDir->Name()}, + {"NO_RANDOM_PORTS", ""}, + }}; NTesting::InitPortManagerFromEnv(); ui16 testPort = 80; // value just must be outside the assignable range @@ -70,11 +88,11 @@ TEST(NetworkTest, GetPortRandom) { } } -TEST(NetworkTest, GetPortNonRandom) { +TEST_F(NetworkTest, GetPortNonRandom) { NTesting::TScopedEnvironment envGuard{{ - {"PORT_SYNC_PATH", TmpDir.Name()}, - {"NO_RANDOM_PORTS", "1"}, - }}; + {"PORT_SYNC_PATH", TmpDir->Name()}, + {"NO_RANDOM_PORTS", "1"}, + }}; NTesting::InitPortManagerFromEnv(); TVector ports(Reserve(100)); // keep integers, we don't need the ports to remain allocated @@ -91,9 +109,36 @@ TEST(NetworkTest, GetPortNonRandom) { } } +TEST_F(NetworkTest, Permissions) { + constexpr ui16 loPort = 3456; + constexpr ui16 hiPort = 7654; + NTesting::TScopedEnvironment envGuard{{ + {"PORT_SYNC_PATH", TmpDir->Name()}, + {"VALID_PORT_RANGE", ToString(loPort) + ":" + ToString(hiPort)}, + }}; + NTesting::InitPortManagerFromEnv(); + TVector ports(Reserve(100)); + for (ui64 port = loPort; port <= hiPort; ++port) { + const TString guardPath = TmpDir->Path() / ToString(static_cast(port)); + TFile f{guardPath, OpenAlways | RdOnly}; + ASSERT_TRUE(f.IsOpen()); + ASSERT_TRUE(Chmod(f.GetName().c_str(), 0444) != -1); + } + Y_DEFER { + Chmod(TmpDir->Path().c_str(), NFs::FP_COMMON_FILE); + }; + ASSERT_TRUE(Chmod(TmpDir->Path().c_str(), 0555) != -1) << errno << " " << strerror(errno); // Lock dir + for (size_t i = 0; i < 100; ++i) { + NTesting::TPortHolder p; + EXPECT_NO_THROW(p = NTesting::GetFreePort()); + ui16 port = p; + ASSERT_GE(port, 3456u); + ASSERT_LE(port, 7654u); + } +} -TEST(FreePortTest, FreePortsRange) { - NTesting::TScopedEnvironment envGuard("PORT_SYNC_PATH", TmpDir.Name()); +TEST_F(FreePortTest, FreePortsRange) { + NTesting::TScopedEnvironment envGuard("PORT_SYNC_PATH", TmpDir->Name()); NTesting::InitPortManagerFromEnv(); for (ui16 i = 2; i < 10; ++i) { From d05af6b0e97386d468a1e8c35f4fa4c51ca09a6e Mon Sep 17 00:00:00 2001 From: cherepashka Date: Thu, 7 Aug 2025 11:47:09 +0300 Subject: [PATCH 05/52] Fix some codestyle issues for rXXXXXX commit_hash:e6899281c7d8eeb73d11a5b63bddc5c111f55f2a --- yt/yt/client/table_client/schema.h | 1 - 1 file changed, 1 deletion(-) diff --git a/yt/yt/client/table_client/schema.h b/yt/yt/client/table_client/schema.h index a42e1e7d6991..908960ffc24c 100644 --- a/yt/yt/client/table_client/schema.h +++ b/yt/yt/client/table_client/schema.h @@ -427,7 +427,6 @@ class TTableSchema final std::vector DeletedColumns; }; - std::shared_ptr ColumnInfo_; int KeyColumnCount_ = 0; bool HasMaterializedComputedColumns_ = false; From acba301ee4565c2e4e621d0d250c5e4c56c91f70 Mon Sep 17 00:00:00 2001 From: rp-1 Date: Thu, 7 Aug 2025 13:29:57 +0300 Subject: [PATCH 06/52] YT-25794: Add parametrizing error format in HTTP proxy commit_hash:0df6e70bee355d9231b80ac4691bcb8bbc5b0e4d --- yt/yt/client/formats/public.h | 1 + yt/yt/core/http/helpers.cpp | 68 ++++++++++++++++++++++++-------- yt/yt/core/http/helpers.h | 25 ++++++++++-- yt/yt/library/formats/format.cpp | 33 ++++++++++++++++ yt/yt/library/formats/format.h | 8 +++- 5 files changed, 113 insertions(+), 22 deletions(-) diff --git a/yt/yt/client/formats/public.h b/yt/yt/client/formats/public.h index baf2ae038b21..555285b44aae 100644 --- a/yt/yt/client/formats/public.h +++ b/yt/yt/client/formats/public.h @@ -83,6 +83,7 @@ DECLARE_REFCOUNTED_STRUCT(TArrowFormatConfig) DECLARE_REFCOUNTED_STRUCT(IYamrConsumer) DECLARE_REFCOUNTED_STRUCT(ISchemalessFormatWriter) +DECLARE_REFCOUNTED_STRUCT(IFormatFactory) DECLARE_REFCOUNTED_STRUCT(TControlAttributesConfig) diff --git a/yt/yt/core/http/helpers.cpp b/yt/yt/core/http/helpers.cpp index 460ae3752bb8..c838eb4ffcbd 100644 --- a/yt/yt/core/http/helpers.cpp +++ b/yt/yt/core/http/helpers.cpp @@ -16,6 +16,8 @@ #include +#include + #include #include @@ -30,6 +32,7 @@ namespace NYT::NHttp { constinit const auto Logger = HttpLogger; using namespace NJson; +using namespace NFormats; using namespace NYson; using namespace NYTree; using namespace NConcurrency; @@ -37,30 +40,56 @@ using namespace NHeaders; //////////////////////////////////////////////////////////////////////////////// -void FillYTError(const THeadersPtr& headers, const TError& error) +std::unique_ptr TJsonFactory::CreateConsumer(IZeroCopyOutput* output) +{ + return CreateJsonConsumer(output); +} + +NYson::TYsonProducer TJsonFactory::CreateProducer(IInputStream* input) +{ + return BIND([=] (IYsonConsumer* consumer) { + ParseJson(input, consumer); + }); +} + +void FillYTError( + const THeadersPtr& headers, + const TError& error, + IFormatFactoryPtr errorFormatFactory) { - TString errorJson; - TStringOutput errorJsonOutput(errorJson); - auto jsonWriter = CreateJsonConsumer(&errorJsonOutput); - Serialize(error, jsonWriter.get()); - jsonWriter->Flush(); + TString errorString; + TStringOutput errorStringOutput(errorString); - headers->Add(XYTErrorHeaderName, errorJson); + auto consumer = errorFormatFactory->CreateConsumer(&errorStringOutput); + + Serialize(error, consumer.get()); + consumer->Flush(); + + headers->Add(XYTErrorHeaderName, errorString); headers->Add(XYTResponseCodeHeaderName, ToString(static_cast(error.GetCode()))); headers->Add(XYTResponseMessageHeaderName, EscapeHeaderValue(error.GetMessage())); } -void FillYTErrorHeaders(const IResponseWriterPtr& rsp, const TError& error) +void FillYTErrorHeaders( + const IResponseWriterPtr& rsp, + const TError& error, + IFormatFactoryPtr errorFormatFactory) { - FillYTError(rsp->GetHeaders(), error); + FillYTError(rsp->GetHeaders(), error, errorFormatFactory); } -void FillYTErrorTrailers(const IResponseWriterPtr& rsp, const TError& error) +void FillYTErrorTrailers( + const IResponseWriterPtr& rsp, + const TError& error, + IFormatFactoryPtr errorFormatFactory) { - FillYTError(rsp->GetTrailers(), error); + FillYTError(rsp->GetTrailers(), error, errorFormatFactory); } -TError ParseYTError(const IResponsePtr& rsp, bool fromTrailers) +TError ParseYTError( + const IResponsePtr& rsp, + bool fromTrailers, + IFormatFactoryPtr errorFormatFactory) { std::string source; const std::string* errorHeader; @@ -74,20 +103,24 @@ TError ParseYTError(const IResponsePtr& rsp, bool fromTrailers) errorHeader = rsp->GetHeaders()->Find(XYTErrorHeaderName); } - TString errorJson; + TString errorString; if (errorHeader) { - errorJson = *errorHeader; + errorString = *errorHeader; } else { static const std::string BodySource("body"); source = BodySource; - errorJson = ToString(rsp->ReadAll()); + errorString = ToString(rsp->ReadAll()); } - TStringInput errorJsonInput(errorJson); + TStringInput errorStringInput(errorString); + std::unique_ptr> buildingConsumer; CreateBuildingYsonConsumer(&buildingConsumer, EYsonType::Node); + + auto producer = errorFormatFactory->CreateProducer(&errorStringInput); + try { - ParseJson(&errorJsonInput, buildingConsumer.get()); + producer.Run(buildingConsumer.get()); } catch (const std::exception& ex) { return TError("Failed to parse error from response") << TErrorAttribute("source", source) @@ -153,6 +186,7 @@ static const auto HeadersWhitelist = JoinSeq(", ", std::vector{ "X-YT-Output-Format", "X-YT-Output-Format0", "X-YT-Output-Format-0", + "X-YT-Error-Format", "X-YT-Header-Format", "X-YT-Suppress-Redirect", "X-YT-Omit-Trailers", diff --git a/yt/yt/core/http/helpers.h b/yt/yt/core/http/helpers.h index 4c1130a1ae35..a38abf7de79d 100644 --- a/yt/yt/core/http/helpers.h +++ b/yt/yt/core/http/helpers.h @@ -11,6 +11,8 @@ #include +#include + #include namespace NYT::NHttp { @@ -65,10 +67,27 @@ inline const std::string XYTTraceIdHeaderName("X-YT-Trace-Id"); //////////////////////////////////////////////////////////////////////////////// -void FillYTErrorHeaders(const IResponseWriterPtr& rsp, const TError& error); -void FillYTErrorTrailers(const IResponseWriterPtr& rsp, const TError& error); +struct TJsonFactory + : public NFormats::IFormatFactory +{ + std::unique_ptr CreateConsumer(IZeroCopyOutput* output) override; + + NYson::TYsonProducer CreateProducer(IInputStream* input) override; +}; + +void FillYTErrorHeaders( + const IResponseWriterPtr& rsp, + const TError& error, + NFormats::IFormatFactoryPtr errorFormatFactory = New()); +void FillYTErrorTrailers( + const IResponseWriterPtr& rsp, + const TError& error, + NFormats::IFormatFactoryPtr errorFormatFactory = New()); -TError ParseYTError(const IResponsePtr& rsp, bool fromTrailers = false); +TError ParseYTError( + const IResponsePtr& rsp, + bool fromTrailers = false, + NFormats::IFormatFactoryPtr errorFormatFactory = New()); //! Catches exception thrown from underlying handler body and //! translates it into HTTP error. diff --git a/yt/yt/library/formats/format.cpp b/yt/yt/library/formats/format.cpp index 6b1733e6bf62..0edbc3526168 100644 --- a/yt/yt/library/formats/format.cpp +++ b/yt/yt/library/formats/format.cpp @@ -499,6 +499,39 @@ TYsonProducer CreateProducerForFormat(const TFormat& format, EDataType dataType, //////////////////////////////////////////////////////////////////////////////// +class TConcreteFactory + : public IFormatFactory +{ +public: + TConcreteFactory(const TFormat& format, EDataType dataType) + : Format_(format) + , DataType_(dataType) + { } + + std::unique_ptr CreateConsumer(IZeroCopyOutput* output) override + { + return CreateConsumerForFormat(Format_, DataType_, output); + } + + NYson::TYsonProducer CreateProducer(IInputStream* input) override + { + return CreateProducerForFormat(Format_, DataType_, input); + } + +private: + TFormat Format_; + EDataType DataType_; +}; + +IFormatFactoryPtr CreateFactoryForFormat( + const TFormat& format, + EDataType dataType) +{ + return New(format, dataType); +} + +//////////////////////////////////////////////////////////////////////////////// + template struct TParserAdapter : public TBase diff --git a/yt/yt/library/formats/format.h b/yt/yt/library/formats/format.h index 7a1b67e688fb..f756e610ad3e 100644 --- a/yt/yt/library/formats/format.h +++ b/yt/yt/library/formats/format.h @@ -44,7 +44,7 @@ DEFINE_REFCOUNTED_TYPE(ISchemalessFormatWriter) //////////////////////////////////////////////////////////////////////////////// -struct IFormatAdapter +struct IFormatFactory : public TRefCounted { virtual std::unique_ptr CreateConsumer(IZeroCopyOutput* output) = 0; @@ -52,7 +52,7 @@ struct IFormatAdapter virtual NYson::TYsonProducer CreateProducer(IInputStream* input) = 0; }; -DEFINE_REFCOUNTED_TYPE(IFormatAdapter) +DEFINE_REFCOUNTED_TYPE(IFormatFactory) //////////////////////////////////////////////////////////////////////////////// @@ -94,6 +94,10 @@ NYson::TYsonProducer CreateProducerForFormat( EDataType dataType, IInputStream* input); +IFormatFactoryPtr CreateFactoryForFormat( + const TFormat& format, + EDataType dataType); + std::unique_ptr CreateParserForFormat( const TFormat& format, EDataType dataType, From cd8fda2d174e97c10e8f59e0286039aa77a5df89 Mon Sep 17 00:00:00 2001 From: aneporada Date: Thu, 7 Aug 2025 14:15:45 +0300 Subject: [PATCH 07/52] Push non-member equality filter expressions over join commit_hash:6bc558d09291d35872e4f2a9da0ada2f6fe5e4f3 --- .../core/common_opt/yql_co_flow2.cpp | 4 +- .../core/common_opt/yql_flatmap_over_join.cpp | 230 ++++++++++++++++++ yql/essentials/core/yql_join.cpp | 9 + yql/essentials/core/yql_join.h | 4 + yql/essentials/core/yql_opt_utils.cpp | 8 +- yql/essentials/core/yql_opt_utils.h | 2 +- .../minirun/part2/canondata/result.json | 14 ++ .../minirun/part3/canondata/result.json | 14 ++ .../NormalizeEqualityFilterOverJoin.yqls | 37 +++ ...rmalizeEqualityFilterOverJoinOptional.yqls | 54 ++++ 10 files changed, 372 insertions(+), 4 deletions(-) create mode 100644 yql/essentials/tests/s-expressions/suites/Optimizers/NormalizeEqualityFilterOverJoin.yqls create mode 100644 yql/essentials/tests/s-expressions/suites/Optimizers/NormalizeEqualityFilterOverJoinOptional.yqls diff --git a/yql/essentials/core/common_opt/yql_co_flow2.cpp b/yql/essentials/core/common_opt/yql_co_flow2.cpp index 8fec283e2757..222c177f5edf 100644 --- a/yql/essentials/core/common_opt/yql_co_flow2.cpp +++ b/yql/essentials/core/common_opt/yql_co_flow2.cpp @@ -1090,7 +1090,7 @@ TExprNode::TPtr PullUpFlatMapOverEquiJoin(const TExprNode::TPtr& node, TExprCont return node; } - static const TStringBuf canaryBaseName = "_yql_canary_"; + static const TStringBuf canaryBaseName = YqlCanaryColumnName; THashMap> joinKeysByLabel = CollectEquiJoinKeyColumnsByLabel(*joinTree); const auto renames = LoadJoinRenameMap(*settings); @@ -1109,7 +1109,7 @@ TExprNode::TPtr PullUpFlatMapOverEquiJoin(const TExprNode::TPtr& node, TExprCont const TTypeAnnotationNode* itemType = input.List().Ref().GetTypeAnn()->Cast()->GetItemType(); auto structType = itemType->Cast(); for (auto& si : structType->GetItems()) { - if (si->GetName().find(canaryBaseName, 0) == 0) { + if (IsNoPullColumn(si->GetName())) { // EquiJoin already processed return node; } diff --git a/yql/essentials/core/common_opt/yql_flatmap_over_join.cpp b/yql/essentials/core/common_opt/yql_flatmap_over_join.cpp index d0f039a8fc71..baa1fe5f05db 100644 --- a/yql/essentials/core/common_opt/yql_flatmap_over_join.cpp +++ b/yql/essentials/core/common_opt/yql_flatmap_over_join.cpp @@ -1177,6 +1177,12 @@ bool IsExtractOrPredicatesOverEquiJoinEnabled(const TTypeAnnotationContext* type return IsOptimizerEnabled(*types) && !IsOptimizerDisabled(*types); } +bool IsNormalizeEqualityFilterOverJoinEnabled(const TTypeAnnotationContext* types) { + YQL_ENSURE(types); + static const char flag[] = "NormalizeEqualityFilterOverJoin"; + return IsOptimizerEnabled(*types) && !IsOptimizerDisabled(*types); +} + struct TExtraInputPredicates { TExprNode::TPtr Row; TExprNodeList Preds; @@ -1343,6 +1349,222 @@ TJoinEqRebuildResult RebuildJoinTreeForEquality(TVector>& } +TExprBase NormalizeEqualityFilterOverJoin(const TCoFlatMapBase& node, const TJoinLabels& labels, + const THashMap& backRenameMap, const TParentsMap& parentsMap, TExprContext& ctx) +{ + TCoEquiJoin equiJoin = node.Input().Cast(); + const TExprNode::TPtr joinTree = equiJoin.Arg(equiJoin.ArgCount() - 2).Ptr(); + const TExprNode::TPtr rowPtr = node.Lambda().Args().Arg(0).Ptr(); + const auto& row = *rowPtr; + const auto body = node.Lambda().Body().Cast(); + + auto predicate = body.Predicate().Ptr(); + + TExprNodeList andComponents; + if (predicate->IsCallable("And")) { + andComponents = predicate->ChildrenList(); + } else { + andComponents.push_back(predicate); + } + + TExprNodeList rest; + TNodeOnNodeOwnedMap remaps; + TVector nodesByInput(equiJoin.ArgCount() - 2); + + for (auto pred : andComponents) { + TExprNodeList sides(2); + // TODO: handle case IsEquality() && !IsMemberEquality() + if (!IsEquality(pred, sides.front(), sides.back()) || HasDependsOn(pred, rowPtr) || !IsStrict(pred)) { + rest.push_back(pred); + continue; + } + + + TVector indexes; + TSet inputs; + TSet usedFields; + + GatherJoinInputs(sides.front(), row, parentsMap, backRenameMap, labels, inputs, usedFields); + if (inputs.size() != 1) { + rest.push_back(pred); + continue; + } + indexes.push_back(*inputs.begin()); + YQL_ENSURE(indexes.back() < nodesByInput.size()); + + inputs.clear(); + GatherJoinInputs(sides.back(), row, parentsMap, backRenameMap, labels, inputs, usedFields); + if (inputs.size() != 1 || *inputs.begin() == indexes.front()) { + rest.push_back(pred); + continue; + } + indexes.push_back(*inputs.begin()); + YQL_ENSURE(indexes.back() < nodesByInput.size()); + + + size_t count = 0; + for (size_t i = 0; i < 2; ++i) { + TExprNode::TPtr side = sides[i]; + if (!side->IsCallable("Member") || &side->Head() != &row) { + ++count; + if (!remaps.contains(side.Get())) { + TExprNode::TPtr output; + TOptimizeExprSettings settings(nullptr); + auto ret = OptimizeExpr(side, output, [&](const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr { + if (node->IsCallable("Member") && &node->Head() == &row) { + const TString originalMemberName{node->Tail().Content()}; + TString memberName = originalMemberName; + if (auto renamed = backRenameMap.FindPtr(memberName)) { + memberName = *renamed; + } + const auto& label = labels.Inputs[indexes[i]]; + TString sourceMemberName = label.MemberName(label.TableName(memberName), label.ColumnName(memberName)); + if (sourceMemberName != originalMemberName) { + return ctx.ChangeChild(*node, TCoMember::idx_Name, ctx.NewAtom(node->Pos(), sourceMemberName)); + } + } + return node; + }, ctx, settings); + YQL_ENSURE(ret != IGraphTransformer::TStatus::Error); + remaps[side.Get()] = output; + } + nodesByInput[indexes[i]].push_back(side); + } + } + + if (!count) { + rest.push_back(pred); + } + } + + if (remaps.empty()) { + return node; + } + + TExprNodeList newJoinArgs; + YQL_ENSURE(labels.Inputs.size() == nodesByInput.size()); + TNodeOnNodeOwnedMap outputRemaps; + TSet outputMembers; + for (size_t i = 0; i < nodesByInput.size(); ++i) { + TCoEquiJoinInput ejInput = equiJoin.Arg(i).Cast(); + if (nodesByInput[i].empty()) { + newJoinArgs.push_back(ejInput.Ptr()); + continue; + } + + const TStructExprType* itemType = ejInput.List().Ref().GetTypeAnn()->Cast()->GetItemType()->Cast(); + const TStructExprType* castedType = itemType; + if (!IsRequiredSide(joinTree, labels, i).first) { + TVector items; + for (auto item : itemType->GetItems()) { + auto type = item->GetItemType(); + if (type->IsOptionalOrNull()) { + items.push_back(item); + } else { + items.push_back(ctx.MakeType(item->GetName(), ctx.MakeType(type))); + } + } + castedType = ctx.MakeType(items); + } + TString prefix; + bool isMultiTableInput = false; + if (labels.Inputs[i].Tables.size() > 1) { + prefix = TStringBuilder() << labels.Inputs[i].Tables[0] << "."; + isMultiTableInput = true; + } + prefix += YqlJoinKeyColumnName; + TVector remappedNames = GenNoClashColumns(*itemType, prefix, nodesByInput[i].size()); + newJoinArgs.push_back(ctx.Builder(equiJoin.Arg(i).Pos()) + .List() + .Callable(0, "OrderedMap") + .Add(0, ejInput.List().Ptr()) + .Lambda(1) + .Param("row") + .Callable("FlattenMembers") + .List(0) + .Atom(0, "") + .Arg(1, "row") + .Seal() + .List(1) + .Atom(0, "") + .Callable(1, "AsStruct") + .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { + for (size_t j = 0; j < nodesByInput[i].size(); ++j) { + auto calcLambda = ctx.ChangeChild(node.Lambda().Ref(), TCoLambda::idx_Body, TExprNode::TPtr(remaps[nodesByInput[i][j].Get()])); + parent + .List(j) + .Atom(0, remappedNames[j]) + .Apply(1, calcLambda) + .With(0) + .Callable("SafeCast") + .Arg(0, "row") + .Add(1, ExpandType(equiJoin.Arg(i).Pos(), *castedType, ctx)) + .Seal() + .Done() + .Seal() + .Seal(); + TString outputMember = isMultiTableInput ? remappedNames[j] : labels.Inputs[i].FullName(remappedNames[j]); + outputRemaps[nodesByInput[i][j].Get()] = ctx.Builder(nodesByInput[i][j]->Pos()) + .Callable("Member") + .Add(0, rowPtr) + .Atom(1, outputMember) + .Seal() + .Build(); + YQL_ENSURE(outputMembers.insert(outputMember).second); + } + return parent; + }) + .Seal() + .Seal() + .Seal() + .Seal() + .Seal() + .Add(1, ejInput.Scope().Ptr()) + .Seal() + .Build()); + } + newJoinArgs.push_back(equiJoin.Arg(equiJoin.ArgCount() - 2).Ptr()); + newJoinArgs.push_back(equiJoin.Arg(equiJoin.ArgCount() - 1).Ptr()); + + auto newJoin = ctx.ChangeChildren(equiJoin.Ref(), std::move(newJoinArgs)); + TExprNode::TPtr newPredicate; + auto status = RemapExpr(body.Predicate().Ptr(), newPredicate, outputRemaps, ctx, TOptimizeExprSettings{nullptr}); + YQL_ENSURE(status != IGraphTransformer::TStatus::Error); + + TExprNode::TPtr newPredicateLambda = ctx.ChangeChild(node.Lambda().Ref(), TCoLambda::idx_Body, std::move(newPredicate)); + TExprNode::TPtr valueLambda = ctx.ChangeChild(node.Lambda().Ref(), TCoLambda::idx_Body, body.Value().Ptr()); + + return TExprBase(ctx.Builder(node.Pos()) + .Callable(node.CallableName()) + .Add(0, newJoin) + .Lambda(1) + .Param("row") + .Callable(body.CallableName()) + .Apply(0, newPredicateLambda) + .With(0, "row") + .Seal() + .Apply(1, valueLambda) + .With(0) + .Callable("RemoveMembers") + .Arg(0, "row") + .List(1) + .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { + size_t i = 0; + for (auto name : outputMembers) { + parent.Atom(i++, name); + } + return parent; + }) + .Seal() + .Seal() + .Done() + .Seal() + .Seal() + .Seal() + .Seal() + .Build()); +} + TExprBase HandleEqualityFilterOverJoin(const TCoFlatMapBase& node, const TJoinLabels& labels, const THashMap& backRenameMap, TExprContext& ctx) { @@ -1671,6 +1893,14 @@ TExprBase FlatMapOverEquiJoin( } } + if (IsNormalizeEqualityFilterOverJoinEnabled(types)) { + auto newNode = NormalizeEqualityFilterOverJoin(node, labels, backRenameMap, parentsMap, ctx); + if (newNode.Raw() != node.Raw()) { + YQL_CLOG(DEBUG, Core) << "NormalizeEqualityFilterOverJoin"; + return newNode; + } + } + if (IsEqualityFilterOverJoinEnabled(types)) { auto newNode = HandleEqualityFilterOverJoin(node, labels, backRenameMap, ctx); if (newNode.Raw() != node.Raw()) { diff --git a/yql/essentials/core/yql_join.cpp b/yql/essentials/core/yql_join.cpp index d08f1b9515f8..085d3316c5b2 100644 --- a/yql/essentials/core/yql_join.cpp +++ b/yql/essentials/core/yql_join.cpp @@ -2280,4 +2280,13 @@ TExprNode::TPtr DropAnyOverJoinInputs(TExprNode::TPtr joinTree, const TJoinLabel return joinTree; } +bool IsNoPullColumn(TStringBuf columnName) { + if (columnName.Contains('.')) { + TStringBuf table, column; + SplitTableName(columnName, table, column); + columnName = column; + } + return columnName.StartsWith(YqlCanaryColumnName) || columnName.StartsWith(YqlJoinKeyColumnName); +} + } // namespace NYql diff --git a/yql/essentials/core/yql_join.h b/yql/essentials/core/yql_join.h index 347ad9b90391..43db2beaec59 100644 --- a/yql/essentials/core/yql_join.h +++ b/yql/essentials/core/yql_join.h @@ -197,4 +197,8 @@ void GetPruneKeysColumnsForJoinLeaves(const NNodes::TCoEquiJoinTuple& joinTree, TExprNode::TPtr DropAnyOverJoinInputs(TExprNode::TPtr joinTree, const TJoinLabels& labels, const THashMap>& keyColumnsByLabel, TExprContext& ctx); +static constexpr TStringBuf YqlCanaryColumnName = "_yql_canary_"; +static constexpr TStringBuf YqlJoinKeyColumnName = "_yql_join_key"; + +bool IsNoPullColumn(TStringBuf columnName); } diff --git a/yql/essentials/core/yql_opt_utils.cpp b/yql/essentials/core/yql_opt_utils.cpp index 8038d15af80b..91f41b80bf45 100644 --- a/yql/essentials/core/yql_opt_utils.cpp +++ b/yql/essentials/core/yql_opt_utils.cpp @@ -1,6 +1,7 @@ #include "yql_opt_utils.h" #include "yql_expr_optimize.h" #include "yql_expr_type_annotation.h" +#include "yql_join.h" #include "yql_type_annotation.h" #include "yql_type_helpers.h" @@ -2393,7 +2394,12 @@ template TPartOfConstraintBase::TSetType GetPathsToKeys(const TExprNode& b template TPartOfConstraintBase::TSetType GetPathsToKeys(const TExprNode& body, const TExprNode& arg); TVector GenNoClashColumns(const TStructExprType& source, TStringBuf prefix, size_t count) { - YQL_ENSURE(prefix.StartsWith("_yql")); + if (!prefix.StartsWith("_yql")) { + YQL_ENSURE(prefix.Contains('.')); + TStringBuf table, column; + SplitTableName(prefix, table, column); + YQL_ENSURE(column.StartsWith("_yql")); + } TSet existing; for (auto& item : source.GetItems()) { TStringBuf column = item->GetName(); diff --git a/yql/essentials/core/yql_opt_utils.h b/yql/essentials/core/yql_opt_utils.h index 1444a45170fa..ed7e07bde12f 100644 --- a/yql/essentials/core/yql_opt_utils.h +++ b/yql/essentials/core/yql_opt_utils.h @@ -181,7 +181,7 @@ template TPartOfConstraintBase::TSetType GetPathsToKeys(const TExprNode& body, const TExprNode& arg); // generates column names with pattern "prefixN" that do not clash with source columns -// prefix should start with "_yql" +// prefix should start with "_yql" or "._yql" TVector GenNoClashColumns(const TStructExprType& source, TStringBuf prefix, size_t count); bool CheckSupportedTypes( diff --git a/yql/essentials/tests/s-expressions/minirun/part2/canondata/result.json b/yql/essentials/tests/s-expressions/minirun/part2/canondata/result.json index 32fe3083d831..30cf9201c515 100644 --- a/yql/essentials/tests/s-expressions/minirun/part2/canondata/result.json +++ b/yql/essentials/tests/s-expressions/minirun/part2/canondata/result.json @@ -391,6 +391,20 @@ "uri": "https://{canondata_backend}/1775319/1b77f6a2ced99cb43171907f99fa05c5976782c8/resource.tar.gz#test.test_Optimizers-MoveExtractMembersOverSort1-default.txt-Results_/results.txt" } ], + "test.test[Optimizers-NormalizeEqualityFilterOverJoinOptional-default.txt-Debug]": [ + { + "checksum": "ca31dd8effb927d18a48d099fdb1e18b", + "size": 1940, + "uri": "https://{canondata_backend}/1881367/22c06fc51a47708d8e7ffe921b07d495560d2fa1/resource.tar.gz#test.test_Optimizers-NormalizeEqualityFilterOverJoinOptional-default.txt-Debug_/opt.yql" + } + ], + "test.test[Optimizers-NormalizeEqualityFilterOverJoinOptional-default.txt-Results]": [ + { + "checksum": "df92f713466329443519bc6326e864a7", + "size": 2163, + "uri": "https://{canondata_backend}/1881367/22c06fc51a47708d8e7ffe921b07d495560d2fa1/resource.tar.gz#test.test_Optimizers-NormalizeEqualityFilterOverJoinOptional-default.txt-Results_/results.txt" + } + ], "test.test[Optimizers-PushDownToFlow-default.txt-Debug]": [ { "checksum": "ed25dda0682388131b4840cbc338e1d3", diff --git a/yql/essentials/tests/s-expressions/minirun/part3/canondata/result.json b/yql/essentials/tests/s-expressions/minirun/part3/canondata/result.json index f941367312d6..751f6b466a01 100644 --- a/yql/essentials/tests/s-expressions/minirun/part3/canondata/result.json +++ b/yql/essentials/tests/s-expressions/minirun/part3/canondata/result.json @@ -447,6 +447,20 @@ "uri": "https://{canondata_backend}/1031349/de1275760530cb37ff3ca2d7aa0c26a1e7476b02/resource.tar.gz#test.test_Optimizers-LengthAfterList-default.txt-Results_/results.txt" } ], + "test.test[Optimizers-NormalizeEqualityFilterOverJoin-default.txt-Debug]": [ + { + "checksum": "a06e166f1839a293a236237622423b8c", + "size": 1612, + "uri": "https://{canondata_backend}/1903885/fdf3d9a48d438f90af2da132ed3e94bcfa6909fe/resource.tar.gz#test.test_Optimizers-NormalizeEqualityFilterOverJoin-default.txt-Debug_/opt.yql" + } + ], + "test.test[Optimizers-NormalizeEqualityFilterOverJoin-default.txt-Results]": [ + { + "checksum": "75a63145bd73f1ceeb0abee9bd7e25db", + "size": 2200, + "uri": "https://{canondata_backend}/1903885/fdf3d9a48d438f90af2da132ed3e94bcfa6909fe/resource.tar.gz#test.test_Optimizers-NormalizeEqualityFilterOverJoin-default.txt-Results_/results.txt" + } + ], "test.test[Result-Pickle-default.txt-Debug]": [ { "checksum": "ff0f81f90265c0afaa74303e7198d058", diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/NormalizeEqualityFilterOverJoin.yqls b/yql/essentials/tests/s-expressions/suites/Optimizers/NormalizeEqualityFilterOverJoin.yqls new file mode 100644 index 000000000000..4b98074ab5ac --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/NormalizeEqualityFilterOverJoin.yqls @@ -0,0 +1,37 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config '"OptimizerFlags" '"NormalizeEqualityFilterOverJoin")) +(let world (Configure! world config '"OptimizerFlags" '"EqualityFilterOverJoin")) + +(let list1 (AsList + (AsStruct '('a.key1 (Just (Int32 '1))) '('c.value1 (Just (String 'A)))) + (AsStruct '('a.key1 (Just (Int32 '7))) '('c.value1 (Just (String 'B)))) + (AsStruct '('a.key1 (Just (Int32 '4))) '('c.value1 (Just (String 'C)))) + (AsStruct '('a.key1 (Just (Int32 '4))) '('c.value1 (Just (String 'D)))) +)) + +(let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('value2 (String 'V))) +)) + +(let joinCross (EquiJoin '(list1 '('a 'c)) '(list2 'b) '('Cross 'c 'b '() '() '()) +'( +'('rename '"a.key1" '"akey1") +'('rename '"c.value1" '"cvalue1") + +) +)) + +(let filtered (Filter joinCross +(lambda '(row) (Coalesce (== (+ (Member row 'akey1) (Int32 '2)) (+ (Member row 'b.key2) (Int32 '0))) (Bool 'false)) +))) + +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink (Key) filtered '('('type)))) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/NormalizeEqualityFilterOverJoinOptional.yqls b/yql/essentials/tests/s-expressions/suites/Optimizers/NormalizeEqualityFilterOverJoinOptional.yqls new file mode 100644 index 000000000000..84a77e7652f9 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/NormalizeEqualityFilterOverJoinOptional.yqls @@ -0,0 +1,54 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config '"OptimizerFlags" '"NormalizeEqualityFilterOverJoin")) +(let world (Configure! world config '"OptimizerFlags" '"EqualityFilterOverJoin")) + +(let list1 (AsList + (AsStruct '('key1 (Int32 '9)) '('value1 (String 'Z))) + (AsStruct '('key1 (Int32 '4)) '('value1 (String 'Y))) + (AsStruct '('key1 (Int32 '3)) '('value1 (String 'X))) + (AsStruct '('key1 (Int32 '4)) '('value1 (String 'W))) + (AsStruct '('key1 (Int32 '8)) '('value1 (String 'V))) +)) + +(let list2 (AsList + (AsStruct '('key2 (Int32 '9)) '('value2 (String 'Z))) + (AsStruct '('key2 (Int32 '4)) '('value2 (String 'Y))) + (AsStruct '('key2 (Int32 '3)) '('value2 (String 'X))) + (AsStruct '('key2 (Int32 '4)) '('value2 (String 'W))) + (AsStruct '('key2 (Int32 '8)) '('value2 (String 'V))) +)) + +(let list3 (AsList + (AsStruct '('key3 (Int32 '9)) '('value3 (String 'Z))) + (AsStruct '('key3 (Int32 '4)) '('value3 (String 'Y))) + (AsStruct '('key3 (Int32 '3)) '('value3 (String 'X))) + (AsStruct '('key3 (Int32 '4)) '('value3 (String 'W))) + (AsStruct '('key3 (Int32 '8)) '('value3 (String 'V))) +)) + +(let joinLeftCross (EquiJoin '(list1 'a) '(list2 'b) '(list3 'c) + +'('Cross '('Left 'a 'b '('a 'key1) '('b 'key2) '()) 'c '() '() '()) + +'( +'('rename '"a.key1" '"akey1") +'('rename '"b.key2" '"bkey2") + +))) + + +(let filtered (Filter joinLeftCross +(lambda '(row) (Coalesce (== (+ (Member row 'bkey2) (Int32 '1)) (+ (Member row 'c.key3) (Int32 '0))) (Bool 'false)) +))) + +(let filtered (Map filtered +(lambda '(row) (AsStruct '('ckey3 (Member row 'c.key3)) '('akey1 (Member row 'akey1)) '('bkey2 (Member row 'bkey2)) '('avalue1 (Member row 'a.value1))) +))) + + +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink (Key) filtered '('('type)))) +(let world (Commit! world res_sink)) +(return world) +) From 17af9437a28a9eaad88044b973d1bd99742d391a Mon Sep 17 00:00:00 2001 From: robot-piglet Date: Thu, 7 Aug 2025 14:39:12 +0300 Subject: [PATCH 08/52] Intermediate changes commit_hash:e4fe01ebc98d78d5cde268f0405c6a3ace7230f8 --- contrib/python/parameterized/py2/ya.make | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/python/parameterized/py2/ya.make b/contrib/python/parameterized/py2/ya.make index 8e92750f331e..4735022fb6fb 100644 --- a/contrib/python/parameterized/py2/ya.make +++ b/contrib/python/parameterized/py2/ya.make @@ -2,7 +2,7 @@ PY2_LIBRARY() -SUBSCRIBER(g:feedback_platform g:python-contrib) +SUBSCRIBER(g:python-contrib) VERSION(0.8.1) From a58c6273c517e8acb896eb7d225091d47deb8e3e Mon Sep 17 00:00:00 2001 From: robot-piglet Date: Thu, 7 Aug 2025 15:17:10 +0300 Subject: [PATCH 09/52] Intermediate changes commit_hash:080b337e4e0f8ace6689336b77ade4ad132ae154 --- yql/essentials/cfg/tests/gateways-experimental.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/yql/essentials/cfg/tests/gateways-experimental.conf b/yql/essentials/cfg/tests/gateways-experimental.conf index fc9754726f7d..6692d8b40ded 100644 --- a/yql/essentials/cfg/tests/gateways-experimental.conf +++ b/yql/essentials/cfg/tests/gateways-experimental.conf @@ -83,6 +83,11 @@ Yt { Name: "_ForbidSensitiveDataInOperationSpec" Value: "true" } + + DefaultSettings { + Name: "_EnforceRegexpProbabilityFail" + Value: "100" + } } Dq { From a246d60d3b3ac79dc02ab9535b4c9aa188ce7e77 Mon Sep 17 00:00:00 2001 From: robot-piglet Date: Thu, 7 Aug 2025 16:52:35 +0300 Subject: [PATCH 10/52] Intermediate changes commit_hash:f0dc5f57e17d0f15ccbb92d7ced5a11008765dc5 --- contrib/libs/tcmalloc/.yandex_meta/build.ym | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/libs/tcmalloc/.yandex_meta/build.ym b/contrib/libs/tcmalloc/.yandex_meta/build.ym index 1b1875257110..299b91fd9355 100644 --- a/contrib/libs/tcmalloc/.yandex_meta/build.ym +++ b/contrib/libs/tcmalloc/.yandex_meta/build.ym @@ -26,7 +26,6 @@ tcmalloc/internal/ya.make SUBSCRIBER( g:cpp-contrib ayles - mikailbag ) SRCS( From d74a3f4e813af8c964567dfd89f725ed81ee141c Mon Sep 17 00:00:00 2001 From: mpereskokova Date: Thu, 7 Aug 2025 16:52:36 +0300 Subject: [PATCH 11/52] Don't push PruneKeys during Evaluate commit_hash:94c4a10f360a253e2a326ae81704500a08ea38d8 --- .../yt/provider/phy_opt/yql_yt_phy_opt_misc.cpp | 4 ++++ .../tests/sql/suites/join/prune_keys_on_evaluate.cfg | 1 + .../tests/sql/suites/join/prune_keys_on_evaluate.sql | 11 +++++++++++ 3 files changed, 16 insertions(+) create mode 100644 yt/yql/tests/sql/suites/join/prune_keys_on_evaluate.cfg create mode 100644 yt/yql/tests/sql/suites/join/prune_keys_on_evaluate.sql diff --git a/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_misc.cpp b/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_misc.cpp index 9852aa0ff328..b2ba2eb2b1ba 100644 --- a/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_misc.cpp +++ b/yt/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_misc.cpp @@ -966,6 +966,10 @@ TMaybeNode TYtPhysicalOptProposalTransformer::UpdateDataSourceCluster } TMaybeNode TYtPhysicalOptProposalTransformer::PushPruneKeysIntoYtOperation(TExprBase node, TExprContext& ctx) const { + if (State_->Types->EvaluationInProgress || State_->PassiveExecution) { + return node; + } + auto op = node.Cast(); auto extractorLambda = op.Extractor(); diff --git a/yt/yql/tests/sql/suites/join/prune_keys_on_evaluate.cfg b/yt/yql/tests/sql/suites/join/prune_keys_on_evaluate.cfg new file mode 100644 index 000000000000..58878f894562 --- /dev/null +++ b/yt/yql/tests/sql/suites/join/prune_keys_on_evaluate.cfg @@ -0,0 +1 @@ +in Input input.txt diff --git a/yt/yql/tests/sql/suites/join/prune_keys_on_evaluate.sql b/yt/yql/tests/sql/suites/join/prune_keys_on_evaluate.sql new file mode 100644 index 000000000000..7f7792394f0b --- /dev/null +++ b/yt/yql/tests/sql/suites/join/prune_keys_on_evaluate.sql @@ -0,0 +1,11 @@ +use plato; + +PRAGMA config.flags('OptimizerFlags', 'EmitPruneKeys'); + +$some_value = ( + SELECT ListLength(AGGREGATE_LIST(`key`)) FROM ( + SELECT * FROM Input AS tbls WHERE key IN (select key from Input) + ) +); + +EVALUATE IF ($some_value > 0) DO EMPTY_ACTION(); From 319c53daa4d3137edd9f6938531c983331ae730e Mon Sep 17 00:00:00 2001 From: robot-piglet Date: Thu, 7 Aug 2025 19:24:44 +0300 Subject: [PATCH 12/52] Intermediate changes commit_hash:da32df476c05ee8928d99ec0d784ed2e3b929624 --- contrib/libs/tcmalloc/.yandex_meta/build.ym | 8 +++++ contrib/libs/tcmalloc/default/ya.make | 4 ++- .../libs/tcmalloc/malloc_extension/ya.make | 5 +-- contrib/libs/tcmalloc/no_percpu_cache/ya.make | 4 ++- contrib/libs/tcmalloc/numa_256k/ya.make | 4 ++- .../libs/tcmalloc/numa_large_pages/ya.make | 4 ++- contrib/libs/tcmalloc/small_but_slow/ya.make | 4 ++- .../libs/tcmalloc/tcmalloc/internal/ya.make | 4 ++- yql/essentials/utils/docs/markdown.cpp | 22 ++++++++---- yql/essentials/utils/docs/markdown_ut.cpp | 34 +++++++++++++++++++ yql/essentials/utils/docs/verification.cpp | 4 ++- 11 files changed, 81 insertions(+), 16 deletions(-) diff --git a/contrib/libs/tcmalloc/.yandex_meta/build.ym b/contrib/libs/tcmalloc/.yandex_meta/build.ym index 299b91fd9355..dc2bd4e9df65 100644 --- a/contrib/libs/tcmalloc/.yandex_meta/build.ym +++ b/contrib/libs/tcmalloc/.yandex_meta/build.ym @@ -56,6 +56,14 @@ IF (NOT DLL_FOR) ENDIF() {% endblock %} +{% block patch_source %} +{{super()}} +find . -type f -name "ya.make" | while read -r file; do + sed -i -e 's/VERSION(.*)/VERSION({{self.date()}})/g' "$file" + sed -i -e 's|ORIGINAL_SOURCE(.*)|ORIGINAL_SOURCE({{self.current_url().strip()}})|g' "$file" +done +{% endblock %} + {% block move_to_output %} {{super()}} cp -R tcmalloc common.inc ${OUTPUT} diff --git a/contrib/libs/tcmalloc/default/ya.make b/contrib/libs/tcmalloc/default/ya.make index 1bc6ab6dacc1..c327f72a16e6 100644 --- a/contrib/libs/tcmalloc/default/ya.make +++ b/contrib/libs/tcmalloc/default/ya.make @@ -2,7 +2,9 @@ LIBRARY() WITHOUT_LICENSE_TEXTS() -VERSION(2021-10-04-45c59ccbc062ac96d83710205033c656e490d376) +VERSION(2025-01-30) + +ORIGINAL_SOURCE(https://github.com/google/tcmalloc/archive/c8dfee3e4c489c5ae0d30c484c92db102a69ec51.tar.gz) LICENSE(Apache-2.0) diff --git a/contrib/libs/tcmalloc/malloc_extension/ya.make b/contrib/libs/tcmalloc/malloc_extension/ya.make index a2044f60c2ab..1b282c2ad5f4 100644 --- a/contrib/libs/tcmalloc/malloc_extension/ya.make +++ b/contrib/libs/tcmalloc/malloc_extension/ya.make @@ -8,8 +8,9 @@ NO_UTIL() NO_COMPILER_WARNINGS() -# https://github.com/google/tcmalloc -VERSION(2020-11-23-a643d89610317be1eff9f7298104eef4c987d8d5) +VERSION(2025-01-30) + +ORIGINAL_SOURCE(https://github.com/google/tcmalloc/archive/c8dfee3e4c489c5ae0d30c484c92db102a69ec51.tar.gz) SRCDIR(contrib/libs/tcmalloc) diff --git a/contrib/libs/tcmalloc/no_percpu_cache/ya.make b/contrib/libs/tcmalloc/no_percpu_cache/ya.make index 928d9092cf88..bab170bea4aa 100644 --- a/contrib/libs/tcmalloc/no_percpu_cache/ya.make +++ b/contrib/libs/tcmalloc/no_percpu_cache/ya.make @@ -2,7 +2,9 @@ LIBRARY() WITHOUT_LICENSE_TEXTS() -VERSION(2021-10-04-45c59ccbc062ac96d83710205033c656e490d376) +VERSION(2025-01-30) + +ORIGINAL_SOURCE(https://github.com/google/tcmalloc/archive/c8dfee3e4c489c5ae0d30c484c92db102a69ec51.tar.gz) LICENSE(Apache-2.0) diff --git a/contrib/libs/tcmalloc/numa_256k/ya.make b/contrib/libs/tcmalloc/numa_256k/ya.make index 5b43065230a0..bc89a03e902c 100644 --- a/contrib/libs/tcmalloc/numa_256k/ya.make +++ b/contrib/libs/tcmalloc/numa_256k/ya.make @@ -2,7 +2,9 @@ LIBRARY() WITHOUT_LICENSE_TEXTS() -VERSION(2021-10-04-45c59ccbc062ac96d83710205033c656e490d376) +VERSION(2025-01-30) + +ORIGINAL_SOURCE(https://github.com/google/tcmalloc/archive/c8dfee3e4c489c5ae0d30c484c92db102a69ec51.tar.gz) LICENSE(Apache-2.0) diff --git a/contrib/libs/tcmalloc/numa_large_pages/ya.make b/contrib/libs/tcmalloc/numa_large_pages/ya.make index 0a94465472dd..a5203798dee1 100644 --- a/contrib/libs/tcmalloc/numa_large_pages/ya.make +++ b/contrib/libs/tcmalloc/numa_large_pages/ya.make @@ -2,7 +2,9 @@ LIBRARY() WITHOUT_LICENSE_TEXTS() -VERSION(2021-10-04-45c59ccbc062ac96d83710205033c656e490d376) +VERSION(2025-01-30) + +ORIGINAL_SOURCE(https://github.com/google/tcmalloc/archive/c8dfee3e4c489c5ae0d30c484c92db102a69ec51.tar.gz) LICENSE(Apache-2.0) diff --git a/contrib/libs/tcmalloc/small_but_slow/ya.make b/contrib/libs/tcmalloc/small_but_slow/ya.make index b0259ab095d9..d17217638457 100644 --- a/contrib/libs/tcmalloc/small_but_slow/ya.make +++ b/contrib/libs/tcmalloc/small_but_slow/ya.make @@ -2,7 +2,9 @@ LIBRARY() WITHOUT_LICENSE_TEXTS() -VERSION(2021-10-04-45c59ccbc062ac96d83710205033c656e490d376) +VERSION(2025-01-30) + +ORIGINAL_SOURCE(https://github.com/google/tcmalloc/archive/c8dfee3e4c489c5ae0d30c484c92db102a69ec51.tar.gz) LICENSE(Apache-2.0) diff --git a/contrib/libs/tcmalloc/tcmalloc/internal/ya.make b/contrib/libs/tcmalloc/tcmalloc/internal/ya.make index 786d44fc25e0..c79e6afc7be6 100644 --- a/contrib/libs/tcmalloc/tcmalloc/internal/ya.make +++ b/contrib/libs/tcmalloc/tcmalloc/internal/ya.make @@ -1,6 +1,8 @@ PROTO_LIBRARY() -VERSION(dummy) +VERSION(2025-01-30) + +ORIGINAL_SOURCE(https://github.com/google/tcmalloc/archive/c8dfee3e4c489c5ae0d30c484c92db102a69ec51.tar.gz) LICENSE(Apache-2.0) diff --git a/yql/essentials/utils/docs/markdown.cpp b/yql/essentials/utils/docs/markdown.cpp index ca0f555a8152..fd13820b8c4b 100644 --- a/yql/essentials/utils/docs/markdown.cpp +++ b/yql/essentials/utils/docs/markdown.cpp @@ -23,20 +23,24 @@ namespace NYql::NDocs { void Parse(IInputStream& markdown, TMarkdownCallback&& onSection) { for (TString line; markdown.ReadLine(line) != 0;) { + size_t depth = HeaderDepth(line); if (IsSkipping_) { - if (IsSectionHeader(line)) { + if (HeaderDepth_ == depth) { ResetSection(std::move(line)); IsSkipping_ = false; } else { // Skip } } else { - if (IsSectionHeader(line)) { + if (HeaderDepth_ == depth) { onSection(std::move(Section_)); ResetSection(std::move(line)); - } else { + } else if (depth == 0 || HeaderDepth_ < depth) { line.append('\n'); Section_.Body.append(std::move(line)); + } else { + onSection(std::move(Section_)); + IsSkipping_ = true; } } } @@ -64,11 +68,15 @@ namespace NYql::NDocs { } } - bool IsSectionHeader(TStringBuf line) const { - return HeaderDepth(line) == HeaderDepth_; - } - size_t HeaderDepth(TStringBuf line) const { + while (line.StartsWith(' ') || line.StartsWith('\t')) { + line.Skip(1); + } + + if (!line.StartsWith('#')) { + return 0; + } + size_t begin = line.find('#'); size_t end = line.find_first_not_of('#', begin); return end != TStringBuf::npos ? (end - begin) : 0; diff --git a/yql/essentials/utils/docs/markdown_ut.cpp b/yql/essentials/utils/docs/markdown_ut.cpp index 8cafa2f22f12..45e45e645a52 100644 --- a/yql/essentials/utils/docs/markdown_ut.cpp +++ b/yql/essentials/utils/docs/markdown_ut.cpp @@ -70,4 +70,38 @@ FROM my_table; UNIT_ASSERT_GE(Count(random.Body, '\n'), 5); } + Y_UNIT_TEST(NestedSections) { + TString markdown = R"( +# Section 1 {#s1} +Section 1 Text. +## Subsection 1 {#s1s1} +Subsection 1.1 Text. +## Subsection 2 {#s1s2} +Subsection 1.2 Text. +# Section 2 {#s2} +Section 2 Text. +## Subsection 1 {#s2s1} +Subsection 2.1 Text. +## Subsection 2 {#s2s2} +Subsection 2.2 Text. +### Subsubsection 1 {#s2s2s1} +Subsection 2.2.1 Text. +# Section 3 {#s3} +Section 3 Text. +)"; + TMarkdownPage page = ParseMarkdownPage(markdown); + { + const TMarkdownSection& section = page.SectionsByAnchor["s1s2"]; + UNIT_ASSERT_STRING_CONTAINS(section.Body, "Subsection 1.2 Text."); + UNIT_ASSERT_C(!section.Body.Contains("Section 1 Text."), section.Body); + UNIT_ASSERT_C(!section.Body.Contains("Section 2 Text."), section.Body); + UNIT_ASSERT_C(!section.Body.Contains("Section 3 Text."), section.Body); + } + { + const TMarkdownSection& section = page.SectionsByAnchor["s2s2s1"]; + UNIT_ASSERT_STRING_CONTAINS(section.Body, "Subsection 2.2.1 Text."); + UNIT_ASSERT_C(!section.Body.Contains("Section 3 Text."), section.Body); + } + } + } // Y_UNIT_TEST_SUITE(MarkdownParserTests) diff --git a/yql/essentials/utils/docs/verification.cpp b/yql/essentials/utils/docs/verification.cpp index d8d68f9f4868..0eaedc2a3b4c 100644 --- a/yql/essentials/utils/docs/verification.cpp +++ b/yql/essentials/utils/docs/verification.cpp @@ -78,7 +78,9 @@ namespace NYql::NDocs { !IsLikelyDocumentedAt(section->Body, name)) { report[EFame::BadLinked][std::move(name)] = TStringBuilder() - << "Absent at section '" << target << "'"; + << "Absent at section '" << target << "', " + << "section header is '" << section->Header.Content << "', " + << "section prefix is '" << TStringBuf(section->Body).SubString(0, 32) << "'"; return; } From d586b7e82dd6c946379bbff3adbe55a169b2c36e Mon Sep 17 00:00:00 2001 From: robot-javacom Date: Thu, 7 Aug 2025 19:25:15 +0300 Subject: [PATCH 13/52] Upload new jdk-17.0.13+11 https://nda.ya.ru/t/6ESaAy__7HMSFA commit_hash:0b98ad6b6e10f45000afc632f68427110c6bac02 --- build/mapping.conf.json | 10 ++++++++++ build/platform/java/jdk/jdk17/jdk.json | 10 +++++----- build/ymake.core.conf | 2 +- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/build/mapping.conf.json b/build/mapping.conf.json index c9511a565c75..da08f5b09cad 100644 --- a/build/mapping.conf.json +++ b/build/mapping.conf.json @@ -1261,6 +1261,7 @@ "7832768362": "{registry_endpoint}/7832768362", "7833244075": "{registry_endpoint}/7833244075", "8342543231": "{registry_endpoint}/8342543231", + "9322842655": "{registry_endpoint}/9322842655", "5776379446": "{registry_endpoint}/5776379446", "5777100597": "{registry_endpoint}/5777100597", "5909067709": "{registry_endpoint}/5909067709", @@ -1277,6 +1278,7 @@ "7832766692": "{registry_endpoint}/7832766692", "7833241952": "{registry_endpoint}/7833241952", "8342542290": "{registry_endpoint}/8342542290", + "9322840030": "{registry_endpoint}/9322840030", "5776375952": "{registry_endpoint}/5776375952", "5777098178": "{registry_endpoint}/5777098178", "5909065014": "{registry_endpoint}/5909065014", @@ -1293,6 +1295,7 @@ "7832762570": "{registry_endpoint}/7832762570", "7833238161": "{registry_endpoint}/7833238161", "8342540244": "{registry_endpoint}/8342540244", + "9322834294": "{registry_endpoint}/9322834294", "5776374505": "{registry_endpoint}/5776374505", "5777096988": "{registry_endpoint}/5777096988", "5909063641": "{registry_endpoint}/5909063641", @@ -1309,6 +1312,7 @@ "7832760150": "{registry_endpoint}/7832760150", "7833236145": "{registry_endpoint}/7833236145", "8342539438": "{registry_endpoint}/8342539438", + "9322830951": "{registry_endpoint}/9322830951", "5776377955": "{registry_endpoint}/5776377955", "5777099502": "{registry_endpoint}/5777099502", "5909066324": "{registry_endpoint}/5909066324", @@ -1325,6 +1329,7 @@ "7832764927": "{registry_endpoint}/7832764927", "7833240023": "{registry_endpoint}/7833240023", "8342541289": "{registry_endpoint}/8342541289", + "9322837247": "{registry_endpoint}/9322837247", "7805449449": "{registry_endpoint}/7805449449", "6391354461": "{registry_endpoint}/6391354461", "7805449010": "{registry_endpoint}/7805449010", @@ -2815,6 +2820,7 @@ "7832768362": "none-none-none-result_resources/jdk-darwin-aarch64.yandex.tgz", "7833244075": "none-none-none-result_resources/jdk-darwin-aarch64.yandex.tgz", "8342543231": "none-none-none-result_resources/jdk-darwin-aarch64.yandex.tgz", + "9322842655": "none-none-none-result_resources/jdk-darwin-aarch64.yandex.tgz", "5776379446": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz", "5777100597": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz", "5909067709": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz", @@ -2831,6 +2837,7 @@ "7832766692": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz", "7833241952": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz", "8342542290": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz", + "9322840030": "none-none-none-result_resources/jdk-darwin-x86_64.yandex.tgz", "5776375952": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz", "5777098178": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz", "5909065014": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz", @@ -2847,6 +2854,7 @@ "7832762570": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz", "7833238161": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz", "8342540244": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz", + "9322834294": "none-none-none-result_resources/jdk-linux-aarch64.yandex.tgz", "5776374505": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz", "5777096988": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz", "5909063641": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz", @@ -2863,6 +2871,7 @@ "7832760150": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz", "7833236145": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz", "8342539438": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz", + "9322830951": "none-none-none-result_resources/jdk-linux-x86_64.yandex.tgz", "5776377955": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz", "5777099502": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz", "5909066324": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz", @@ -2879,6 +2888,7 @@ "7832764927": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz", "7833240023": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz", "8342541289": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz", + "9322837247": "none-none-none-result_resources/jdk-windows-amd64.yandex.tgz", "7805449449": "none-none-none-result_resources/protoc-linux-aarch_64.tgz", "6391354461": "none-none-none-result_resources/protoc-linux-x86_64.tgz", "7805449010": "none-none-none-result_resources/protoc-linux-x86_64.tgz", diff --git a/build/platform/java/jdk/jdk17/jdk.json b/build/platform/java/jdk/jdk17/jdk.json index dd4db71a476c..3d1f00d97959 100644 --- a/build/platform/java/jdk/jdk17/jdk.json +++ b/build/platform/java/jdk/jdk17/jdk.json @@ -1,19 +1,19 @@ { "by_platform": { "darwin-arm64": { - "uri": "sbr:7832768362" + "uri": "sbr:9322842655" }, "darwin-x86_64": { - "uri": "sbr:7832766692" + "uri": "sbr:9322840030" }, "linux-x86_64": { - "uri": "sbr:7832760150" + "uri": "sbr:9322830951" }, "linux-aarch64": { - "uri": "sbr:7832762570" + "uri": "sbr:9322834294" }, "win32-x86_64": { - "uri": "sbr:7832764927" + "uri": "sbr:9322837247" } } } diff --git a/build/ymake.core.conf b/build/ymake.core.conf index 50f21b64029f..da6a220a0da5 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -21,7 +21,7 @@ DOCS_FAKEID=1 # Change of this value will invalidate some parts of configure cache # but will not affect builds anyhow (except tests referring build/ directory) -JSON_CACHE_FAKE_ID=20241225 +JSON_CACHE_FAKE_ID=20241226 STRUCT_CMD=yes _DBG_EXPR_ERROR_DETAILS=one From ad8ab8c9dae9d3ec464c953f51adc748c2d5a6ee Mon Sep 17 00:00:00 2001 From: mikhnenko Date: Thu, 7 Aug 2025 19:33:05 +0300 Subject: [PATCH 14/52] Move under ifdef strange methods commit_hash:bebc8b7e2622b045bbe7bf63caa356b1c59ab17a --- contrib/libs/antlr3_cpp_runtime/include/antlr3cyclicdfa.inl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/libs/antlr3_cpp_runtime/include/antlr3cyclicdfa.inl b/contrib/libs/antlr3_cpp_runtime/include/antlr3cyclicdfa.inl index 61d15bfcf209..abac02268218 100644 --- a/contrib/libs/antlr3_cpp_runtime/include/antlr3cyclicdfa.inl +++ b/contrib/libs/antlr3_cpp_runtime/include/antlr3cyclicdfa.inl @@ -22,6 +22,7 @@ CyclicDFA::CyclicDFA( ANTLR_INT32 decisionNumber m_description = description; } +#if ANTLR3_ALLOW_CHANGES_IN_CONST_FIELDS template CyclicDFA::CyclicDFA( const CyclicDFA& dfa ) { @@ -50,6 +51,7 @@ CyclicDFA& CyclicDFA::operator=( const m_transition = dfa.m_transition; return *this; } +#endif template ANTLR_INT32 CyclicDFA::specialStateTransition(CtxType * , From 6857498fe18b2fad4593cc48ca55189592b33316 Mon Sep 17 00:00:00 2001 From: baymer Date: Thu, 7 Aug 2025 20:01:17 +0300 Subject: [PATCH 15/52] =?UTF-8?q?nots/builder:=20=D0=94=D0=BE=D1=81=D1=8B?= =?UTF-8?q?=D0=BF=D0=B0=D1=82=D1=8C=20=D0=BB=D0=BE=D0=B3=D0=B8=D1=80=D0=BE?= =?UTF-8?q?=D0=B2=D0=B0=D0=BD=D0=B8=D1=8F=20=D0=BE=D0=B1=20=D1=83=D1=81?= =?UTF-8?q?=D1=82=D0=B0=D0=BD=D0=BE=D0=B2=D0=BA=D0=B5=20=D0=B7=D0=B0=D0=B2?= =?UTF-8?q?=D0=B8=D1=81=D0=B8=D0=BC=D0=BE=D1=81=D1=82=D0=B5=D0=B9=20commit?= =?UTF-8?q?=5Fhash:bb0e0e0fbba15b82c707868efb18f399a4b3e82d?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../lib/nots/package_manager/base/package_manager.py | 7 ++++++- build/plugins/lib/nots/package_manager/base/timeit.py | 8 ++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/build/plugins/lib/nots/package_manager/base/package_manager.py b/build/plugins/lib/nots/package_manager/base/package_manager.py index 5a6483fe0eaf..ae1a705e765f 100644 --- a/build/plugins/lib/nots/package_manager/base/package_manager.py +++ b/build/plugins/lib/nots/package_manager/base/package_manager.py @@ -6,7 +6,7 @@ from .constants import NPM_REGISTRY_URL from .package_json import PackageJson from .utils import build_nm_path, build_pj_path -from .timeit import timeit +from .timeit import timeit, is_timeit_enabled class PackageManagerError(RuntimeError): @@ -137,6 +137,11 @@ def _exec_command(self, args, cwd: str, include_defaults=True, script_path=None, p = subprocess.Popen(cmd, cwd=cwd, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) stdout, stderr = p.communicate() + if is_timeit_enabled(): + print(f'cd {cwd} && {" ".join(cmd)}', file=sys.stderr) + print(f'stdout: {stdout.decode("utf-8")}', file=sys.stderr) + print(f'stderr: {stderr.decode("utf-8")}', file=sys.stderr) + if p.returncode != 0: self._dump_debug_log() diff --git a/build/plugins/lib/nots/package_manager/base/timeit.py b/build/plugins/lib/nots/package_manager/base/timeit.py index c0b25daee4df..697d9efe6147 100644 --- a/build/plugins/lib/nots/package_manager/base/timeit.py +++ b/build/plugins/lib/nots/package_manager/base/timeit.py @@ -27,3 +27,11 @@ def timeit(func): return logging.timeit(func) else: return func + + +def is_timeit_enabled(): + logging = import_optional_module("devtools.frontend_build_platform.libraries.logging") + if logging: + return logging.timeit_options.enabled + else: + return False From 07cc9eeeb63d52ed22a83beeb7be109a6544560c Mon Sep 17 00:00:00 2001 From: vpozdyayev Date: Thu, 7 Aug 2025 20:32:47 +0300 Subject: [PATCH 16/52] Switch RESOURCES_LIBRARY() to the new command engine commit_hash:39a82309a21a5abe5bd7a8a7d54a7415eae5e125 --- build/conf/compilers/msvc_compiler.conf | 2 +- build/platform/local_so/ya.make | 2 +- build/ymake.core.conf | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/build/conf/compilers/msvc_compiler.conf b/build/conf/compilers/msvc_compiler.conf index 37e8f8406112..e495f3fbe1c8 100644 --- a/build/conf/compilers/msvc_compiler.conf +++ b/build/conf/compilers/msvc_compiler.conf @@ -85,4 +85,4 @@ _SRC_C_NODEPS_CMD=${TOOLCHAIN_ENV} ${CL_WRAPPER} ${C_COMPILER} /c /Fo${OUTFILE} _SRC_CPP_CMD_NEW=${TOOLCHAIN_ENV} ${CL_WRAPPER} ${CXX_COMPILER} /c /Fo$_COMPILE_OUTPUTS ${input:SRC} ${EXTRA_C_FLAGS} ${pre=/I :_C__INCLUDE} ${CXXFLAGS} ${SRCFLAGS} ${_LANG_CFLAGS_VALUE_NEW} ${hide;kv:"soe"} ${hide;kv:_C_CPP_KV_STYLE_P} ${hide;kv:_C_CPP_KV_STYLE_PC} _SRC_C_CMD_NEW=${TOOLCHAIN_ENV} ${CL_WRAPPER} ${C_COMPILER} /c /Fo$_COMPILE_OUTPUTS ${input:SRC} ${EXTRA_C_FLAGS} ${pre=/I :_C__INCLUDE} ${CFLAGS} ${CONLYFLAGS} ${SRCFLAGS} ${hide;kv:"soe"} ${hide;kv:_C_CPP_KV_STYLE_P} ${hide;kv:_C_CPP_KV_STYLE_PC} _SRC_M_CMD=$_EMPTY_CMD -_SRC_MASM_CMD=${cwd:ARCADIA_BUILD_ROOT} ${TOOLCHAIN_ENV} ${ML_WRAPPER} ${MASM_COMPILER_OLD} ${MASMFLAGS} ${SRCFLAGS} $_MASM_IO ${hide;kv:"p AS"} ${hide;kv:"pc yellow"} +_SRC_MASM_CMD=${cwd:ARCADIA_BUILD_ROOT} ${TOOLCHAIN_ENV} ${ML_WRAPPER} ${MASM_COMPILER} ${MASMFLAGS} ${SRCFLAGS} $_MASM_IO ${hide;kv:"p AS"} ${hide;kv:"pc yellow"} diff --git a/build/platform/local_so/ya.make b/build/platform/local_so/ya.make index 547ed3f99187..83891f5e80a2 100644 --- a/build/platform/local_so/ya.make +++ b/build/platform/local_so/ya.make @@ -1,7 +1,7 @@ RESOURCES_LIBRARY() IF (OS_LINUX) - SET_APPEND(RPATH_GLOBAL '-Wl,-rpath,$ORIGIN') + SET_APPEND(RPATH_GLOBAL '-Wl,-rpath,${"$"}ORIGIN') ENDIF() END() diff --git a/build/ymake.core.conf b/build/ymake.core.conf index da6a220a0da5..551ab661f3d4 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -1249,7 +1249,7 @@ macro EXPORT_ALL_DYNAMIC_SYMBOLS() { ### ### Enable linking of PROGRAM with dynamic CUDA. By default CUDA uses static linking macro USE_DYNAMIC_CUDA() { - LDFLAGS(-Wl,-rpath,$ORIGIN) + LDFLAGS(-Wl,-rpath,${"$"}ORIGIN) ENABLE(USE_DYNAMIC_CUDA) } @@ -1907,6 +1907,7 @@ _RESOURCES_LIBRARY_SEM=IGNORED ### @see: [DECLARE_EXTERNAL_RESOURCE()](#macro_DECLARE_EXTERNAL_RESOURCE) module RESOURCES_LIBRARY: _BARE_UNIT { .CMD=RESOURCES_LIBRARY_LINK + .STRUCT_CMD=yes .SEM=_RESOURCES_LIBRARY_SEM .ALLOWED=DECLARE_EXTERNAL_RESOURCE EXTRALIBS OBJADDE_GLOBAL .RESTRICTED=ALLOCATOR SIZE TAG DATA TEST_DATA DEPENDS FORK_TESTS FORK_SUBTESTS SPLIT_FACTOR TEST_CWD RUN TIMEOUT SRC SRCS SPLIT_DWARF From 204564c71a58d1100bc31609f93ce5513eae3dae Mon Sep 17 00:00:00 2001 From: robot-ydb-importer Date: Thu, 7 Aug 2025 21:50:34 +0300 Subject: [PATCH 17/52] YDB Import 827 commit_hash:634b938c375e2e06d1f496a1fad234d4a19914b7 --- yql/essentials/data/language/pragmas_opensource.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/yql/essentials/data/language/pragmas_opensource.json b/yql/essentials/data/language/pragmas_opensource.json index 9b1e6cb682ed..7e20d69bf7ab 100644 --- a/yql/essentials/data/language/pragmas_opensource.json +++ b/yql/essentials/data/language/pragmas_opensource.json @@ -800,6 +800,9 @@ { "name": "dq.UseWideChannels" }, + { + "name": "dq.ValuePackerVersion" + }, { "name": "dq.WatermarksEnableIdlePartitions" }, From b5000342d74c4765c905d3c8174663232270b661 Mon Sep 17 00:00:00 2001 From: robot-ratatosk Date: Fri, 8 Aug 2025 02:13:12 +0300 Subject: [PATCH 18/52] New version of the tld SKIP_CHECK SKIP_REVIEW commit_hash:24155c32e2dc3095a246a478c3e73f65e8b91455 --- library/cpp/tld/tlds-alpha-by-domain.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/cpp/tld/tlds-alpha-by-domain.txt b/library/cpp/tld/tlds-alpha-by-domain.txt index e46cf0ba9749..c902ec00d1c0 100644 --- a/library/cpp/tld/tlds-alpha-by-domain.txt +++ b/library/cpp/tld/tlds-alpha-by-domain.txt @@ -1,4 +1,4 @@ -# Version 2025080100, Last Updated Fri Aug 1 07:07:01 2025 UTC +# Version 2025080700, Last Updated Thu Aug 7 07:07:01 2025 UTC AAA AARP ABB From 608ba11a8d33ec0bc1b8a69c91f9628a800471cd Mon Sep 17 00:00:00 2001 From: Oleg Doronin Date: Fri, 8 Aug 2025 07:01:10 +0200 Subject: [PATCH 19/52] portion write controller fix (#22545) --- ydb/core/tx/columnshard/operations/slice_builder/builder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/tx/columnshard/operations/slice_builder/builder.cpp b/ydb/core/tx/columnshard/operations/slice_builder/builder.cpp index 12af80633ab3..4193c8e046d0 100644 --- a/ydb/core/tx/columnshard/operations/slice_builder/builder.cpp +++ b/ydb/core/tx/columnshard/operations/slice_builder/builder.cpp @@ -45,7 +45,7 @@ void TBuildSlicesTask::ReplyError(const TString& message, const NColumnShard::TE } class TPortionWriteController: public NColumnShard::IWriteController, - public NColumnShard::TMonitoringObjectsCounter { + public NColumnShard::TMonitoringObjectsCounter { public: class TInsertPortion { private: From 2323c401fbb90440647b584956f94263d01752a6 Mon Sep 17 00:00:00 2001 From: uzhastik Date: Fri, 8 Aug 2025 09:36:38 +0300 Subject: [PATCH 20/52] remove self include (#22533) --- ydb/core/mon/mon.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/ydb/core/mon/mon.h b/ydb/core/mon/mon.h index e27fece95954..0025862cb02f 100644 --- a/ydb/core/mon/mon.h +++ b/ydb/core/mon/mon.h @@ -15,8 +15,6 @@ #include #include -#include "mon.h" - namespace NActors { void MakeJsonErrorReply(NJson::TJsonValue& jsonResponse, TString& message, const NYdb::TStatus& status); From 830dbae708148f1a5aaed1c34b284cc46cfc55e5 Mon Sep 17 00:00:00 2001 From: Oleg Doronin Date: Fri, 8 Aug 2025 08:40:13 +0200 Subject: [PATCH 21/52] sources aggregation has been disabled (#22540) --- .../reader/simple_reader/iterator/context.h | 12 +++++---- .../ut_rw/ut_columnshard_read_write.cpp | 27 +++++++++++-------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/context.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/context.h index 9df894b473fc..8aae206207f3 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/context.h +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/context.h @@ -47,11 +47,13 @@ class TSpecialReadContext: public NCommon::TSpecialReadContext, TNonCopyable { if (!aggrProc) { SourcesAggregationScript = nullptr; } else { - NCommon::TFetchingScriptBuilder builder(*this); - builder.AddStep(std::make_shared()); - builder.AddStep(std::make_shared(aggrProc)); - builder.AddStep(std::make_shared(aggrProc)); - SourcesAggregationScript = std::move(builder).Build(); + // TODO: fix me, temporary disabled + SourcesAggregationScript = nullptr; + // NCommon::TFetchingScriptBuilder builder(*this); + // builder.AddStep(std::make_shared()); + // builder.AddStep(std::make_shared(aggrProc)); + // builder.AddStep(std::make_shared(aggrProc)); + // SourcesAggregationScript = std::move(builder).Build(); } } return *SourcesAggregationScript; diff --git a/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp b/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp index c196740ce08e..3e2ec4f475a4 100644 --- a/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp +++ b/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -1430,12 +1431,16 @@ struct TReadAggregateResult { std::vector Counts = { 100 }; }; -void TestReadAggregate(const std::vector& ydbSchema, const TString& testDataBlob, bool addProjection, +void TestReadAggregate(const std::vector& ydbSchema, const TString& testDataBlob, bool addProjection, const bool simpleReader, const std::vector& aggKeys = {}, const TReadAggregateResult& expectedResult = {}, const TReadAggregateResult& expectedFiltered = { 1, { 1 }, { 1 }, { 1 } }) { addProjection = true; TTestBasicRuntime runtime; TTester::Setup(runtime); + if (simpleReader) { + runtime.GetAppData(0).ColumnShardConfig.SetReaderClassName("SIMPLE"); + runtime.GetAppData(0).ColumnShardConfig.SetDeduplicationEnabled(true); + } auto csDefaultControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard(); TActorId sender = runtime.AllocateEdgeActor(); @@ -1992,15 +1997,15 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { TestReadWithProgramNoProjection(); } - Y_UNIT_TEST(ReadAggregate) { + Y_UNIT_TEST_DUO(ReadAggregate, SimpleReader) { auto schema = TTestSchema::YdbAllTypesSchema(); auto testBlob = MakeTestBlob({ 0, 100 }, schema); - TestReadAggregate(schema, testBlob, false); - TestReadAggregate(schema, testBlob, true); + TestReadAggregate(schema, testBlob, false, SimpleReader); + TestReadAggregate(schema, testBlob, true, SimpleReader); } - Y_UNIT_TEST(ReadGroupBy) { + Y_UNIT_TEST_DUO(ReadGroupBy, SimpleReader) { auto schema = TTestSchema::YdbAllTypesSchema(); auto testBlob = MakeTestBlob({ 0, 100 }, schema); @@ -2022,26 +2027,26 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { // the type has the same values in test batch so result would be grouped in one row if (sameValTypes.contains(schema[key].GetType().GetTypeId())) { - TestReadAggregate(schema, testBlob, (key % 2), { key }, resGrouped, resFiltered); + TestReadAggregate(schema, testBlob, (key % 2), SimpleReader, { key }, resGrouped, resFiltered); } else { - TestReadAggregate(schema, testBlob, (key % 2), { key }, resDefault, resFiltered); + TestReadAggregate(schema, testBlob, (key % 2), SimpleReader, { key }, resDefault, resFiltered); } } for (ui32 key = 0; key < schema.size() - 1; ++key) { Cerr << "-- group by key: " << key << ", " << key + 1 << "\n"; if (sameValTypes.contains(schema[key].GetType().GetTypeId()) && sameValTypes.contains(schema[key + 1].GetType().GetTypeId())) { - TestReadAggregate(schema, testBlob, (key % 2), { key, key + 1 }, resGrouped, resFiltered); + TestReadAggregate(schema, testBlob, SimpleReader, (key % 2), { key, key + 1 }, resGrouped, resFiltered); } else { - TestReadAggregate(schema, testBlob, (key % 2), { key, key + 1 }, resDefault, resFiltered); + TestReadAggregate(schema, testBlob, SimpleReader, (key % 2), { key, key + 1 }, resDefault, resFiltered); } } for (ui32 key = 0; key < schema.size() - 2; ++key) { Cerr << "-- group by key: " << key << ", " << key + 1 << ", " << key + 2 << "\n"; if (sameValTypes.contains(schema[key].GetType().GetTypeId()) && sameValTypes.contains(schema[key + 1].GetType().GetTypeId()) && sameValTypes.contains(schema[key + 1].GetType().GetTypeId())) { - TestReadAggregate(schema, testBlob, (key % 2), { key, key + 1, key + 2 }, resGrouped, resFiltered); + TestReadAggregate(schema, testBlob, (key % 2), SimpleReader, { key, key + 1, key + 2 }, resGrouped, resFiltered); } else { - TestReadAggregate(schema, testBlob, (key % 2), { key, key + 1, key + 2 }, resDefault, resFiltered); + TestReadAggregate(schema, testBlob, (key % 2), SimpleReader, { key, key + 1, key + 2 }, resDefault, resFiltered); } } } From d18580f3e9dcec09858c127a6b1ff0f8c2b7eaab Mon Sep 17 00:00:00 2001 From: Alexey Efimov Date: Fri, 8 Aug 2025 13:56:53 +0700 Subject: [PATCH 22/52] restrict database users access (#22513) --- ydb/core/viewer/json_pipe_req.cpp | 13 + ydb/core/viewer/json_pipe_req.h | 2 + ydb/core/viewer/storage_groups.h | 63 +- ydb/core/viewer/tests/canondata/result.json | 7843 +++++++++++++------ ydb/core/viewer/tests/test.py | 13 +- ydb/core/viewer/viewer.cpp | 7 +- ydb/core/viewer/viewer_nodelist.h | 59 +- 7 files changed, 5454 insertions(+), 2546 deletions(-) diff --git a/ydb/core/viewer/json_pipe_req.cpp b/ydb/core/viewer/json_pipe_req.cpp index 8263a2c90c3d..2978d316ee1c 100644 --- a/ydb/core/viewer/json_pipe_req.cpp +++ b/ydb/core/viewer/json_pipe_req.cpp @@ -940,6 +940,19 @@ std::vector TViewerPipeClient::GetNodesFromBoardReply(TEvStateStorage:: return GetNodesFromBoardReply(*ev->Get()); } +std::vector TViewerPipeClient::GetDatabaseNodes() { + if (DatabaseBoardInfoResponse && DatabaseBoardInfoResponse->IsOk()) { + return GetNodesFromBoardReply(DatabaseBoardInfoResponse->GetRef()); + } else if (ResourceBoardInfoResponse && ResourceBoardInfoResponse->IsOk()) { + return GetNodesFromBoardReply(ResourceBoardInfoResponse->GetRef()); + } + return {0}; +} + +bool TViewerPipeClient::IsDatabaseRequest() { + return DatabaseBoardInfoResponse || ResourceBoardInfoResponse; +} + void TViewerPipeClient::InitConfig(const TCgiParameters& params) { Followers = FromStringWithDefault(params.Get("followers"), Followers); Metrics = FromStringWithDefault(params.Get("metrics"), Metrics); diff --git a/ydb/core/viewer/json_pipe_req.h b/ydb/core/viewer/json_pipe_req.h index 70104fbe5446..8b121118bd2f 100644 --- a/ydb/core/viewer/json_pipe_req.h +++ b/ydb/core/viewer/json_pipe_req.h @@ -333,6 +333,8 @@ class TViewerPipeClient : public TActorBootstrapped { TRequestResponse MakeRequestStateStorageEndpointsLookup(const TString& path, ui64 cookie = 0); std::vector GetNodesFromBoardReply(TEvStateStorage::TEvBoardInfo::TPtr& ev); std::vector GetNodesFromBoardReply(const TEvStateStorage::TEvBoardInfo& ev); + std::vector GetDatabaseNodes(); + bool IsDatabaseRequest(); void InitConfig(const TCgiParameters& params); void InitConfig(const TRequestSettings& settings); void BuildParamsFromJson(TStringBuf data); diff --git a/ydb/core/viewer/storage_groups.h b/ydb/core/viewer/storage_groups.h index dbe165db2e89..8fdc1bb15336 100644 --- a/ydb/core/viewer/storage_groups.h +++ b/ydb/core/viewer/storage_groups.h @@ -859,6 +859,20 @@ class TStorageGroups : public TViewerPipeClient { NeedSort = false; NeedLimit = false; } + + } + +public: + void Bootstrap() override { + if (NeedToRedirect()) { + return; + } + if (!Viewer->CheckAccessViewer(TBase::GetRequest())) { + FieldsRequired.reset(+EGroupFields::NodeId); // fields that are not available for database users + FieldsRequired.reset(+EGroupFields::PDiskId); + FieldsRequired.reset(+EGroupFields::PDisk); + FieldsRequired.reset(+EGroupFields::PileName); + } FieldsRequested = FieldsRequired; // no dependent fields for (auto field = +EGroupFields::GroupId; field != +EGroupFields::COUNT; ++field) { if (FieldsRequired.test(field)) { @@ -868,13 +882,6 @@ class TStorageGroups : public TViewerPipeClient { } } } - } - -public: - void Bootstrap() override { - if (NeedToRedirect()) { - return; - } if (Database) { if (!DatabaseNavigateResponse) { DatabaseNavigateResponse = MakeRequestSchemeCacheNavigate(Database, 0); @@ -2057,26 +2064,28 @@ class TStorageGroups : public TViewerPipeClient { } } - auto itPDisk = PDisks.find(vdisk.VSlotId); - if (itPDisk != PDisks.end()) { - const TPDisk& pdisk = itPDisk->second; - NKikimrViewer::TStoragePDisk& jsonPDisk = *jsonVDisk.MutablePDisk(); - jsonPDisk.SetPDiskId(pdisk.GetPDiskId()); - jsonPDisk.SetPath(pdisk.Path); - jsonPDisk.SetType(pdisk.Type); - jsonPDisk.SetGuid(::ToString(pdisk.Guid)); - jsonPDisk.SetCategory(pdisk.Category); - jsonPDisk.SetTotalSize(pdisk.TotalSize); - jsonPDisk.SetAvailableSize(pdisk.AvailableSize); - jsonPDisk.SetStatus(pdisk.Status); - jsonPDisk.SetDecommitStatus(pdisk.DecommitStatus); - jsonPDisk.SetSlotSize(pdisk.GetSlotTotalSize()); - if (pdisk.DiskSpace != NKikimrViewer::Grey) { - jsonPDisk.SetDiskSpace(pdisk.DiskSpace); - } - auto itPDiskByPDiskId = PDisksByPDiskId.find(vdisk.VSlotId); - if (itPDiskByPDiskId != PDisksByPDiskId.end()) { - jsonPDisk.MutableWhiteboard()->CopyFrom(*(itPDiskByPDiskId->second)); + if (FieldsRequested.test(+EGroupFields::PDisk)) { + auto itPDisk = PDisks.find(vdisk.VSlotId); + if (itPDisk != PDisks.end()) { + const TPDisk& pdisk = itPDisk->second; + NKikimrViewer::TStoragePDisk& jsonPDisk = *jsonVDisk.MutablePDisk(); + jsonPDisk.SetPDiskId(pdisk.GetPDiskId()); + jsonPDisk.SetPath(pdisk.Path); + jsonPDisk.SetType(pdisk.Type); + jsonPDisk.SetGuid(::ToString(pdisk.Guid)); + jsonPDisk.SetCategory(pdisk.Category); + jsonPDisk.SetTotalSize(pdisk.TotalSize); + jsonPDisk.SetAvailableSize(pdisk.AvailableSize); + jsonPDisk.SetStatus(pdisk.Status); + jsonPDisk.SetDecommitStatus(pdisk.DecommitStatus); + jsonPDisk.SetSlotSize(pdisk.GetSlotTotalSize()); + if (pdisk.DiskSpace != NKikimrViewer::Grey) { + jsonPDisk.SetDiskSpace(pdisk.DiskSpace); + } + auto itPDiskByPDiskId = PDisksByPDiskId.find(vdisk.VSlotId); + if (itPDiskByPDiskId != PDisksByPDiskId.end()) { + jsonPDisk.MutableWhiteboard()->CopyFrom(*(itPDiskByPDiskId->second)); + } } } if (!vdisk.Donors.empty()) { diff --git a/ydb/core/viewer/tests/canondata/result.json b/ydb/core/viewer/tests/canondata/result.json index 82e8ed97f6a4..874608a124d3 100644 --- a/ydb/core/viewer/tests/canondata/result.json +++ b/ydb/core/viewer/tests/canondata/result.json @@ -486,7 +486,7 @@ }, "restart_pdisk_viewer": { "status_code": 403, - "text": "

403 Forbidden

: Error: SID is not allowed\n

" + "text": "

403 Forbidden

" } }, "test.test_storage_groups": { @@ -1943,6 +1943,172 @@ "Owner": "root", "Path": "/Root" } + }, + "no-database": { + "Common": { + "ACL": [ + { + "AccessRights": [ + "ConnectDatabase" + ], + "AccessType": "Allow", + "InheritanceType": [ + "Inherit" + ], + "Subject": "USERS" + }, + { + "AccessRights": [ + "ReadAttributes", + "DescribeSchema" + ], + "AccessType": "Allow", + "InheritanceType": [ + "Inherit" + ], + "Subject": "METADATA-READERS" + }, + { + "AccessRights": [ + "SelectRow" + ], + "AccessType": "Allow", + "InheritanceType": [ + "Inherit" + ], + "Subject": "DATA-READERS" + }, + { + "AccessRights": [ + "UpdateRow", + "EraseRow" + ], + "AccessType": "Allow", + "InheritanceType": [ + "Inherit" + ], + "Subject": "DATA-WRITERS" + }, + { + "AccessRights": [ + "WriteAttributes", + "CreateDirectory", + "CreateTable", + "CreateQueue", + "RemoveSchema", + "AlterSchema" + ], + "AccessType": "Allow", + "InheritanceType": [ + "Inherit" + ], + "Subject": "DDL-ADMINS" + }, + { + "AccessRights": [ + "GrantAccessRights" + ], + "AccessType": "Allow", + "InheritanceType": [ + "Inherit" + ], + "Subject": "ACCESS-ADMINS" + }, + { + "AccessRights": [ + "CreateDatabase", + "DropDatabase" + ], + "AccessType": "Allow", + "InheritanceType": [ + "Inherit" + ], + "Subject": "DATABASE-ADMINS" + } + ], + "EffectiveACL": [ + { + "AccessRights": [ + "ConnectDatabase" + ], + "AccessType": "Allow", + "InheritanceType": [ + "Inherit" + ], + "Subject": "USERS" + }, + { + "AccessRights": [ + "ReadAttributes", + "DescribeSchema" + ], + "AccessType": "Allow", + "InheritanceType": [ + "Inherit" + ], + "Subject": "METADATA-READERS" + }, + { + "AccessRights": [ + "SelectRow" + ], + "AccessType": "Allow", + "InheritanceType": [ + "Inherit" + ], + "Subject": "DATA-READERS" + }, + { + "AccessRights": [ + "UpdateRow", + "EraseRow" + ], + "AccessType": "Allow", + "InheritanceType": [ + "Inherit" + ], + "Subject": "DATA-WRITERS" + }, + { + "AccessRights": [ + "WriteAttributes", + "CreateDirectory", + "CreateTable", + "CreateQueue", + "RemoveSchema", + "AlterSchema" + ], + "AccessType": "Allow", + "InheritanceType": [ + "Inherit" + ], + "Subject": "DDL-ADMINS" + }, + { + "AccessRights": [ + "GrantAccessRights" + ], + "AccessType": "Allow", + "InheritanceType": [ + "Inherit" + ], + "Subject": "ACCESS-ADMINS" + }, + { + "AccessRights": [ + "CreateDatabase", + "DropDatabase" + ], + "AccessType": "Allow", + "InheritanceType": [ + "Inherit" + ], + "Subject": "DATABASE-ADMINS" + } + ], + "Owner": "root", + "Path": "/Root" + } } }, "test.test_viewer_acl_write": [ @@ -2306,6 +2472,27 @@ }, "Success": true, "Version": 2 + }, + "no-database": { + "Result": { + "Entities": [ + { + "Name": "/Root/shared_db", + "Type": "ext_sub_domain" + }, + { + "Name": "/Root/dedicated_db", + "Type": "ext_sub_domain" + }, + { + "Name": "/Root/serverless_db", + "Type": "ext_sub_domain" + } + ], + "Total": 3 + }, + "Success": true, + "Version": 2 } }, "test.test_viewer_bsgroupinfo": { @@ -2422,6 +2609,9 @@ }, "/Root/shared_db": { "read": true + }, + "no-database": { + "read": false } }, "test.test_viewer_cluster": { @@ -3152,8 +3342,133 @@ "port": "not-zero-number" }, "self_check_result": "accepted-value" + }, + "no-database": { + "location": { + "host": "text", + "id": 1, + "port": "not-zero-number" + }, + "self_check_result": "accepted-value" } }, + "test.test_viewer_nodelist": { + "/Root": [ + { + "Address": "", + "Host": "text", + "Id": 1, + "PhysicalLocation": { + "Body": 1, + "DataCenter": 49, + "DataCenterId": "1", + "Location": "DC=1/R=1/U=1/", + "Rack": 1, + "Room": 0 + }, + "Port": "not-zero-number", + "ResolveHost": "text" + } + ], + "/Root/dedicated_db": [ + { + "Address": "", + "Host": "text", + "Id": 50000, + "PhysicalLocation": { + "Body": 0, + "DataCenter": 0, + "DataCenterId": "", + "Location": "DC=/R=/U=0/", + "Rack": 0, + "Room": 0 + }, + "Port": "not-zero-number", + "ResolveHost": "text" + } + ], + "/Root/serverless_db": [ + { + "Address": "", + "Host": "text", + "Id": 50001, + "PhysicalLocation": { + "Body": 0, + "DataCenter": 0, + "DataCenterId": "", + "Location": "DC=/R=/U=0/", + "Rack": 0, + "Room": 0 + }, + "Port": "not-zero-number", + "ResolveHost": "text" + } + ], + "/Root/shared_db": [ + { + "Address": "", + "Host": "text", + "Id": 50001, + "PhysicalLocation": { + "Body": 0, + "DataCenter": 0, + "DataCenterId": "", + "Location": "DC=/R=/U=0/", + "Rack": 0, + "Room": 0 + }, + "Port": "not-zero-number", + "ResolveHost": "text" + } + ], + "no-database": [ + { + "Address": "", + "Host": "text", + "Id": 1, + "PhysicalLocation": { + "Body": 1, + "DataCenter": 49, + "DataCenterId": "1", + "Location": "DC=1/R=1/U=1/", + "Rack": 1, + "Room": 0 + }, + "Port": "not-zero-number", + "ResolveHost": "text" + }, + { + "Address": "", + "Host": "text", + "Id": 50001, + "PhysicalLocation": { + "Body": 0, + "DataCenter": 0, + "DataCenterId": "", + "Location": "DC=/R=/U=0/", + "Rack": 0, + "Room": 0 + }, + "Port": "not-zero-number", + "ResolveHost": "text" + }, + { + "Address": "", + "Host": "text", + "Id": 50000, + "PhysicalLocation": { + "Body": 0, + "DataCenter": 0, + "DataCenterId": "", + "Location": "DC=/R=/U=0/", + "Rack": 0, + "Room": 0 + }, + "Port": "not-zero-number", + "ResolveHost": "text" + } + ] + }, "test.test_viewer_nodes": { "/Root": { "FieldsAvailable": "10000000010000110111111100000111", @@ -3463,37 +3778,251 @@ } ], "TotalNodes": "1" - } - }, - "test.test_viewer_nodes_all": { - "/Root": { - "FieldsAvailable": "11111111110111111111111111111111", - "FieldsRequired": "11111111111111111111111111111111", - "FoundNodes": "1", - "MaximumDisksPerNode": "1", - "MaximumSlotsPerDisk": "5", + }, + "no-database": { + "FieldsAvailable": "10000000010100110111111100000111", + "FieldsRequired": "00000000000100000000000000000101", + "FoundNodes": "3", "Nodes": [ { - "ClockSkewMaxUs": "text", - "ClockSkewMinUs": "text", - "ClockSkewUs": "text", - "ConnectStatus": "Green", - "Connections": "not-zero-number", - "CpuUsage": "not-zero-number", - "Database": "/Root", - "DiskSpaceUsage": "not-zero-number", - "NetworkUtilization": "number", - "NetworkUtilizationMax": "number", - "NetworkUtilizationMin": "number", "NodeId": 1, - "PDisks": [ - { - "AvailableSize": "text", - "Category": "0", - "ChangeTime": "not-zero-number-text", - "Device": "Green", - "EnforcedDynamicSlotSize": "not-zero-number-text", - "Guid": "1", + "SystemState": { + "ChangeTime": "not-zero-number-text", + "CoresTotal": "not-zero-number", + "CoresUsed": "not-zero-number", + "Endpoints": [ + { + "Address": "text", + "Name": "grpc" + }, + { + "Address": "text", + "Name": "http-mon" + }, + { + "Address": "text", + "Name": "ic" + } + ], + "Host": "text", + "LoadAverage": "not-empty-array", + "Location": { + "DataCenter": "1", + "Rack": "1", + "Unit": "1" + }, + "MaxDiskUsage": "not-zero-number", + "MemoryLimit": "not-zero-number-text", + "NodeId": 1, + "NumberOfCpus": "not-zero-number", + "PoolStats": [ + { + "Limit": "not-zero-number", + "Name": "System", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "User", + "Threads": 3, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "Batch", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IO", + "Threads": 1, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IC", + "Threads": 1, + "Usage": "number" + } + ], + "RealNumberOfCpus": "not-zero-number", + "Roles": "not-empty-array", + "StartTime": "not-zero-number-text", + "SystemState": "Green", + "TotalSessions": "number" + } + }, + { + "NodeId": 50001, + "SystemState": { + "ChangeTime": "not-zero-number-text", + "CoresTotal": "not-zero-number", + "CoresUsed": "not-zero-number", + "Endpoints": [ + { + "Address": "text", + "Name": "grpc" + }, + { + "Address": "text", + "Name": "http-mon" + }, + { + "Address": "text", + "Name": "ic" + } + ], + "Host": "text", + "LoadAverage": "not-empty-array", + "Location": {}, + "MemoryLimit": "not-zero-number-text", + "NodeId": 50001, + "NumberOfCpus": "not-zero-number", + "PoolStats": [ + { + "Limit": "not-zero-number", + "Name": "System", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "User", + "Threads": 3, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "Batch", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IO", + "Threads": 1, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IC", + "Threads": 1, + "Usage": "number" + } + ], + "RealNumberOfCpus": "not-zero-number", + "Roles": "not-empty-array", + "StartTime": "not-zero-number-text", + "SystemState": "Green", + "Tenants": [ + "/Root/shared_db" + ], + "TotalSessions": "number" + } + }, + { + "NodeId": 50000, + "SystemState": { + "ChangeTime": "not-zero-number-text", + "CoresTotal": "not-zero-number", + "CoresUsed": "not-zero-number", + "Endpoints": [ + { + "Address": "text", + "Name": "grpc" + }, + { + "Address": "text", + "Name": "http-mon" + }, + { + "Address": "text", + "Name": "ic" + } + ], + "Host": "text", + "LoadAverage": "not-empty-array", + "Location": {}, + "MemoryLimit": "not-zero-number-text", + "NodeId": 50000, + "NumberOfCpus": "not-zero-number", + "PoolStats": [ + { + "Limit": "not-zero-number", + "Name": "System", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "User", + "Threads": 3, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "Batch", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IO", + "Threads": 1, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IC", + "Threads": 1, + "Usage": "number" + } + ], + "RealNumberOfCpus": "not-zero-number", + "Roles": "not-empty-array", + "StartTime": "not-zero-number-text", + "SystemState": "Green", + "Tenants": [ + "/Root/dedicated_db" + ], + "TotalSessions": "number" + } + } + ], + "TotalNodes": "3" + } + }, + "test.test_viewer_nodes_all": { + "/Root": { + "FieldsAvailable": "11111111110111111111111111111111", + "FieldsRequired": "11111111111111111111111111111111", + "FoundNodes": "1", + "MaximumDisksPerNode": "1", + "MaximumSlotsPerDisk": "5", + "Nodes": [ + { + "ClockSkewMaxUs": "text", + "ClockSkewMinUs": "text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "Connections": "not-zero-number", + "CpuUsage": "not-zero-number", + "Database": "/Root", + "DiskSpaceUsage": "not-zero-number", + "NetworkUtilization": "number", + "NetworkUtilizationMax": "number", + "NetworkUtilizationMin": "number", + "NodeId": 1, + "PDisks": [ + { + "AvailableSize": "text", + "Category": "0", + "ChangeTime": "not-zero-number-text", + "Device": "Green", + "EnforcedDynamicSlotSize": "not-zero-number-text", + "Guid": "1", "LogTotalSize": "not-zero-number-text", "LogUsedSize": "not-zero-number-text", "NodeId": 1, @@ -4443,157 +4972,126 @@ } ], "TotalNodes": "1" - } - }, - "test.test_viewer_nodes_issue_14992": { - "response_group": { - "FieldsAvailable": "10000000110111110111111100000111", - "FieldsRequired": "00000000001100000010000000000101", + }, + "no-database": { + "FieldsAvailable": "11111111110111111111111111111111", + "FieldsRequired": "11111111111111111111111111111111", "FoundNodes": "3", "MaximumDisksPerNode": "1", + "MaximumSlotsPerDisk": "5", "Nodes": [ { - "NodeId": 1, - "SystemState": { - "ChangeTime": "not-zero-number-text", - "CoresTotal": "not-zero-number", - "CoresUsed": "not-zero-number", - "Endpoints": [ - { - "Address": "text", - "Name": "grpc" - }, - { - "Address": "text", - "Name": "http-mon" + "ClockSkewMaxUs": "text", + "ClockSkewMinUs": "text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "Connections": "not-zero-number", + "CpuUsage": "not-zero-number", + "DiskSpaceUsage": "not-zero-number", + "NetworkUtilization": "number", + "NetworkUtilizationMax": "number", + "NetworkUtilizationMin": "number", + "NodeId": 1, + "PDisks": [ + { + "AvailableSize": "text", + "Category": "0", + "ChangeTime": "not-zero-number-text", + "Device": "Green", + "EnforcedDynamicSlotSize": "not-zero-number-text", + "Guid": "1", + "LogTotalSize": "not-zero-number-text", + "LogUsedSize": "not-zero-number-text", + "NodeId": 1, + "NumActiveSlots": 5, + "PDiskId": 1, + "Path": "SectorMap:1:64", + "Realtime": "Green", + "SerialNumber": "", + "State": "Normal", + "SystemSize": "not-zero-number-text", + "TotalSize": "not-zero-number-text" + } + ], + "Peers": [ + { + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 1, + "PeerName": "text", + "PeerNodeId": 50000, + "PingTimeUs": "text", + "ScopeId": { + "X1": "72057594046678944", + "X2": "6" }, - { - "Address": "text", - "Name": "ic" - } - ], - "Host": "text", - "LoadAverage": "not-empty-array", - "Location": { - "DataCenter": "1", - "Rack": "1", - "Unit": "1" + "Utilization": "number" }, - "MaxDiskUsage": "not-zero-number", - "MemoryLimit": "not-zero-number-text", - "NodeId": 1, - "NumberOfCpus": "not-zero-number", - "PoolStats": [ - { - "Limit": "not-zero-number", - "Name": "System", - "Threads": 2, - "Usage": "number" - }, - { - "Limit": "not-zero-number", - "Name": "User", - "Threads": 3, - "Usage": "number" - }, - { - "Limit": "not-zero-number", - "Name": "Batch", - "Threads": 2, - "Usage": "number" - }, - { - "Limit": "not-zero-number", - "Name": "IO", - "Threads": 1, - "Usage": "number" - }, - { - "Limit": "not-zero-number", - "Name": "IC", - "Threads": 1, - "Usage": "number" - } - ], - "RealNumberOfCpus": "not-zero-number", - "Roles": "not-empty-array", - "StartTime": "not-zero-number-text", - "SystemState": "Green", - "TotalSessions": "number" - }, - "UptimeSeconds": "number" - }, - { - "NodeId": 50001, - "SystemState": { - "ChangeTime": "not-zero-number-text", - "CoresTotal": "not-zero-number", - "CoresUsed": "not-zero-number", - "Endpoints": [ - { - "Address": "text", - "Name": "grpc" - }, - { - "Address": "text", - "Name": "http-mon" - }, - { - "Address": "text", - "Name": "ic" - } - ], - "Host": "text", - "LoadAverage": "not-empty-array", - "Location": {}, - "MemoryLimit": "not-zero-number-text", - "NodeId": 50001, - "NumberOfCpus": "not-zero-number", - "PoolStats": [ - { - "Limit": "not-zero-number", - "Name": "System", - "Threads": 2, - "Usage": "number" - }, - { - "Limit": "not-zero-number", - "Name": "User", - "Threads": 3, - "Usage": "number" + { + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 1, + "PeerName": "text", + "PeerNodeId": 50001, + "PingTimeUs": "text", + "ScopeId": { + "X1": "72057594046678944", + "X2": "7" }, - { - "Limit": "not-zero-number", - "Name": "Batch", - "Threads": 2, - "Usage": "number" + "Utilization": "number" + } + ], + "PingTimeMaxUs": "text", + "PingTimeMinUs": "text", + "PingTimeUs": "text", + "ReceiveThroughput": "text", + "ReverseClockSkewUs": "text", + "ReversePeers": [ + { + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 50000, + "PeerName": "text", + "PeerNodeId": 1, + "PingTimeUs": "text", + "ScopeId": { + "X1": "0", + "X2": "1" }, - { - "Limit": "not-zero-number", - "Name": "IO", - "Threads": 1, - "Usage": "number" + "Utilization": "number" + }, + { + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 50001, + "PeerName": "text", + "PeerNodeId": 1, + "PingTimeUs": "text", + "ScopeId": { + "X1": "0", + "X2": "1" }, - { - "Limit": "not-zero-number", - "Name": "IC", - "Threads": 1, - "Usage": "number" - } - ], - "RealNumberOfCpus": "not-zero-number", - "Roles": "not-empty-array", - "StartTime": "not-zero-number-text", - "SystemState": "Green", - "Tenants": [ - "/Root/shared_db" - ], - "TotalSessions": "number" - }, - "UptimeSeconds": "number" - }, - { - "NodeId": 50000, + "Utilization": "number" + } + ], + "ReversePingTimeUs": "text", + "SendThroughput": "text", "SystemState": { "ChangeTime": "not-zero-number-text", "CoresTotal": "not-zero-number", @@ -4614,9 +5112,15 @@ ], "Host": "text", "LoadAverage": "not-empty-array", - "Location": {}, + "Location": { + "DataCenter": "1", + "Rack": "1", + "Unit": "1" + }, + "MaxDiskUsage": "not-zero-number", "MemoryLimit": "not-zero-number-text", - "NodeId": 50000, + "MemoryStats": "not-empty-object", + "NodeId": 1, "NumberOfCpus": "not-zero-number", "PoolStats": [ { @@ -4654,723 +5158,641 @@ "Roles": "not-empty-array", "StartTime": "not-zero-number-text", "SystemState": "Green", - "Tenants": [ - "/Root/dedicated_db" - ], "TotalSessions": "number" }, - "UptimeSeconds": "number" - } - ], - "TotalNodes": "3" - }, - "response_group_by": { - "FieldsAvailable": "10000000110111110111111100000111", - "FieldsRequired": "00000000001100000010000000000101", - "FoundNodes": "3", - "MaximumDisksPerNode": "1", - "NodeGroups": [ - { - "GroupName": "up <10m", - "NodeCount": "3" - } - ], - "TotalNodes": "3" - } - }, - "test.test_viewer_pdiskinfo": { - "PDiskStateInfo": [ - { - "AvailableSize": "text", - "Category": "0", - "ChangeTime": "not-zero-number-text", - "Device": 1, - "EnforcedDynamicSlotSize": "not-zero-number-text", - "Guid": "1", - "LogTotalSize": "not-zero-number-text", - "LogUsedSize": "not-zero-number-text", - "NodeId": 1, - "NumActiveSlots": 5, - "PDiskId": 1, - "Path": "SectorMap:1:64", - "Realtime": 1, - "SerialNumber": "", - "State": 10, - "SystemSize": "not-zero-number-text", - "TotalSize": "not-zero-number-text" - } - ], - "ResponseDuration": "not-zero-number", - "ResponseTime": "not-zero-number-text" - }, - "test.test_viewer_query": { - "/Root": { - "result": [ - { - "columns": [ + "Tablets": [ { - "name": "column0", - "type": "Int32" - } - ], - "rows": [ - [ - 42 - ] - ] - } - ], - "version": 9 - }, - "/Root/dedicated_db": { - "result": [ - { - "columns": [ + "Count": 1, + "State": "Green", + "Type": "BSController" + }, { - "name": "column0", - "type": "Int32" - } - ], - "rows": [ - [ - 42 - ] - ] - } - ], - "version": 9 - }, - "/Root/serverless_db": { - "result": [ - { - "columns": [ + "Count": 1, + "State": "Green", + "Type": "Cms" + }, { - "name": "column0", - "type": "Int32" - } - ], - "rows": [ - [ - 42 - ] - ] - } - ], - "version": 9 - }, - "/Root/shared_db": { - "result": [ - { - "columns": [ + "Count": 1, + "State": "Green", + "Type": "Console" + }, { - "name": "column0", - "type": "Int32" - } - ], - "rows": [ - [ - 42 - ] - ] - } - ], - "version": 9 - } - }, - "test.test_viewer_query_from_table": { - "/Root/dedicated_db": { - "result": [ - { - "columns": [ + "Count": 1, + "State": "Green", + "Type": "Coordinator" + }, { - "name": "id", - "type": "Int64?" + "Count": 1, + "State": "Green", + "Type": "Hive" }, { - "name": "name", - "type": "Utf8?" - } - ], - "rows": [ - [ - "1", - "one" - ], - [ - "2", - "two" - ], - [ - "3", - "three" - ] - ] - } - ], - "version": 9 - }, - "/Root/serverless_db": { - "result": [ - { - "columns": [ + "Count": 1, + "State": "Green", + "Type": "Mediator" + }, { - "name": "id", - "type": "Int64?" + "Count": 1, + "State": "Green", + "Type": "NodeBroker" }, { - "name": "name", - "type": "Utf8?" - } - ], - "rows": [ - [ - "1", - "one" - ], - [ - "2", - "two" - ], - [ - "3", - "three" - ] - ] - } - ], - "version": 9 - }, - "/Root/shared_db": { - "result": [ - { - "columns": [ + "Count": 1, + "State": "Green", + "Type": "SchemeShard" + }, { - "name": "id", - "type": "Int64?" + "Count": 1, + "State": "Green", + "Type": "TenantSlotBroker" }, { - "name": "name", - "type": "Utf8?" + "Count": 1, + "State": "Green", + "Type": "TxAllocator" } ], - "rows": [ - [ - "1", - "one" - ], - [ - "2", - "two" - ], - [ - "3", - "three" - ] - ] - } - ], - "version": 9 - } - }, - "test.test_viewer_query_from_table_different_schemas": { - "classic": [ - { - "id": "1", - "name": "one" - }, - { - "id": "2", - "name": "two" - }, - { - "id": "3", - "name": "three" - } - ], - "modern": { - "columns": [ - { - "name": "id", - "type": "Int64?" - }, - { - "name": "name", - "type": "Utf8?" - } - ], - "result": [ - [ - "1", - "one" - ], - [ - "2", - "two" - ], - [ - "3", - "three" - ] - ], - "version": 9 - }, - "multi": { - "result": [ - { - "columns": [ + "UptimeSeconds": "number", + "VDisks": [ { - "name": "id", - "type": "Int64?" + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "static", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 1, + "GroupID": 0, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 0, + "VDiskState": "OK" }, { - "name": "name", - "type": "Utf8?" - } - ], - "rows": [ - [ - "1", - "one" - ], - [ - "2", - "two" - ], - [ - "3", - "three" - ] - ] - } - ], - "version": 9 - }, - "ydb": { - "result": [ - { - "id": 1, - "name": "one" - }, - { - "id": 2, - "name": "two" - }, - { - "id": 3, - "name": "three" - } - ], - "version": 9 - }, - "ydb2": { - "result": [ - { - "columns": [ - { - "name": "id", - "type": "Int64?" + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "dynamic_storage_pool:1", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 1, + "GroupID": 2181038080, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 1000, + "VDiskState": "OK" }, { - "name": "name", - "type": "Utf8?" - } - ], - "rows": [ - { - "id": 1, - "name": "one" + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "dynamic_storage_pool:1", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 1, + "GroupID": 2181038081, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 1001, + "VDiskState": "OK" }, { - "id": 2, - "name": "two" + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "/Root/dedicated_db:hdd", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 2, + "GroupID": 2181038082, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 1002, + "VDiskState": "OK" }, { - "id": 3, - "name": "three" + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "/Root/shared_db:hdd", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 2, + "GroupID": 2181038083, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 1003, + "VDiskState": "OK" } ] - } - ], - "version": 9 - } - }, - "test.test_viewer_query_issue_13757": { - "/Root": { - "result": [ + }, { - "columns": [ - { - "name": "column0", - "type": "Struct<'three':Date?,'two':Utf8>" - } - ], - "rows": [ - [ - { - "three": null, - "two": "42" - } - ] - ] - } - ], - "version": 9 - }, - "/Root/dedicated_db": { - "result": [ - { - "columns": [ + "ClockSkewMaxUs": "text", + "ClockSkewMinUs": "text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "Connections": "not-zero-number", + "CpuUsage": "not-zero-number", + "Database": "/Root/shared_db", + "NetworkUtilization": "number", + "NetworkUtilizationMax": "number", + "NetworkUtilizationMin": "number", + "NodeId": 50001, + "Peers": [ { - "name": "column0", - "type": "Struct<'three':Date?,'two':Utf8>" + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 50001, + "PeerName": "text", + "PeerNodeId": 1, + "PingTimeUs": "text", + "ScopeId": { + "X1": "0", + "X2": "1" + }, + "Utilization": "number" + }, + { + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 50001, + "PeerName": "text", + "PeerNodeId": 50000, + "PingTimeUs": "text", + "ScopeId": { + "X1": "72057594046678944", + "X2": "6" + }, + "Utilization": "number" } ], - "rows": [ - [ - { - "three": null, - "two": "42" - } - ] - ] - } - ], - "version": 9 - }, - "/Root/serverless_db": { - "result": [ - { - "columns": [ + "PingTimeMaxUs": "text", + "PingTimeMinUs": "text", + "PingTimeUs": "text", + "ReceiveThroughput": "text", + "ReverseClockSkewUs": "text", + "ReversePeers": [ { - "name": "column0", - "type": "Struct<'three':Date?,'two':Utf8>" + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 1, + "PeerName": "text", + "PeerNodeId": 50001, + "PingTimeUs": "text", + "ScopeId": { + "X1": "72057594046678944", + "X2": "7" + }, + "Utilization": "number" + }, + { + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 50000, + "PeerName": "text", + "PeerNodeId": 50001, + "PingTimeUs": "text", + "ScopeId": { + "X1": "72057594046678944", + "X2": "7" + }, + "Utilization": "number" } ], - "rows": [ - [ + "ReversePingTimeUs": "text", + "SendThroughput": "text", + "SystemState": { + "ChangeTime": "not-zero-number-text", + "CoresTotal": "not-zero-number", + "CoresUsed": "not-zero-number", + "Endpoints": [ { - "three": null, - "two": "42" + "Address": "text", + "Name": "grpc" + }, + { + "Address": "text", + "Name": "http-mon" + }, + { + "Address": "text", + "Name": "ic" } - ] - ] - } - ], - "version": 9 - }, - "/Root/shared_db": { - "result": [ - { - "columns": [ - { - "name": "column0", - "type": "Struct<'three':Date?,'two':Utf8>" - } - ], - "rows": [ - [ + ], + "Host": "text", + "LoadAverage": "not-empty-array", + "Location": {}, + "MemoryLimit": "not-zero-number-text", + "MemoryStats": "not-empty-object", + "NodeId": 50001, + "NumberOfCpus": "not-zero-number", + "PoolStats": [ { - "three": null, - "two": "42" + "Limit": "not-zero-number", + "Name": "System", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "User", + "Threads": 3, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "Batch", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IO", + "Threads": 1, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IC", + "Threads": 1, + "Usage": "number" } - ] - ] - } - ], - "version": 9 - } - }, - "test.test_viewer_query_issue_13945": { - "/Root": { - "result": [ - { - "columns": [ + ], + "RealNumberOfCpus": "not-zero-number", + "Roles": "not-empty-array", + "StartTime": "not-zero-number-text", + "SystemState": "Green", + "Tenants": [ + "/Root/shared_db" + ], + "TotalSessions": "number" + }, + "Tablets": [ { - "name": "column0", - "type": "EmptyList" + "Count": 4, + "State": "Green", + "Type": "Coordinator" + }, + { + "Count": 2, + "State": "Green", + "Type": "DataShard" + }, + { + "Count": 1, + "State": "Green", + "Type": "Hive" + }, + { + "Count": 6, + "State": "Green", + "Type": "Mediator" + }, + { + "Count": 2, + "State": "Green", + "Type": "SchemeShard" + }, + { + "Count": 1, + "State": "Green", + "Type": "StatisticsAggregator" + }, + { + "Count": 2, + "State": "Green", + "Type": "SysViewProcessor" } ], - "rows": [ - [ - [] - ] - ] - } - ], - "version": 9 - }, - "/Root/dedicated_db": { - "result": [ + "UptimeSeconds": "number" + }, { - "columns": [ - { - "name": "column0", - "type": "EmptyList" - } - ], - "rows": [ - [ - [] - ] - ] - } - ], - "version": 9 - }, - "/Root/serverless_db": { - "result": [ - { - "columns": [ + "ClockSkewMaxUs": "text", + "ClockSkewMinUs": "text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "Connections": "not-zero-number", + "CpuUsage": "not-zero-number", + "Database": "/Root/dedicated_db", + "NetworkUtilization": "number", + "NetworkUtilizationMax": "number", + "NetworkUtilizationMin": "number", + "NodeId": 50000, + "Peers": [ { - "name": "column0", - "type": "EmptyList" - } - ], - "rows": [ - [ - [] - ] - ] - } - ], - "version": 9 - }, - "/Root/shared_db": { - "result": [ - { - "columns": [ + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 50000, + "PeerName": "text", + "PeerNodeId": 1, + "PingTimeUs": "text", + "ScopeId": { + "X1": "0", + "X2": "1" + }, + "Utilization": "number" + }, { - "name": "column0", - "type": "EmptyList" + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 50000, + "PeerName": "text", + "PeerNodeId": 50001, + "PingTimeUs": "text", + "ScopeId": { + "X1": "72057594046678944", + "X2": "7" + }, + "Utilization": "number" } ], - "rows": [ - [ - [] - ] - ] - } - ], - "version": 9 - } - }, - "test.test_viewer_query_long": { - "execute_response": { - "execution_id": "text", - "operation_id": "text", - "result": [ - { - "columns": [ + "PingTimeMaxUs": "text", + "PingTimeMinUs": "text", + "PingTimeUs": "text", + "ReceiveThroughput": "text", + "ReverseClockSkewUs": "text", + "ReversePeers": [ { - "name": "id", - "type": "Int64?" + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 1, + "PeerName": "text", + "PeerNodeId": 50000, + "PingTimeUs": "text", + "ScopeId": { + "X1": "72057594046678944", + "X2": "6" + }, + "Utilization": "number" }, { - "name": "name", - "type": "Utf8?" + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 50001, + "PeerName": "text", + "PeerNodeId": 50000, + "PingTimeUs": "text", + "ScopeId": { + "X1": "72057594046678944", + "X2": "6" + }, + "Utilization": "number" } ], - "rows": [ - [ - "1", - "one" + "ReversePingTimeUs": "text", + "SendThroughput": "text", + "SystemState": { + "ChangeTime": "not-zero-number-text", + "CoresTotal": "not-zero-number", + "CoresUsed": "not-zero-number", + "Endpoints": [ + { + "Address": "text", + "Name": "grpc" + }, + { + "Address": "text", + "Name": "http-mon" + }, + { + "Address": "text", + "Name": "ic" + } ], - [ - "2", - "two" + "Host": "text", + "LoadAverage": "not-empty-array", + "Location": {}, + "MemoryLimit": "not-zero-number-text", + "MemoryStats": "not-empty-object", + "NodeId": 50000, + "NumberOfCpus": "not-zero-number", + "PoolStats": [ + { + "Limit": "not-zero-number", + "Name": "System", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "User", + "Threads": 3, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "Batch", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IO", + "Threads": 1, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IC", + "Threads": 1, + "Usage": "number" + } ], - [ - "3", - "three" - ] - ] - } - ], - "version": 9 - }, - "fetch_by_execution_id": { - "execution_id": "text", - "result": [ - { - "columns": [ + "RealNumberOfCpus": "not-zero-number", + "Roles": "not-empty-array", + "StartTime": "not-zero-number-text", + "SystemState": "Green", + "Tenants": [ + "/Root/dedicated_db" + ], + "TotalSessions": "number" + }, + "Tablets": [ { - "name": "id", - "type": "Int64?" + "Count": 3, + "State": "Green", + "Type": "Coordinator" }, { - "name": "name", - "type": "Utf8?" - } - ], - "rows": [ - [ - "1", - "one" - ], - [ - "2", - "two" - ], - [ - "3", - "three" - ] - ] - } - ], - "version": 9 - }, - "fetch_by_operation_id": { - "execution_id": "text", - "operation_id": "text", - "result": [ - { - "columns": [ + "Count": 1, + "State": "Green", + "Type": "DataShard" + }, { - "name": "id", - "type": "Int64?" + "Count": 1, + "State": "Green", + "Type": "Hive" }, { - "name": "name", - "type": "Utf8?" + "Count": 3, + "State": "Green", + "Type": "Mediator" + }, + { + "Count": 1, + "State": "Green", + "Type": "SchemeShard" + }, + { + "Count": 1, + "State": "Green", + "Type": "StatisticsAggregator" + }, + { + "Count": 1, + "State": "Green", + "Type": "SysViewProcessor" } ], - "rows": [ - [ - "1", - "one" - ], - [ - "2", - "two" - ], - [ - "3", - "three" - ] - ] + "UptimeSeconds": "number" } ], - "version": 9 - }, - "fetch_error_no_id": { - "status_code": 400, - "text": "operation_id or execution_id required for fetch-long-query" - }, - "fetch_invalid_operation_id": { - "status_code": 400, - "text": "operation_id or execution_id required for fetch-long-query" - } - }, - "test.test_viewer_query_long_multipart": { - "execute_stream_response": { - "multipart_parts": [ - { - "exec_mode": "EXEC_MODE_EXECUTE", - "exec_status": "EXEC_STATUS_STARTING", - "execution_id": "text", - "meta": { - "event": "ScriptResponse" - }, - "operation_id": "text", - "status": "SUCCESS" - }, - { - "id": "text", - "meta": { - "event": "OperationResponse" - }, - "metadata": { - "exec_mode": "EXEC_MODE_EXECUTE", - "exec_stats": { - "query_plan": "{}" - }, - "exec_status": "EXEC_STATUS_COMPLETED", - "execution_id": "text", - "result_sets_meta": [ - { - "columns": [ - { - "name": "id", - "type": { - "optional_type": { - "item": { - "type_id": "INT64" - } - } - } - }, - { - "name": "name", - "type": { - "optional_type": { - "item": { - "type_id": "UTF8" - } - } - } - } - ], - "finished": true, - "number_rows": "3" - } - ], - "script_content": { - "text": "SELECT * FROM table1 LIMIT 3;" - } - }, - "ready": true - }, - { - "meta": { - "event": "StreamData", - "result_index": 0, - "seq_no": 0 - }, - "result": { - "columns": [ - { - "name": "id", - "type": "Int64?" - }, - { - "name": "name", - "type": "Utf8?" - } - ], - "rows": [ - [ - "1", - "one" - ], - [ - "2", - "two" - ], - [ - "3", - "three" - ] - ] - } - } - ] - }, - "fetch_error_stream_response": { - "status_code": 400, - "text": "operation_id or execution_id required for fetch-long-query" - }, - "fetch_invalid_stream_response": { - "status_code": 400, - "text": "operation_id or execution_id required for fetch-long-query" + "TotalNodes": "3" } }, - "test.test_viewer_storage_nodes": { - "/Root": { - "FieldsAvailable": "10000000110000110111111100000111", - "FieldsRequired": "00000000000000000000000000000111", - "FoundNodes": "1", - "MaximumSlotsPerDisk": "5", + "test.test_viewer_nodes_issue_14992": { + "response_group": { + "FieldsAvailable": "10000000110111110111111100000111", + "FieldsRequired": "00000000001100000010000000000101", + "FoundNodes": "3", + "MaximumDisksPerNode": "1", "Nodes": [ { "NodeId": 1, @@ -5440,19 +5862,11 @@ "StartTime": "not-zero-number-text", "SystemState": "Green", "TotalSessions": "number" - } - } - ], - "TotalNodes": "1" - }, - "/Root/dedicated_db": { - "FieldsAvailable": "10000000110000110111111100000111", - "FieldsRequired": "00000000000000000000000000000111", - "FoundNodes": "1", - "MaximumSlotsPerDisk": "5", - "Nodes": [ + }, + "UptimeSeconds": "number" + }, { - "NodeId": 1, + "NodeId": 50001, "SystemState": { "ChangeTime": "not-zero-number-text", "CoresTotal": "not-zero-number", @@ -5473,14 +5887,9 @@ ], "Host": "text", "LoadAverage": "not-empty-array", - "Location": { - "DataCenter": "1", - "Rack": "1", - "Unit": "1" - }, - "MaxDiskUsage": "not-zero-number", + "Location": {}, "MemoryLimit": "not-zero-number-text", - "NodeId": 1, + "NodeId": 50001, "NumberOfCpus": "not-zero-number", "PoolStats": [ { @@ -5518,20 +5927,15 @@ "Roles": "not-empty-array", "StartTime": "not-zero-number-text", "SystemState": "Green", + "Tenants": [ + "/Root/shared_db" + ], "TotalSessions": "number" - } - } - ], - "TotalNodes": "1" - }, - "/Root/serverless_db": { - "FieldsAvailable": "10000000110000110111111100000111", - "FieldsRequired": "00000000000000000000000000000111", - "FoundNodes": "1", - "MaximumSlotsPerDisk": "5", - "Nodes": [ + }, + "UptimeSeconds": "number" + }, { - "NodeId": 1, + "NodeId": 50000, "SystemState": { "ChangeTime": "not-zero-number-text", "CoresTotal": "not-zero-number", @@ -5552,14 +5956,9 @@ ], "Host": "text", "LoadAverage": "not-empty-array", - "Location": { - "DataCenter": "1", - "Rack": "1", - "Unit": "1" - }, - "MaxDiskUsage": "not-zero-number", + "Location": {}, "MemoryLimit": "not-zero-number-text", - "NodeId": 1, + "NodeId": 50000, "NumberOfCpus": "not-zero-number", "PoolStats": [ { @@ -5597,1323 +5996,862 @@ "Roles": "not-empty-array", "StartTime": "not-zero-number-text", "SystemState": "Green", + "Tenants": [ + "/Root/dedicated_db" + ], "TotalSessions": "number" - } + }, + "UptimeSeconds": "number" } ], - "Problems": [ - "no-database-board-info" - ], - "TotalNodes": "1" + "TotalNodes": "3" }, - "/Root/shared_db": { - "FieldsAvailable": "10000000110000110111111100000111", - "FieldsRequired": "00000000000000000000000000000111", - "FoundNodes": "1", - "MaximumSlotsPerDisk": "5", - "Nodes": [ + "response_group_by": { + "FieldsAvailable": "10000000110111110111111100000111", + "FieldsRequired": "00000000001100000010000000000101", + "FoundNodes": "3", + "MaximumDisksPerNode": "1", + "NodeGroups": [ { - "NodeId": 1, - "SystemState": { - "ChangeTime": "not-zero-number-text", - "CoresTotal": "not-zero-number", - "CoresUsed": "not-zero-number", - "Endpoints": [ - { - "Address": "text", - "Name": "grpc" - }, - { - "Address": "text", - "Name": "http-mon" - }, - { - "Address": "text", - "Name": "ic" - } - ], - "Host": "text", - "LoadAverage": "not-empty-array", - "Location": { - "DataCenter": "1", - "Rack": "1", - "Unit": "1" - }, - "MaxDiskUsage": "not-zero-number", - "MemoryLimit": "not-zero-number-text", - "NodeId": 1, - "NumberOfCpus": "not-zero-number", - "PoolStats": [ - { - "Limit": "not-zero-number", - "Name": "System", - "Threads": 2, - "Usage": "number" - }, - { - "Limit": "not-zero-number", - "Name": "User", - "Threads": 3, - "Usage": "number" - }, - { - "Limit": "not-zero-number", - "Name": "Batch", - "Threads": 2, - "Usage": "number" - }, - { - "Limit": "not-zero-number", - "Name": "IO", - "Threads": 1, - "Usage": "number" - }, - { - "Limit": "not-zero-number", - "Name": "IC", - "Threads": 1, - "Usage": "number" - } - ], - "RealNumberOfCpus": "not-zero-number", - "Roles": "not-empty-array", - "StartTime": "not-zero-number-text", - "SystemState": "Green", - "TotalSessions": "number" - } + "GroupName": "up <10m", + "NodeCount": "3" } ], - "TotalNodes": "1" + "TotalNodes": "3" } }, - "test.test_viewer_storage_nodes_all": { + "test.test_viewer_pdiskinfo": { + "PDiskStateInfo": [ + { + "AvailableSize": "text", + "Category": "0", + "ChangeTime": "not-zero-number-text", + "Device": 1, + "EnforcedDynamicSlotSize": "not-zero-number-text", + "Guid": "1", + "LogTotalSize": "not-zero-number-text", + "LogUsedSize": "not-zero-number-text", + "NodeId": 1, + "NumActiveSlots": 5, + "PDiskId": 1, + "Path": "SectorMap:1:64", + "Realtime": 1, + "SerialNumber": "", + "State": 10, + "SystemSize": "not-zero-number-text", + "TotalSize": "not-zero-number-text" + } + ], + "ResponseDuration": "not-zero-number", + "ResponseTime": "not-zero-number-text" + }, + "test.test_viewer_query": { "/Root": { - "FieldsAvailable": "11111111110111111111111111111111", - "FieldsRequired": "11111111111111111111111111111111", - "FoundNodes": "1", - "MaximumDisksPerNode": "1", - "MaximumSlotsPerDisk": "5", - "Nodes": [ + "result": [ { - "ClockSkewMaxUs": "text", - "ClockSkewMinUs": "text", - "ClockSkewUs": "text", - "ConnectStatus": "Green", - "Connections": "not-zero-number", - "CpuUsage": "not-zero-number", - "Database": "/Root", - "DiskSpaceUsage": "not-zero-number", - "NetworkUtilization": "number", - "NetworkUtilizationMax": "number", - "NetworkUtilizationMin": "number", - "NodeId": 1, - "PDisks": [ + "columns": [ { - "AvailableSize": "text", - "Category": "0", - "ChangeTime": "not-zero-number-text", - "Device": "Green", - "EnforcedDynamicSlotSize": "not-zero-number-text", - "Guid": "1", - "LogTotalSize": "not-zero-number-text", - "LogUsedSize": "not-zero-number-text", - "NodeId": 1, - "NumActiveSlots": 5, - "PDiskId": 1, - "Path": "SectorMap:1:64", - "Realtime": "Green", - "SerialNumber": "", - "State": "Normal", - "SystemSize": "not-zero-number-text", - "TotalSize": "not-zero-number-text" + "name": "column0", + "type": "Int32" } ], - "Peers": [ - { - "BytesWritten": "text", - "ChangeTime": "not-zero-number-text", - "ClockSkewUs": "text", - "ConnectStatus": "Green", - "ConnectTime": "not-zero-number-text", - "Connected": true, - "NodeId": 1, - "PeerName": "text", - "PeerNodeId": 50000, - "PingTimeUs": "text", - "ScopeId": { - "X1": "72057594046678944", - "X2": "6" - }, - "Utilization": "number" - }, + "rows": [ + [ + 42 + ] + ] + } + ], + "version": 9 + }, + "/Root/dedicated_db": { + "result": [ + { + "columns": [ { - "BytesWritten": "text", - "ChangeTime": "not-zero-number-text", - "ClockSkewUs": "text", - "ConnectStatus": "Green", - "ConnectTime": "not-zero-number-text", - "Connected": true, - "NodeId": 1, - "PeerName": "text", - "PeerNodeId": 50001, - "PingTimeUs": "text", - "ScopeId": { - "X1": "72057594046678944", - "X2": "7" - }, - "Utilization": "number" + "name": "column0", + "type": "Int32" } ], - "PingTimeMaxUs": "text", - "PingTimeMinUs": "text", - "PingTimeUs": "text", - "ReceiveThroughput": "text", - "ReverseClockSkewUs": "text", - "ReversePeers": [ + "rows": [ + [ + 42 + ] + ] + } + ], + "version": 9 + }, + "/Root/serverless_db": { + "result": [ + { + "columns": [ { - "BytesWritten": "text", - "ChangeTime": "not-zero-number-text", - "ClockSkewUs": "text", - "ConnectStatus": "Green", - "ConnectTime": "not-zero-number-text", - "Connected": true, - "NodeId": 50000, - "PeerName": "text", - "PeerNodeId": 1, - "PingTimeUs": "text", - "ScopeId": { - "X1": "0", - "X2": "1" - }, - "Utilization": "number" - }, + "name": "column0", + "type": "Int32" + } + ], + "rows": [ + [ + 42 + ] + ] + } + ], + "version": 9 + }, + "/Root/shared_db": { + "result": [ + { + "columns": [ { - "BytesWritten": "text", - "ChangeTime": "not-zero-number-text", - "ClockSkewUs": "text", - "ConnectStatus": "Green", - "ConnectTime": "not-zero-number-text", - "Connected": true, - "NodeId": 50001, - "PeerName": "text", - "PeerNodeId": 1, - "PingTimeUs": "text", - "ScopeId": { - "X1": "0", - "X2": "1" - }, - "Utilization": "number" + "name": "column0", + "type": "Int32" } ], - "ReversePingTimeUs": "text", - "SendThroughput": "text", - "SystemState": { - "ChangeTime": "not-zero-number-text", - "CoresTotal": "not-zero-number", - "CoresUsed": "not-zero-number", - "Endpoints": [ - { - "Address": "text", - "Name": "grpc" - }, - { - "Address": "text", - "Name": "http-mon" - }, - { - "Address": "text", - "Name": "ic" - } - ], - "Host": "text", - "LoadAverage": "not-empty-array", - "Location": { - "DataCenter": "1", - "Rack": "1", - "Unit": "1" + "rows": [ + [ + 42 + ] + ] + } + ], + "version": 9 + }, + "no-database": { + "result": [ + { + "columns": [ + { + "name": "column0", + "type": "Int32" + } + ], + "rows": [ + [ + 42 + ] + ] + } + ], + "version": 9 + } + }, + "test.test_viewer_query_from_table": { + "/Root/dedicated_db": { + "result": [ + { + "columns": [ + { + "name": "id", + "type": "Int64?" }, - "MaxDiskUsage": "not-zero-number", - "MemoryLimit": "not-zero-number-text", - "MemoryStats": "not-empty-object", - "NodeId": 1, - "NumberOfCpus": "not-zero-number", - "PoolStats": [ - { - "Limit": "not-zero-number", - "Name": "System", - "Threads": 2, - "Usage": "number" - }, - { - "Limit": "not-zero-number", - "Name": "User", - "Threads": 3, - "Usage": "number" - }, - { - "Limit": "not-zero-number", - "Name": "Batch", - "Threads": 2, - "Usage": "number" - }, - { - "Limit": "not-zero-number", - "Name": "IO", - "Threads": 1, - "Usage": "number" - }, - { - "Limit": "not-zero-number", - "Name": "IC", - "Threads": 1, - "Usage": "number" - } + { + "name": "name", + "type": "Utf8?" + } + ], + "rows": [ + [ + "1", + "one" ], - "RealNumberOfCpus": "not-zero-number", - "Roles": "not-empty-array", - "StartTime": "not-zero-number-text", - "SystemState": "Green", - "TotalSessions": "number" - }, - "UptimeSeconds": "number", - "VDisks": [ + [ + "2", + "two" + ], + [ + "3", + "three" + ] + ] + } + ], + "version": 9 + }, + "/Root/serverless_db": { + "result": [ + { + "columns": [ { - "AllocatedSize": "0", - "AvailableSize": "text", - "ChangeTime": "not-zero-number-text", - "DiskSpace": "Green", - "FrontQueues": "Green", - "Guid": "1", - "HasUnreadableBlobs": false, - "IncarnationGuid": "not-zero-number-text", - "InstanceGuid": "not-zero-number-text", - "Kind": "0", - "NodeId": 1, - "PDiskId": 1, - "Replicated": true, - "ReplicationProgress": 1, - "ReplicationSecondsRemaining": 0, - "SatisfactionRank": { - "FreshRank": { - "Flag": "Green" - }, - "LevelRank": { - "Flag": "Green" - } - }, - "StoragePoolName": "static", - "VDiskId": { - "Domain": 0, - "GroupGeneration": 1, - "GroupID": 0, - "Ring": 0, - "VDisk": 0 - }, - "VDiskSlotId": 0, - "VDiskState": "OK" + "name": "id", + "type": "Int64?" }, { - "AllocatedSize": "0", - "AvailableSize": "text", - "ChangeTime": "not-zero-number-text", - "DiskSpace": "Green", - "FrontQueues": "Green", - "Guid": "1", - "HasUnreadableBlobs": false, - "IncarnationGuid": "not-zero-number-text", - "InstanceGuid": "not-zero-number-text", - "Kind": "0", - "NodeId": 1, - "PDiskId": 1, - "Replicated": true, - "ReplicationProgress": 1, - "ReplicationSecondsRemaining": 0, - "SatisfactionRank": { - "FreshRank": { - "Flag": "Green" - }, - "LevelRank": { - "Flag": "Green" - } - }, - "StoragePoolName": "dynamic_storage_pool:1", - "VDiskId": { - "Domain": 0, - "GroupGeneration": 1, - "GroupID": 2181038080, - "Ring": 0, - "VDisk": 0 - }, - "VDiskSlotId": 1000, - "VDiskState": "OK" - }, + "name": "name", + "type": "Utf8?" + } + ], + "rows": [ + [ + "1", + "one" + ], + [ + "2", + "two" + ], + [ + "3", + "three" + ] + ] + } + ], + "version": 9 + }, + "/Root/shared_db": { + "result": [ + { + "columns": [ { - "AllocatedSize": "0", - "AvailableSize": "text", - "ChangeTime": "not-zero-number-text", - "DiskSpace": "Green", - "FrontQueues": "Green", - "Guid": "1", - "HasUnreadableBlobs": false, - "IncarnationGuid": "not-zero-number-text", - "InstanceGuid": "not-zero-number-text", - "Kind": "0", - "NodeId": 1, - "PDiskId": 1, - "Replicated": true, - "ReplicationProgress": 1, - "ReplicationSecondsRemaining": 0, - "SatisfactionRank": { - "FreshRank": { - "Flag": "Green" - }, - "LevelRank": { - "Flag": "Green" - } - }, - "StoragePoolName": "dynamic_storage_pool:1", - "VDiskId": { - "Domain": 0, - "GroupGeneration": 1, - "GroupID": 2181038081, - "Ring": 0, - "VDisk": 0 - }, - "VDiskSlotId": 1001, - "VDiskState": "OK" + "name": "id", + "type": "Int64?" }, { - "AllocatedSize": "0", - "AvailableSize": "text", - "ChangeTime": "not-zero-number-text", - "DiskSpace": "Green", - "FrontQueues": "Green", - "Guid": "1", - "HasUnreadableBlobs": false, - "IncarnationGuid": "not-zero-number-text", - "InstanceGuid": "not-zero-number-text", - "Kind": "0", - "NodeId": 1, - "PDiskId": 1, - "Replicated": true, - "ReplicationProgress": 1, - "ReplicationSecondsRemaining": 0, - "SatisfactionRank": { - "FreshRank": { - "Flag": "Green" - }, - "LevelRank": { - "Flag": "Green" - } - }, - "StoragePoolName": "/Root/dedicated_db:hdd", - "VDiskId": { - "Domain": 0, - "GroupGeneration": 2, - "GroupID": 2181038082, - "Ring": 0, - "VDisk": 0 - }, - "VDiskSlotId": 1002, - "VDiskState": "OK" - }, - { - "AllocatedSize": "0", - "AvailableSize": "text", - "ChangeTime": "not-zero-number-text", - "DiskSpace": "Green", - "FrontQueues": "Green", - "Guid": "1", - "HasUnreadableBlobs": false, - "IncarnationGuid": "not-zero-number-text", - "InstanceGuid": "not-zero-number-text", - "Kind": "0", - "NodeId": 1, - "PDiskId": 1, - "Replicated": true, - "ReplicationProgress": 1, - "ReplicationSecondsRemaining": 0, - "SatisfactionRank": { - "FreshRank": { - "Flag": "Green" - }, - "LevelRank": { - "Flag": "Green" - } - }, - "StoragePoolName": "/Root/shared_db:hdd", - "VDiskId": { - "Domain": 0, - "GroupGeneration": 2, - "GroupID": 2181038083, - "Ring": 0, - "VDisk": 0 - }, - "VDiskSlotId": 1003, - "VDiskState": "OK" + "name": "name", + "type": "Utf8?" } + ], + "rows": [ + [ + "1", + "one" + ], + [ + "2", + "two" + ], + [ + "3", + "three" + ] ] } ], - "TotalNodes": "1" + "version": 9 + } + }, + "test.test_viewer_query_from_table_different_schemas": { + "classic": [ + { + "id": "1", + "name": "one" + }, + { + "id": "2", + "name": "two" + }, + { + "id": "3", + "name": "three" + } + ], + "modern": { + "columns": [ + { + "name": "id", + "type": "Int64?" + }, + { + "name": "name", + "type": "Utf8?" + } + ], + "result": [ + [ + "1", + "one" + ], + [ + "2", + "two" + ], + [ + "3", + "three" + ] + ], + "version": 9 }, - "/Root/dedicated_db": { - "FieldsAvailable": "11111111110111111111111111111111", - "FieldsRequired": "11111111111111111111111111111111", - "FoundNodes": "1", - "MaximumDisksPerNode": "1", - "MaximumSlotsPerDisk": "5", - "Nodes": [ + "multi": { + "result": [ { - "ClockSkewMaxUs": "text", - "ClockSkewMinUs": "text", - "ClockSkewUs": "text", - "ConnectStatus": "Green", - "Connections": "not-zero-number", - "CpuUsage": "not-zero-number", - "DiskSpaceUsage": "not-zero-number", - "NetworkUtilization": "number", - "NetworkUtilizationMax": "number", - "NetworkUtilizationMin": "number", - "NodeId": 1, - "PDisks": [ + "columns": [ { - "AvailableSize": "text", - "Category": "0", - "ChangeTime": "not-zero-number-text", - "Device": "Green", - "EnforcedDynamicSlotSize": "not-zero-number-text", - "Guid": "1", - "LogTotalSize": "not-zero-number-text", - "LogUsedSize": "not-zero-number-text", - "NodeId": 1, - "NumActiveSlots": 5, - "PDiskId": 1, - "Path": "SectorMap:1:64", - "Realtime": "Green", - "SerialNumber": "", - "State": "Normal", - "SystemSize": "not-zero-number-text", - "TotalSize": "not-zero-number-text" + "name": "id", + "type": "Int64?" + }, + { + "name": "name", + "type": "Utf8?" } ], - "Peers": [ + "rows": [ + [ + "1", + "one" + ], + [ + "2", + "two" + ], + [ + "3", + "three" + ] + ] + } + ], + "version": 9 + }, + "ydb": { + "result": [ + { + "id": 1, + "name": "one" + }, + { + "id": 2, + "name": "two" + }, + { + "id": 3, + "name": "three" + } + ], + "version": 9 + }, + "ydb2": { + "result": [ + { + "columns": [ { - "BytesWritten": "text", - "ChangeTime": "not-zero-number-text", - "ClockSkewUs": "text", - "ConnectStatus": "Green", - "ConnectTime": "not-zero-number-text", - "Connected": true, - "NodeId": 1, - "PeerName": "text", - "PeerNodeId": 50000, - "PingTimeUs": "text", - "ScopeId": { - "X1": "72057594046678944", - "X2": "6" - }, - "Utilization": "number" + "name": "id", + "type": "Int64?" }, { - "BytesWritten": "text", - "ChangeTime": "not-zero-number-text", - "ClockSkewUs": "text", - "ConnectStatus": "Green", - "ConnectTime": "not-zero-number-text", - "Connected": true, - "NodeId": 1, - "PeerName": "text", - "PeerNodeId": 50001, - "PingTimeUs": "text", - "ScopeId": { - "X1": "72057594046678944", - "X2": "7" - }, - "Utilization": "number" + "name": "name", + "type": "Utf8?" } ], - "PingTimeMaxUs": "text", - "PingTimeMinUs": "text", - "PingTimeUs": "text", - "ReceiveThroughput": "text", - "ReverseClockSkewUs": "text", - "ReversePeers": [ + "rows": [ { - "BytesWritten": "text", - "ChangeTime": "not-zero-number-text", - "ClockSkewUs": "text", - "ConnectStatus": "Green", - "ConnectTime": "not-zero-number-text", - "Connected": true, - "NodeId": 50000, - "PeerName": "text", - "PeerNodeId": 1, - "PingTimeUs": "text", - "ScopeId": { - "X1": "0", - "X2": "1" - }, - "Utilization": "number" + "id": 1, + "name": "one" }, { - "BytesWritten": "text", - "ChangeTime": "not-zero-number-text", - "ClockSkewUs": "text", - "ConnectStatus": "Green", - "ConnectTime": "not-zero-number-text", - "Connected": true, - "NodeId": 50001, - "PeerName": "text", - "PeerNodeId": 1, - "PingTimeUs": "text", - "ScopeId": { - "X1": "0", - "X2": "1" - }, - "Utilization": "number" + "id": 2, + "name": "two" + }, + { + "id": 3, + "name": "three" + } + ] + } + ], + "version": 9 + } + }, + "test.test_viewer_query_issue_13757": { + "/Root": { + "result": [ + { + "columns": [ + { + "name": "column0", + "type": "Struct<'three':Date?,'two':Utf8>" } ], - "ReversePingTimeUs": "text", - "SendThroughput": "text", - "SystemState": { - "ChangeTime": "not-zero-number-text", - "CoresTotal": "not-zero-number", - "CoresUsed": "not-zero-number", - "Endpoints": [ - { - "Address": "text", - "Name": "grpc" - }, - { - "Address": "text", - "Name": "http-mon" - }, + "rows": [ + [ { - "Address": "text", - "Name": "ic" + "three": null, + "two": "42" } - ], - "Host": "text", - "LoadAverage": "not-empty-array", - "Location": { - "DataCenter": "1", - "Rack": "1", - "Unit": "1" - }, - "MaxDiskUsage": "not-zero-number", - "MemoryLimit": "not-zero-number-text", - "MemoryStats": "not-empty-object", - "NodeId": 1, - "NumberOfCpus": "not-zero-number", - "PoolStats": [ - { - "Limit": "not-zero-number", - "Name": "System", - "Threads": 2, - "Usage": "number" - }, - { - "Limit": "not-zero-number", - "Name": "User", - "Threads": 3, - "Usage": "number" - }, + ] + ] + } + ], + "version": 9 + }, + "/Root/dedicated_db": { + "result": [ + { + "columns": [ + { + "name": "column0", + "type": "Struct<'three':Date?,'two':Utf8>" + } + ], + "rows": [ + [ { - "Limit": "not-zero-number", - "Name": "Batch", - "Threads": 2, - "Usage": "number" - }, + "three": null, + "two": "42" + } + ] + ] + } + ], + "version": 9 + }, + "/Root/serverless_db": { + "result": [ + { + "columns": [ + { + "name": "column0", + "type": "Struct<'three':Date?,'two':Utf8>" + } + ], + "rows": [ + [ { - "Limit": "not-zero-number", - "Name": "IO", - "Threads": 1, - "Usage": "number" - }, + "three": null, + "two": "42" + } + ] + ] + } + ], + "version": 9 + }, + "/Root/shared_db": { + "result": [ + { + "columns": [ + { + "name": "column0", + "type": "Struct<'three':Date?,'two':Utf8>" + } + ], + "rows": [ + [ { - "Limit": "not-zero-number", - "Name": "IC", - "Threads": 1, - "Usage": "number" + "three": null, + "two": "42" } - ], - "RealNumberOfCpus": "not-zero-number", - "Roles": "not-empty-array", - "StartTime": "not-zero-number-text", - "SystemState": "Green", - "TotalSessions": "number" - }, - "UptimeSeconds": "number", - "VDisks": [ + ] + ] + } + ], + "version": 9 + }, + "no-database": { + "result": [ + { + "columns": [ { - "AllocatedSize": "0", - "AvailableSize": "text", - "ChangeTime": "not-zero-number-text", - "DiskSpace": "Green", - "FrontQueues": "Green", - "Guid": "1", - "HasUnreadableBlobs": false, - "IncarnationGuid": "not-zero-number-text", - "InstanceGuid": "not-zero-number-text", - "Kind": "0", - "NodeId": 1, - "PDiskId": 1, - "Replicated": true, - "ReplicationProgress": 1, - "ReplicationSecondsRemaining": 0, - "SatisfactionRank": { - "FreshRank": { - "Flag": "Green" - }, - "LevelRank": { - "Flag": "Green" - } - }, - "StoragePoolName": "static", - "VDiskId": { - "Domain": 0, - "GroupGeneration": 1, - "GroupID": 0, - "Ring": 0, - "VDisk": 0 - }, - "VDiskSlotId": 0, - "VDiskState": "OK" - }, + "name": "column0", + "type": "Struct<'three':Date?,'two':Utf8>" + } + ], + "rows": [ + [ + { + "three": null, + "two": "42" + } + ] + ] + } + ], + "version": 9 + } + }, + "test.test_viewer_query_issue_13945": { + "/Root": { + "result": [ + { + "columns": [ { - "AllocatedSize": "0", - "AvailableSize": "text", - "ChangeTime": "not-zero-number-text", - "DiskSpace": "Green", - "FrontQueues": "Green", - "Guid": "1", - "HasUnreadableBlobs": false, - "IncarnationGuid": "not-zero-number-text", - "InstanceGuid": "not-zero-number-text", - "Kind": "0", - "NodeId": 1, - "PDiskId": 1, - "Replicated": true, - "ReplicationProgress": 1, - "ReplicationSecondsRemaining": 0, - "SatisfactionRank": { - "FreshRank": { - "Flag": "Green" - }, - "LevelRank": { - "Flag": "Green" - } - }, - "StoragePoolName": "dynamic_storage_pool:1", - "VDiskId": { - "Domain": 0, - "GroupGeneration": 1, - "GroupID": 2181038080, - "Ring": 0, - "VDisk": 0 - }, - "VDiskSlotId": 1000, - "VDiskState": "OK" - }, - { - "AllocatedSize": "0", - "AvailableSize": "text", - "ChangeTime": "not-zero-number-text", - "DiskSpace": "Green", - "FrontQueues": "Green", - "Guid": "1", - "HasUnreadableBlobs": false, - "IncarnationGuid": "not-zero-number-text", - "InstanceGuid": "not-zero-number-text", - "Kind": "0", - "NodeId": 1, - "PDiskId": 1, - "Replicated": true, - "ReplicationProgress": 1, - "ReplicationSecondsRemaining": 0, - "SatisfactionRank": { - "FreshRank": { - "Flag": "Green" - }, - "LevelRank": { - "Flag": "Green" - } - }, - "StoragePoolName": "dynamic_storage_pool:1", - "VDiskId": { - "Domain": 0, - "GroupGeneration": 1, - "GroupID": 2181038081, - "Ring": 0, - "VDisk": 0 - }, - "VDiskSlotId": 1001, - "VDiskState": "OK" - }, - { - "AllocatedSize": "0", - "AvailableSize": "text", - "ChangeTime": "not-zero-number-text", - "DiskSpace": "Green", - "FrontQueues": "Green", - "Guid": "1", - "HasUnreadableBlobs": false, - "IncarnationGuid": "not-zero-number-text", - "InstanceGuid": "not-zero-number-text", - "Kind": "0", - "NodeId": 1, - "PDiskId": 1, - "Replicated": true, - "ReplicationProgress": 1, - "ReplicationSecondsRemaining": 0, - "SatisfactionRank": { - "FreshRank": { - "Flag": "Green" - }, - "LevelRank": { - "Flag": "Green" - } - }, - "StoragePoolName": "/Root/dedicated_db:hdd", - "VDiskId": { - "Domain": 0, - "GroupGeneration": 2, - "GroupID": 2181038082, - "Ring": 0, - "VDisk": 0 - }, - "VDiskSlotId": 1002, - "VDiskState": "OK" - }, + "name": "column0", + "type": "EmptyList" + } + ], + "rows": [ + [ + [] + ] + ] + } + ], + "version": 9 + }, + "/Root/dedicated_db": { + "result": [ + { + "columns": [ { - "AllocatedSize": "0", - "AvailableSize": "text", - "ChangeTime": "not-zero-number-text", - "DiskSpace": "Green", - "FrontQueues": "Green", - "Guid": "1", - "HasUnreadableBlobs": false, - "IncarnationGuid": "not-zero-number-text", - "InstanceGuid": "not-zero-number-text", - "Kind": "0", - "NodeId": 1, - "PDiskId": 1, - "Replicated": true, - "ReplicationProgress": 1, - "ReplicationSecondsRemaining": 0, - "SatisfactionRank": { - "FreshRank": { - "Flag": "Green" - }, - "LevelRank": { - "Flag": "Green" - } - }, - "StoragePoolName": "/Root/shared_db:hdd", - "VDiskId": { - "Domain": 0, - "GroupGeneration": 2, - "GroupID": 2181038083, - "Ring": 0, - "VDisk": 0 - }, - "VDiskSlotId": 1003, - "VDiskState": "OK" + "name": "column0", + "type": "EmptyList" } + ], + "rows": [ + [ + [] + ] ] } ], - "TotalNodes": "1" + "version": 9 }, "/Root/serverless_db": { - "FieldsAvailable": "11111111110111111111111111111111", - "FieldsRequired": "11111111111111111111111111111111", - "FoundNodes": "1", - "MaximumDisksPerNode": "1", - "MaximumSlotsPerDisk": "5", - "Nodes": [ + "result": [ { - "ClockSkewMaxUs": "text", - "ClockSkewMinUs": "text", - "ClockSkewUs": "text", - "ConnectStatus": "Green", - "Connections": "not-zero-number", - "CpuUsage": "not-zero-number", - "DiskSpaceUsage": "not-zero-number", - "NetworkUtilization": "number", - "NetworkUtilizationMax": "number", - "NetworkUtilizationMin": "number", - "NodeId": 1, - "PDisks": [ + "columns": [ { - "AvailableSize": "text", - "Category": "0", - "ChangeTime": "not-zero-number-text", - "Device": "Green", - "EnforcedDynamicSlotSize": "not-zero-number-text", - "Guid": "1", - "LogTotalSize": "not-zero-number-text", - "LogUsedSize": "not-zero-number-text", - "NodeId": 1, - "NumActiveSlots": 5, - "PDiskId": 1, - "Path": "SectorMap:1:64", - "Realtime": "Green", - "SerialNumber": "", - "State": "Normal", - "SystemSize": "not-zero-number-text", - "TotalSize": "not-zero-number-text" + "name": "column0", + "type": "EmptyList" } ], - "Peers": [ - { - "BytesWritten": "text", - "ChangeTime": "not-zero-number-text", - "ClockSkewUs": "text", - "ConnectStatus": "Green", - "ConnectTime": "not-zero-number-text", - "Connected": true, - "NodeId": 1, - "PeerName": "text", - "PeerNodeId": 50000, - "PingTimeUs": "text", - "ScopeId": { - "X1": "72057594046678944", - "X2": "6" - }, - "Utilization": "number" - }, + "rows": [ + [ + [] + ] + ] + } + ], + "version": 9 + }, + "/Root/shared_db": { + "result": [ + { + "columns": [ { - "BytesWritten": "text", - "ChangeTime": "not-zero-number-text", - "ClockSkewUs": "text", - "ConnectStatus": "Green", - "ConnectTime": "not-zero-number-text", - "Connected": true, - "NodeId": 1, - "PeerName": "text", - "PeerNodeId": 50001, - "PingTimeUs": "text", - "ScopeId": { - "X1": "72057594046678944", - "X2": "7" - }, - "Utilization": "number" + "name": "column0", + "type": "EmptyList" } ], - "PingTimeMaxUs": "text", - "PingTimeMinUs": "text", - "PingTimeUs": "text", - "ReceiveThroughput": "text", - "ReverseClockSkewUs": "text", - "ReversePeers": [ + "rows": [ + [ + [] + ] + ] + } + ], + "version": 9 + }, + "no-database": { + "result": [ + { + "columns": [ { - "BytesWritten": "text", - "ChangeTime": "not-zero-number-text", - "ClockSkewUs": "text", - "ConnectStatus": "Green", - "ConnectTime": "not-zero-number-text", - "Connected": true, - "NodeId": 50000, - "PeerName": "text", - "PeerNodeId": 1, - "PingTimeUs": "text", - "ScopeId": { - "X1": "0", - "X2": "1" - }, - "Utilization": "number" + "name": "column0", + "type": "EmptyList" + } + ], + "rows": [ + [ + [] + ] + ] + } + ], + "version": 9 + } + }, + "test.test_viewer_query_long": { + "execute_response": { + "execution_id": "text", + "operation_id": "text", + "result": [ + { + "columns": [ + { + "name": "id", + "type": "Int64?" }, { - "BytesWritten": "text", - "ChangeTime": "not-zero-number-text", - "ClockSkewUs": "text", - "ConnectStatus": "Green", - "ConnectTime": "not-zero-number-text", - "Connected": true, - "NodeId": 50001, - "PeerName": "text", - "PeerNodeId": 1, - "PingTimeUs": "text", - "ScopeId": { - "X1": "0", - "X2": "1" - }, - "Utilization": "number" + "name": "name", + "type": "Utf8?" } ], - "ReversePingTimeUs": "text", - "SendThroughput": "text", - "SystemState": { - "ChangeTime": "not-zero-number-text", - "CoresTotal": "not-zero-number", - "CoresUsed": "not-zero-number", - "Endpoints": [ - { - "Address": "text", - "Name": "grpc" - }, - { - "Address": "text", - "Name": "http-mon" - }, - { - "Address": "text", - "Name": "ic" - } + "rows": [ + [ + "1", + "one" ], - "Host": "text", - "LoadAverage": "not-empty-array", - "Location": { - "DataCenter": "1", - "Rack": "1", - "Unit": "1" - }, - "MaxDiskUsage": "not-zero-number", - "MemoryLimit": "not-zero-number-text", - "MemoryStats": "not-empty-object", - "NodeId": 1, - "NumberOfCpus": "not-zero-number", - "PoolStats": [ - { - "Limit": "not-zero-number", - "Name": "System", - "Threads": 2, - "Usage": "number" - }, - { - "Limit": "not-zero-number", - "Name": "User", - "Threads": 3, - "Usage": "number" - }, - { - "Limit": "not-zero-number", - "Name": "Batch", - "Threads": 2, - "Usage": "number" - }, - { - "Limit": "not-zero-number", - "Name": "IO", - "Threads": 1, - "Usage": "number" - }, - { - "Limit": "not-zero-number", - "Name": "IC", - "Threads": 1, - "Usage": "number" - } + [ + "2", + "two" ], - "RealNumberOfCpus": "not-zero-number", - "Roles": "not-empty-array", - "StartTime": "not-zero-number-text", - "SystemState": "Green", - "TotalSessions": "number" - }, - "UptimeSeconds": "number", - "VDisks": [ + [ + "3", + "three" + ] + ] + } + ], + "version": 9 + }, + "fetch_by_execution_id": { + "execution_id": "text", + "result": [ + { + "columns": [ { - "AllocatedSize": "0", - "AvailableSize": "text", - "ChangeTime": "not-zero-number-text", - "DiskSpace": "Green", - "FrontQueues": "Green", - "Guid": "1", - "HasUnreadableBlobs": false, - "IncarnationGuid": "not-zero-number-text", - "InstanceGuid": "not-zero-number-text", - "Kind": "0", - "NodeId": 1, - "PDiskId": 1, - "Replicated": true, - "ReplicationProgress": 1, - "ReplicationSecondsRemaining": 0, - "SatisfactionRank": { - "FreshRank": { - "Flag": "Green" - }, - "LevelRank": { - "Flag": "Green" - } - }, - "StoragePoolName": "static", - "VDiskId": { - "Domain": 0, - "GroupGeneration": 1, - "GroupID": 0, - "Ring": 0, - "VDisk": 0 - }, - "VDiskSlotId": 0, - "VDiskState": "OK" + "name": "id", + "type": "Int64?" }, { - "AllocatedSize": "0", - "AvailableSize": "text", - "ChangeTime": "not-zero-number-text", - "DiskSpace": "Green", - "FrontQueues": "Green", - "Guid": "1", - "HasUnreadableBlobs": false, - "IncarnationGuid": "not-zero-number-text", - "InstanceGuid": "not-zero-number-text", - "Kind": "0", - "NodeId": 1, - "PDiskId": 1, - "Replicated": true, - "ReplicationProgress": 1, - "ReplicationSecondsRemaining": 0, - "SatisfactionRank": { - "FreshRank": { - "Flag": "Green" - }, - "LevelRank": { - "Flag": "Green" - } - }, - "StoragePoolName": "dynamic_storage_pool:1", - "VDiskId": { - "Domain": 0, - "GroupGeneration": 1, - "GroupID": 2181038080, - "Ring": 0, - "VDisk": 0 - }, - "VDiskSlotId": 1000, - "VDiskState": "OK" + "name": "name", + "type": "Utf8?" + } + ], + "rows": [ + [ + "1", + "one" + ], + [ + "2", + "two" + ], + [ + "3", + "three" + ] + ] + } + ], + "version": 9 + }, + "fetch_by_operation_id": { + "execution_id": "text", + "operation_id": "text", + "result": [ + { + "columns": [ + { + "name": "id", + "type": "Int64?" }, { - "AllocatedSize": "0", - "AvailableSize": "text", - "ChangeTime": "not-zero-number-text", - "DiskSpace": "Green", - "FrontQueues": "Green", - "Guid": "1", - "HasUnreadableBlobs": false, - "IncarnationGuid": "not-zero-number-text", - "InstanceGuid": "not-zero-number-text", - "Kind": "0", - "NodeId": 1, - "PDiskId": 1, - "Replicated": true, - "ReplicationProgress": 1, - "ReplicationSecondsRemaining": 0, - "SatisfactionRank": { - "FreshRank": { - "Flag": "Green" - }, - "LevelRank": { - "Flag": "Green" - } - }, - "StoragePoolName": "dynamic_storage_pool:1", - "VDiskId": { - "Domain": 0, - "GroupGeneration": 1, - "GroupID": 2181038081, - "Ring": 0, - "VDisk": 0 - }, - "VDiskSlotId": 1001, - "VDiskState": "OK" - }, - { - "AllocatedSize": "0", - "AvailableSize": "text", - "ChangeTime": "not-zero-number-text", - "DiskSpace": "Green", - "FrontQueues": "Green", - "Guid": "1", - "HasUnreadableBlobs": false, - "IncarnationGuid": "not-zero-number-text", - "InstanceGuid": "not-zero-number-text", - "Kind": "0", - "NodeId": 1, - "PDiskId": 1, - "Replicated": true, - "ReplicationProgress": 1, - "ReplicationSecondsRemaining": 0, - "SatisfactionRank": { - "FreshRank": { - "Flag": "Green" - }, - "LevelRank": { - "Flag": "Green" - } - }, - "StoragePoolName": "/Root/dedicated_db:hdd", - "VDiskId": { - "Domain": 0, - "GroupGeneration": 2, - "GroupID": 2181038082, - "Ring": 0, - "VDisk": 0 - }, - "VDiskSlotId": 1002, - "VDiskState": "OK" - }, - { - "AllocatedSize": "0", - "AvailableSize": "text", - "ChangeTime": "not-zero-number-text", - "DiskSpace": "Green", - "FrontQueues": "Green", - "Guid": "1", - "HasUnreadableBlobs": false, - "IncarnationGuid": "not-zero-number-text", - "InstanceGuid": "not-zero-number-text", - "Kind": "0", - "NodeId": 1, - "PDiskId": 1, - "Replicated": true, - "ReplicationProgress": 1, - "ReplicationSecondsRemaining": 0, - "SatisfactionRank": { - "FreshRank": { - "Flag": "Green" - }, - "LevelRank": { - "Flag": "Green" - } - }, - "StoragePoolName": "/Root/shared_db:hdd", - "VDiskId": { - "Domain": 0, - "GroupGeneration": 2, - "GroupID": 2181038083, - "Ring": 0, - "VDisk": 0 - }, - "VDiskSlotId": 1003, - "VDiskState": "OK" + "name": "name", + "type": "Utf8?" } + ], + "rows": [ + [ + "1", + "one" + ], + [ + "2", + "two" + ], + [ + "3", + "three" + ] ] } ], - "TotalNodes": "1" + "version": 9 }, - "/Root/shared_db": { - "FieldsAvailable": "11111111110111111111111111111111", - "FieldsRequired": "11111111111111111111111111111111", + "fetch_error_no_id": { + "status_code": 400, + "text": "operation_id or execution_id required for fetch-long-query" + }, + "fetch_invalid_operation_id": { + "status_code": 400, + "text": "operation_id or execution_id required for fetch-long-query" + } + }, + "test.test_viewer_query_long_multipart": { + "execute_stream_response": { + "multipart_parts": [ + { + "exec_mode": "EXEC_MODE_EXECUTE", + "exec_status": "EXEC_STATUS_STARTING", + "execution_id": "text", + "meta": { + "event": "ScriptResponse" + }, + "operation_id": "text", + "status": "SUCCESS" + }, + { + "id": "text", + "meta": { + "event": "OperationResponse" + }, + "metadata": { + "exec_mode": "EXEC_MODE_EXECUTE", + "exec_stats": { + "query_plan": "{}" + }, + "exec_status": "EXEC_STATUS_COMPLETED", + "execution_id": "text", + "result_sets_meta": [ + { + "columns": [ + { + "name": "id", + "type": { + "optional_type": { + "item": { + "type_id": "INT64" + } + } + } + }, + { + "name": "name", + "type": { + "optional_type": { + "item": { + "type_id": "UTF8" + } + } + } + } + ], + "finished": true, + "number_rows": "3" + } + ], + "script_content": { + "text": "SELECT * FROM table1 LIMIT 3;" + } + }, + "ready": true + }, + { + "meta": { + "event": "StreamData", + "result_index": 0, + "seq_no": 0 + }, + "result": { + "columns": [ + { + "name": "id", + "type": "Int64?" + }, + { + "name": "name", + "type": "Utf8?" + } + ], + "rows": [ + [ + "1", + "one" + ], + [ + "2", + "two" + ], + [ + "3", + "three" + ] + ] + } + } + ] + }, + "fetch_error_stream_response": { + "status_code": 400, + "text": "operation_id or execution_id required for fetch-long-query" + }, + "fetch_invalid_stream_response": { + "status_code": 400, + "text": "operation_id or execution_id required for fetch-long-query" + } + }, + "test.test_viewer_storage_nodes": { + "/Root": { + "FieldsAvailable": "10000000110000110111111100000111", + "FieldsRequired": "00000000000000000000000000000111", "FoundNodes": "1", - "MaximumDisksPerNode": "1", "MaximumSlotsPerDisk": "5", "Nodes": [ { - "ClockSkewMaxUs": "text", - "ClockSkewMinUs": "text", - "ClockSkewUs": "text", - "ConnectStatus": "Green", - "Connections": "not-zero-number", - "CpuUsage": "not-zero-number", - "DiskSpaceUsage": "not-zero-number", - "NetworkUtilization": "number", - "NetworkUtilizationMax": "number", - "NetworkUtilizationMin": "number", "NodeId": 1, - "PDisks": [ - { - "AvailableSize": "text", - "Category": "0", - "ChangeTime": "not-zero-number-text", - "Device": "Green", - "EnforcedDynamicSlotSize": "not-zero-number-text", - "Guid": "1", - "LogTotalSize": "not-zero-number-text", - "LogUsedSize": "not-zero-number-text", - "NodeId": 1, - "NumActiveSlots": 5, - "PDiskId": 1, - "Path": "SectorMap:1:64", - "Realtime": "Green", - "SerialNumber": "", - "State": "Normal", - "SystemSize": "not-zero-number-text", - "TotalSize": "not-zero-number-text" - } - ], - "Peers": [ - { - "BytesWritten": "text", - "ChangeTime": "not-zero-number-text", - "ClockSkewUs": "text", - "ConnectStatus": "Green", - "ConnectTime": "not-zero-number-text", - "Connected": true, - "NodeId": 1, - "PeerName": "text", - "PeerNodeId": 50000, - "PingTimeUs": "text", - "ScopeId": { - "X1": "72057594046678944", - "X2": "6" + "SystemState": { + "ChangeTime": "not-zero-number-text", + "CoresTotal": "not-zero-number", + "CoresUsed": "not-zero-number", + "Endpoints": [ + { + "Address": "text", + "Name": "grpc" }, - "Utilization": "number" + { + "Address": "text", + "Name": "http-mon" + }, + { + "Address": "text", + "Name": "ic" + } + ], + "Host": "text", + "LoadAverage": "not-empty-array", + "Location": { + "DataCenter": "1", + "Rack": "1", + "Unit": "1" }, - { - "BytesWritten": "text", - "ChangeTime": "not-zero-number-text", - "ClockSkewUs": "text", - "ConnectStatus": "Green", - "ConnectTime": "not-zero-number-text", - "Connected": true, - "NodeId": 1, - "PeerName": "text", - "PeerNodeId": 50001, - "PingTimeUs": "text", - "ScopeId": { - "X1": "72057594046678944", - "X2": "7" + "MaxDiskUsage": "not-zero-number", + "MemoryLimit": "not-zero-number-text", + "NodeId": 1, + "NumberOfCpus": "not-zero-number", + "PoolStats": [ + { + "Limit": "not-zero-number", + "Name": "System", + "Threads": 2, + "Usage": "number" }, - "Utilization": "number" - } - ], - "PingTimeMaxUs": "text", - "PingTimeMinUs": "text", - "PingTimeUs": "text", - "ReceiveThroughput": "text", - "ReverseClockSkewUs": "text", - "ReversePeers": [ - { - "BytesWritten": "text", - "ChangeTime": "not-zero-number-text", - "ClockSkewUs": "text", - "ConnectStatus": "Green", - "ConnectTime": "not-zero-number-text", - "Connected": true, - "NodeId": 50000, - "PeerName": "text", - "PeerNodeId": 1, - "PingTimeUs": "text", - "ScopeId": { - "X1": "0", - "X2": "1" + { + "Limit": "not-zero-number", + "Name": "User", + "Threads": 3, + "Usage": "number" }, - "Utilization": "number" - }, - { - "BytesWritten": "text", - "ChangeTime": "not-zero-number-text", - "ClockSkewUs": "text", - "ConnectStatus": "Green", - "ConnectTime": "not-zero-number-text", - "Connected": true, - "NodeId": 50001, - "PeerName": "text", - "PeerNodeId": 1, - "PingTimeUs": "text", - "ScopeId": { - "X1": "0", - "X2": "1" + { + "Limit": "not-zero-number", + "Name": "Batch", + "Threads": 2, + "Usage": "number" }, - "Utilization": "number" - } - ], - "ReversePingTimeUs": "text", - "SendThroughput": "text", + { + "Limit": "not-zero-number", + "Name": "IO", + "Threads": 1, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IC", + "Threads": 1, + "Usage": "number" + } + ], + "RealNumberOfCpus": "not-zero-number", + "Roles": "not-empty-array", + "StartTime": "not-zero-number-text", + "SystemState": "Green", + "TotalSessions": "number" + } + } + ], + "TotalNodes": "1" + }, + "/Root/dedicated_db": { + "FieldsAvailable": "10000000110000110111111100000111", + "FieldsRequired": "00000000000000000000000000000111", + "FoundNodes": "1", + "MaximumSlotsPerDisk": "5", + "Nodes": [ + { + "NodeId": 1, "SystemState": { "ChangeTime": "not-zero-number-text", "CoresTotal": "not-zero-number", @@ -6941,7 +6879,6 @@ }, "MaxDiskUsage": "not-zero-number", "MemoryLimit": "not-zero-number-text", - "MemoryStats": "not-empty-object", "NodeId": 1, "NumberOfCpus": "not-zero-number", "PoolStats": [ @@ -6981,395 +6918,3095 @@ "StartTime": "not-zero-number-text", "SystemState": "Green", "TotalSessions": "number" - }, - "UptimeSeconds": "number", - "VDisks": [ - { - "AllocatedSize": "0", - "AvailableSize": "text", - "ChangeTime": "not-zero-number-text", - "DiskSpace": "Green", - "FrontQueues": "Green", - "Guid": "1", - "HasUnreadableBlobs": false, - "IncarnationGuid": "not-zero-number-text", - "InstanceGuid": "not-zero-number-text", - "Kind": "0", - "NodeId": 1, - "PDiskId": 1, - "Replicated": true, - "ReplicationProgress": 1, - "ReplicationSecondsRemaining": 0, - "SatisfactionRank": { - "FreshRank": { - "Flag": "Green" - }, - "LevelRank": { - "Flag": "Green" - } + } + } + ], + "TotalNodes": "1" + }, + "/Root/serverless_db": { + "FieldsAvailable": "10000000110000110111111100000111", + "FieldsRequired": "00000000000000000000000000000111", + "FoundNodes": "1", + "MaximumSlotsPerDisk": "5", + "Nodes": [ + { + "NodeId": 1, + "SystemState": { + "ChangeTime": "not-zero-number-text", + "CoresTotal": "not-zero-number", + "CoresUsed": "not-zero-number", + "Endpoints": [ + { + "Address": "text", + "Name": "grpc" }, - "StoragePoolName": "static", - "VDiskId": { - "Domain": 0, - "GroupGeneration": 1, - "GroupID": 0, - "Ring": 0, - "VDisk": 0 + { + "Address": "text", + "Name": "http-mon" }, - "VDiskSlotId": 0, - "VDiskState": "OK" + { + "Address": "text", + "Name": "ic" + } + ], + "Host": "text", + "LoadAverage": "not-empty-array", + "Location": { + "DataCenter": "1", + "Rack": "1", + "Unit": "1" + }, + "MaxDiskUsage": "not-zero-number", + "MemoryLimit": "not-zero-number-text", + "NodeId": 1, + "NumberOfCpus": "not-zero-number", + "PoolStats": [ + { + "Limit": "not-zero-number", + "Name": "System", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "User", + "Threads": 3, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "Batch", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IO", + "Threads": 1, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IC", + "Threads": 1, + "Usage": "number" + } + ], + "RealNumberOfCpus": "not-zero-number", + "Roles": "not-empty-array", + "StartTime": "not-zero-number-text", + "SystemState": "Green", + "TotalSessions": "number" + } + } + ], + "Problems": [ + "no-database-board-info" + ], + "TotalNodes": "1" + }, + "/Root/shared_db": { + "FieldsAvailable": "10000000110000110111111100000111", + "FieldsRequired": "00000000000000000000000000000111", + "FoundNodes": "1", + "MaximumSlotsPerDisk": "5", + "Nodes": [ + { + "NodeId": 1, + "SystemState": { + "ChangeTime": "not-zero-number-text", + "CoresTotal": "not-zero-number", + "CoresUsed": "not-zero-number", + "Endpoints": [ + { + "Address": "text", + "Name": "grpc" + }, + { + "Address": "text", + "Name": "http-mon" + }, + { + "Address": "text", + "Name": "ic" + } + ], + "Host": "text", + "LoadAverage": "not-empty-array", + "Location": { + "DataCenter": "1", + "Rack": "1", + "Unit": "1" + }, + "MaxDiskUsage": "not-zero-number", + "MemoryLimit": "not-zero-number-text", + "NodeId": 1, + "NumberOfCpus": "not-zero-number", + "PoolStats": [ + { + "Limit": "not-zero-number", + "Name": "System", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "User", + "Threads": 3, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "Batch", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IO", + "Threads": 1, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IC", + "Threads": 1, + "Usage": "number" + } + ], + "RealNumberOfCpus": "not-zero-number", + "Roles": "not-empty-array", + "StartTime": "not-zero-number-text", + "SystemState": "Green", + "TotalSessions": "number" + } + } + ], + "TotalNodes": "1" + }, + "no-database": { + "FieldsAvailable": "10000000010100110111111100000111", + "FieldsRequired": "00000000000100000000000000000111", + "FoundNodes": "3", + "NeedFilter": true, + "Nodes": [ + { + "NodeId": 1, + "SystemState": { + "ChangeTime": "not-zero-number-text", + "CoresTotal": "not-zero-number", + "CoresUsed": "not-zero-number", + "Endpoints": [ + { + "Address": "text", + "Name": "grpc" + }, + { + "Address": "text", + "Name": "http-mon" + }, + { + "Address": "text", + "Name": "ic" + } + ], + "Host": "text", + "LoadAverage": "not-empty-array", + "Location": { + "DataCenter": "1", + "Rack": "1", + "Unit": "1" + }, + "MaxDiskUsage": "not-zero-number", + "MemoryLimit": "not-zero-number-text", + "NodeId": 1, + "NumberOfCpus": "not-zero-number", + "PoolStats": [ + { + "Limit": "not-zero-number", + "Name": "System", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "User", + "Threads": 3, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "Batch", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IO", + "Threads": 1, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IC", + "Threads": 1, + "Usage": "number" + } + ], + "RealNumberOfCpus": "not-zero-number", + "Roles": "not-empty-array", + "StartTime": "not-zero-number-text", + "SystemState": "Green", + "TotalSessions": "number" + } + }, + { + "NodeId": 50001, + "SystemState": { + "ChangeTime": "not-zero-number-text", + "CoresTotal": "not-zero-number", + "CoresUsed": "not-zero-number", + "Endpoints": [ + { + "Address": "text", + "Name": "grpc" + }, + { + "Address": "text", + "Name": "http-mon" + }, + { + "Address": "text", + "Name": "ic" + } + ], + "Host": "text", + "LoadAverage": "not-empty-array", + "Location": {}, + "MemoryLimit": "not-zero-number-text", + "NodeId": 50001, + "NumberOfCpus": "not-zero-number", + "PoolStats": [ + { + "Limit": "not-zero-number", + "Name": "System", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "User", + "Threads": 3, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "Batch", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IO", + "Threads": 1, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IC", + "Threads": 1, + "Usage": "number" + } + ], + "RealNumberOfCpus": "not-zero-number", + "Roles": "not-empty-array", + "StartTime": "not-zero-number-text", + "SystemState": "Green", + "Tenants": [ + "/Root/shared_db" + ], + "TotalSessions": "number" + } + }, + { + "NodeId": 50000, + "SystemState": { + "ChangeTime": "not-zero-number-text", + "CoresTotal": "not-zero-number", + "CoresUsed": "not-zero-number", + "Endpoints": [ + { + "Address": "text", + "Name": "grpc" + }, + { + "Address": "text", + "Name": "http-mon" + }, + { + "Address": "text", + "Name": "ic" + } + ], + "Host": "text", + "LoadAverage": "not-empty-array", + "Location": {}, + "MemoryLimit": "not-zero-number-text", + "NodeId": 50000, + "NumberOfCpus": "not-zero-number", + "PoolStats": [ + { + "Limit": "not-zero-number", + "Name": "System", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "User", + "Threads": 3, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "Batch", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IO", + "Threads": 1, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IC", + "Threads": 1, + "Usage": "number" + } + ], + "RealNumberOfCpus": "not-zero-number", + "Roles": "not-empty-array", + "StartTime": "not-zero-number-text", + "SystemState": "Green", + "Tenants": [ + "/Root/dedicated_db" + ], + "TotalSessions": "number" + } + } + ], + "TotalNodes": "3" + } + }, + "test.test_viewer_storage_nodes_all": { + "/Root": { + "FieldsAvailable": "11111111110111111111111111111111", + "FieldsRequired": "11111111111111111111111111111111", + "FoundNodes": "1", + "MaximumDisksPerNode": "1", + "MaximumSlotsPerDisk": "5", + "Nodes": [ + { + "ClockSkewMaxUs": "text", + "ClockSkewMinUs": "text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "Connections": "not-zero-number", + "CpuUsage": "not-zero-number", + "Database": "/Root", + "DiskSpaceUsage": "not-zero-number", + "NetworkUtilization": "number", + "NetworkUtilizationMax": "number", + "NetworkUtilizationMin": "number", + "NodeId": 1, + "PDisks": [ + { + "AvailableSize": "text", + "Category": "0", + "ChangeTime": "not-zero-number-text", + "Device": "Green", + "EnforcedDynamicSlotSize": "not-zero-number-text", + "Guid": "1", + "LogTotalSize": "not-zero-number-text", + "LogUsedSize": "not-zero-number-text", + "NodeId": 1, + "NumActiveSlots": 5, + "PDiskId": 1, + "Path": "SectorMap:1:64", + "Realtime": "Green", + "SerialNumber": "", + "State": "Normal", + "SystemSize": "not-zero-number-text", + "TotalSize": "not-zero-number-text" + } + ], + "Peers": [ + { + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 1, + "PeerName": "text", + "PeerNodeId": 50000, + "PingTimeUs": "text", + "ScopeId": { + "X1": "72057594046678944", + "X2": "6" + }, + "Utilization": "number" + }, + { + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 1, + "PeerName": "text", + "PeerNodeId": 50001, + "PingTimeUs": "text", + "ScopeId": { + "X1": "72057594046678944", + "X2": "7" + }, + "Utilization": "number" + } + ], + "PingTimeMaxUs": "text", + "PingTimeMinUs": "text", + "PingTimeUs": "text", + "ReceiveThroughput": "text", + "ReverseClockSkewUs": "text", + "ReversePeers": [ + { + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 50000, + "PeerName": "text", + "PeerNodeId": 1, + "PingTimeUs": "text", + "ScopeId": { + "X1": "0", + "X2": "1" + }, + "Utilization": "number" + }, + { + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 50001, + "PeerName": "text", + "PeerNodeId": 1, + "PingTimeUs": "text", + "ScopeId": { + "X1": "0", + "X2": "1" + }, + "Utilization": "number" + } + ], + "ReversePingTimeUs": "text", + "SendThroughput": "text", + "SystemState": { + "ChangeTime": "not-zero-number-text", + "CoresTotal": "not-zero-number", + "CoresUsed": "not-zero-number", + "Endpoints": [ + { + "Address": "text", + "Name": "grpc" + }, + { + "Address": "text", + "Name": "http-mon" + }, + { + "Address": "text", + "Name": "ic" + } + ], + "Host": "text", + "LoadAverage": "not-empty-array", + "Location": { + "DataCenter": "1", + "Rack": "1", + "Unit": "1" + }, + "MaxDiskUsage": "not-zero-number", + "MemoryLimit": "not-zero-number-text", + "MemoryStats": "not-empty-object", + "NodeId": 1, + "NumberOfCpus": "not-zero-number", + "PoolStats": [ + { + "Limit": "not-zero-number", + "Name": "System", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "User", + "Threads": 3, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "Batch", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IO", + "Threads": 1, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IC", + "Threads": 1, + "Usage": "number" + } + ], + "RealNumberOfCpus": "not-zero-number", + "Roles": "not-empty-array", + "StartTime": "not-zero-number-text", + "SystemState": "Green", + "TotalSessions": "number" + }, + "UptimeSeconds": "number", + "VDisks": [ + { + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "static", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 1, + "GroupID": 0, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 0, + "VDiskState": "OK" + }, + { + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "dynamic_storage_pool:1", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 1, + "GroupID": 2181038080, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 1000, + "VDiskState": "OK" + }, + { + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "dynamic_storage_pool:1", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 1, + "GroupID": 2181038081, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 1001, + "VDiskState": "OK" + }, + { + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "/Root/dedicated_db:hdd", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 2, + "GroupID": 2181038082, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 1002, + "VDiskState": "OK" + }, + { + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "/Root/shared_db:hdd", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 2, + "GroupID": 2181038083, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 1003, + "VDiskState": "OK" + } + ] + } + ], + "TotalNodes": "1" + }, + "/Root/dedicated_db": { + "FieldsAvailable": "11111111110111111111111111111111", + "FieldsRequired": "11111111111111111111111111111111", + "FoundNodes": "1", + "MaximumDisksPerNode": "1", + "MaximumSlotsPerDisk": "5", + "Nodes": [ + { + "ClockSkewMaxUs": "text", + "ClockSkewMinUs": "text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "Connections": "not-zero-number", + "CpuUsage": "not-zero-number", + "DiskSpaceUsage": "not-zero-number", + "NetworkUtilization": "number", + "NetworkUtilizationMax": "number", + "NetworkUtilizationMin": "number", + "NodeId": 1, + "PDisks": [ + { + "AvailableSize": "text", + "Category": "0", + "ChangeTime": "not-zero-number-text", + "Device": "Green", + "EnforcedDynamicSlotSize": "not-zero-number-text", + "Guid": "1", + "LogTotalSize": "not-zero-number-text", + "LogUsedSize": "not-zero-number-text", + "NodeId": 1, + "NumActiveSlots": 5, + "PDiskId": 1, + "Path": "SectorMap:1:64", + "Realtime": "Green", + "SerialNumber": "", + "State": "Normal", + "SystemSize": "not-zero-number-text", + "TotalSize": "not-zero-number-text" + } + ], + "Peers": [ + { + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 1, + "PeerName": "text", + "PeerNodeId": 50000, + "PingTimeUs": "text", + "ScopeId": { + "X1": "72057594046678944", + "X2": "6" + }, + "Utilization": "number" + }, + { + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 1, + "PeerName": "text", + "PeerNodeId": 50001, + "PingTimeUs": "text", + "ScopeId": { + "X1": "72057594046678944", + "X2": "7" + }, + "Utilization": "number" + } + ], + "PingTimeMaxUs": "text", + "PingTimeMinUs": "text", + "PingTimeUs": "text", + "ReceiveThroughput": "text", + "ReverseClockSkewUs": "text", + "ReversePeers": [ + { + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 50000, + "PeerName": "text", + "PeerNodeId": 1, + "PingTimeUs": "text", + "ScopeId": { + "X1": "0", + "X2": "1" + }, + "Utilization": "number" + }, + { + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 50001, + "PeerName": "text", + "PeerNodeId": 1, + "PingTimeUs": "text", + "ScopeId": { + "X1": "0", + "X2": "1" + }, + "Utilization": "number" + } + ], + "ReversePingTimeUs": "text", + "SendThroughput": "text", + "SystemState": { + "ChangeTime": "not-zero-number-text", + "CoresTotal": "not-zero-number", + "CoresUsed": "not-zero-number", + "Endpoints": [ + { + "Address": "text", + "Name": "grpc" + }, + { + "Address": "text", + "Name": "http-mon" + }, + { + "Address": "text", + "Name": "ic" + } + ], + "Host": "text", + "LoadAverage": "not-empty-array", + "Location": { + "DataCenter": "1", + "Rack": "1", + "Unit": "1" + }, + "MaxDiskUsage": "not-zero-number", + "MemoryLimit": "not-zero-number-text", + "MemoryStats": "not-empty-object", + "NodeId": 1, + "NumberOfCpus": "not-zero-number", + "PoolStats": [ + { + "Limit": "not-zero-number", + "Name": "System", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "User", + "Threads": 3, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "Batch", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IO", + "Threads": 1, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IC", + "Threads": 1, + "Usage": "number" + } + ], + "RealNumberOfCpus": "not-zero-number", + "Roles": "not-empty-array", + "StartTime": "not-zero-number-text", + "SystemState": "Green", + "TotalSessions": "number" + }, + "UptimeSeconds": "number", + "VDisks": [ + { + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "static", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 1, + "GroupID": 0, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 0, + "VDiskState": "OK" + }, + { + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "dynamic_storage_pool:1", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 1, + "GroupID": 2181038080, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 1000, + "VDiskState": "OK" + }, + { + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "dynamic_storage_pool:1", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 1, + "GroupID": 2181038081, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 1001, + "VDiskState": "OK" + }, + { + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "/Root/dedicated_db:hdd", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 2, + "GroupID": 2181038082, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 1002, + "VDiskState": "OK" + }, + { + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "/Root/shared_db:hdd", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 2, + "GroupID": 2181038083, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 1003, + "VDiskState": "OK" + } + ] + } + ], + "TotalNodes": "1" + }, + "/Root/serverless_db": { + "FieldsAvailable": "11111111110111111111111111111111", + "FieldsRequired": "11111111111111111111111111111111", + "FoundNodes": "1", + "MaximumDisksPerNode": "1", + "MaximumSlotsPerDisk": "5", + "Nodes": [ + { + "ClockSkewMaxUs": "text", + "ClockSkewMinUs": "text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "Connections": "not-zero-number", + "CpuUsage": "not-zero-number", + "DiskSpaceUsage": "not-zero-number", + "NetworkUtilization": "number", + "NetworkUtilizationMax": "number", + "NetworkUtilizationMin": "number", + "NodeId": 1, + "PDisks": [ + { + "AvailableSize": "text", + "Category": "0", + "ChangeTime": "not-zero-number-text", + "Device": "Green", + "EnforcedDynamicSlotSize": "not-zero-number-text", + "Guid": "1", + "LogTotalSize": "not-zero-number-text", + "LogUsedSize": "not-zero-number-text", + "NodeId": 1, + "NumActiveSlots": 5, + "PDiskId": 1, + "Path": "SectorMap:1:64", + "Realtime": "Green", + "SerialNumber": "", + "State": "Normal", + "SystemSize": "not-zero-number-text", + "TotalSize": "not-zero-number-text" + } + ], + "Peers": [ + { + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 1, + "PeerName": "text", + "PeerNodeId": 50000, + "PingTimeUs": "text", + "ScopeId": { + "X1": "72057594046678944", + "X2": "6" + }, + "Utilization": "number" + }, + { + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 1, + "PeerName": "text", + "PeerNodeId": 50001, + "PingTimeUs": "text", + "ScopeId": { + "X1": "72057594046678944", + "X2": "7" + }, + "Utilization": "number" + } + ], + "PingTimeMaxUs": "text", + "PingTimeMinUs": "text", + "PingTimeUs": "text", + "ReceiveThroughput": "text", + "ReverseClockSkewUs": "text", + "ReversePeers": [ + { + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 50000, + "PeerName": "text", + "PeerNodeId": 1, + "PingTimeUs": "text", + "ScopeId": { + "X1": "0", + "X2": "1" + }, + "Utilization": "number" + }, + { + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 50001, + "PeerName": "text", + "PeerNodeId": 1, + "PingTimeUs": "text", + "ScopeId": { + "X1": "0", + "X2": "1" + }, + "Utilization": "number" + } + ], + "ReversePingTimeUs": "text", + "SendThroughput": "text", + "SystemState": { + "ChangeTime": "not-zero-number-text", + "CoresTotal": "not-zero-number", + "CoresUsed": "not-zero-number", + "Endpoints": [ + { + "Address": "text", + "Name": "grpc" + }, + { + "Address": "text", + "Name": "http-mon" + }, + { + "Address": "text", + "Name": "ic" + } + ], + "Host": "text", + "LoadAverage": "not-empty-array", + "Location": { + "DataCenter": "1", + "Rack": "1", + "Unit": "1" + }, + "MaxDiskUsage": "not-zero-number", + "MemoryLimit": "not-zero-number-text", + "MemoryStats": "not-empty-object", + "NodeId": 1, + "NumberOfCpus": "not-zero-number", + "PoolStats": [ + { + "Limit": "not-zero-number", + "Name": "System", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "User", + "Threads": 3, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "Batch", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IO", + "Threads": 1, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IC", + "Threads": 1, + "Usage": "number" + } + ], + "RealNumberOfCpus": "not-zero-number", + "Roles": "not-empty-array", + "StartTime": "not-zero-number-text", + "SystemState": "Green", + "TotalSessions": "number" + }, + "UptimeSeconds": "number", + "VDisks": [ + { + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "static", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 1, + "GroupID": 0, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 0, + "VDiskState": "OK" + }, + { + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "dynamic_storage_pool:1", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 1, + "GroupID": 2181038080, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 1000, + "VDiskState": "OK" + }, + { + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "dynamic_storage_pool:1", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 1, + "GroupID": 2181038081, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 1001, + "VDiskState": "OK" + }, + { + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "/Root/dedicated_db:hdd", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 2, + "GroupID": 2181038082, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 1002, + "VDiskState": "OK" + }, + { + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "/Root/shared_db:hdd", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 2, + "GroupID": 2181038083, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 1003, + "VDiskState": "OK" + } + ] + } + ], + "TotalNodes": "1" + }, + "/Root/shared_db": { + "FieldsAvailable": "11111111110111111111111111111111", + "FieldsRequired": "11111111111111111111111111111111", + "FoundNodes": "1", + "MaximumDisksPerNode": "1", + "MaximumSlotsPerDisk": "5", + "Nodes": [ + { + "ClockSkewMaxUs": "text", + "ClockSkewMinUs": "text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "Connections": "not-zero-number", + "CpuUsage": "not-zero-number", + "DiskSpaceUsage": "not-zero-number", + "NetworkUtilization": "number", + "NetworkUtilizationMax": "number", + "NetworkUtilizationMin": "number", + "NodeId": 1, + "PDisks": [ + { + "AvailableSize": "text", + "Category": "0", + "ChangeTime": "not-zero-number-text", + "Device": "Green", + "EnforcedDynamicSlotSize": "not-zero-number-text", + "Guid": "1", + "LogTotalSize": "not-zero-number-text", + "LogUsedSize": "not-zero-number-text", + "NodeId": 1, + "NumActiveSlots": 5, + "PDiskId": 1, + "Path": "SectorMap:1:64", + "Realtime": "Green", + "SerialNumber": "", + "State": "Normal", + "SystemSize": "not-zero-number-text", + "TotalSize": "not-zero-number-text" + } + ], + "Peers": [ + { + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 1, + "PeerName": "text", + "PeerNodeId": 50000, + "PingTimeUs": "text", + "ScopeId": { + "X1": "72057594046678944", + "X2": "6" + }, + "Utilization": "number" + }, + { + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 1, + "PeerName": "text", + "PeerNodeId": 50001, + "PingTimeUs": "text", + "ScopeId": { + "X1": "72057594046678944", + "X2": "7" + }, + "Utilization": "number" + } + ], + "PingTimeMaxUs": "text", + "PingTimeMinUs": "text", + "PingTimeUs": "text", + "ReceiveThroughput": "text", + "ReverseClockSkewUs": "text", + "ReversePeers": [ + { + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 50000, + "PeerName": "text", + "PeerNodeId": 1, + "PingTimeUs": "text", + "ScopeId": { + "X1": "0", + "X2": "1" + }, + "Utilization": "number" + }, + { + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 50001, + "PeerName": "text", + "PeerNodeId": 1, + "PingTimeUs": "text", + "ScopeId": { + "X1": "0", + "X2": "1" + }, + "Utilization": "number" + } + ], + "ReversePingTimeUs": "text", + "SendThroughput": "text", + "SystemState": { + "ChangeTime": "not-zero-number-text", + "CoresTotal": "not-zero-number", + "CoresUsed": "not-zero-number", + "Endpoints": [ + { + "Address": "text", + "Name": "grpc" + }, + { + "Address": "text", + "Name": "http-mon" + }, + { + "Address": "text", + "Name": "ic" + } + ], + "Host": "text", + "LoadAverage": "not-empty-array", + "Location": { + "DataCenter": "1", + "Rack": "1", + "Unit": "1" + }, + "MaxDiskUsage": "not-zero-number", + "MemoryLimit": "not-zero-number-text", + "MemoryStats": "not-empty-object", + "NodeId": 1, + "NumberOfCpus": "not-zero-number", + "PoolStats": [ + { + "Limit": "not-zero-number", + "Name": "System", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "User", + "Threads": 3, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "Batch", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IO", + "Threads": 1, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IC", + "Threads": 1, + "Usage": "number" + } + ], + "RealNumberOfCpus": "not-zero-number", + "Roles": "not-empty-array", + "StartTime": "not-zero-number-text", + "SystemState": "Green", + "TotalSessions": "number" + }, + "UptimeSeconds": "number", + "VDisks": [ + { + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "static", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 1, + "GroupID": 0, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 0, + "VDiskState": "OK" + }, + { + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "dynamic_storage_pool:1", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 1, + "GroupID": 2181038080, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 1000, + "VDiskState": "OK" + }, + { + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "dynamic_storage_pool:1", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 1, + "GroupID": 2181038081, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 1001, + "VDiskState": "OK" + }, + { + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "/Root/dedicated_db:hdd", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 2, + "GroupID": 2181038082, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 1002, + "VDiskState": "OK" + }, + { + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "/Root/shared_db:hdd", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 2, + "GroupID": 2181038083, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 1003, + "VDiskState": "OK" + } + ] + } + ], + "TotalNodes": "1" + }, + "no-database": { + "FieldsAvailable": "11111111110111111111111111111111", + "FieldsRequired": "11111111111111111111111111111111", + "FoundNodes": "1", + "MaximumDisksPerNode": "1", + "MaximumSlotsPerDisk": "5", + "Nodes": [ + { + "ClockSkewMaxUs": "text", + "ClockSkewMinUs": "text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "Connections": "not-zero-number", + "CpuUsage": "not-zero-number", + "DiskSpaceUsage": "not-zero-number", + "NetworkUtilization": "number", + "NetworkUtilizationMax": "number", + "NetworkUtilizationMin": "number", + "NodeId": 1, + "PDisks": [ + { + "AvailableSize": "text", + "Category": "0", + "ChangeTime": "not-zero-number-text", + "Device": "Green", + "EnforcedDynamicSlotSize": "not-zero-number-text", + "Guid": "1", + "LogTotalSize": "not-zero-number-text", + "LogUsedSize": "not-zero-number-text", + "NodeId": 1, + "NumActiveSlots": 5, + "PDiskId": 1, + "Path": "SectorMap:1:64", + "Realtime": "Green", + "SerialNumber": "", + "State": "Normal", + "SystemSize": "not-zero-number-text", + "TotalSize": "not-zero-number-text" + } + ], + "Peers": [ + { + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 1, + "PeerName": "text", + "PeerNodeId": 50000, + "PingTimeUs": "text", + "ScopeId": { + "X1": "72057594046678944", + "X2": "6" + }, + "Utilization": "number" + }, + { + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 1, + "PeerName": "text", + "PeerNodeId": 50001, + "PingTimeUs": "text", + "ScopeId": { + "X1": "72057594046678944", + "X2": "7" + }, + "Utilization": "number" + } + ], + "PingTimeMaxUs": "text", + "PingTimeMinUs": "text", + "PingTimeUs": "text", + "ReceiveThroughput": "text", + "ReverseClockSkewUs": "text", + "ReversePeers": [ + { + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 50000, + "PeerName": "text", + "PeerNodeId": 1, + "PingTimeUs": "text", + "ScopeId": { + "X1": "0", + "X2": "1" + }, + "Utilization": "number" + }, + { + "BytesWritten": "text", + "ChangeTime": "not-zero-number-text", + "ClockSkewUs": "text", + "ConnectStatus": "Green", + "ConnectTime": "not-zero-number-text", + "Connected": true, + "NodeId": 50001, + "PeerName": "text", + "PeerNodeId": 1, + "PingTimeUs": "text", + "ScopeId": { + "X1": "0", + "X2": "1" + }, + "Utilization": "number" + } + ], + "ReversePingTimeUs": "text", + "SendThroughput": "text", + "SystemState": { + "ChangeTime": "not-zero-number-text", + "CoresTotal": "not-zero-number", + "CoresUsed": "not-zero-number", + "Endpoints": [ + { + "Address": "text", + "Name": "grpc" + }, + { + "Address": "text", + "Name": "http-mon" + }, + { + "Address": "text", + "Name": "ic" + } + ], + "Host": "text", + "LoadAverage": "not-empty-array", + "Location": { + "DataCenter": "1", + "Rack": "1", + "Unit": "1" + }, + "MaxDiskUsage": "not-zero-number", + "MemoryLimit": "not-zero-number-text", + "MemoryStats": "not-empty-object", + "NodeId": 1, + "NumberOfCpus": "not-zero-number", + "PoolStats": [ + { + "Limit": "not-zero-number", + "Name": "System", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "User", + "Threads": 3, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "Batch", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IO", + "Threads": 1, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IC", + "Threads": 1, + "Usage": "number" + } + ], + "RealNumberOfCpus": "not-zero-number", + "Roles": "not-empty-array", + "StartTime": "not-zero-number-text", + "SystemState": "Green", + "TotalSessions": "number" + }, + "Tablets": [ + { + "Count": 1, + "State": "Green", + "Type": "BSController" + }, + { + "Count": 1, + "State": "Green", + "Type": "Cms" + }, + { + "Count": 1, + "State": "Green", + "Type": "Console" + }, + { + "Count": 1, + "State": "Green", + "Type": "Coordinator" + }, + { + "Count": 1, + "State": "Green", + "Type": "Hive" + }, + { + "Count": 1, + "State": "Green", + "Type": "Mediator" + }, + { + "Count": 1, + "State": "Green", + "Type": "NodeBroker" + }, + { + "Count": 1, + "State": "Green", + "Type": "SchemeShard" + }, + { + "Count": 1, + "State": "Green", + "Type": "TenantSlotBroker" + }, + { + "Count": 1, + "State": "Green", + "Type": "TxAllocator" + } + ], + "UptimeSeconds": "number", + "VDisks": [ + { + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "static", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 1, + "GroupID": 0, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 0, + "VDiskState": "OK" + }, + { + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "dynamic_storage_pool:1", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 1, + "GroupID": 2181038080, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 1000, + "VDiskState": "OK" + }, + { + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "dynamic_storage_pool:1", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 1, + "GroupID": 2181038081, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 1001, + "VDiskState": "OK" + }, + { + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "/Root/dedicated_db:hdd", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 2, + "GroupID": 2181038082, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 1002, + "VDiskState": "OK" + }, + { + "AllocatedSize": "0", + "AvailableSize": "text", + "ChangeTime": "not-zero-number-text", + "DiskSpace": "Green", + "FrontQueues": "Green", + "Guid": "1", + "HasUnreadableBlobs": false, + "IncarnationGuid": "not-zero-number-text", + "InstanceGuid": "not-zero-number-text", + "Kind": "0", + "NodeId": 1, + "PDiskId": 1, + "Replicated": true, + "ReplicationProgress": 1, + "ReplicationSecondsRemaining": 0, + "SatisfactionRank": { + "FreshRank": { + "Flag": "Green" + }, + "LevelRank": { + "Flag": "Green" + } + }, + "StoragePoolName": "/Root/shared_db:hdd", + "VDiskId": { + "Domain": 0, + "GroupGeneration": 2, + "GroupID": 2181038083, + "Ring": 0, + "VDisk": 0 + }, + "VDiskSlotId": 1003, + "VDiskState": "OK" + } + ] + } + ], + "TotalNodes": "1" + } + }, + "test.test_viewer_sysinfo": { + "ResponseDuration": "not-zero-number", + "ResponseTime": "not-zero-number-text", + "SystemStateInfo": [ + { + "ChangeTime": "not-zero-number-text", + "Endpoints": [ + { + "Address": "text", + "Name": "grpc" + }, + { + "Address": "text", + "Name": "http-mon" + }, + { + "Address": "text", + "Name": "ic" + } + ], + "Host": "text", + "LoadAverage": "not-empty-array", + "Location": { + "DataCenter": "", + "Rack": "", + "Unit": "0" + }, + "MemoryLimit": "not-zero-number-text", + "NodeId": 50001, + "NumberOfCpus": "not-zero-number", + "PoolStats": [ + { + "Limit": "not-zero-number", + "Name": "System", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "User", + "Threads": 3, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "Batch", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IO", + "Threads": 1, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IC", + "Threads": 1, + "Usage": "number" + } + ], + "Roles": "not-empty-array", + "StartTime": "not-zero-number-text", + "SystemState": 1, + "Tenants": [ + "/Root/shared_db" + ], + "TotalSessions": "number" + }, + { + "ChangeTime": "not-zero-number-text", + "Endpoints": [ + { + "Address": "text", + "Name": "grpc" + }, + { + "Address": "text", + "Name": "http-mon" + }, + { + "Address": "text", + "Name": "ic" + } + ], + "Host": "text", + "LoadAverage": "not-empty-array", + "Location": { + "DataCenter": "", + "Rack": "", + "Unit": "0" + }, + "MemoryLimit": "not-zero-number-text", + "NodeId": 50000, + "NumberOfCpus": "not-zero-number", + "PoolStats": [ + { + "Limit": "not-zero-number", + "Name": "System", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "User", + "Threads": 3, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "Batch", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IO", + "Threads": 1, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IC", + "Threads": 1, + "Usage": "number" + } + ], + "Roles": "not-empty-array", + "StartTime": "not-zero-number-text", + "SystemState": 1, + "Tenants": [ + "/Root/dedicated_db" + ], + "TotalSessions": "number" + }, + { + "ChangeTime": "not-zero-number-text", + "Endpoints": [ + { + "Address": "text", + "Name": "grpc" + }, + { + "Address": "text", + "Name": "http-mon" + }, + { + "Address": "text", + "Name": "ic" + } + ], + "Host": "text", + "LoadAverage": "not-empty-array", + "Location": { + "DataCenter": "1", + "Rack": "1", + "Unit": "1" + }, + "MaxDiskUsage": "not-zero-number", + "MemoryLimit": "not-zero-number-text", + "NodeId": 1, + "NumberOfCpus": "not-zero-number", + "PoolStats": [ + { + "Limit": "not-zero-number", + "Name": "System", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "User", + "Threads": 3, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "Batch", + "Threads": 2, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IO", + "Threads": 1, + "Usage": "number" + }, + { + "Limit": "not-zero-number", + "Name": "IC", + "Threads": 1, + "Usage": "number" + } + ], + "Roles": "not-empty-array", + "StartTime": "not-zero-number-text", + "SystemState": 1, + "TotalSessions": "number" + } + ] + }, + "test.test_viewer_tabletinfo": { + "detailed": { + "/Root": { + "ProcessDuration": "not-zero-number-text", + "ResponseDuration": "not-zero-number-text", + "ResponseTime": "not-zero-number-text", + "TabletStateInfo": [ + { + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 2, + "Leader": true, + "NodeId": 1, + "Overall": 1, + "State": 10, + "TabletId": "72057594037932033", + "Type": 15 + }, + { + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 2, + "Leader": true, + "NodeId": 1, + "Overall": 1, + "State": 10, + "TabletId": "72057594037936128", + "Type": 21 + }, + { + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 2, + "Leader": true, + "NodeId": 1, + "Overall": 1, + "State": 10, + "TabletId": "72057594037936129", + "Type": 22 + }, + { + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 2, + "Leader": true, + "NodeId": 1, + "Overall": 1, + "State": 10, + "TabletId": "72057594037936130", + "Type": 27 + }, + { + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 2, + "Leader": true, + "NodeId": 1, + "Overall": 1, + "State": 10, + "TabletId": "72057594037936131", + "Type": 28 + }, + { + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 2, + "Leader": true, + "NodeId": 1, + "Overall": 1, + "State": 10, + "TabletId": "72057594037968897", + "Type": 14 + }, + { + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 2, + "Leader": true, + "NodeId": 1, + "Overall": 1, + "State": 10, + "TabletId": "72057594046316545", + "Type": 13 + }, + { + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 2, + "Leader": true, + "NodeId": 1, + "Overall": 1, + "State": 10, + "TabletId": "72057594046382081", + "Type": 5 + }, + { + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 2, + "Leader": true, + "NodeId": 1, + "Overall": 1, + "State": 10, + "TabletId": "72057594046447617", + "Type": 23 + }, + { + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 2, + "Leader": true, + "NodeId": 1, + "Overall": 1, + "State": 10, + "TabletId": "72057594046678944", + "Type": 16 + } + ] + }, + "/Root/dedicated_db": { + "ProcessDuration": "not-zero-number-text", + "ResponseDuration": "not-zero-number-text", + "ResponseTime": "not-zero-number-text", + "TabletStateInfo": [ + { + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72057594037968897", + "Leader": true, + "NodeId": 50000, + "Overall": 1, + "State": 10, + "TabletId": "72075186224037888", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" + }, + "Type": 14 + }, + { + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72075186224037888", + "Leader": true, + "NodeId": 50000, + "Overall": 1, + "State": 10, + "TabletId": "72075186224037889", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" + }, + "Type": 5 + }, + { + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72075186224037888", + "Leader": true, + "NodeId": 50000, + "Overall": 1, + "State": 10, + "TabletId": "72075186224037890", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" + }, + "Type": 5 + }, + { + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72075186224037888", + "Leader": true, + "NodeId": 50000, + "Overall": 1, + "State": 10, + "TabletId": "72075186224037891", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" + }, + "Type": 32 + }, + { + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72075186224037888", + "Leader": true, + "NodeId": 50000, + "Overall": 1, + "State": 10, + "TabletId": "72075186224037892", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" }, - { - "AllocatedSize": "0", - "AvailableSize": "text", - "ChangeTime": "not-zero-number-text", - "DiskSpace": "Green", - "FrontQueues": "Green", - "Guid": "1", - "HasUnreadableBlobs": false, - "IncarnationGuid": "not-zero-number-text", - "InstanceGuid": "not-zero-number-text", - "Kind": "0", - "NodeId": 1, - "PDiskId": 1, - "Replicated": true, - "ReplicationProgress": 1, - "ReplicationSecondsRemaining": 0, - "SatisfactionRank": { - "FreshRank": { - "Flag": "Green" - }, - "LevelRank": { - "Flag": "Green" - } - }, - "StoragePoolName": "dynamic_storage_pool:1", - "VDiskId": { - "Domain": 0, - "GroupGeneration": 1, - "GroupID": 2181038080, - "Ring": 0, - "VDisk": 0 - }, - "VDiskSlotId": 1000, - "VDiskState": "OK" + "Type": 5 + }, + { + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72075186224037888", + "Leader": true, + "NodeId": 50000, + "Overall": 1, + "State": 10, + "TabletId": "72075186224037893", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" }, - { - "AllocatedSize": "0", - "AvailableSize": "text", - "ChangeTime": "not-zero-number-text", - "DiskSpace": "Green", - "FrontQueues": "Green", - "Guid": "1", - "HasUnreadableBlobs": false, - "IncarnationGuid": "not-zero-number-text", - "InstanceGuid": "not-zero-number-text", - "Kind": "0", - "NodeId": 1, - "PDiskId": 1, - "Replicated": true, - "ReplicationProgress": 1, - "ReplicationSecondsRemaining": 0, - "SatisfactionRank": { - "FreshRank": { - "Flag": "Green" - }, - "LevelRank": { - "Flag": "Green" - } - }, - "StoragePoolName": "dynamic_storage_pool:1", - "VDiskId": { - "Domain": 0, - "GroupGeneration": 1, - "GroupID": 2181038081, - "Ring": 0, - "VDisk": 0 - }, - "VDiskSlotId": 1001, - "VDiskState": "OK" + "Type": 13 + }, + { + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72075186224037888", + "Leader": true, + "NodeId": 50000, + "Overall": 1, + "State": 10, + "TabletId": "72075186224037894", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" }, - { - "AllocatedSize": "0", - "AvailableSize": "text", - "ChangeTime": "not-zero-number-text", - "DiskSpace": "Green", - "FrontQueues": "Green", - "Guid": "1", - "HasUnreadableBlobs": false, - "IncarnationGuid": "not-zero-number-text", - "InstanceGuid": "not-zero-number-text", - "Kind": "0", - "NodeId": 1, - "PDiskId": 1, - "Replicated": true, - "ReplicationProgress": 1, - "ReplicationSecondsRemaining": 0, - "SatisfactionRank": { - "FreshRank": { - "Flag": "Green" - }, - "LevelRank": { - "Flag": "Green" - } - }, - "StoragePoolName": "/Root/dedicated_db:hdd", - "VDiskId": { - "Domain": 0, - "GroupGeneration": 2, - "GroupID": 2181038082, - "Ring": 0, - "VDisk": 0 - }, - "VDiskSlotId": 1002, - "VDiskState": "OK" + "Type": 40 + }, + { + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72075186224037888", + "Leader": true, + "NodeId": 50000, + "Overall": 1, + "State": 10, + "TabletId": "72075186224037895", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" }, - { - "AllocatedSize": "0", - "AvailableSize": "text", - "ChangeTime": "not-zero-number-text", - "DiskSpace": "Green", - "FrontQueues": "Green", - "Guid": "1", - "HasUnreadableBlobs": false, - "IncarnationGuid": "not-zero-number-text", - "InstanceGuid": "not-zero-number-text", - "Kind": "0", - "NodeId": 1, - "PDiskId": 1, - "Replicated": true, - "ReplicationProgress": 1, - "ReplicationSecondsRemaining": 0, - "SatisfactionRank": { - "FreshRank": { - "Flag": "Green" - }, - "LevelRank": { - "Flag": "Green" - } - }, - "StoragePoolName": "/Root/shared_db:hdd", - "VDiskId": { - "Domain": 0, - "GroupGeneration": 2, - "GroupID": 2181038083, - "Ring": 0, - "VDisk": 0 - }, - "VDiskSlotId": 1003, - "VDiskState": "OK" - } - ] - } - ], - "TotalNodes": "1" - } - }, - "test.test_viewer_sysinfo": { - "ResponseDuration": "not-zero-number", - "ResponseTime": "not-zero-number-text", - "SystemStateInfo": [ - { - "ChangeTime": "not-zero-number-text", - "Endpoints": [ + "Type": 13 + }, { - "Address": "text", - "Name": "grpc" + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72075186224037888", + "Leader": true, + "NodeId": 50000, + "Overall": 1, + "State": 10, + "TabletId": "72075186224037896", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" + }, + "Type": 13 }, { - "Address": "text", - "Name": "http-mon" + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72075186224037888", + "Leader": true, + "NodeId": 50000, + "Overall": 1, + "State": 10, + "TabletId": "72075186224037897", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" + }, + "Type": 16 }, { - "Address": "text", - "Name": "ic" - } - ], - "Host": "text", - "LoadAverage": "not-empty-array", - "Location": { - "DataCenter": "", - "Rack": "", - "Unit": "0" - }, - "MemoryLimit": "not-zero-number-text", - "NodeId": 50001, - "NumberOfCpus": "not-zero-number", - "PoolStats": [ + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72075186224037888", + "Leader": true, + "NodeId": 50000, + "Overall": 1, + "State": 10, + "TabletId": "72075186224037898", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" + }, + "Type": 18 + } + ] + }, + "/Root/serverless_db": { + "ProcessDuration": "not-zero-number-text", + "ResponseDuration": "not-zero-number-text", + "ResponseTime": "not-zero-number-text", + "TabletStateInfo": [ { - "Limit": "not-zero-number", - "Name": "System", - "Threads": 2, - "Usage": "number" + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72075186224038889", + "Leader": true, + "NodeId": 50001, + "Overall": 1, + "State": 10, + "TabletId": "72075186224038899", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" + }, + "Type": 16 }, { - "Limit": "not-zero-number", - "Name": "User", - "Threads": 3, - "Usage": "number" + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72075186224038889", + "Leader": true, + "NodeId": 50001, + "Overall": 1, + "State": 10, + "TabletId": "72075186224038900", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" + }, + "Type": 13 }, { - "Limit": "not-zero-number", - "Name": "Batch", - "Threads": 2, - "Usage": "number" + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72075186224038889", + "Leader": true, + "NodeId": 50001, + "Overall": 1, + "State": 10, + "TabletId": "72075186224038901", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" + }, + "Type": 5 }, { - "Limit": "not-zero-number", - "Name": "IO", - "Threads": 1, - "Usage": "number" + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72075186224038889", + "Leader": true, + "NodeId": 50001, + "Overall": 1, + "State": 10, + "TabletId": "72075186224038902", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" + }, + "Type": 5 }, { - "Limit": "not-zero-number", - "Name": "IC", - "Threads": 1, - "Usage": "number" - } - ], - "Roles": "not-empty-array", - "StartTime": "not-zero-number-text", - "SystemState": 1, - "Tenants": [ - "/Root/shared_db" - ], - "TotalSessions": "number" - }, - { - "ChangeTime": "not-zero-number-text", - "Endpoints": [ - { - "Address": "text", - "Name": "grpc" + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72075186224038889", + "Leader": true, + "NodeId": 50001, + "Overall": 1, + "State": 10, + "TabletId": "72075186224038903", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" + }, + "Type": 5 }, { - "Address": "text", - "Name": "http-mon" + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72075186224038889", + "Leader": true, + "NodeId": 50001, + "Overall": 1, + "State": 10, + "TabletId": "72075186224038904", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" + }, + "Type": 32 }, { - "Address": "text", - "Name": "ic" + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72075186224038889", + "Leader": true, + "NodeId": 50001, + "Overall": 1, + "State": 10, + "TabletId": "72075186224038906", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" + }, + "Type": 18 } - ], - "Host": "text", - "LoadAverage": "not-empty-array", - "Location": { - "DataCenter": "", - "Rack": "", - "Unit": "0" - }, - "MemoryLimit": "not-zero-number-text", - "NodeId": 50000, - "NumberOfCpus": "not-zero-number", - "PoolStats": [ - { - "Limit": "not-zero-number", - "Name": "System", - "Threads": 2, - "Usage": "number" - }, + ] + }, + "/Root/shared_db": { + "ProcessDuration": "not-zero-number-text", + "ResponseDuration": "not-zero-number-text", + "ResponseTime": "not-zero-number-text", + "TabletStateInfo": [ { - "Limit": "not-zero-number", - "Name": "User", - "Threads": 3, - "Usage": "number" + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72057594037968897", + "Leader": true, + "NodeId": 50001, + "Overall": 1, + "State": 10, + "TabletId": "72075186224038889", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" + }, + "Type": 14 }, { - "Limit": "not-zero-number", - "Name": "Batch", - "Threads": 2, - "Usage": "number" + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72075186224038889", + "Leader": true, + "NodeId": 50001, + "Overall": 1, + "State": 10, + "TabletId": "72075186224038890", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" + }, + "Type": 40 }, { - "Limit": "not-zero-number", - "Name": "IO", - "Threads": 1, - "Usage": "number" + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72075186224038889", + "Leader": true, + "NodeId": 50001, + "Overall": 1, + "State": 10, + "TabletId": "72075186224038891", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" + }, + "Type": 5 }, { - "Limit": "not-zero-number", - "Name": "IC", - "Threads": 1, - "Usage": "number" - } - ], - "Roles": "not-empty-array", - "StartTime": "not-zero-number-text", - "SystemState": 1, - "Tenants": [ - "/Root/dedicated_db" - ], - "TotalSessions": "number" - }, - { - "ChangeTime": "not-zero-number-text", - "Endpoints": [ - { - "Address": "text", - "Name": "grpc" + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72075186224038889", + "Leader": true, + "NodeId": 50001, + "Overall": 1, + "State": 10, + "TabletId": "72075186224038892", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" + }, + "Type": 5 }, { - "Address": "text", - "Name": "http-mon" + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72075186224038889", + "Leader": true, + "NodeId": 50001, + "Overall": 1, + "State": 10, + "TabletId": "72075186224038893", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" + }, + "Type": 32 }, { - "Address": "text", - "Name": "ic" - } - ], - "Host": "text", - "LoadAverage": "not-empty-array", - "Location": { - "DataCenter": "1", - "Rack": "1", - "Unit": "1" - }, - "MaxDiskUsage": "not-zero-number", - "MemoryLimit": "not-zero-number-text", - "NodeId": 1, - "NumberOfCpus": "not-zero-number", - "PoolStats": [ + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72075186224038889", + "Leader": true, + "NodeId": 50001, + "Overall": 1, + "State": 10, + "TabletId": "72075186224038894", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" + }, + "Type": 13 + }, { - "Limit": "not-zero-number", - "Name": "System", - "Threads": 2, - "Usage": "number" + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72075186224038889", + "Leader": true, + "NodeId": 50001, + "Overall": 1, + "State": 10, + "TabletId": "72075186224038895", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" + }, + "Type": 13 }, { - "Limit": "not-zero-number", - "Name": "User", - "Threads": 3, - "Usage": "number" + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72075186224038889", + "Leader": true, + "NodeId": 50001, + "Overall": 1, + "State": 10, + "TabletId": "72075186224038896", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" + }, + "Type": 13 }, { - "Limit": "not-zero-number", - "Name": "Batch", - "Threads": 2, - "Usage": "number" + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72075186224038889", + "Leader": true, + "NodeId": 50001, + "Overall": 1, + "State": 10, + "TabletId": "72075186224038897", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" + }, + "Type": 5 }, { - "Limit": "not-zero-number", - "Name": "IO", - "Threads": 1, - "Usage": "number" + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72075186224038889", + "Leader": true, + "NodeId": 50001, + "Overall": 1, + "State": 10, + "TabletId": "72075186224038898", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" + }, + "Type": 16 }, { - "Limit": "not-zero-number", - "Name": "IC", - "Threads": 1, - "Usage": "number" + "ChangeTime": "not-zero-number-text", + "FollowerId": 0, + "Generation": 1, + "HiveId": "72075186224038889", + "Leader": true, + "NodeId": 50001, + "Overall": 1, + "State": 10, + "TabletId": "72075186224038905", + "TenantId": { + "PathId": "not-zero-number-text", + "SchemeShard": "72057594046678944" + }, + "Type": 18 } - ], - "Roles": "not-empty-array", - "StartTime": "not-zero-number-text", - "SystemState": 1, - "TotalSessions": "number" - } - ] - }, - "test.test_viewer_tabletinfo": { - "detailed": { - "/Root": { + ] + }, + "no-database": { "ProcessDuration": "not-zero-number-text", "ResponseDuration": "not-zero-number-text", "ResponseTime": "not-zero-number-text", @@ -7483,14 +10120,7 @@ "State": 10, "TabletId": "72057594046678944", "Type": 16 - } - ] - }, - "/Root/dedicated_db": { - "ProcessDuration": "not-zero-number-text", - "ResponseDuration": "not-zero-number-text", - "ResponseTime": "not-zero-number-text", - "TabletStateInfo": [ + }, { "ChangeTime": "not-zero-number-text", "FollowerId": 0, @@ -7666,29 +10296,22 @@ "SchemeShard": "72057594046678944" }, "Type": 18 - } - ] - }, - "/Root/serverless_db": { - "ProcessDuration": "not-zero-number-text", - "ResponseDuration": "not-zero-number-text", - "ResponseTime": "not-zero-number-text", - "TabletStateInfo": [ + }, { "ChangeTime": "not-zero-number-text", "FollowerId": 0, "Generation": 1, - "HiveId": "72075186224038889", + "HiveId": "72057594037968897", "Leader": true, "NodeId": 50001, "Overall": 1, "State": 10, - "TabletId": "72075186224038899", + "TabletId": "72075186224038889", "TenantId": { "PathId": "not-zero-number-text", "SchemeShard": "72057594046678944" }, - "Type": 16 + "Type": 14 }, { "ChangeTime": "not-zero-number-text", @@ -7699,12 +10322,12 @@ "NodeId": 50001, "Overall": 1, "State": 10, - "TabletId": "72075186224038900", + "TabletId": "72075186224038890", "TenantId": { "PathId": "not-zero-number-text", "SchemeShard": "72057594046678944" }, - "Type": 13 + "Type": 40 }, { "ChangeTime": "not-zero-number-text", @@ -7715,7 +10338,7 @@ "NodeId": 50001, "Overall": 1, "State": 10, - "TabletId": "72075186224038901", + "TabletId": "72075186224038891", "TenantId": { "PathId": "not-zero-number-text", "SchemeShard": "72057594046678944" @@ -7731,7 +10354,7 @@ "NodeId": 50001, "Overall": 1, "State": 10, - "TabletId": "72075186224038902", + "TabletId": "72075186224038892", "TenantId": { "PathId": "not-zero-number-text", "SchemeShard": "72057594046678944" @@ -7747,12 +10370,12 @@ "NodeId": 50001, "Overall": 1, "State": 10, - "TabletId": "72075186224038903", + "TabletId": "72075186224038893", "TenantId": { "PathId": "not-zero-number-text", "SchemeShard": "72057594046678944" }, - "Type": 5 + "Type": 32 }, { "ChangeTime": "not-zero-number-text", @@ -7763,12 +10386,12 @@ "NodeId": 50001, "Overall": 1, "State": 10, - "TabletId": "72075186224038904", + "TabletId": "72075186224038894", "TenantId": { "PathId": "not-zero-number-text", "SchemeShard": "72057594046678944" }, - "Type": 32 + "Type": 13 }, { "ChangeTime": "not-zero-number-text", @@ -7779,35 +10402,28 @@ "NodeId": 50001, "Overall": 1, "State": 10, - "TabletId": "72075186224038906", + "TabletId": "72075186224038895", "TenantId": { "PathId": "not-zero-number-text", "SchemeShard": "72057594046678944" }, - "Type": 18 - } - ] - }, - "/Root/shared_db": { - "ProcessDuration": "not-zero-number-text", - "ResponseDuration": "not-zero-number-text", - "ResponseTime": "not-zero-number-text", - "TabletStateInfo": [ + "Type": 13 + }, { "ChangeTime": "not-zero-number-text", "FollowerId": 0, "Generation": 1, - "HiveId": "72057594037968897", + "HiveId": "72075186224038889", "Leader": true, "NodeId": 50001, "Overall": 1, "State": 10, - "TabletId": "72075186224038889", + "TabletId": "72075186224038896", "TenantId": { "PathId": "not-zero-number-text", "SchemeShard": "72057594046678944" }, - "Type": 14 + "Type": 13 }, { "ChangeTime": "not-zero-number-text", @@ -7818,12 +10434,12 @@ "NodeId": 50001, "Overall": 1, "State": 10, - "TabletId": "72075186224038890", + "TabletId": "72075186224038897", "TenantId": { "PathId": "not-zero-number-text", "SchemeShard": "72057594046678944" }, - "Type": 40 + "Type": 5 }, { "ChangeTime": "not-zero-number-text", @@ -7834,12 +10450,12 @@ "NodeId": 50001, "Overall": 1, "State": 10, - "TabletId": "72075186224038891", + "TabletId": "72075186224038898", "TenantId": { "PathId": "not-zero-number-text", "SchemeShard": "72057594046678944" }, - "Type": 5 + "Type": 16 }, { "ChangeTime": "not-zero-number-text", @@ -7850,12 +10466,12 @@ "NodeId": 50001, "Overall": 1, "State": 10, - "TabletId": "72075186224038892", + "TabletId": "72075186224038899", "TenantId": { "PathId": "not-zero-number-text", "SchemeShard": "72057594046678944" }, - "Type": 5 + "Type": 16 }, { "ChangeTime": "not-zero-number-text", @@ -7866,12 +10482,12 @@ "NodeId": 50001, "Overall": 1, "State": 10, - "TabletId": "72075186224038893", + "TabletId": "72075186224038900", "TenantId": { "PathId": "not-zero-number-text", "SchemeShard": "72057594046678944" }, - "Type": 32 + "Type": 13 }, { "ChangeTime": "not-zero-number-text", @@ -7882,12 +10498,12 @@ "NodeId": 50001, "Overall": 1, "State": 10, - "TabletId": "72075186224038894", + "TabletId": "72075186224038901", "TenantId": { "PathId": "not-zero-number-text", "SchemeShard": "72057594046678944" }, - "Type": 13 + "Type": 5 }, { "ChangeTime": "not-zero-number-text", @@ -7898,12 +10514,12 @@ "NodeId": 50001, "Overall": 1, "State": 10, - "TabletId": "72075186224038895", + "TabletId": "72075186224038902", "TenantId": { "PathId": "not-zero-number-text", "SchemeShard": "72057594046678944" }, - "Type": 13 + "Type": 5 }, { "ChangeTime": "not-zero-number-text", @@ -7914,12 +10530,12 @@ "NodeId": 50001, "Overall": 1, "State": 10, - "TabletId": "72075186224038896", + "TabletId": "72075186224038903", "TenantId": { "PathId": "not-zero-number-text", "SchemeShard": "72057594046678944" }, - "Type": 13 + "Type": 5 }, { "ChangeTime": "not-zero-number-text", @@ -7930,12 +10546,12 @@ "NodeId": 50001, "Overall": 1, "State": 10, - "TabletId": "72075186224038897", + "TabletId": "72075186224038904", "TenantId": { "PathId": "not-zero-number-text", "SchemeShard": "72057594046678944" }, - "Type": 5 + "Type": 32 }, { "ChangeTime": "not-zero-number-text", @@ -7946,12 +10562,12 @@ "NodeId": 50001, "Overall": 1, "State": 10, - "TabletId": "72075186224038898", + "TabletId": "72075186224038905", "TenantId": { "PathId": "not-zero-number-text", "SchemeShard": "72057594046678944" }, - "Type": 16 + "Type": 18 }, { "ChangeTime": "not-zero-number-text", @@ -7962,7 +10578,7 @@ "NodeId": 50001, "Overall": 1, "State": 10, - "TabletId": "72075186224038905", + "TabletId": "72075186224038906", "TenantId": { "PathId": "not-zero-number-text", "SchemeShard": "72057594046678944" @@ -8108,6 +10724,63 @@ "Type": "SysViewProcessor" } ] + }, + "no-database": { + "ResponseTime": "not-zero-number-text", + "TabletStateInfo": [ + { + "Count": 1, + "Type": "BSController" + }, + { + "Count": 1, + "Type": "Cms" + }, + { + "Count": 1, + "Type": "Console" + }, + { + "Count": 8, + "Type": "Coordinator" + }, + { + "Count": 3, + "Type": "DataShard" + }, + { + "Count": 3, + "Type": "Hive" + }, + { + "Count": 10, + "Type": "Mediator" + }, + { + "Count": 1, + "Type": "NodeBroker" + }, + { + "Count": 4, + "Type": "SchemeShard" + }, + { + "Count": 2, + "Type": "StatisticsAggregator" + }, + { + "Count": 3, + "Type": "SysViewProcessor" + }, + { + "Count": 1, + "Type": "TenantSlotBroker" + }, + { + "Count": 1, + "Type": "TxAllocator" + } + ] } } }, @@ -8522,6 +11195,206 @@ "Type": "Shared" } ] + }, + "no-database": { + "TenantInfo": [ + { + "AliveNodes": 1, + "CoresTotal": "not-zero-number", + "CoresUsed": "not-zero-number", + "CreateTime": "not-zero-number-text", + "Id": "72057594046678944-1", + "MemoryLimit": "not-zero-number-text", + "MemoryStats": {}, + "Metrics": {}, + "Name": "/Root", + "NetworkUtilization": "number", + "NetworkWriteThroughput": "text", + "NodeIds": [ + 1 + ], + "Overall": "Green", + "Owner": "root", + "PoolStats": [ + { + "Name": "System", + "Threads": 2, + "Usage": "number" + }, + { + "Name": "User", + "Threads": 3, + "Usage": "number" + }, + { + "Name": "Batch", + "Threads": 2, + "Usage": "number" + }, + { + "Name": "IO", + "Threads": 1, + "Usage": "number" + }, + { + "Name": "IC", + "Threads": 1, + "Usage": "number" + } + ], + "State": "RUNNING", + "Type": "Domain" + }, + { + "AliveNodes": 1, + "CoresTotal": "not-zero-number", + "CoresUsed": "not-zero-number", + "DatabaseQuotas": {}, + "Id": "72057594046678944-6", + "MemoryLimit": "not-zero-number-text", + "MemoryStats": {}, + "Metrics": "not-empty-object", + "Name": "/Root/dedicated_db", + "NetworkUtilization": "number", + "NetworkWriteThroughput": "text", + "NodeIds": [ + 50000 + ], + "Overall": "Green", + "Owner": "root", + "PoolStats": [ + { + "Name": "System", + "Threads": 2, + "Usage": "number" + }, + { + "Name": "User", + "Threads": 3, + "Usage": "number" + }, + { + "Name": "Batch", + "Threads": 2, + "Usage": "number" + }, + { + "Name": "IO", + "Threads": 1, + "Usage": "number" + }, + { + "Name": "IC", + "Threads": 1, + "Usage": "number" + } + ], + "Resources": { + "Allocated": [ + { + "Count": 1, + "Kind": "hdd", + "Type": "storage" + } + ], + "Required": [ + { + "Count": 1, + "Kind": "hdd", + "Type": "storage" + } + ] + }, + "State": "RUNNING", + "StateStats": [ + { + "Count": 10, + "VolatileState": "TABLET_VOLATILE_STATE_RUNNING" + } + ], + "Type": "Dedicated" + }, + { + "CoresUsed": "not-zero-number", + "DatabaseQuotas": {}, + "Id": "72057594046678944-8", + "Metrics": "not-empty-object", + "Name": "/Root/serverless_db", + "Overall": "Green", + "Owner": "root", + "ResourceId": "72057594046678944-7", + "State": "RUNNING", + "StateStats": [ + { + "Count": 7, + "VolatileState": "TABLET_VOLATILE_STATE_RUNNING" + } + ], + "Type": "Serverless" + }, + { + "AliveNodes": 1, + "CoresTotal": "not-zero-number", + "CoresUsed": "not-zero-number", + "DatabaseQuotas": {}, + "Id": "72057594046678944-7", + "MemoryLimit": "not-zero-number-text", + "MemoryStats": {}, + "Metrics": "not-empty-object", + "Name": "/Root/shared_db", + "NetworkUtilization": "number", + "NetworkWriteThroughput": "text", + "NodeIds": [ + 50001 + ], + "Overall": "Green", + "Owner": "root", + "PoolStats": [ + { + "Name": "System", + "Threads": 2, + "Usage": "number" + }, + { + "Name": "User", + "Threads": 3, + "Usage": "number" + }, + { + "Name": "Batch", + "Threads": 2, + "Usage": "number" + }, + { + "Name": "IO", + "Threads": 1, + "Usage": "number" + }, + { + "Name": "IC", + "Threads": 1, + "Usage": "number" + } + ], + "Resources": { + "Allocated": [ + { + "Count": 1, + "Kind": "hdd", + "Type": "storage" + } + ] + }, + "State": "RUNNING", + "StateStats": [ + { + "Count": 10, + "VolatileState": "TABLET_VOLATILE_STATE_RUNNING" + } + ], + "Type": "Shared" + } + ] } }, "test.test_viewer_vdiskinfo": { diff --git a/ydb/core/viewer/tests/test.py b/ydb/core/viewer/tests/test.py index f7af905ade0f..c32e607e8289 100644 --- a/ydb/core/viewer/tests/test.py +++ b/ydb/core/viewer/tests/test.py @@ -59,6 +59,7 @@ def login_user(request): cluster.create_serverless_database(serverless_db, shared_db, token=root_token) cluster.wait_tenant_up(serverless_db, token=root_token) databases = [domain_name, dedicated_db, shared_db, serverless_db] +databases_and_no_database = ['no-database', domain_name, dedicated_db, shared_db, serverless_db] default_headers = { 'Cookie': 'ydb_session_id=' + root_session_id, } @@ -107,6 +108,7 @@ def call_viewer_db(url, params=None): if params is None: params = {} result = {} + result["no-database"] = call_viewer(url, params) for name in databases: params["database"] = name result[name] = call_viewer(url, params) @@ -504,6 +506,7 @@ def normalize_result_nodes(result): 'Roles', 'ConnectTime', 'Connections', + 'ResolveHost', ]) @@ -585,6 +588,12 @@ def get_viewer_db_normalized(url, params=None): return normalize_result(get_viewer_db(url, params)) +def test_viewer_nodelist(): + result = get_viewer_db_normalized("/viewer/nodelist", { + }) + return result + + def test_viewer_nodes(): result = get_viewer_db_normalized("/viewer/nodes", { }) @@ -642,10 +651,10 @@ def test_viewer_tabletinfo(): 'group': 'Type', 'enums': 'true', }) - for name in databases: + for name in databases_and_no_database: result['totals'][name]['TabletStateInfo'].sort(key=lambda x: x['Type']) result['detailed'] = get_viewer_db_normalized("/viewer/tabletinfo") - for name in databases: + for name in databases_and_no_database: result['detailed'][name]['TabletStateInfo'].sort(key=lambda x: x['TabletId']) return result diff --git a/ydb/core/viewer/viewer.cpp b/ydb/core/viewer/viewer.cpp index bcc36eae28a8..3ebbacf364b1 100644 --- a/ydb/core/viewer/viewer.cpp +++ b/ydb/core/viewer/viewer.cpp @@ -133,21 +133,22 @@ class TViewer : public TActorBootstrapped, public IViewer { .RelPath = "healthcheck", .ActorSystem = ctx.ActorSystem(), .ActorId = ctx.SelfID, - .UseAuth = false, + .UseAuth = true, + .AllowedSIDs = databaseAllowedSIDs, }); mon->RegisterActorPage({ .RelPath = "vdisk", .ActorSystem = ctx.ActorSystem(), .ActorId = ctx.SelfID, .UseAuth = true, - .AllowedSIDs = databaseAllowedSIDs, + .AllowedSIDs = viewerAllowedSIDs, }); mon->RegisterActorPage({ .RelPath = "pdisk", .ActorSystem = ctx.ActorSystem(), .ActorId = ctx.SelfID, .UseAuth = true, - .AllowedSIDs = monitoringAllowedSIDs, + .AllowedSIDs = viewerAllowedSIDs, }); mon->RegisterActorPage({ .RelPath = "operation", diff --git a/ydb/core/viewer/viewer_nodelist.h b/ydb/core/viewer/viewer_nodelist.h index d3b2963a1f4a..bb26b5a06f95 100644 --- a/ydb/core/viewer/viewer_nodelist.h +++ b/ydb/core/viewer/viewer_nodelist.h @@ -1,18 +1,15 @@ #pragma once -#include "json_handlers.h" -#include "viewer.h" -#include -#include +#include "json_pipe_req.h" #include namespace NKikimr::NViewer { using namespace NActors; -class TJsonNodeList : public TActorBootstrapped { - IViewer* Viewer; - NMon::TEvHttpInfo::TPtr Event; - TAutoPtr NodesInfo; +class TJsonNodeList : public TViewerPipeClient { + TRequestResponse NodesInfo; + using TBase = TViewerPipeClient; + using TThis = TJsonNodeList; public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { @@ -20,34 +17,43 @@ class TJsonNodeList : public TActorBootstrapped { } TJsonNodeList(IViewer* viewer, NMon::TEvHttpInfo::TPtr &ev) - : Viewer(viewer) - , Event(ev) + : TBase(viewer, ev, "/viewer/nodelist") {} - void Bootstrap(const TActorContext& ctx) { - const TActorId nameserviceId = GetNameserviceActorId(); - ctx.Send(nameserviceId, new TEvInterconnect::TEvListNodes()); - ctx.Schedule(TDuration::Seconds(10), new TEvents::TEvWakeup()); - Become(&TThis::StateRequestedBrowse); + void Bootstrap() override { + if (TBase::NeedToRedirect()) { + return; + } + NodesInfo = MakeRequest(GetNameserviceActorId(), new TEvInterconnect::TEvListNodes()); + Become(&TThis::StateRequestedBrowse, TDuration::Seconds(10), new TEvents::TEvWakeup()); } - STFUNC(StateRequestedBrowse) { + STATEFN(StateRequestedBrowse) { switch (ev->GetTypeRewrite()) { - HFunc(TEvInterconnect::TEvNodesInfo, Handle); - CFunc(TEvents::TSystem::Wakeup, Timeout); + hFunc(TEvInterconnect::TEvNodesInfo, Handle); + cFunc(TEvents::TSystem::Wakeup, TBase::HandleTimeout); } } - void Handle(TEvInterconnect::TEvNodesInfo::TPtr &ev, const TActorContext &ctx) { - NodesInfo = ev->Release(); - ReplyAndDie(ctx); + void Handle(TEvInterconnect::TEvNodesInfo::TPtr& ev) { + if (NodesInfo.Set(std::move(ev))) { + RequestDone(); + } } - void ReplyAndDie(const TActorContext &ctx) { + void ReplyAndPassAway() override { NJson::TJsonValue json; json.SetType(NJson::EJsonValueType::JSON_ARRAY); - if (NodesInfo != nullptr) { + std::optional> databaseNodeIds; + if (IsDatabaseRequest()) { + auto nodes = GetDatabaseNodes(); + databaseNodeIds = std::unordered_set(nodes.begin(), nodes.end()); + } + if (NodesInfo.IsOk()) { for (auto it = NodesInfo->Nodes.begin(); it != NodesInfo->Nodes.end(); ++it) { + if (databaseNodeIds && databaseNodeIds->count(it->NodeId) == 0) { + continue; // skip nodes that are not in the database + } const TEvInterconnect::TNodeInfo& nodeInfo = *it; NJson::TJsonValue& jsonNodeInfo = json.AppendValue(NJson::TJsonValue()); jsonNodeInfo["Id"] = nodeInfo.NodeId; @@ -71,12 +77,7 @@ class TJsonNodeList : public TActorBootstrapped { } } } - ctx.Send(Event->Sender, new NMon::TEvHttpInfoRes(Viewer->GetHTTPOKJSON(Event->Get(), NJson::WriteJson(json, false)), 0, NMon::IEvHttpInfoRes::EContentType::Custom)); - Die(ctx); - } - - void Timeout(const TActorContext &ctx) { - ReplyAndDie(ctx); + TBase::ReplyAndPassAway(GetHTTPOKJSON(json)); } static YAML::Node GetSwagger() { From 546769d2a38a1e3e51225a5977422f6bd0ceb189 Mon Sep 17 00:00:00 2001 From: Evgenik2 Date: Fri, 8 Aug 2025 10:19:26 +0300 Subject: [PATCH 23/52] Keep current config in disconnected DC (#22443) --- ydb/core/blobstorage/nodewarden/distconf.h | 3 +- .../nodewarden/distconf_generate.cpp | 7 ++- .../distconf_invoke_state_storage.cpp | 8 +-- ...distconf_statestorage_config_generator.cpp | 55 ++++++++++++++++++- .../distconf_statestorage_config_generator.h | 5 +- .../blobstorage/nodewarden/distconf_ut.cpp | 24 +++++++- 6 files changed, 89 insertions(+), 13 deletions(-) diff --git a/ydb/core/blobstorage/nodewarden/distconf.h b/ydb/core/blobstorage/nodewarden/distconf.h index b92cc26900be..e3831cf0c18f 100644 --- a/ydb/core/blobstorage/nodewarden/distconf.h +++ b/ydb/core/blobstorage/nodewarden/distconf.h @@ -468,7 +468,8 @@ namespace NKikimr::NStorage { bool GenerateStateStorageConfig(NKikimrConfig::TDomainsConfig::TStateStorage *ss , const NKikimrBlobStorage::TStorageConfig& baseConfig - , std::unordered_set& usedNodes); + , std::unordered_set& usedNodes + , const NKikimrConfig::TDomainsConfig::TStateStorage& oldConfig = {}); //////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Bridge ops diff --git a/ydb/core/blobstorage/nodewarden/distconf_generate.cpp b/ydb/core/blobstorage/nodewarden/distconf_generate.cpp index 684fe725ddf8..1dced30ba3bb 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_generate.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_generate.cpp @@ -575,8 +575,9 @@ namespace NKikimr::NStorage { }); } - bool TDistributedConfigKeeper::GenerateStateStorageConfig(NKikimrConfig::TDomainsConfig::TStateStorage *ss, - const NKikimrBlobStorage::TStorageConfig& baseConfig, std::unordered_set& usedNodes) { + bool TDistributedConfigKeeper::GenerateStateStorageConfig(NKikimrConfig::TDomainsConfig::TStateStorage *ss + , const NKikimrBlobStorage::TStorageConfig& baseConfig, std::unordered_set& usedNodes + , const NKikimrConfig::TDomainsConfig::TStateStorage& oldConfig) { std::map>>> nodes; bool goodConfig = true; for (const auto& node : baseConfig.GetAllNodes()) { @@ -585,7 +586,7 @@ namespace NKikimr::NStorage { nodes[pileId][location.GetDataCenterId()].emplace_back(node.GetNodeId(), location); } for (auto& [pileId, nodesByDataCenter] : nodes) { - TStateStoragePerPileGenerator generator(nodesByDataCenter, SelfHealNodesState, pileId, usedNodes); + TStateStoragePerPileGenerator generator(nodesByDataCenter, SelfHealNodesState, pileId, usedNodes, oldConfig); generator.AddRingGroup(ss); goodConfig &= generator.IsGoodConfig(); } diff --git a/ydb/core/blobstorage/nodewarden/distconf_invoke_state_storage.cpp b/ydb/core/blobstorage/nodewarden/distconf_invoke_state_storage.cpp index 25db25107c86..52c6e6ad1a41 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_invoke_state_storage.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_invoke_state_storage.cpp @@ -7,12 +7,12 @@ namespace NKikimr::NStorage { using TInvokeRequestHandlerActor = TDistributedConfigKeeper::TInvokeRequestHandlerActor; bool TInvokeRequestHandlerActor::GetRecommendedStateStorageConfig(NKikimrBlobStorage::TStateStorageConfig* currentConfig) { - const NKikimrBlobStorage::TStorageConfig &config = *Self->StorageConfig; + const NKikimrBlobStorage::TStorageConfig& config = *Self->StorageConfig; bool result = true; std::unordered_set usedNodes; - result &= Self->GenerateStateStorageConfig(currentConfig->MutableStateStorageConfig(), config, usedNodes); - result &= Self->GenerateStateStorageConfig(currentConfig->MutableStateStorageBoardConfig(), config, usedNodes); - result &= Self->GenerateStateStorageConfig(currentConfig->MutableSchemeBoardConfig(), config, usedNodes); + result &= Self->GenerateStateStorageConfig(currentConfig->MutableStateStorageConfig(), config, usedNodes, config.GetStateStorageConfig()); + result &= Self->GenerateStateStorageConfig(currentConfig->MutableStateStorageBoardConfig(), config, usedNodes, config.GetStateStorageBoardConfig()); + result &= Self->GenerateStateStorageConfig(currentConfig->MutableSchemeBoardConfig(), config, usedNodes, config.GetSchemeBoardConfig()); return result; } diff --git a/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.cpp b/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.cpp index 6385d52048af..8d7e47d17c3d 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.cpp @@ -1,6 +1,7 @@ #include "distconf.h" #include "distconf_statestorage_config_generator.h" +#include #include #include #include @@ -12,10 +13,12 @@ namespace NKikimr::NStorage { , const std::unordered_map& selfHealNodesState , TBridgePileId pileId , std::unordered_set& usedNodes + , const NKikimrConfig::TDomainsConfig::TStateStorage& oldConfig ) : PileId(pileId) , SelfHealNodesState(selfHealNodesState) , UsedNodes(usedNodes) + , OldConfig(oldConfig) { FillNodeGroups(nodes); CalculateRingsParameters(); @@ -45,7 +48,7 @@ namespace NKikimr::NStorage { } Y_ABORT_UNLESS(NodeGroups.size() > 0 && NodeGroups[0].Nodes.size() > 0); for (auto& ng : NodeGroups) { - ng.Disconnected = ng.State[0] + ng.State[1] < ng.Nodes.size() / 2; + ng.Disconnected = ng.State[0] + ng.State[1] <= ng.Nodes.size() / 2; } } @@ -150,7 +153,57 @@ namespace NKikimr::NStorage { return true; } + bool TStateStoragePerPileGenerator::PickOldNodesStrategy(TNodeGroup& group) { + if (!group.Disconnected || (OldConfig.RingGroupsSize() == 0 && !OldConfig.HasRing())) { + return false; + } + std::unordered_set nodesInGroup; + for (auto& n : group.Nodes) { + nodesInGroup.insert(std::get<0>(n)); + } + ui32 ringGroupIdx = 0; + for (ui32 i : xrange(OldConfig.RingGroupsSize())) { + if (PileId == TBridgePileId::FromProto(&OldConfig.GetRingGroups(i), &NKikimrConfig::TDomainsConfig::TStateStorage::TRing::GetBridgePileId)) { + ringGroupIdx = i; + break; + } + } + std::vector> result; + TIntrusivePtr info = BuildStateStorageInfo(OldConfig); + if (info->RingGroups.size() <= ringGroupIdx) { + return false; + } + if (info->RingGroups[ringGroupIdx].NToSelect != NToSelect) { + return false; + } + ui32 ringsSize = info->RingGroups[ringGroupIdx].Rings.size(); + for (ui32 ringIdx : xrange(ringsSize)) { + if (nodesInGroup.contains(info->RingGroups[ringGroupIdx].Rings[ringIdx].Replicas.front().NodeId())) { + result.push_back({}); + for (auto& r : info->RingGroups[ringGroupIdx].Rings[ringIdx].Replicas) { + if (!nodesInGroup.contains(r.NodeId())) { + return false; + } + result.back().emplace_back(r.NodeId()); + } + if (result.back().size() != ReplicasInRingCount) { + return false; + } + } + } + if (result.size() != RingsInGroupCount) { + return false; + } + for (auto& ring : result) { + Rings.emplace_back(ring); + } + return true; + } + void TStateStoragePerPileGenerator::PickNodes(TNodeGroup& group) { + if (PickOldNodesStrategy(group)) { + return; + } std::unordered_map> rackStates; for (auto& n : group.Nodes) { auto rack = std::get<1>(n).GetRackId(); diff --git a/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.h b/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.h index f462d286f565..00f9499b8568 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.h +++ b/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.h @@ -10,7 +10,8 @@ namespace NKikimr::NStorage { TStateStoragePerPileGenerator(THashMap>>& nodes, const std::unordered_map& selfHealNodesState, TBridgePileId pileId, - std::unordered_set& usedNodes); + std::unordered_set& usedNodes, + const NKikimrConfig::TDomainsConfig::TStateStorage& oldConfig); bool IsGoodConfig() const; void AddRingGroup(NKikimrConfig::TDomainsConfig::TStateStorage *ss); @@ -26,6 +27,7 @@ namespace NKikimr::NStorage { void FillNodeGroups(THashMap>>& nodes); void CalculateRingsParameters(); bool PickNodesSimpleStrategy(TNodeGroup& group, ui32 stateLimit, bool ignoreRacks); + bool PickOldNodesStrategy(TNodeGroup& group); void PickNodes(TNodeGroup& group); ui32 CalcNodeState(ui32 nodeId, bool disconnected) const; @@ -33,6 +35,7 @@ namespace NKikimr::NStorage { const std::unordered_map& SelfHealNodesState; std::vector NodeGroups; std::unordered_set& UsedNodes; + const NKikimrConfig::TDomainsConfig::TStateStorage& OldConfig; std::vector> Rings; ui32 RingsInGroupCount = 1; ui32 ReplicasInRingCount = 1; diff --git a/ydb/core/blobstorage/nodewarden/distconf_ut.cpp b/ydb/core/blobstorage/nodewarden/distconf_ut.cpp index b23e934cec3a..2d2dfd5049bf 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_ut.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_ut.cpp @@ -35,7 +35,7 @@ Y_UNIT_TEST_SUITE(TDistconfGenerateConfigTest) { return ss; } - NKikimrConfig::TDomainsConfig::TStateStorage GenerateDCStateStorage(ui32 dcCnt, ui32 racksCnt, ui32 nodesInRack, std::unordered_map nodesState = {}, std::unordered_set usedNodes = {}) { + NKikimrConfig::TDomainsConfig::TStateStorage GenerateDCStateStorage(ui32 dcCnt, ui32 racksCnt, ui32 nodesInRack, std::unordered_map nodesState = {}, std::unordered_set usedNodes = {}, std::vector oldConfig = {}, ui32 oldNToSelect = 9) { NKikimrBlobStorage::TStorageConfig config; ui32 nodeId = 1; NKikimr::NStorage::TDistributedConfigKeeper keeper(nullptr, nullptr, true); @@ -50,11 +50,20 @@ Y_UNIT_TEST_SUITE(TDistconfGenerateConfigTest) { } } } + NKikimrConfig::TDomainsConfig::TStateStorage oldSS; + if (!oldConfig.empty()) { + auto* rg = oldSS.AddRingGroups(); + rg->SetNToSelect(oldNToSelect); + for (ui32 node : oldConfig) { + auto* ssRing = rg->AddRing(); + ssRing->AddNode(node); + } + } NKikimrConfig::TDomainsConfig::TStateStorage ss; - for(auto [nodeId, state] : nodesState) { + for (auto [nodeId, state] : nodesState) { keeper.SelfHealNodesState[nodeId] = state; } - keeper.GenerateStateStorageConfig(&ss, config, usedNodes); + keeper.GenerateStateStorageConfig(&ss, config, usedNodes, oldSS); return ss; } @@ -153,7 +162,16 @@ Y_UNIT_TEST_SUITE(TDistconfGenerateConfigTest) { CheckStateStorage(GenerateDCStateStorage(1, 1, 20, { {3, 2} }, { 1, 2, 3, 4, 9 }), 5, {5, 6, 7, 8, 10, 11, 12, 13}); CheckStateStorage(GenerateDCStateStorage(3, 3, 3, { {13, 2} }, { 4, 16 }), 9, {1, 5, 7, 10, 14, 17, 19, 22, 25}); CheckStateStorage(GenerateDCStateStorage(4, 3, 1, { {3, 2} }, { 1 }), 9, {2, 4, 5, 6, 7, 8, 9, 10, 11}); + } + + Y_UNIT_TEST(UseOldNodesInDisconnectedDC) { + // DC is connected, not enough bad nodes in DC - normak config generation + CheckStateStorage(GenerateDCStateStorage(3, 3, 3, { {10, 2}, {11, 4}, {13, 3}, {14, 4} }, {}, {1, 5, 8, 10, 14, 17, 19, 22, 25}), 9, {1, 4, 7, 12, 15, 16, 19, 22, 25}); + // Disconnected DC, but current config is invalid, build new config without usage of node statuses in disconnected DC + CheckStateStorage(GenerateDCStateStorage(3, 3, 3, { {10, 2}, {11, 4}, {13, 3}, {14, 4}, {15, 2} }, {}, {1, 5, 8, 10, 14, 17, 19, 22, 25}, 5), 9, {1, 4, 7, 10, 13, 16, 19, 22, 25}); + // DC disconnected - use previous config for this DC + CheckStateStorage(GenerateDCStateStorage(3, 3, 3, { {10, 2}, {11, 4}, {13, 3}, {14, 4}, {15, 2} }, {}, {1, 5, 8, 10, 14, 17, 19, 22, 25}, 9), 9, {1, 4, 7, 10, 14, 17, 19, 22, 25}); } } } From b34e8a07a260c6bdbf9803fc01bdbbfc8e25e160 Mon Sep 17 00:00:00 2001 From: qyryq Date: Fri, 8 Aug 2025 10:45:16 +0300 Subject: [PATCH 24/52] Kafka: Fix defaults (#22518) --- ydb/core/kafka_proxy/kafka_records.h | 130 +++++++++---------- ydb/core/kafka_proxy/ut/ut_protocol.cpp | 6 +- ydb/core/kafka_proxy/ut/ut_serialization.cpp | 42 +++--- 3 files changed, 90 insertions(+), 88 deletions(-) diff --git a/ydb/core/kafka_proxy/kafka_records.h b/ydb/core/kafka_proxy/kafka_records.h index e94c6888affd..f40f081e7af7 100644 --- a/ydb/core/kafka_proxy/kafka_records.h +++ b/ydb/core/kafka_proxy/kafka_records.h @@ -32,7 +32,7 @@ class TKafkaHeader: public TMessage { struct KeyMeta { using Type = TKafkaBytes; using TypeDesc = NPrivate::TKafkaBytesDesc; - + static constexpr const char* Name = "key"; static constexpr const char* About = ""; static const Type Default; // = {""}; @@ -48,7 +48,7 @@ class TKafkaHeader: public TMessage { struct ValueMeta { using Type = TKafkaBytes; using TypeDesc = NPrivate::TKafkaBytesDesc; - + static constexpr const char* Name = "value"; static constexpr const char* About = ""; static constexpr NPrivate::ESizeFormat SizeFormat = NPrivate::ESizeFormat::Varint; @@ -63,7 +63,7 @@ class TKafkaHeader: public TMessage { i32 Size(TKafkaVersion version) const override; void Read(TKafkaReadable& readable, TKafkaVersion version) override; void Write(TKafkaWritable& writable, TKafkaVersion version) const override; - + bool operator==(const TKafkaHeader& other) const = default; }; @@ -82,11 +82,11 @@ class TKafkaRecord: public TMessage { struct LengthMeta { using Type = TKafkaInt32; using TypeDesc = NPrivate::TKafkaVarintDesc; - + static constexpr const char* Name = "length"; static constexpr const char* About = ""; static constexpr Type Default = 0; - + static constexpr TKafkaVersions PresentVersions = VersionsAlways; static constexpr TKafkaVersions TaggedVersions = VersionsNever; static constexpr TKafkaVersions NullableVersions = VersionsNever; @@ -97,11 +97,11 @@ class TKafkaRecord: public TMessage { struct AttributesMeta { using Type = TKafkaInt8; using TypeDesc = NPrivate::TKafkaIntDesc; - + static constexpr const char* Name = "attributes"; static constexpr const char* About = ""; static constexpr Type Default = 0; - + static constexpr TKafkaVersions PresentVersions = VersionsAlways; static constexpr TKafkaVersions TaggedVersions = VersionsNever; static constexpr TKafkaVersions NullableVersions = VersionsNever; @@ -112,11 +112,11 @@ class TKafkaRecord: public TMessage { struct TimestampDeltaMeta { using Type = TKafkaInt64; using TypeDesc = NPrivate::TKafkaVarintDesc; - + static constexpr const char* Name = "timestampDelta"; static constexpr const char* About = ""; static constexpr Type Default = 0; - + static constexpr TKafkaVersions PresentVersions = VersionsAlways; static constexpr TKafkaVersions TaggedVersions = VersionsNever; static constexpr TKafkaVersions NullableVersions = VersionsNever; @@ -127,11 +127,11 @@ class TKafkaRecord: public TMessage { struct OffsetDeltaMeta { using Type = TKafkaInt64; using TypeDesc = NPrivate::TKafkaVarintDesc; - + static constexpr const char* Name = "offsetDelta"; static constexpr const char* About = ""; static constexpr Type Default = 0; - + static constexpr TKafkaVersions PresentVersions = VersionsAlways; static constexpr TKafkaVersions TaggedVersions = VersionsNever; static constexpr TKafkaVersions NullableVersions = VersionsNever; @@ -142,12 +142,12 @@ class TKafkaRecord: public TMessage { struct KeyMeta { using Type = TKafkaBytes; using TypeDesc = NPrivate::TKafkaBytesDesc; - + static constexpr const char* Name = "key"; static constexpr const char* About = ""; static const Type Default; // = {""}; static constexpr NPrivate::ESizeFormat SizeFormat = NPrivate::ESizeFormat::Varint; - + static constexpr TKafkaVersions PresentVersions = VersionsAlways; static constexpr TKafkaVersions TaggedVersions = VersionsNever; static constexpr TKafkaVersions NullableVersions = VersionsAlways; @@ -190,7 +190,7 @@ class TKafkaRecord: public TMessage { i32 Size(TKafkaVersion version) const override; void Read(TKafkaReadable& readable, TKafkaVersion version) override; void Write(TKafkaWritable& writable, TKafkaVersion version) const override; - + bool operator==(const TKafkaRecord& other) const = default; NKikimrPQClient::TDataChunk DataChunk; @@ -211,11 +211,11 @@ class TKafkaRecordBatch: public TMessage { struct BaseOffsetMeta { using Type = TKafkaInt64; using TypeDesc = NPrivate::TKafkaIntDesc; - + static constexpr const char* Name = "baseOffset"; static constexpr const char* About = ""; static constexpr Type Default = 0; - + static constexpr TKafkaVersions PresentVersions = VersionsAlways; static constexpr TKafkaVersions TaggedVersions = VersionsNever; static constexpr TKafkaVersions NullableVersions = VersionsNever; @@ -226,11 +226,11 @@ class TKafkaRecordBatch: public TMessage { struct BatchLengthMeta { using Type = TKafkaInt32; using TypeDesc = NPrivate::TKafkaIntDesc; - + static constexpr const char* Name = "batchLength"; static constexpr const char* About = ""; static constexpr Type Default = 0; - + static constexpr TKafkaVersions PresentVersions = VersionsAlways; static constexpr TKafkaVersions TaggedVersions = VersionsNever; static constexpr TKafkaVersions NullableVersions = VersionsNever; @@ -241,11 +241,11 @@ class TKafkaRecordBatch: public TMessage { struct PartitionLeaderEpochMeta { using Type = TKafkaInt32; using TypeDesc = NPrivate::TKafkaIntDesc; - + static constexpr const char* Name = "partitionLeaderEpoch"; static constexpr const char* About = ""; - static constexpr Type Default = 0; - + static constexpr Type Default = -1; + static constexpr TKafkaVersions PresentVersions = VersionsAlways; static constexpr TKafkaVersions TaggedVersions = VersionsNever; static constexpr TKafkaVersions NullableVersions = VersionsNever; @@ -256,11 +256,11 @@ class TKafkaRecordBatch: public TMessage { struct MagicMeta { using Type = TKafkaInt8; using TypeDesc = NPrivate::TKafkaIntDesc; - + static constexpr const char* Name = "magic"; static constexpr const char* About = "current magic value is 2"; static constexpr Type Default = 2; - + static constexpr TKafkaVersions PresentVersions = VersionsAlways; static constexpr TKafkaVersions TaggedVersions = VersionsNever; static constexpr TKafkaVersions NullableVersions = VersionsNever; @@ -271,11 +271,11 @@ class TKafkaRecordBatch: public TMessage { struct CrcMeta { using Type = TKafkaInt32; using TypeDesc = NPrivate::TKafkaIntDesc; - + static constexpr const char* Name = "crc"; static constexpr const char* About = "The CRC covers the data from the attributes to the end of the batch (i.e. all the bytes that follow the CRC)"; static constexpr Type Default = 0; - + static constexpr TKafkaVersions PresentVersions = VersionsAlways; static constexpr TKafkaVersions TaggedVersions = VersionsNever; static constexpr TKafkaVersions NullableVersions = VersionsNever; @@ -286,11 +286,11 @@ class TKafkaRecordBatch: public TMessage { struct AttributesMeta { using Type = TKafkaInt16; using TypeDesc = NPrivate::TKafkaIntDesc; - + static constexpr const char* Name = "attributes"; static constexpr const char* About = ""; static constexpr Type Default = 0; - + static constexpr TKafkaVersions PresentVersions = VersionsAlways; static constexpr TKafkaVersions TaggedVersions = VersionsNever; static constexpr TKafkaVersions NullableVersions = VersionsNever; @@ -301,11 +301,11 @@ class TKafkaRecordBatch: public TMessage { struct LastOffsetDeltaMeta { using Type = TKafkaInt32; using TypeDesc = NPrivate::TKafkaIntDesc; - + static constexpr const char* Name = "lastOffsetDelta"; static constexpr const char* About = ""; static constexpr Type Default = 0; - + static constexpr TKafkaVersions PresentVersions = VersionsAlways; static constexpr TKafkaVersions TaggedVersions = VersionsNever; static constexpr TKafkaVersions NullableVersions = VersionsNever; @@ -316,11 +316,11 @@ class TKafkaRecordBatch: public TMessage { struct BaseTimestampMeta { using Type = TKafkaInt64; using TypeDesc = NPrivate::TKafkaIntDesc; - + static constexpr const char* Name = "baseTimestamp"; static constexpr const char* About = ""; - static constexpr Type Default = 0; - + static constexpr Type Default = -1; + static constexpr TKafkaVersions PresentVersions = VersionsAlways; static constexpr TKafkaVersions TaggedVersions = VersionsNever; static constexpr TKafkaVersions NullableVersions = VersionsNever; @@ -331,11 +331,11 @@ class TKafkaRecordBatch: public TMessage { struct MaxTimestampMeta { using Type = TKafkaInt64; using TypeDesc = NPrivate::TKafkaIntDesc; - + static constexpr const char* Name = "maxTimestamp"; static constexpr const char* About = ""; - static constexpr Type Default = 0; - + static constexpr Type Default = -1; + static constexpr TKafkaVersions PresentVersions = VersionsAlways; static constexpr TKafkaVersions TaggedVersions = VersionsNever; static constexpr TKafkaVersions NullableVersions = VersionsNever; @@ -346,11 +346,11 @@ class TKafkaRecordBatch: public TMessage { struct ProducerIdMeta { using Type = TKafkaInt64; using TypeDesc = NPrivate::TKafkaIntDesc; - + static constexpr const char* Name = "producerId"; static constexpr const char* About = ""; - static constexpr Type Default = 0; - + static constexpr Type Default = -1; + static constexpr TKafkaVersions PresentVersions = VersionsAlways; static constexpr TKafkaVersions TaggedVersions = VersionsNever; static constexpr TKafkaVersions NullableVersions = VersionsNever; @@ -361,11 +361,11 @@ class TKafkaRecordBatch: public TMessage { struct ProducerEpochMeta { using Type = TKafkaInt16; using TypeDesc = NPrivate::TKafkaIntDesc; - + static constexpr const char* Name = "producerEpoch"; static constexpr const char* About = ""; - static constexpr Type Default = 0; - + static constexpr Type Default = -1; + static constexpr TKafkaVersions PresentVersions = VersionsAlways; static constexpr TKafkaVersions TaggedVersions = VersionsNever; static constexpr TKafkaVersions NullableVersions = VersionsNever; @@ -376,11 +376,11 @@ class TKafkaRecordBatch: public TMessage { struct BaseSequenceMeta { using Type = TKafkaInt32; using TypeDesc = NPrivate::TKafkaIntDesc; - + static constexpr const char* Name = "baseSequence"; static constexpr const char* About = ""; - static constexpr Type Default = 0; - + static constexpr Type Default = -1; + static constexpr TKafkaVersions PresentVersions = VersionsAlways; static constexpr TKafkaVersions TaggedVersions = VersionsNever; static constexpr TKafkaVersions NullableVersions = VersionsNever; @@ -393,10 +393,10 @@ class TKafkaRecordBatch: public TMessage { using ItemTypeDesc = NPrivate::TKafkaStructDesc; using Type = std::vector; using TypeDesc = NPrivate::TKafkaArrayDesc; - + static constexpr const char* Name = "records"; static constexpr const char* About = ""; - + static constexpr TKafkaVersions PresentVersions = VersionsAlways; static constexpr TKafkaVersions TaggedVersions = VersionsNever; static constexpr TKafkaVersions NullableVersions = VersionsNever; @@ -407,7 +407,7 @@ class TKafkaRecordBatch: public TMessage { i32 Size(TKafkaVersion version) const override; void Read(TKafkaReadable& readable, TKafkaVersion version) override; void Write(TKafkaWritable& writable, TKafkaVersion version) const override; - + bool operator==(const TKafkaRecordBatch& other) const = default; ECompressionType CompressionType(); @@ -431,11 +431,11 @@ class TKafkaRecordV0: public TMessage { struct MessageSizeMeta { using Type = TKafkaInt32; using TypeDesc = NPrivate::TKafkaIntDesc; - + static constexpr const char* Name = "messageSize"; static constexpr const char* About = ""; static constexpr Type Default = 0; - + static constexpr TKafkaVersions PresentVersions = VersionsAlways; static constexpr TKafkaVersions TaggedVersions = VersionsNever; static constexpr TKafkaVersions NullableVersions = VersionsNever; @@ -446,11 +446,11 @@ class TKafkaRecordV0: public TMessage { struct CrcMeta { using Type = TKafkaInt32; using TypeDesc = NPrivate::TKafkaIntDesc; - + static constexpr const char* Name = "CRC"; static constexpr const char* About = ""; static constexpr Type Default = 0; - + static constexpr TKafkaVersions PresentVersions = VersionsAlways; static constexpr TKafkaVersions TaggedVersions = VersionsNever; static constexpr TKafkaVersions NullableVersions = VersionsNever; @@ -461,11 +461,11 @@ class TKafkaRecordV0: public TMessage { struct MagicMeta { using Type = TKafkaInt8; using TypeDesc = NPrivate::TKafkaIntDesc; - + static constexpr const char* Name = "magic"; static constexpr const char* About = ""; static constexpr Type Default = 0; - + static constexpr TKafkaVersions PresentVersions = VersionsAlways; static constexpr TKafkaVersions TaggedVersions = VersionsNever; static constexpr TKafkaVersions NullableVersions = VersionsNever; @@ -476,11 +476,11 @@ class TKafkaRecordV0: public TMessage { struct AttributesMeta { using Type = TKafkaInt8; using TypeDesc = NPrivate::TKafkaIntDesc; - + static constexpr const char* Name = "attributes"; static constexpr const char* About = ""; static constexpr Type Default = 0; - + static constexpr TKafkaVersions PresentVersions = VersionsAlways; static constexpr TKafkaVersions TaggedVersions = VersionsNever; static constexpr TKafkaVersions NullableVersions = VersionsNever; @@ -491,11 +491,11 @@ class TKafkaRecordV0: public TMessage { struct TimestampMeta { using Type = TKafkaInt64; using TypeDesc = NPrivate::TKafkaIntDesc; - + static constexpr const char* Name = "timestamp"; static constexpr const char* About = ""; static constexpr Type Default = 0; - + static constexpr TKafkaVersions PresentVersions = TKafkaVersions(1, Max()); static constexpr TKafkaVersions TaggedVersions = VersionsNever; static constexpr TKafkaVersions NullableVersions = VersionsNever; @@ -506,11 +506,11 @@ class TKafkaRecordV0: public TMessage { struct KeyMeta { using Type = TKafkaBytes; using TypeDesc = NPrivate::TKafkaBytesDesc; - + static constexpr const char* Name = "key"; static constexpr const char* About = ""; static const Type Default; // = {""}; - + static constexpr TKafkaVersions PresentVersions = VersionsAlways; static constexpr TKafkaVersions TaggedVersions = VersionsNever; static constexpr TKafkaVersions NullableVersions = VersionsAlways; @@ -535,7 +535,7 @@ class TKafkaRecordV0: public TMessage { i32 Size(TKafkaVersion version) const override; void Read(TKafkaReadable& readable, TKafkaVersion version) override; void Write(TKafkaWritable& writable, TKafkaVersion version) const override; - + bool operator==(const TKafkaRecordV0& other) const = default; }; @@ -552,11 +552,11 @@ class TKafkaRecordBatchV0: public TMessage { struct OffsetMeta { using Type = TKafkaInt64; using TypeDesc = NPrivate::TKafkaIntDesc; - + static constexpr const char* Name = "offset"; static constexpr const char* About = ""; static constexpr Type Default = 0; - + static constexpr TKafkaVersions PresentVersions = VersionsAlways; static constexpr TKafkaVersions TaggedVersions = VersionsNever; static constexpr TKafkaVersions NullableVersions = VersionsNever; @@ -567,10 +567,10 @@ class TKafkaRecordBatchV0: public TMessage { struct RecordMeta { using Type = TKafkaRecordV0; using TypeDesc = NPrivate::TKafkaStructDesc; - + static constexpr const char* Name = "records"; static constexpr const char* About = ""; - + static constexpr TKafkaVersions PresentVersions = VersionsAlways; static constexpr TKafkaVersions TaggedVersions = VersionsNever; static constexpr TKafkaVersions NullableVersions = VersionsNever; @@ -581,7 +581,7 @@ class TKafkaRecordBatchV0: public TMessage { i32 Size(TKafkaVersion version) const override; void Read(TKafkaReadable& readable, TKafkaVersion version) override; void Write(TKafkaWritable& writable, TKafkaVersion version) const override; - + bool operator==(const TKafkaRecordBatchV0& other) const = default; }; diff --git a/ydb/core/kafka_proxy/ut/ut_protocol.cpp b/ydb/core/kafka_proxy/ut/ut_protocol.cpp index 018cd822d369..1909e27ca21a 100644 --- a/ydb/core/kafka_proxy/ut/ut_protocol.cpp +++ b/ydb/core/kafka_proxy/ut/ut_protocol.cpp @@ -403,12 +403,14 @@ Y_UNIT_TEST_SUITE(KafkaProtocol) { auto readHeaderValueStr = TString(readHeaderValue.data(), readHeaderValue.size()); UNIT_ASSERT_VALUES_EQUAL(readHeaderValueStr, headerValue); } - + + auto msg2 = client.Produce(topicName, 0, batch); + // read by logbroker protocol auto readMessages = Read(topicReader); UNIT_ASSERT_EQUAL(readMessages.size(), 1); - UNIT_ASSERT_EQUAL(readMessages[0].GetMessages().size(), 1); + UNIT_ASSERT_EQUAL(readMessages[0].GetMessages().size(), 2); auto& readMessage = readMessages[0].GetMessages()[0]; readMessage.Commit(); diff --git a/ydb/core/kafka_proxy/ut/ut_serialization.cpp b/ydb/core/kafka_proxy/ut/ut_serialization.cpp index f2abee013c0c..987803a5f406 100644 --- a/ydb/core/kafka_proxy/ut/ut_serialization.cpp +++ b/ydb/core/kafka_proxy/ut/ut_serialization.cpp @@ -72,7 +72,7 @@ Y_UNIT_TEST(ApiVersionsRequest) { } Y_UNIT_TEST(ApiVersionsResponse) { - TString longString = "long-string-value-0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + TString longString = "long-string-value-0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; TWritableBuf sb(nullptr, BUFFER_SIZE); @@ -304,7 +304,7 @@ struct Meta_TKafkaInt8 { static constexpr const char* About = "The test field."; static constexpr const TKafkaInt32 Tag = 31; static const Type Default; // = 7; - + static constexpr TKafkaVersions PresentVersions{3, 97}; static constexpr TKafkaVersions TaggedVersions{11, 17}; static constexpr TKafkaVersions NullableVersions{5, 19}; @@ -367,7 +367,7 @@ struct Meta_TKafkaStruct { static constexpr const char* About = "The test field."; static constexpr const TKafkaInt32 Tag = 31; static const Type Default; // = 7; - + static constexpr TKafkaVersions PresentVersions{3, 97}; static constexpr TKafkaVersions TaggedVersions{11, 17}; static constexpr TKafkaVersions NullableVersions{5, 19}; @@ -390,7 +390,7 @@ struct Meta_TKafkaString { static constexpr const char* About = "The test field."; static constexpr const TKafkaInt32 Tag = 31; static const Type Default; // = 7; - + static constexpr TKafkaVersions PresentVersions{3, 97}; static constexpr TKafkaVersions TaggedVersions{11, 17}; static constexpr TKafkaVersions NullableVersions{5, 19}; @@ -455,7 +455,7 @@ struct Meta_TKafkaArray { static constexpr const char* Name = "value"; static constexpr const char* About = "The test field."; static constexpr const TKafkaInt32 Tag = 31; - + static constexpr TKafkaVersions PresentVersions{3, 97}; static constexpr TKafkaVersions TaggedVersions{11, 17}; static constexpr TKafkaVersions NullableVersions{5, 19}; @@ -518,7 +518,7 @@ struct Meta_TKafkaBytes { static constexpr const char* Name = "value"; static constexpr const char* About = "The test field."; static constexpr const TKafkaInt32 Tag = 31; - + static constexpr TKafkaVersions PresentVersions{3, 97}; static constexpr TKafkaVersions TaggedVersions{11, 17}; static constexpr TKafkaVersions NullableVersions{5, 19}; @@ -616,7 +616,7 @@ struct Meta_TKafkaFloat64 { static constexpr const char* About = "The test field."; static constexpr const TKafkaInt32 Tag = 31; static const Type Default; // = 7; - + static constexpr TKafkaVersions PresentVersions{3, 97}; static constexpr TKafkaVersions TaggedVersions{11, 17}; static constexpr TKafkaVersions NullableVersions{5, 19}; @@ -644,7 +644,7 @@ Y_UNIT_TEST(TKafkaFloat64_PresentVersion_NotTaggedVersion) { } Y_UNIT_TEST(RequestHeader_reference) { - ui8 reference[] = {0x00, 0x12, 0x00, 0x00, 0x7F, 0x6F, 0x6F, 0x68, 0x00, 0x0A, 0x70, 0x72, 0x6F, 0x64, 0x75, 0x63, + ui8 reference[] = {0x00, 0x12, 0x00, 0x00, 0x7F, 0x6F, 0x6F, 0x68, 0x00, 0x0A, 0x70, 0x72, 0x6F, 0x64, 0x75, 0x63, 0x65, 0x72, 0x2D, 0x31}; TWritableBuf sb(nullptr, BUFFER_SIZE); @@ -667,9 +667,9 @@ Y_UNIT_TEST(ProduceRequestData) { 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x01, 0x89, 0x0C, 0x95, 0xAF, 0x25, 0x00, 0x00, 0x01, 0x89, 0x0C, 0x95, 0xB2, 0x19, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x5C, 0x00, 0x00, 0x00, 0x06, 0x6B, 0x65, - 0x79, 0x0E, 0x6D, 0x73, 0x67, 0x2D, 0x31, 0x2D, 0x31, 0x06, 0x06, 0x68, 0x2D, 0x31, 0x0A, 0x76, - 0x2D, 0x31, 0x2D, 0x31, 0x06, 0x68, 0x2D, 0x32, 0x0A, 0x76, 0x2D, 0x32, 0x2D, 0x31, 0x06, 0x68, - 0x2D, 0x33, 0x0A, 0x76, 0x2D, 0x33, 0x2D, 0x31, 0x5E, 0x00, 0xE6, 0x0B, 0x02, 0x06, 0x6B, 0x65, + 0x79, 0x0E, 0x6D, 0x73, 0x67, 0x2D, 0x31, 0x2D, 0x31, 0x06, 0x06, 0x68, 0x2D, 0x31, 0x0A, 0x76, + 0x2D, 0x31, 0x2D, 0x31, 0x06, 0x68, 0x2D, 0x32, 0x0A, 0x76, 0x2D, 0x32, 0x2D, 0x31, 0x06, 0x68, + 0x2D, 0x33, 0x0A, 0x76, 0x2D, 0x33, 0x2D, 0x31, 0x5E, 0x00, 0xE6, 0x0B, 0x02, 0x06, 0x6B, 0x65, 0x79, 0x0E, 0x6D, 0x73, 0x67, 0x2D, 0x31, 0x2D, 0x32, 0x06, 0x06, 0x68, 0x2D, 0x31, 0x0A, 0x76, 0x2D, 0x31, 0x2D, 0x32, 0x06, 0x68, 0x2D, 0x32, 0x0A, 0x76, 0x2D, 0x32, 0x2D, 0x32, 0x06, 0x68, 0x2D, 0x33, 0x0A, 0x76, 0x2D, 0x33, 0x2D, 0x32, 0x5E, 0x00, 0xE8, 0x0B, 0x04, 0x06, 0x6B, 0x65, @@ -706,7 +706,7 @@ Y_UNIT_TEST(ProduceRequestData) { UNIT_ASSERT_EQUAL(r0.BaseSequence, 0); UNIT_ASSERT_EQUAL(r0.Records.size(), (size_t)3); - + UNIT_ASSERT_EQUAL(r0.Records[0].Key, TKafkaRawBytes("key", 3)); UNIT_ASSERT_EQUAL(r0.Records[0].Value, TKafkaRawBytes("msg-1-1", 7)); UNIT_ASSERT_EQUAL(r0.Records[0].Headers.size(), (size_t)3); @@ -716,7 +716,7 @@ Y_UNIT_TEST(ProduceRequestData) { UNIT_ASSERT_EQUAL(r0.Records[0].Headers[1].Value, TKafkaRawBytes("v-2-1", 5)); UNIT_ASSERT_EQUAL(r0.Records[0].Headers[2].Key, TKafkaRawBytes("h-3", 3)); UNIT_ASSERT_EQUAL(r0.Records[0].Headers[2].Value, TKafkaRawBytes("v-3-1", 5)); - + UNIT_ASSERT_EQUAL(r0.Records[1].Key, TKafkaRawBytes("key", 3)); UNIT_ASSERT_EQUAL(r0.Records[1].Value, TKafkaRawBytes("msg-1-2", 7)); UNIT_ASSERT_EQUAL(r0.Records[1].Headers.size(), (size_t)3); @@ -726,7 +726,7 @@ Y_UNIT_TEST(ProduceRequestData) { UNIT_ASSERT_EQUAL(r0.Records[1].Headers[1].Value, TKafkaRawBytes("v-2-2", 5)); UNIT_ASSERT_EQUAL(r0.Records[1].Headers[2].Key, TKafkaRawBytes("h-3", 3)); UNIT_ASSERT_EQUAL(r0.Records[1].Headers[2].Value, TKafkaRawBytes("v-3-2", 5)); - + UNIT_ASSERT_EQUAL(r0.Records[2].Key, TKafkaRawBytes("key", 3)); UNIT_ASSERT_EQUAL(r0.Records[2].Value, TKafkaRawBytes("msg-1-3", 7)); UNIT_ASSERT_EQUAL(r0.Records[2].Headers.size(), (size_t)3); @@ -775,19 +775,19 @@ Y_UNIT_TEST(ProduceRequestData_Record_v0) { auto& r0 = *result.TopicData[0].PartitionData[0].Records; UNIT_ASSERT_EQUAL(r0.BaseOffset, 0); UNIT_ASSERT_EQUAL(r0.BatchLength, 0); - UNIT_ASSERT_EQUAL(r0.PartitionLeaderEpoch, 0); + UNIT_ASSERT_EQUAL(r0.PartitionLeaderEpoch, -1); UNIT_ASSERT_EQUAL(r0.Magic, 0); UNIT_ASSERT_EQUAL(r0.Crc, 544167206); UNIT_ASSERT_EQUAL(r0.Attributes, 0); UNIT_ASSERT_EQUAL(r0.LastOffsetDelta, 0); - UNIT_ASSERT_EQUAL(r0.BaseTimestamp, 0); - UNIT_ASSERT_EQUAL(r0.MaxTimestamp, 0); - UNIT_ASSERT_EQUAL(r0.ProducerId, 0); - UNIT_ASSERT_EQUAL(r0.ProducerEpoch, 0); - UNIT_ASSERT_EQUAL(r0.BaseSequence, 0); + UNIT_ASSERT_EQUAL(r0.BaseTimestamp, -1); + UNIT_ASSERT_EQUAL(r0.MaxTimestamp, -1); + UNIT_ASSERT_EQUAL(r0.ProducerId, -1); + UNIT_ASSERT_EQUAL(r0.ProducerEpoch, -1); + UNIT_ASSERT_EQUAL(r0.BaseSequence, -1); UNIT_ASSERT_EQUAL(r0.Records.size(), (size_t)1); - + UNIT_ASSERT_EQUAL(r0.Records[0].Key, TKafkaRawBytes("key-1", 5)); UNIT_ASSERT_EQUAL(r0.Records[0].Value, TKafkaRawBytes("test message", 12)); UNIT_ASSERT_EQUAL(r0.Records[0].Headers.size(), (size_t)0); From b51b4d098b8dd0a796846081ac95ae5b8ae71dc3 Mon Sep 17 00:00:00 2001 From: Semyon Date: Fri, 8 Aug 2025 11:22:45 +0300 Subject: [PATCH 25/52] deduplication fixes (#22497) --- .../engines/reader/simple_reader/duplicates/manager.cpp | 2 +- .../engines/reader/simple_reader/duplicates/splitter.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.cpp index 9b13bbb82945..7a161831c342 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.cpp @@ -109,7 +109,7 @@ class TColumnDataAccessorFetching: public IDataAccessorRequestsSubscriber { ui64 mem = 0; for (const auto& accessor : result.ExtractPortionsVector()) { - mem += accessor->GetColumnRawBytes(Columns); + mem += accessor->GetColumnRawBytes(Columns, false); } NGroupedMemoryManager::TDeduplicationMemoryLimiterOperator::SendToAllocation(Callback->GetContext()->GetMemoryProcessId(), diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/splitter.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/splitter.h index c94b5afd6a69..67004eeb3292 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/splitter.h +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/splitter.h @@ -41,7 +41,7 @@ class TColumnDataSplitter { } TString DebugString() const { - return TStringBuilder() << (IsLast ? "Last:" : "First:") << Key.GetData().DebugJson(0); + return TStringBuilder() << (IsLast ? "Last:" : "First:") << Key.GetSorting()->DebugJson(0); } }; From 6966047948c42f095fadcace4eb7aaac681560fa Mon Sep 17 00:00:00 2001 From: Nikolay Shestakov Date: Fri, 8 Aug 2025 13:34:00 +0500 Subject: [PATCH 26/52] =?UTF-8?q?Added=20ability=20to=20add=20a=20transfer?= =?UTF-8?q?=20and=20a=20async=20replication=20to=20subdirecto=E2=80=A6=20(?= =?UTF-8?q?#22503)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ydb/core/transfer/ut/common/utils.h | 12 +++++-- .../transfer/ut/functional/transfer_ut.cpp | 31 +++++++++++++++++++ ...emeshard__operation_create_replication.cpp | 2 +- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/ydb/core/transfer/ut/common/utils.h b/ydb/core/transfer/ut/common/utils.h index 9e18ddf516f7..15b63071c534 100644 --- a/ydb/core/transfer/ut/common/utils.h +++ b/ydb/core/transfer/ut/common/utils.h @@ -426,6 +426,10 @@ struct MainTestCase { }; void CreateTransfer(const std::string& lambda, const CreateTransferSettings& settings = CreateTransferSettings()) { + CreateTransfer(TransferName, lambda, settings); + } + + void CreateTransfer(const std::string& name, const std::string& lambda, const CreateTransferSettings& settings = CreateTransferSettings()) { std::vector options; if (!settings.LocalTopic) { options.push_back(TStringBuilder() << "CONNECTION_STRING = 'grpc://" << ConnectionString << "'"); @@ -457,7 +461,7 @@ struct MainTestCase { WITH ( %s ); - )", lambda.data(), TransferName.data(), topicName.data(), TableName.data(), optionsStr.data()); + )", lambda.data(), name.data(), topicName.data(), TableName.data(), optionsStr.data()); ExecuteDDL(ddl, true, settings.ExpectedError); } @@ -497,6 +501,10 @@ struct MainTestCase { } void AlterTransfer(const AlterTransferSettings& settings, bool success = true) { + AlterTransfer(TransferName, settings, success); + } + + void AlterTransfer(const std::string& name, const AlterTransferSettings& settings, bool success = true) { std::string lambda = settings.TransformLambda ? *settings.TransformLambda : ""; std::string setLambda = settings.TransformLambda ? "SET USING $l" : ""; @@ -523,7 +531,7 @@ struct MainTestCase { ALTER TRANSFER `%s` %s %s; - )", lambda.data(), TransferName.data(), setLambda.data(), setOptions.data()), success); + )", lambda.data(), name.data(), setLambda.data(), setOptions.data()), success); } void DropTransfer() { diff --git a/ydb/core/transfer/ut/functional/transfer_ut.cpp b/ydb/core/transfer/ut/functional/transfer_ut.cpp index f9ce907cd7b7..44a6443ee5c5 100644 --- a/ydb/core/transfer/ut/functional/transfer_ut.cpp +++ b/ydb/core/transfer/ut/functional/transfer_ut.cpp @@ -1311,5 +1311,36 @@ Y_UNIT_TEST_SUITE(Transfer) testCase.DropTopic(); testCase.DropTable(); } + + Y_UNIT_TEST(CreateAndAlterTransferInDirectory) + { + MainTestCase testCase; + testCase.CreateTable(R"( + CREATE TABLE `%s` ( + Key Uint64 NOT NULL, + Message Utf8, + PRIMARY KEY (Key) + ) WITH ( + STORE = %s + ); + )"); + testCase.CreateTopic(1); + + testCase.CreateDirectory("/local/subdir"); + testCase.CreateTransfer(TStringBuilder() << "subdir/" << testCase.TransferName, Sprintf(R"( + $l = ($x) -> { + return [ + <| + Key:CAST($x._offset AS Uint64), + Message:CAST($x._data AS Utf8) + |> + ]; + }; + )", testCase.TableName.data())); + + testCase.AlterTransfer(TStringBuilder() << "subdir/" << testCase.TransferName, + MainTestCase::AlterTransferSettings::WithBatching(TDuration::Seconds(1), 1)); + } + } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_replication.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_replication.cpp index 7a5ea4b63865..d495eb058f96 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_replication.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_replication.cpp @@ -382,7 +382,7 @@ class TCreateReplication: public TSubOperation { return result; } - auto path = parentPath.Child(name); + auto path = parentPath.Child(name, TPath::TSplitChildTag{}); { const auto checks = path.Check(); checks From 47bf508c51117520988357fe6a5905938dae93ad Mon Sep 17 00:00:00 2001 From: Nikolay Shestakov Date: Fri, 8 Aug 2025 13:54:59 +0500 Subject: [PATCH 27/52] Added _create_timestamp and _write_timestamp fields to the topic message (#22480) --- ydb/core/transfer/purecalc.h | 2 + ydb/core/transfer/purecalc_input.cpp | 28 ++++++-- ydb/core/transfer/transfer_writer.cpp | 2 + .../column_table/transfer_columntable_ut.cpp | 10 +++ .../transfer/ut/common/transfer_common.cpp | 66 +++++++++++++++++++ ydb/core/transfer/ut/common/transfer_common.h | 2 + ydb/core/transfer/ut/common/utils.h | 40 +++++++++-- .../ut/row_table/transfer_rowtable_ut.cpp | 10 +++ .../replication/ydb_proxy/topic_message.cpp | 5 ++ .../tx/replication/ydb_proxy/topic_message.h | 1 + 10 files changed, 155 insertions(+), 11 deletions(-) diff --git a/ydb/core/transfer/purecalc.h b/ydb/core/transfer/purecalc.h index ff54c1ae3944..fe6a56d0cb97 100644 --- a/ydb/core/transfer/purecalc.h +++ b/ydb/core/transfer/purecalc.h @@ -13,12 +13,14 @@ using namespace NYql::NPureCalc; using namespace NKikimr::NMiniKQL; struct TMessage { + TInstant CreateTimestamp; TString Data; TString MessageGroupId; ui64 Offset = 0; ui32 Partition = 0; TString ProducerId; ui64 SeqNo = 0; + TInstant WriteTimestamp; }; class TMessageInputSpec: public TInputSpecBase { diff --git a/ydb/core/transfer/purecalc_input.cpp b/ydb/core/transfer/purecalc_input.cpp index a2573a549ff0..8d3d0f2bdb25 100644 --- a/ydb/core/transfer/purecalc_input.cpp +++ b/ydb/core/transfer/purecalc_input.cpp @@ -12,14 +12,16 @@ namespace { using namespace NYql::NUdf; using namespace NKikimr::NMiniKQL; +constexpr const char* CreateTimestampFieldName = "_create_timestamp"; constexpr const char* DataFieldName = "_data"; constexpr const char* MessageGroupIdFieldName = "_message_group_id"; constexpr const char* OffsetFieldName = "_offset"; constexpr const char* PartitionFieldName = "_partition"; constexpr const char* ProducerIdFieldName = "_producer_id"; constexpr const char* SeqNoFieldName = "_seq_no"; +constexpr const char* WriteTimestampFieldName = "_write_timestamp"; -constexpr const size_t FieldCount = 6; // Change it when change fields +constexpr const size_t FieldCount = 8; // Change it when change fields NYT::TNode CreateTypeNode(const TString& fieldType) { @@ -39,12 +41,14 @@ void AddField(NYT::TNode& node, const TString& fieldName, const TString& fieldTy NYT::TNode CreateMessageScheme() { auto structMembers = NYT::TNode::CreateList(); + AddField(structMembers, CreateTimestampFieldName, "Timestamp"); AddField(structMembers, DataFieldName, "String"); AddField(structMembers, MessageGroupIdFieldName, "String"); AddField(structMembers, OffsetFieldName, "Uint64"); AddField(structMembers, PartitionFieldName, "Uint32"); AddField(structMembers, ProducerIdFieldName, "String"); AddField(structMembers, SeqNoFieldName, "Uint64"); + AddField(structMembers, WriteTimestampFieldName, "Timestamp"); return NYT::TNode::CreateList() .Add("StructType") @@ -56,6 +60,10 @@ static const TVector InputSchema{ CreateMessageScheme() }; struct TMessageWrapper { const TMessage& Message; + NYql::NUdf::TUnboxedValuePod GetCreateTimestamp() const { + return NYql::NUdf::TUnboxedValuePod(Message.CreateTimestamp.MilliSeconds()); + } + NYql::NUdf::TUnboxedValuePod GetData() const { return NKikimr::NMiniKQL::MakeString(Message.Data); } @@ -79,6 +87,10 @@ struct TMessageWrapper { NYql::NUdf::TUnboxedValuePod GetSeqNo() const { return NYql::NUdf::TUnboxedValuePod(Message.SeqNo); } + + NYql::NUdf::TUnboxedValuePod GetWriteTimestamp() const { + return NYql::NUdf::TUnboxedValuePod(Message.WriteTimestamp.MilliSeconds()); + } }; class TInputConverter { @@ -100,12 +112,14 @@ class TInputConverter { TMessageWrapper wrap {*message}; // lex order by field name - items[0] = wrap.GetData(); - items[1] = wrap.GetMessageGroupId(); - items[2] = wrap.GetOffset(); - items[3] = wrap.GetPartition(); - items[4] = wrap.GetProducerId(); - items[5] = wrap.GetSeqNo(); + items[0] = wrap.GetCreateTimestamp(); + items[1] = wrap.GetData(); + items[2] = wrap.GetMessageGroupId(); + items[3] = wrap.GetOffset(); + items[4] = wrap.GetPartition(); + items[5] = wrap.GetProducerId(); + items[6] = wrap.GetSeqNo(); + items[7] = wrap.GetWriteTimestamp(); } void ClearCache() { diff --git a/ydb/core/transfer/transfer_writer.cpp b/ydb/core/transfer/transfer_writer.cpp index 10da7930c8d6..e57bab1dd497 100644 --- a/ydb/core/transfer/transfer_writer.cpp +++ b/ydb/core/transfer/transfer_writer.cpp @@ -266,12 +266,14 @@ class TTransferWriter for (auto& message : records) { TMessage input; + input.CreateTimestamp = message.GetCreateTime(); input.Data = std::move(message.GetData()); input.MessageGroupId = std::move(message.GetMessageGroupId()); input.Partition = partitionId; input.ProducerId = std::move(message.GetProducerId()); input.Offset = message.GetOffset(); input.SeqNo = message.GetSeqNo(); + input.WriteTimestamp = message.GetWriteTime(); auto setError = [&](const auto& msg) { ProcessingErrorStatus = TEvWorker::TEvGone::EStatus::SCHEME_ERROR; diff --git a/ydb/core/transfer/ut/column_table/transfer_columntable_ut.cpp b/ydb/core/transfer/ut/column_table/transfer_columntable_ut.cpp index 4f5e5d942829..8155a4062c28 100644 --- a/ydb/core/transfer/ut/column_table/transfer_columntable_ut.cpp +++ b/ydb/core/transfer/ut/column_table/transfer_columntable_ut.cpp @@ -83,6 +83,11 @@ Y_UNIT_TEST_SUITE(Transfer_ColumnTable) } + Y_UNIT_TEST(MessageField_CreateTimestamp) + { + MessageField_CreateTimestamp(TableType); + } + Y_UNIT_TEST(MessageField_Partition) { MessageField_Partition(TableType); @@ -103,6 +108,11 @@ Y_UNIT_TEST_SUITE(Transfer_ColumnTable) MessageField_ProducerId(TableType); } + Y_UNIT_TEST(MessageField_WriteTimestamp) + { + MessageField_WriteTimestamp(TableType); + } + Y_UNIT_TEST(ProcessingJsonMessage) { diff --git a/ydb/core/transfer/ut/common/transfer_common.cpp b/ydb/core/transfer/ut/common/transfer_common.cpp index 487f57c93fba..b367dba4de1d 100644 --- a/ydb/core/transfer/ut/common/transfer_common.cpp +++ b/ydb/core/transfer/ut/common/transfer_common.cpp @@ -601,6 +601,72 @@ void MessageField_MessageGroupId(const std::string& tableType) { }); } +void MessageField_CreateTimestamp(const std::string& tableType) { + TInstant timestamp = TInstant::Now() - TDuration::Minutes(1); + + MainTestCase(std::nullopt, tableType).Run({ + .TableDDL = R"( + CREATE TABLE `%s` ( + Offset Uint64 NOT NULL, + CreateTimestamp Timestamp64, + PRIMARY KEY (Offset) + ) WITH ( + STORE = %s + ); + )", + + .Lambda = R"( + $l = ($x) -> { + return [ + <| + Offset:CAST($x._offset AS Uint64), + CreateTimestamp:$x._create_timestamp + |> + ]; + }; + )", + + .Messages = {_withCreateTimestamp(timestamp)}, + + .Expectations = {{ + _T("CreateTimestamp", std::move(timestamp), TDuration::MilliSeconds(1)), + }} + }); +} + +void MessageField_WriteTimestamp(const std::string& tableType) { + TInstant timestamp = TInstant::Now(); + + MainTestCase(std::nullopt, tableType).Run({ + .TableDDL = R"( + CREATE TABLE `%s` ( + Offset Uint64 NOT NULL, + WriteTimestamp Timestamp64, + PRIMARY KEY (Offset) + ) WITH ( + STORE = %s + ); + )", + + .Lambda = R"( + $l = ($x) -> { + return [ + <| + Offset:CAST($x._offset AS Uint64), + WriteTimestamp:$x._write_timestamp + |> + ]; + }; + )", + + .Messages = {{ "Message-1" }}, + + .Expectations = {{ + _T("WriteTimestamp", std::move(timestamp), TDuration::Seconds(5)), + }} + }); +} + void WriteNullToKeyColumn(const std::string& tableType) { MainTestCase testCase(std::nullopt, tableType); diff --git a/ydb/core/transfer/ut/common/transfer_common.h b/ydb/core/transfer/ut/common/transfer_common.h index 911bc6f91e68..00044e13e33a 100644 --- a/ydb/core/transfer/ut/common/transfer_common.h +++ b/ydb/core/transfer/ut/common/transfer_common.h @@ -25,10 +25,12 @@ void ColumnType_Double(const std::string& tableType); void ColumnType_Utf8_LongValue(const std::string& tableType); void ColumnType_Uuid(const std::string& tableType); +void MessageField_CreateTimestamp(const std::string& tableType); void MessageField_MessageGroupId(const std::string& tableType); void MessageField_Partition(const std::string& tableType); void MessageField_ProducerId(const std::string& tableType); void MessageField_SeqNo(const std::string& tableType); +void MessageField_WriteTimestamp(const std::string& tableType); void ProcessingJsonMessage(const std::string& tableType); void ProcessingCDCMessage(const std::string& tableType); diff --git a/ydb/core/transfer/ut/common/utils.h b/ydb/core/transfer/ut/common/utils.h index 15b63071c534..a36974d73f17 100644 --- a/ydb/core/transfer/ut/common/utils.h +++ b/ydb/core/transfer/ut/common/utils.h @@ -64,6 +64,26 @@ struct DateTimeChecker : public Checker { } }; +struct Timestamp64Checker : public Checker { + Timestamp64Checker(TInstant&& expected, TDuration&& precision) + : Checker(std::move(expected)) + , Precision(precision) { + } + + TInstant Get(const ::Ydb::Value& value) override { + return TInstant::MilliSeconds(value.int64_value()); + } + + void Assert(const std::string& msg, const ::Ydb::Value& value) override { + auto v = Get(value); + if (Expected - Precision > v || Expected + Precision < v) { + UNIT_ASSERT_VALUES_EQUAL_C(Get(value), Expected, msg); + } + } + + TDuration Precision; +}; + template<> inline bool Checker::Get(const ::Ydb::Value& value) { return value.bool_value(); @@ -127,11 +147,11 @@ std::pair> _C(std::string&& name, T&& expecte }; } -template -std::pair> _T(std::string&& name, T&& expected) { +template +std::pair> _T(std::string&& name, T&&... expected) { return { std::move(name), - std::make_shared(std::move(expected)) + std::make_shared(std::forward(expected)...) }; } @@ -141,6 +161,7 @@ struct TMessage { std::optional ProducerId = std::nullopt; std::optional MessageGroupId = std::nullopt; std::optional SeqNo = std::nullopt; + std::optional CreateTimestamp = std::nullopt; }; inline TMessage _withSeqNo(ui64 seqNo) { @@ -173,6 +194,17 @@ inline TMessage _withMessageGroupId(const std::string& messageGroupId) { }; } +inline TMessage _withCreateTimestamp(const TInstant& timestamp) { + return { + .Message = TStringBuilder() << "Message-" << timestamp, + .Partition = 0, + .ProducerId = std::nullopt, + .MessageGroupId = std::nullopt, + .SeqNo = std::nullopt, + .CreateTimestamp = timestamp, + }; +} + using TExpectations = TVector>>>; struct TConfig { @@ -685,7 +717,7 @@ struct MainTestCase { } auto writeSession = TopicClient.CreateSimpleBlockingWriteSession(writeSettings); - UNIT_ASSERT(writeSession->Write(message.Message, message.SeqNo)); + UNIT_ASSERT(writeSession->Write(message.Message, message.SeqNo, message.CreateTimestamp)); writeSession->Close(TDuration::Seconds(1)); } diff --git a/ydb/core/transfer/ut/row_table/transfer_rowtable_ut.cpp b/ydb/core/transfer/ut/row_table/transfer_rowtable_ut.cpp index 044425b4d26c..241c8b34e69f 100644 --- a/ydb/core/transfer/ut/row_table/transfer_rowtable_ut.cpp +++ b/ydb/core/transfer/ut/row_table/transfer_rowtable_ut.cpp @@ -93,6 +93,11 @@ Y_UNIT_TEST_SUITE(Transfer_RowTable) } + Y_UNIT_TEST(MessageField_CreateTimestamp) + { + MessageField_CreateTimestamp(TableType); + } + Y_UNIT_TEST(MessageField_Partition) { MessageField_Partition(TableType); @@ -113,6 +118,11 @@ Y_UNIT_TEST_SUITE(Transfer_RowTable) MessageField_ProducerId(TableType); } + Y_UNIT_TEST(MessageField_WriteTimestamp) + { + MessageField_WriteTimestamp(TableType); + } + Y_UNIT_TEST(ProcessingJsonMessage) { diff --git a/ydb/core/tx/replication/ydb_proxy/topic_message.cpp b/ydb/core/tx/replication/ydb_proxy/topic_message.cpp index 22df8133492c..3b9a9184375b 100644 --- a/ydb/core/tx/replication/ydb_proxy/topic_message.cpp +++ b/ydb/core/tx/replication/ydb_proxy/topic_message.cpp @@ -63,6 +63,10 @@ TInstant TTopicMessage::GetCreateTime() const { return CreateTime; } +TInstant TTopicMessage::GetWriteTime() const { + return WriteTime; +} + TString TTopicMessage::GetMessageGroupId() const { return TString(MessageGroupId); } @@ -78,6 +82,7 @@ void TTopicMessage::Out(IOutputStream& out) const { << " Offset: " << Offset << " SeqNo: " << SeqNo << " CreateTime: " << CreateTime + << " WriteTime: " << WriteTime << " MessageGroupId: " << MessageGroupId << " ProducerId: " << ProducerId << " }"; diff --git a/ydb/core/tx/replication/ydb_proxy/topic_message.h b/ydb/core/tx/replication/ydb_proxy/topic_message.h index 786002a57956..dcccf9da0c34 100644 --- a/ydb/core/tx/replication/ydb_proxy/topic_message.h +++ b/ydb/core/tx/replication/ydb_proxy/topic_message.h @@ -21,6 +21,7 @@ class TTopicMessage: protected NYdb::NTopic::TReadSessionEvent::TDataReceivedEve ui64 GetOffset() const; ui64 GetSeqNo() const; TInstant GetCreateTime() const; + TInstant GetWriteTime() const; TString GetMessageGroupId() const; TString GetProducerId() const; void Out(IOutputStream& out) const; From 213c5a4c707acf99b63d5e173be18fdb732548e7 Mon Sep 17 00:00:00 2001 From: Andrey Zaspa Date: Fri, 8 Aug 2025 13:24:26 +0400 Subject: [PATCH 28/52] Refactored sysview registry and added compatibility tests (#22296) --- .../kqp/executer_actor/kqp_table_resolver.cpp | 4 +- ydb/core/kqp/gateway/kqp_metadata_loader.cpp | 6 +- ydb/core/kqp/gateway/kqp_metadata_loader.h | 2 +- .../kqp/proxy_service/kqp_proxy_service.cpp | 2 +- .../kqp/proxy_service/kqp_session_info.cpp | 2 +- ydb/core/sys_view/auth/auth_scan_base.h | 1 - ydb/core/sys_view/auth/group_members.cpp | 2 +- ydb/core/sys_view/auth/groups.cpp | 2 +- ydb/core/sys_view/auth/owners.cpp | 2 +- ydb/core/sys_view/auth/permissions.cpp | 2 +- ydb/core/sys_view/auth/users.cpp | 2 +- ydb/core/sys_view/common/events.h | 5 +- ydb/core/sys_view/common/processor_scan.h | 10 +- ydb/core/sys_view/common/registry.cpp | 104 +++ .../sys_view/common/{schema.h => registry.h} | 551 +++++------ .../common/{schema.cpp => resolver.cpp} | 249 ++--- ydb/core/sys_view/common/resolver.h | 52 ++ .../sys_view/common/scan_actor_base_impl.h | 3 +- ydb/core/sys_view/common/ya.make | 4 +- .../sys_view/compile_cache/compile_cache.cpp | 2 +- ydb/core/sys_view/nodes/nodes.cpp | 2 +- .../partition_stats/partition_stats.cpp | 2 +- .../partition_stats/top_partitions.cpp | 1 + ydb/core/sys_view/pg_tables/pg_tables.cpp | 2 +- ydb/core/sys_view/pg_tables/pg_tables.h | 2 +- .../sys_view/query_stats/query_metrics.cpp | 1 + ydb/core/sys_view/query_stats/query_stats.cpp | 2 +- .../resource_pool_classifiers.cpp | 2 +- .../resource_pools/resource_pools.cpp | 2 +- ydb/core/sys_view/scan.cpp | 58 +- ydb/core/sys_view/sessions/sessions.cpp | 2 +- ydb/core/sys_view/show_create/show_create.cpp | 4 +- ydb/core/sys_view/storage/base.h | 2 +- ydb/core/sys_view/tablets/tablets.cpp | 2 +- ydb/core/sys_view/ut/ya.make | 1 + ydb/core/sys_view/ut_registry.cpp | 864 ++++++++++++++++++ ydb/core/tx/columnshard/common/portion.cpp | 1 - .../iterator/sys_view/chunks/schema.cpp | 2 + .../iterator/sys_view/chunks/source.cpp | 1 + .../iterator/sys_view/granules/schema.cpp | 2 + .../iterator/sys_view/granules/source.cpp | 1 + .../iterator/sys_view/optimizer/schema.cpp | 2 + .../iterator/sys_view/optimizer/source.cpp | 1 + .../iterator/sys_view/portions/schema.cpp | 2 + .../iterator/sys_view/portions/source.cpp | 1 + .../iterator/sys_view/schemas/schema.cpp | 2 + .../iterator/sys_view/schemas/source.cpp | 1 + .../reader/transaction/tx_internal_scan.cpp | 1 - .../engines/reader/transaction/tx_scan.cpp | 1 - .../engines/scheme/abstract/index_info.cpp | 3 +- .../columnshard/engines/scheme/index_info.cpp | 1 - .../columnshard/engines/scheme/index_info.h | 1 - .../test_helper/columnshard_ut_common.cpp | 2 + ydb/core/tx/scheme_board/cache.cpp | 27 +- ydb/core/tx/schemeshard/schemeshard_impl.cpp | 6 +- ydb/core/tx/tx_proxy/describe.cpp | 3 +- ydb/tests/compatibility/test_system_views.py | 114 +++ ydb/tests/compatibility/ya.make | 1 + ydb/tests/library/compatibility/fixtures.py | 18 +- 59 files changed, 1562 insertions(+), 588 deletions(-) create mode 100644 ydb/core/sys_view/common/registry.cpp rename ydb/core/sys_view/common/{schema.h => registry.h} (55%) rename ydb/core/sys_view/common/{schema.cpp => resolver.cpp} (51%) create mode 100644 ydb/core/sys_view/common/resolver.h create mode 100644 ydb/core/sys_view/ut_registry.cpp create mode 100644 ydb/tests/compatibility/test_system_views.py diff --git a/ydb/core/kqp/executer_actor/kqp_table_resolver.cpp b/ydb/core/kqp/executer_actor/kqp_table_resolver.cpp index 6cefe30fc997..b7f38c99b089 100644 --- a/ydb/core/kqp/executer_actor/kqp_table_resolver.cpp +++ b/ydb/core/kqp/executer_actor/kqp_table_resolver.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include namespace NKikimr::NKqp { @@ -407,7 +407,7 @@ class TKqpTableResolver : public TActorBootstrapped { } } } else if (!ResolvingNamesFinished) { - // CTAS + // CTAS AFL_ENSURE(!stageInfo.Meta.TableId); AFL_ENSURE(stageInfo.Meta.TablePath); const auto splittedPath = SplitPath(stageInfo.Meta.TablePath); diff --git a/ydb/core/kqp/gateway/kqp_metadata_loader.cpp b/ydb/core/kqp/gateway/kqp_metadata_loader.cpp index 33db919122cd..630c7b629e52 100644 --- a/ydb/core/kqp/gateway/kqp_metadata_loader.cpp +++ b/ydb/core/kqp/gateway/kqp_metadata_loader.cpp @@ -658,7 +658,7 @@ NThreading::TFuture TKqpTableMetadat ) { TNavigate::TEntry entry; - auto schema = SystemViewRewrittenResolver->GetSystemViewSchema(sysViewPath.ViewName, NSysView::ISystemViewResolver::ETarget::Domain); + auto schema = SystemViewRewrittenResolver->GetSystemViewSchema(sysViewPath.ViewName, NSysView::ISystemViewResolver::ESource::Domain); entry.Kind = TNavigate::KindTable; entry.Columns = std::move(schema->Columns); entry.TableId = TTableId(TSysTables::SysSchemeShard, 0, sysViewPath.ViewName); @@ -967,7 +967,7 @@ NThreading::TFuture TKqpTableMetadataLoader::LoadTableMeta promise.SetValue(externalDataSourceMetadata); return; } - + auto loadDynamicMetadata = [promise, externalDataSourceMetadata, settings, table, database, externalPath] () mutable { NExternalSource::IExternalSource::TPtr externalSource; if (settings.ExternalSourceFactory) { @@ -1007,7 +1007,7 @@ NThreading::TFuture TKqpTableMetadataLoader::LoadTableMeta if (externalDataSourceMetadata.Metadata->ExternalSource.Type == ToString(NYql::EDatabaseType::Ydb) && externalPath) { auto& source = externalDataSourceMetadata.Metadata->ExternalSource; THashMap properties = {source.Properties.GetProperties().begin(), source.Properties.GetProperties().end()}; - + auto token = source.Token; auto secretName = source.DataSourceAuth.GetToken().GetTokenSecretName(); auto structuredTokenJson = NYql::ComposeStructuredTokenJsonForTokenAuthWithSecret(secretName, token); diff --git a/ydb/core/kqp/gateway/kqp_metadata_loader.h b/ydb/core/kqp/gateway/kqp_metadata_loader.h index 8f45e1fae9e7..91a0aef61f13 100644 --- a/ydb/core/kqp/gateway/kqp_metadata_loader.h +++ b/ydb/core/kqp/gateway/kqp_metadata_loader.h @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include diff --git a/ydb/core/kqp/proxy_service/kqp_proxy_service.cpp b/ydb/core/kqp/proxy_service/kqp_proxy_service.cpp index 769dfb5ffe28..c82ed30b84f1 100644 --- a/ydb/core/kqp/proxy_service/kqp_proxy_service.cpp +++ b/ydb/core/kqp/proxy_service/kqp_proxy_service.cpp @@ -31,7 +31,7 @@ #include #include #include -#include +#include #include #include diff --git a/ydb/core/kqp/proxy_service/kqp_session_info.cpp b/ydb/core/kqp/proxy_service/kqp_session_info.cpp index 383fd6690244..861afb95ef07 100644 --- a/ydb/core/kqp/proxy_service/kqp_session_info.cpp +++ b/ydb/core/kqp/proxy_service/kqp_session_info.cpp @@ -1,6 +1,6 @@ #include "kqp_proxy_service_impl.h" -#include +#include namespace NKikimr::NKqp { diff --git a/ydb/core/sys_view/auth/auth_scan_base.h b/ydb/core/sys_view/auth/auth_scan_base.h index af4ad61584b9..1b0f386d05eb 100644 --- a/ydb/core/sys_view/auth/auth_scan_base.h +++ b/ydb/core/sys_view/auth/auth_scan_base.h @@ -4,7 +4,6 @@ #include #include -#include #include #include #include diff --git a/ydb/core/sys_view/auth/group_members.cpp b/ydb/core/sys_view/auth/group_members.cpp index d7c4f188132a..c4d1b814c4af 100644 --- a/ydb/core/sys_view/auth/group_members.cpp +++ b/ydb/core/sys_view/auth/group_members.cpp @@ -2,7 +2,7 @@ #include "group_members.h" #include -#include +#include #include #include #include diff --git a/ydb/core/sys_view/auth/groups.cpp b/ydb/core/sys_view/auth/groups.cpp index 8b9b7d936723..54403a5cc2cf 100644 --- a/ydb/core/sys_view/auth/groups.cpp +++ b/ydb/core/sys_view/auth/groups.cpp @@ -2,7 +2,7 @@ #include "groups.h" #include -#include +#include #include #include #include diff --git a/ydb/core/sys_view/auth/owners.cpp b/ydb/core/sys_view/auth/owners.cpp index 83b9e1f36a35..8a31e772cc78 100644 --- a/ydb/core/sys_view/auth/owners.cpp +++ b/ydb/core/sys_view/auth/owners.cpp @@ -2,7 +2,7 @@ #include "owners.h" #include -#include +#include #include #include #include diff --git a/ydb/core/sys_view/auth/permissions.cpp b/ydb/core/sys_view/auth/permissions.cpp index ed8b036d3607..dcedd5198ff7 100644 --- a/ydb/core/sys_view/auth/permissions.cpp +++ b/ydb/core/sys_view/auth/permissions.cpp @@ -2,7 +2,7 @@ #include "permissions.h" #include -#include +#include #include #include #include diff --git a/ydb/core/sys_view/auth/users.cpp b/ydb/core/sys_view/auth/users.cpp index 3e2d5788372d..d3fc381e715a 100644 --- a/ydb/core/sys_view/auth/users.cpp +++ b/ydb/core/sys_view/auth/users.cpp @@ -3,7 +3,7 @@ #include #include -#include +#include #include #include #include diff --git a/ydb/core/sys_view/common/events.h b/ydb/core/sys_view/common/events.h index 72774e9e3480..3ab8fb57e74b 100644 --- a/ydb/core/sys_view/common/events.h +++ b/ydb/core/sys_view/common/events.h @@ -1,13 +1,12 @@ #pragma once -#include "utils.h" -#include "db_counters.h" - #include #include #include #include #include +#include +#include namespace NKikimr { namespace NSysView { diff --git a/ydb/core/sys_view/common/processor_scan.h b/ydb/core/sys_view/common/processor_scan.h index 9ddba223d667..6d86329968e6 100644 --- a/ydb/core/sys_view/common/processor_scan.h +++ b/ydb/core/sys_view/common/processor_scan.h @@ -1,10 +1,10 @@ -#include "common.h" -#include "events.h" -#include "keys.h" -#include "schema.h" -#include "scan_actor_base_impl.h" +#pragma once #include +#include +#include +#include +#include #include diff --git a/ydb/core/sys_view/common/registry.cpp b/ydb/core/sys_view/common/registry.cpp new file mode 100644 index 000000000000..fa15a3ec055c --- /dev/null +++ b/ydb/core/sys_view/common/registry.cpp @@ -0,0 +1,104 @@ +#include "registry.h" + +#include +#include + +namespace NKikimr { +namespace NSysView { + +namespace { + +TVector GetPgStaticTableColumns(const TString& schema, const TString& tableName) { + TVector res; + auto columns = NYql::NPg::GetStaticColumns().FindPtr(NYql::NPg::TTableInfoKey{schema, tableName}); + res.reserve(columns->size()); + for (size_t i = 0; i < columns->size(); i++) { + const auto& column = columns->at(i); + res.emplace_back(i, column.UdtType, column.Name); + } + return res; +} + +} + +Schema::PgColumn::PgColumn(NIceDb::TColumnId columnId, TStringBuf columnTypeName, TStringBuf columnName) + : _ColumnId(columnId) + , _ColumnTypeInfo(NPg::TypeDescFromPgTypeId(NYql::NPg::LookupType(TString(columnTypeName)).TypeId)) + , _ColumnName(columnName) +{} + +const TVector& Schema::PgTablesSchemaProvider::GetColumns(TStringBuf tableName) const { + TString key(tableName); + Y_ENSURE(columnsStorage.contains(key)); + return columnsStorage.at(key); +} + +Schema::PgTablesSchemaProvider::PgTablesSchemaProvider() { + columnsStorage[TString(PgTablesName)] = GetPgStaticTableColumns("pg_catalog", "pg_tables"); + columnsStorage[TString(InformationSchemaTablesName)] = GetPgStaticTableColumns("information_schema", "tables"); + columnsStorage[TString(PgClassName)] = GetPgStaticTableColumns("pg_catalog", "pg_class"); +} + +const TVector SysViewsRegistry::SysViews = { + {"partition_stats", ESysViewType::EPartitionStats, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"nodes", ESysViewType::ENodes, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + + {"top_queries_by_duration_one_minute", ESysViewType::ETopQueriesByDurationOneMinute, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"top_queries_by_duration_one_hour", ESysViewType::ETopQueriesByDurationOneHour, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"top_queries_by_read_bytes_one_minute", ESysViewType::ETopQueriesByReadBytesOneMinute, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"top_queries_by_read_bytes_one_hour", ESysViewType::ETopQueriesByReadBytesOneHour, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"top_queries_by_cpu_time_one_minute", ESysViewType::ETopQueriesByCpuTimeOneMinute, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"top_queries_by_cpu_time_one_hour", ESysViewType::ETopQueriesByCpuTimeOneHour, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"top_queries_by_request_units_one_minute", ESysViewType::ETopQueriesByRequestUnitsOneMinute, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"top_queries_by_request_units_one_hour", ESysViewType::ETopQueriesByRequestUnitsOneHour, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + + {"query_sessions", ESysViewType::EQuerySessions, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"query_metrics_one_minute", ESysViewType::EQueryMetricsOneMinute, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + // don't have approved schema yet + // {"compile_cache_queries", ESysViewType::ECompileCacheQueries, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + + {"ds_pdisks", ESysViewType::EPDisks, {ESource::Domain}, &FillSchema}, + {"ds_vslots", ESysViewType::EVSlots, {ESource::Domain}, &FillSchema}, + {"ds_groups", ESysViewType::EGroups, {ESource::Domain}, &FillSchema}, + {"ds_storage_pools", ESysViewType::EStoragePools, {ESource::Domain}, &FillSchema}, + {"ds_storage_stats", ESysViewType::EStorageStats, {ESource::Domain}, &FillSchema}, + {"hive_tablets", ESysViewType::ETablets, {ESource::Domain}, &FillSchema}, + + {"top_partitions_one_minute", ESysViewType::ETopPartitionsByCpuOneMinute, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"top_partitions_one_hour", ESysViewType::ETopPartitionsByCpuOneHour, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"top_partitions_by_tli_one_minute", ESysViewType::ETopPartitionsByTliOneMinute, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"top_partitions_by_tli_one_hour", ESysViewType::ETopPartitionsByTliOneHour, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + + {"resource_pool_classifiers", ESysViewType::EResourcePoolClassifiers, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"resource_pools", ESysViewType::EResourcePools, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + + {"auth_users", ESysViewType::EAuthUsers, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"auth_groups", ESysViewType::EAuthGroups, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"auth_group_members", ESysViewType::EAuthGroupMembers, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"auth_owners", ESysViewType::EAuthOwners, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"auth_permissions", ESysViewType::EAuthPermissions, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"auth_effective_permissions", ESysViewType::EAuthEffectivePermissions, {ESource::Domain, ESource::SubDomain}, &FillSchema}, +}; + +const TVector SysViewsRegistry::RewrittenSysViews = { + {"show_create", ESysViewType::EShowCreate, {}, &FillSchema}, +}; + +SysViewsRegistry::SysViewsRegistry() { + for (const auto& registryRecord : SysViews) { + SysViewTypesMap.emplace(registryRecord.Name, registryRecord.Type); + } + + for (const auto& registryRecord : RewrittenSysViews) { + SysViewTypesMap.emplace(registryRecord.Name, registryRecord.Type); + } + + SysViewTypesMap.emplace(PgTablesName, ESysViewType::EPgTables); + SysViewTypesMap.emplace(InformationSchemaTablesName, ESysViewType::EInformationSchemaTables); + SysViewTypesMap.emplace(PgClassName, ESysViewType::EPgClass); +} + +const SysViewsRegistry Registry; + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/common/schema.h b/ydb/core/sys_view/common/registry.h similarity index 55% rename from ydb/core/sys_view/common/schema.h rename to ydb/core/sys_view/common/registry.h index 2e8766729b32..84997468af67 100644 --- a/ydb/core/sys_view/common/schema.h +++ b/ydb/core/sys_view/common/registry.h @@ -1,40 +1,82 @@ #pragma once -#include "path.h" - #include -#include -#include -#include -#include +#include namespace NKikimr { namespace NSysView { -constexpr TStringBuf PartitionStatsName = "partition_stats"; -constexpr TStringBuf NodesName = "nodes"; -constexpr TStringBuf QuerySessions = "query_sessions"; -constexpr TStringBuf CompileCacheQueries = "compile_cache_queries"; -constexpr TStringBuf ResourcePoolsName = "resource_pools"; +template +struct TSchemaFiller { + + using TSchema = ISystemViewResolver::TSchema; + + template + struct TFiller; + + template + struct TFiller { + static void Fill(TSchema& schema) { + schema.Columns[Column::ColumnId] = TSysTables::TTableColumnInfo( + Table::template TableColumns::GetColumnName(), + Column::ColumnId, NScheme::TTypeInfo(Column::ColumnType), "", -1); + } + }; + + template + struct TFiller { + static void Fill(TSchema& schema) { + TFiller::Fill(schema); + TFiller::Fill(schema); + } + }; -constexpr TStringBuf TopQueriesByDuration1MinuteName = "top_queries_by_duration_one_minute"; -constexpr TStringBuf TopQueriesByDuration1HourName = "top_queries_by_duration_one_hour"; -constexpr TStringBuf TopQueriesByReadBytes1MinuteName = "top_queries_by_read_bytes_one_minute"; -constexpr TStringBuf TopQueriesByReadBytes1HourName = "top_queries_by_read_bytes_one_hour"; -constexpr TStringBuf TopQueriesByCpuTime1MinuteName = "top_queries_by_cpu_time_one_minute"; -constexpr TStringBuf TopQueriesByCpuTime1HourName = "top_queries_by_cpu_time_one_hour"; -constexpr TStringBuf TopQueriesByRequestUnits1MinuteName = "top_queries_by_request_units_one_minute"; -constexpr TStringBuf TopQueriesByRequestUnits1HourName = "top_queries_by_request_units_one_hour"; + template + using TColumnsType = typename Table::template TableColumns; -constexpr TStringBuf PDisksName = "ds_pdisks"; -constexpr TStringBuf VSlotsName = "ds_vslots"; -constexpr TStringBuf GroupsName = "ds_groups"; -constexpr TStringBuf StoragePoolsName = "ds_storage_pools"; -constexpr TStringBuf StorageStatsName = "ds_storage_stats"; + template + static void FillColumns(TSchema& schema, TColumnsType) { + TFiller::Fill(schema); + } -constexpr TStringBuf TabletsName = "hive_tablets"; + template + struct TKeyFiller; -constexpr TStringBuf QueryMetricsName = "query_metrics_one_minute"; + template + struct TKeyFiller { + static void Fill(TSchema& schema, i32 index) { + auto& column = schema.Columns[Key::ColumnId]; + column.KeyOrder = index; + schema.KeyColumnTypes.push_back(column.PType); + } + }; + + template + struct TKeyFiller { + static void Fill(TSchema& schema, i32 index) { + TKeyFiller::Fill(schema, index); + TKeyFiller::Fill(schema, index + 1); + } + }; + + template + using TKeysType = typename Table::template TableKey; + + template + static void FillKeys(TSchema& schema, TKeysType) { + TKeyFiller::Fill(schema, 0); + } + + static void Fill(TSchema& schema) { + FillColumns(schema, typename Table::TColumns()); + FillKeys(schema, typename Table::TKey()); + } +}; + +template +void FillSchema(ISystemViewResolver::TSchema& schema) { + TSchemaFiller::Fill(schema); +} constexpr TStringBuf StorePrimaryIndexStatsName = "store_primary_index_stats"; constexpr TStringBuf StorePrimaryIndexSchemaStatsName = "store_primary_index_schema_stats"; @@ -47,62 +89,42 @@ constexpr TStringBuf TablePrimaryIndexPortionStatsName = "primary_index_portion_ constexpr TStringBuf TablePrimaryIndexGranuleStatsName = "primary_index_granule_stats"; constexpr TStringBuf TablePrimaryIndexOptimizerStatsName = "primary_index_optimizer_stats"; -constexpr TStringBuf TopPartitionsByCpu1MinuteName = "top_partitions_one_minute"; -constexpr TStringBuf TopPartitionsByCpu1HourName = "top_partitions_one_hour"; - -constexpr TStringBuf TopPartitionsByTli1MinuteName = "top_partitions_by_tli_one_minute"; -constexpr TStringBuf TopPartitionsByTli1HourName = "top_partitions_by_tli_one_hour"; - constexpr TStringBuf PgTablesName = "pg_tables"; constexpr TStringBuf InformationSchemaTablesName = "tables"; constexpr TStringBuf PgClassName = "pg_class"; -constexpr TStringBuf ResourcePoolClassifiersName = "resource_pool_classifiers"; - -constexpr TStringBuf ShowCreateName = "show_create"; - -namespace NAuth { - constexpr TStringBuf UsersName = "auth_users"; - constexpr TStringBuf GroupsName = "auth_groups"; - constexpr TStringBuf GroupMembersName = "auth_group_members"; - constexpr TStringBuf OwnersName = "auth_owners"; - constexpr TStringBuf PermissionsName = "auth_permissions"; - constexpr TStringBuf EffectivePermissionsName = "auth_effective_permissions"; -} - - struct Schema : NIceDb::Schema { struct PartitionStats : Table<1> { - struct OwnerId : Column<1, NScheme::NTypeIds::Uint64> {}; - struct PathId : Column<2, NScheme::NTypeIds::Uint64> {}; - struct PartIdx : Column<3, NScheme::NTypeIds::Uint64> {}; - struct DataSize : Column<4, NScheme::NTypeIds::Uint64> {}; - struct RowCount : Column<5, NScheme::NTypeIds::Uint64> {}; - struct IndexSize : Column<6, NScheme::NTypeIds::Uint64> {}; - struct CPUCores : Column<7, NScheme::NTypeIds::Double> {}; - struct TabletId : Column<8, NScheme::NTypeIds::Uint64> {}; - struct Path : Column<9, NScheme::NTypeIds::Utf8> {}; - struct NodeId : Column<10, NScheme::NTypeIds::Uint32> {}; - struct StartTime : Column<11, NScheme::NTypeIds::Timestamp> {}; - struct AccessTime : Column<12, NScheme::NTypeIds::Timestamp> {}; - struct UpdateTime : Column<13, NScheme::NTypeIds::Timestamp> {}; - struct InFlightTxCount : Column<14, NScheme::NTypeIds::Uint32> {}; - struct RowUpdates : Column<15, NScheme::NTypeIds::Uint64> {}; - struct RowDeletes : Column<16, NScheme::NTypeIds::Uint64> {}; - struct RowReads : Column<17, NScheme::NTypeIds::Uint64> {}; - struct RangeReads : Column<18, NScheme::NTypeIds::Uint64> {}; - struct RangeReadRows : Column<19, NScheme::NTypeIds::Uint64> {}; - struct ImmediateTxCompleted : Column<20, NScheme::NTypeIds::Uint64> {}; - struct CoordinatedTxCompleted : Column<21, NScheme::NTypeIds::Uint64> {}; - struct TxRejectedByOverload : Column<22, NScheme::NTypeIds::Uint64> {}; + struct OwnerId : Column<1, NScheme::NTypeIds::Uint64> {}; + struct PathId : Column<2, NScheme::NTypeIds::Uint64> {}; + struct PartIdx : Column<3, NScheme::NTypeIds::Uint64> {}; + struct DataSize : Column<4, NScheme::NTypeIds::Uint64> {}; + struct RowCount : Column<5, NScheme::NTypeIds::Uint64> {}; + struct IndexSize : Column<6, NScheme::NTypeIds::Uint64> {}; + struct CPUCores : Column<7, NScheme::NTypeIds::Double> {}; + struct TabletId : Column<8, NScheme::NTypeIds::Uint64> {}; + struct Path : Column<9, NScheme::NTypeIds::Utf8> {}; + struct NodeId : Column<10, NScheme::NTypeIds::Uint32> {}; + struct StartTime : Column<11, NScheme::NTypeIds::Timestamp> {}; + struct AccessTime : Column<12, NScheme::NTypeIds::Timestamp> {}; + struct UpdateTime : Column<13, NScheme::NTypeIds::Timestamp> {}; + struct InFlightTxCount : Column<14, NScheme::NTypeIds::Uint32> {}; + struct RowUpdates : Column<15, NScheme::NTypeIds::Uint64> {}; + struct RowDeletes : Column<16, NScheme::NTypeIds::Uint64> {}; + struct RowReads : Column<17, NScheme::NTypeIds::Uint64> {}; + struct RangeReads : Column<18, NScheme::NTypeIds::Uint64> {}; + struct RangeReadRows : Column<19, NScheme::NTypeIds::Uint64> {}; + struct ImmediateTxCompleted : Column<20, NScheme::NTypeIds::Uint64> {}; + struct CoordinatedTxCompleted : Column<21, NScheme::NTypeIds::Uint64> {}; + struct TxRejectedByOverload : Column<22, NScheme::NTypeIds::Uint64> {}; struct TxRejectedByOutOfStorage : Column<23, NScheme::NTypeIds::Uint64> {}; - struct LastTtlRunTime : Column<24, NScheme::NTypeIds::Timestamp> {}; - struct LastTtlRowsProcessed : Column<25, NScheme::NTypeIds::Uint64> {}; - struct LastTtlRowsErased : Column<26, NScheme::NTypeIds::Uint64> {}; - struct FollowerId : Column<27, NScheme::NTypeIds::Uint32> {}; - struct LocksAcquired : Column<28, NScheme::NTypeIds::Uint64> {}; - struct LocksWholeShard : Column<29, NScheme::NTypeIds::Uint64> {}; - struct LocksBroken : Column<30, NScheme::NTypeIds::Uint64> {}; + struct LastTtlRunTime : Column<24, NScheme::NTypeIds::Timestamp> {}; + struct LastTtlRowsProcessed : Column<25, NScheme::NTypeIds::Uint64> {}; + struct LastTtlRowsErased : Column<26, NScheme::NTypeIds::Uint64> {}; + struct FollowerId : Column<27, NScheme::NTypeIds::Uint32> {}; + struct LocksAcquired : Column<28, NScheme::NTypeIds::Uint64> {}; + struct LocksWholeShard : Column<29, NScheme::NTypeIds::Uint64> {}; + struct LocksBroken : Column<30, NScheme::NTypeIds::Uint64> {}; using TKey = TableKey; using TColumns = TableColumns< @@ -140,15 +162,15 @@ struct Schema : NIceDb::Schema { }; struct Nodes : Table<2> { - struct NodeId : Column<1, NScheme::NTypeIds::Uint32> {}; - struct Address : Column<2, NScheme::NTypeIds::Utf8> {}; - struct Host : Column<3, NScheme::NTypeIds::Utf8> {}; - struct Port : Column<4, NScheme::NTypeIds::Uint32> {}; - struct StartTime : Column<5, NScheme::NTypeIds::Timestamp> {}; - struct UpTime : Column<6, NScheme::NTypeIds::Interval> {}; - struct CpuThreads: Column<7, NScheme::NTypeIds::Uint32> {}; - struct CpuUsage : Column<8, NScheme::NTypeIds::Double> {}; - struct CpuIdle : Column<9, NScheme::NTypeIds::Double> {}; + struct NodeId : Column<1, NScheme::NTypeIds::Uint32> {}; + struct Address : Column<2, NScheme::NTypeIds::Utf8> {}; + struct Host : Column<3, NScheme::NTypeIds::Utf8> {}; + struct Port : Column<4, NScheme::NTypeIds::Uint32> {}; + struct StartTime : Column<5, NScheme::NTypeIds::Timestamp> {}; + struct UpTime : Column<6, NScheme::NTypeIds::Interval> {}; + struct CpuThreads : Column<7, NScheme::NTypeIds::Uint32> {}; + struct CpuUsage : Column<8, NScheme::NTypeIds::Double> {}; + struct CpuIdle : Column<9, NScheme::NTypeIds::Double> {}; using TKey = TableKey; using TColumns = TableColumns< @@ -228,25 +250,25 @@ struct Schema : NIceDb::Schema { }; struct PDisks : Table<4> { - struct NodeId : Column<1, NScheme::NTypeIds::Uint32> {}; - struct PDiskId : Column<2, NScheme::NTypeIds::Uint32> {}; - struct TypeCol : Column<3, NScheme::NTypeIds::Utf8> { static TString GetColumnName(const TString&) { return "Type"; } }; - struct Kind : Column<4, NScheme::NTypeIds::Uint64> {}; - struct Path : Column<5, NScheme::NTypeIds::Utf8> {}; - struct Guid : Column<6, NScheme::NTypeIds::Uint64> {}; - struct BoxId : Column<7, NScheme::NTypeIds::Uint32> {}; - struct SharedWithOS : Column<8, NScheme::NTypeIds::Bool> {}; - struct ReadCentric : Column<9, NScheme::NTypeIds::Bool> {}; - struct AvailableSize : Column<10, NScheme::NTypeIds::Uint64> {}; - struct TotalSize : Column<11, NScheme::NTypeIds::Uint64> {}; - struct Status : Column<12, NScheme::NTypeIds::Utf8> {}; - //struct StopFactor : Column<13, NScheme::NTypeIds::Double> {}; - struct StatusChangeTimestamp : Column<14, NScheme::NTypeIds::Timestamp> {}; - struct ExpectedSlotCount : Column<15, NScheme::NTypeIds::Uint32> {}; - struct NumActiveSlots : Column<16, NScheme::NTypeIds::Uint32> {}; - struct DecommitStatus : Column<17, NScheme::NTypeIds::Utf8> {}; - struct State : Column<18, NScheme::NTypeIds::Utf8> {}; - struct SlotSizeInUnits : Column<19, NScheme::NTypeIds::Uint32> {}; + struct NodeId : Column<1, NScheme::NTypeIds::Uint32> {}; + struct PDiskId : Column<2, NScheme::NTypeIds::Uint32> {}; + struct TypeCol : Column<3, NScheme::NTypeIds::Utf8> { static TString GetColumnName(const TString&) { return "Type"; } }; + struct Kind : Column<4, NScheme::NTypeIds::Uint64> {}; + struct Path : Column<5, NScheme::NTypeIds::Utf8> {}; + struct Guid : Column<6, NScheme::NTypeIds::Uint64> {}; + struct BoxId : Column<7, NScheme::NTypeIds::Uint32> {}; + struct SharedWithOS : Column<8, NScheme::NTypeIds::Bool> {}; + struct ReadCentric : Column<9, NScheme::NTypeIds::Bool> {}; + struct AvailableSize : Column<10, NScheme::NTypeIds::Uint64> {}; + struct TotalSize : Column<11, NScheme::NTypeIds::Uint64> {}; + struct Status : Column<12, NScheme::NTypeIds::Utf8> {}; + //struct StopFactor : Column<13, NScheme::NTypeIds::Double> {}; + struct StatusChangeTimestamp : Column<14, NScheme::NTypeIds::Timestamp> {}; + struct ExpectedSlotCount : Column<15, NScheme::NTypeIds::Uint32> {}; + struct NumActiveSlots : Column<16, NScheme::NTypeIds::Uint32> {}; + struct DecommitStatus : Column<17, NScheme::NTypeIds::Utf8> {}; + struct State : Column<18, NScheme::NTypeIds::Utf8> {}; + struct SlotSizeInUnits : Column<19, NScheme::NTypeIds::Uint32> {}; struct InferPDiskSlotCountFromUnitSize : Column<20, NScheme::NTypeIds::Uint64> {}; using TKey = TableKey; @@ -273,24 +295,24 @@ struct Schema : NIceDb::Schema { }; struct VSlots : Table<5> { - struct NodeId : Column<1, NScheme::NTypeIds::Uint32> {}; - struct PDiskId : Column<2, NScheme::NTypeIds::Uint32> {}; - struct VSlotId : Column<3, NScheme::NTypeIds::Uint32> {}; - //struct Category : Column<4, NScheme::NTypeIds::Uint64> {}; - struct GroupId : Column<5, NScheme::NTypeIds::Uint32> {}; + struct NodeId : Column<1, NScheme::NTypeIds::Uint32> {}; + struct PDiskId : Column<2, NScheme::NTypeIds::Uint32> {}; + struct VSlotId : Column<3, NScheme::NTypeIds::Uint32> {}; + //struct Category : Column<4, NScheme::NTypeIds::Uint64> {}; + struct GroupId : Column<5, NScheme::NTypeIds::Uint32> {}; struct GroupGeneration : Column<6, NScheme::NTypeIds::Uint32> {}; - //struct Ring : Column<7, NScheme::NTypeIds::Uint32> {}; - struct FailDomain : Column<8, NScheme::NTypeIds::Uint32> {}; - struct VDisk : Column<9, NScheme::NTypeIds::Uint32> {}; - struct AllocatedSize : Column<10, NScheme::NTypeIds::Uint64> {}; - struct AvailableSize : Column<11, NScheme::NTypeIds::Uint64> {}; - struct Status : Column<12, NScheme::NTypeIds::Utf8> {}; - //struct StopFactor : Column<13, NScheme::NTypeIds::Double> {}; - struct Kind : Column<14, NScheme::NTypeIds::Utf8> {}; - struct FailRealm : Column<15, NScheme::NTypeIds::Uint32> {}; - struct Replicated : Column<16, NScheme::NTypeIds::Bool> {}; - struct DiskSpace : Column<17, NScheme::NTypeIds::Utf8> {}; - struct State : Column <18, NScheme::NTypeIds::Utf8> {}; + //struct Ring : Column<7, NScheme::NTypeIds::Uint32> {}; + struct FailDomain : Column<8, NScheme::NTypeIds::Uint32> {}; + struct VDisk : Column<9, NScheme::NTypeIds::Uint32> {}; + struct AllocatedSize : Column<10, NScheme::NTypeIds::Uint64> {}; + struct AvailableSize : Column<11, NScheme::NTypeIds::Uint64> {}; + struct Status : Column<12, NScheme::NTypeIds::Utf8> {}; + //struct StopFactor : Column<13, NScheme::NTypeIds::Double> {}; + struct Kind : Column<14, NScheme::NTypeIds::Utf8> {}; + struct FailRealm : Column<15, NScheme::NTypeIds::Uint32> {}; + struct Replicated : Column<16, NScheme::NTypeIds::Bool> {}; + struct DiskSpace : Column<17, NScheme::NTypeIds::Utf8> {}; + struct State : Column <18, NScheme::NTypeIds::Utf8> {}; using TKey = TableKey; using TColumns = TableColumns< @@ -312,27 +334,27 @@ struct Schema : NIceDb::Schema { }; struct Groups : Table<6> { - struct GroupId : Column<1, NScheme::NTypeIds::Uint32> {}; - struct Generation : Column<2, NScheme::NTypeIds::Uint32> {}; - struct ErasureSpecies : Column<3, NScheme::NTypeIds::Utf8> {}; - struct BoxId : Column<4, NScheme::NTypeIds::Uint64> {}; - struct StoragePoolId : Column<5, NScheme::NTypeIds::Uint64> {}; - struct EncryptionMode : Column<6, NScheme::NTypeIds::Uint32> {}; - struct LifeCyclePhase : Column<7, NScheme::NTypeIds::Uint32> {}; - struct AllocatedSize : Column<8, NScheme::NTypeIds::Uint64> {}; - struct AvailableSize : Column<9, NScheme::NTypeIds::Uint64> {}; - //struct Usage : Column<10, NScheme::NTypeIds::Double> {}; - //struct StopFactor : Column<11, NScheme::NTypeIds::Double> {}; - struct SeenOperational : Column<12, NScheme::NTypeIds::Bool> {}; + struct GroupId : Column<1, NScheme::NTypeIds::Uint32> {}; + struct Generation : Column<2, NScheme::NTypeIds::Uint32> {}; + struct ErasureSpecies : Column<3, NScheme::NTypeIds::Utf8> {}; + struct BoxId : Column<4, NScheme::NTypeIds::Uint64> {}; + struct StoragePoolId : Column<5, NScheme::NTypeIds::Uint64> {}; + struct EncryptionMode : Column<6, NScheme::NTypeIds::Uint32> {}; + struct LifeCyclePhase : Column<7, NScheme::NTypeIds::Uint32> {}; + struct AllocatedSize : Column<8, NScheme::NTypeIds::Uint64> {}; + struct AvailableSize : Column<9, NScheme::NTypeIds::Uint64> {}; + //struct Usage : Column<10, NScheme::NTypeIds::Double> {}; + //struct StopFactor : Column<11, NScheme::NTypeIds::Double> {}; + struct SeenOperational : Column<12, NScheme::NTypeIds::Bool> {}; struct PutTabletLogLatency : Column<13, NScheme::NTypeIds::Interval> {}; - struct PutUserDataLatency : Column<14, NScheme::NTypeIds::Interval> {}; - struct GetFastLatency : Column<15, NScheme::NTypeIds::Interval> {}; - struct LayoutCorrect : Column<16, NScheme::NTypeIds::Bool> {}; - struct OperatingStatus : Column<17, NScheme::NTypeIds::Utf8> {}; - struct ExpectedStatus : Column<18, NScheme::NTypeIds::Utf8> {}; - struct ProxyGroupId : Column<19, NScheme::NTypeIds::Uint32> {}; - struct BridgePileId : Column<20, NScheme::NTypeIds::Uint32> {}; - struct GroupSizeInUnits : Column<21, NScheme::NTypeIds::Uint32> {}; + struct PutUserDataLatency : Column<14, NScheme::NTypeIds::Interval> {}; + struct GetFastLatency : Column<15, NScheme::NTypeIds::Interval> {}; + struct LayoutCorrect : Column<16, NScheme::NTypeIds::Bool> {}; + struct OperatingStatus : Column<17, NScheme::NTypeIds::Utf8> {}; + struct ExpectedStatus : Column<18, NScheme::NTypeIds::Utf8> {}; + struct ProxyGroupId : Column<19, NScheme::NTypeIds::Uint32> {}; + struct BridgePileId : Column<20, NScheme::NTypeIds::Uint32> {}; + struct GroupSizeInUnits : Column<21, NScheme::NTypeIds::Uint32> {}; using TKey = TableKey; using TColumns = TableColumns< @@ -358,17 +380,17 @@ struct Schema : NIceDb::Schema { }; struct StoragePools : Table<7> { - struct BoxId : Column<1, NScheme::NTypeIds::Uint64> {}; - struct StoragePoolId : Column<2, NScheme::NTypeIds::Uint64> {}; - struct Name : Column<3, NScheme::NTypeIds::Utf8> {}; - struct Generation : Column<4, NScheme::NTypeIds::Uint64> {}; - struct ErasureSpecies : Column<5, NScheme::NTypeIds::Utf8> {}; - struct VDiskKind : Column<6, NScheme::NTypeIds::Utf8> {}; - struct Kind : Column<7, NScheme::NTypeIds::Utf8> {}; - struct NumGroups : Column<8, NScheme::NTypeIds::Uint32> {}; - struct EncryptionMode : Column<9, NScheme::NTypeIds::Uint32> {}; - struct SchemeshardId : Column<10, NScheme::NTypeIds::Uint64> {}; - struct PathId : Column<11, NScheme::NTypeIds::Uint64> {}; + struct BoxId : Column<1, NScheme::NTypeIds::Uint64> {}; + struct StoragePoolId : Column<2, NScheme::NTypeIds::Uint64> {}; + struct Name : Column<3, NScheme::NTypeIds::Utf8> {}; + struct Generation : Column<4, NScheme::NTypeIds::Uint64> {}; + struct ErasureSpecies : Column<5, NScheme::NTypeIds::Utf8> {}; + struct VDiskKind : Column<6, NScheme::NTypeIds::Utf8> {}; + struct Kind : Column<7, NScheme::NTypeIds::Utf8> {}; + struct NumGroups : Column<8, NScheme::NTypeIds::Uint32> {}; + struct EncryptionMode : Column<9, NScheme::NTypeIds::Uint32> {}; + struct SchemeshardId : Column<10, NScheme::NTypeIds::Uint64> {}; + struct PathId : Column<11, NScheme::NTypeIds::Uint64> {}; struct DefaultGroupSizeInUnits : Column<12, NScheme::NTypeIds::Uint32> {}; using TKey = TableKey; @@ -388,17 +410,17 @@ struct Schema : NIceDb::Schema { }; struct Tablets : Table<8> { - struct TabletId : Column<1, NScheme::NTypeIds::Uint64> {}; - struct FollowerId : Column<2, NScheme::NTypeIds::Uint32> {}; - struct TypeCol : Column<3, NScheme::NTypeIds::Utf8> { static TString GetColumnName(const TString&) { return "Type"; } }; - struct State : Column<4, NScheme::NTypeIds::Utf8> {}; + struct TabletId : Column<1, NScheme::NTypeIds::Uint64> {}; + struct FollowerId : Column<2, NScheme::NTypeIds::Uint32> {}; + struct TypeCol : Column<3, NScheme::NTypeIds::Utf8> { static TString GetColumnName(const TString&) { return "Type"; } }; + struct State : Column<4, NScheme::NTypeIds::Utf8> {}; struct VolatileState : Column<5, NScheme::NTypeIds::Utf8> {}; - struct BootState : Column<6, NScheme::NTypeIds::Utf8> {}; - struct Generation : Column<7, NScheme::NTypeIds::Uint32> {}; - struct NodeId : Column<8, NScheme::NTypeIds::Uint32> {}; - struct CPU : Column<9, NScheme::NTypeIds::Double> {}; - struct Memory : Column<10, NScheme::NTypeIds::Uint64> {}; - struct Network : Column<11, NScheme::NTypeIds::Uint64> {}; + struct BootState : Column<6, NScheme::NTypeIds::Utf8> {}; + struct Generation : Column<7, NScheme::NTypeIds::Uint32> {}; + struct NodeId : Column<8, NScheme::NTypeIds::Uint32> {}; + struct CPU : Column<9, NScheme::NTypeIds::Double> {}; + struct Memory : Column<10, NScheme::NTypeIds::Uint64> {}; + struct Network : Column<11, NScheme::NTypeIds::Uint64> {}; using TKey = TableKey; using TColumns = TableColumns< @@ -416,34 +438,34 @@ struct Schema : NIceDb::Schema { }; struct QueryMetrics : Table<9> { - struct IntervalEnd : Column<1, NScheme::NTypeIds::Timestamp> {}; - struct Rank : Column<2, NScheme::NTypeIds::Uint32> {}; - struct QueryText : Column<3, NScheme::NTypeIds::Utf8> {}; - struct Count : Column<4, NScheme::NTypeIds::Uint64> {}; - struct SumCPUTime : Column<5, NScheme::NTypeIds::Uint64> {}; - struct MinCPUTime : Column<6, NScheme::NTypeIds::Uint64> {}; - struct MaxCPUTime : Column<7, NScheme::NTypeIds::Uint64> {}; - struct SumDuration : Column<8, NScheme::NTypeIds::Interval> {}; - struct MinDuration : Column<9, NScheme::NTypeIds::Interval> {}; - struct MaxDuration : Column<10, NScheme::NTypeIds::Interval> {}; - struct MinReadRows : Column<11, NScheme::NTypeIds::Uint64> {}; - struct MaxReadRows : Column<12, NScheme::NTypeIds::Uint64> {}; - struct SumReadRows : Column<13, NScheme::NTypeIds::Uint64> {}; - struct MinReadBytes : Column<14, NScheme::NTypeIds::Uint64> {}; - struct MaxReadBytes : Column<15, NScheme::NTypeIds::Uint64> {}; - struct SumReadBytes : Column<16, NScheme::NTypeIds::Uint64> {}; - struct MinUpdateRows : Column<17, NScheme::NTypeIds::Uint64> {}; - struct MaxUpdateRows : Column<18, NScheme::NTypeIds::Uint64> {}; - struct SumUpdateRows : Column<19, NScheme::NTypeIds::Uint64> {}; - struct MinUpdateBytes : Column<20, NScheme::NTypeIds::Uint64> {}; - struct MaxUpdateBytes : Column<21, NScheme::NTypeIds::Uint64> {}; - struct SumUpdateBytes : Column<22, NScheme::NTypeIds::Uint64> {}; - struct MinDeleteRows : Column<23, NScheme::NTypeIds::Uint64> {}; - struct MaxDeleteRows : Column<24, NScheme::NTypeIds::Uint64> {}; - struct SumDeleteRows : Column<25, NScheme::NTypeIds::Uint64> {}; - struct MinRequestUnits: Column<26, NScheme::NTypeIds::Uint64> {}; - struct MaxRequestUnits: Column<27, NScheme::NTypeIds::Uint64> {}; - struct SumRequestUnits: Column<28, NScheme::NTypeIds::Uint64> {}; + struct IntervalEnd : Column<1, NScheme::NTypeIds::Timestamp> {}; + struct Rank : Column<2, NScheme::NTypeIds::Uint32> {}; + struct QueryText : Column<3, NScheme::NTypeIds::Utf8> {}; + struct Count : Column<4, NScheme::NTypeIds::Uint64> {}; + struct SumCPUTime : Column<5, NScheme::NTypeIds::Uint64> {}; + struct MinCPUTime : Column<6, NScheme::NTypeIds::Uint64> {}; + struct MaxCPUTime : Column<7, NScheme::NTypeIds::Uint64> {}; + struct SumDuration : Column<8, NScheme::NTypeIds::Interval> {}; + struct MinDuration : Column<9, NScheme::NTypeIds::Interval> {}; + struct MaxDuration : Column<10, NScheme::NTypeIds::Interval> {}; + struct MinReadRows : Column<11, NScheme::NTypeIds::Uint64> {}; + struct MaxReadRows : Column<12, NScheme::NTypeIds::Uint64> {}; + struct SumReadRows : Column<13, NScheme::NTypeIds::Uint64> {}; + struct MinReadBytes : Column<14, NScheme::NTypeIds::Uint64> {}; + struct MaxReadBytes : Column<15, NScheme::NTypeIds::Uint64> {}; + struct SumReadBytes : Column<16, NScheme::NTypeIds::Uint64> {}; + struct MinUpdateRows : Column<17, NScheme::NTypeIds::Uint64> {}; + struct MaxUpdateRows : Column<18, NScheme::NTypeIds::Uint64> {}; + struct SumUpdateRows : Column<19, NScheme::NTypeIds::Uint64> {}; + struct MinUpdateBytes : Column<20, NScheme::NTypeIds::Uint64> {}; + struct MaxUpdateBytes : Column<21, NScheme::NTypeIds::Uint64> {}; + struct SumUpdateBytes : Column<22, NScheme::NTypeIds::Uint64> {}; + struct MinDeleteRows : Column<23, NScheme::NTypeIds::Uint64> {}; + struct MaxDeleteRows : Column<24, NScheme::NTypeIds::Uint64> {}; + struct SumDeleteRows : Column<25, NScheme::NTypeIds::Uint64> {}; + struct MinRequestUnits : Column<26, NScheme::NTypeIds::Uint64> {}; + struct MaxRequestUnits : Column<27, NScheme::NTypeIds::Uint64> {}; + struct SumRequestUnits : Column<28, NScheme::NTypeIds::Uint64> {}; using TKey = TableKey; using TColumns = TableColumns< @@ -501,13 +523,13 @@ struct Schema : NIceDb::Schema { }; struct StorageStats : Table<11> { - struct PDiskFilter : Column<2, NScheme::NTypeIds::Utf8> {}; - struct ErasureSpecies : Column<3, NScheme::NTypeIds::Utf8> {}; - struct CurrentGroupsCreated : Column<4, NScheme::NTypeIds::Uint32> {}; - struct CurrentAllocatedSize : Column<5, NScheme::NTypeIds::Uint64> {}; - struct CurrentAvailableSize : Column<6, NScheme::NTypeIds::Uint64> {}; + struct PDiskFilter : Column<2, NScheme::NTypeIds::Utf8> {}; + struct ErasureSpecies : Column<3, NScheme::NTypeIds::Utf8> {}; + struct CurrentGroupsCreated : Column<4, NScheme::NTypeIds::Uint32> {}; + struct CurrentAllocatedSize : Column<5, NScheme::NTypeIds::Uint64> {}; + struct CurrentAvailableSize : Column<6, NScheme::NTypeIds::Uint64> {}; struct AvailableGroupsToCreate : Column<7, NScheme::NTypeIds::Uint32> {}; - struct AvailableSizeToCreate : Column<8, NScheme::NTypeIds::Uint64> {}; + struct AvailableSizeToCreate : Column<8, NScheme::NTypeIds::Uint64> {}; using TKey = TableKey; using TColumns = TableColumns { - struct SessionId : Column<1, NScheme::NTypeIds::Utf8> {}; - struct NodeId : Column<2, NScheme::NTypeIds::Uint32> {}; - struct State : Column<3, NScheme::NTypeIds::Utf8> {}; - struct Query : Column<4, NScheme::NTypeIds::Utf8> {}; - struct QueryCount : Column<5, NScheme::NTypeIds::Uint32> {}; - struct ClientAddress : Column<6, NScheme::NTypeIds::Utf8> {}; - struct ClientPID : Column<7, NScheme::NTypeIds::Utf8> {}; - struct ClientUserAgent : Column<8, NScheme::NTypeIds::Utf8> {}; + struct SessionId : Column<1, NScheme::NTypeIds::Utf8> {}; + struct NodeId : Column<2, NScheme::NTypeIds::Uint32> {}; + struct State : Column<3, NScheme::NTypeIds::Utf8> {}; + struct Query : Column<4, NScheme::NTypeIds::Utf8> {}; + struct QueryCount : Column<5, NScheme::NTypeIds::Uint32> {}; + struct ClientAddress : Column<6, NScheme::NTypeIds::Utf8> {}; + struct ClientPID : Column<7, NScheme::NTypeIds::Utf8> {}; + struct ClientUserAgent : Column<8, NScheme::NTypeIds::Utf8> {}; struct ClientSdkBuildInfo : Column<9, NScheme::NTypeIds::Utf8> {}; - struct ApplicationName : Column<10, NScheme::NTypeIds::Utf8> {}; - struct SessionStartAt : Column<11, NScheme::NTypeIds::Timestamp> {}; - struct QueryStartAt : Column<12, NScheme::NTypeIds::Timestamp> {}; - struct StateChangeAt : Column<13, NScheme::NTypeIds::Timestamp> {}; - struct UserSID : Column<14, NScheme::NTypeIds::Utf8> {}; + struct ApplicationName : Column<10, NScheme::NTypeIds::Utf8> {}; + struct SessionStartAt : Column<11, NScheme::NTypeIds::Timestamp> {}; + struct QueryStartAt : Column<12, NScheme::NTypeIds::Timestamp> {}; + struct StateChangeAt : Column<13, NScheme::NTypeIds::Timestamp> {}; + struct UserSID : Column<14, NScheme::NTypeIds::Utf8> {}; using TKey = TableKey; using TColumns = TableColumns< @@ -662,14 +684,14 @@ struct Schema : NIceDb::Schema { }; struct AuthUsers : Table<15> { - struct Sid: Column<1, NScheme::NTypeIds::Utf8> {}; - struct IsEnabled: Column<2, NScheme::NTypeIds::Bool> {}; - struct IsLockedOut: Column<3, NScheme::NTypeIds::Bool> {}; - struct CreatedAt: Column<4, NScheme::NTypeIds::Timestamp> {}; - struct LastSuccessfulAttemptAt: Column<5, NScheme::NTypeIds::Timestamp> {}; - struct LastFailedAttemptAt: Column<6, NScheme::NTypeIds::Timestamp> {}; - struct FailedAttemptCount: Column<7, NScheme::NTypeIds::Uint32> {}; - struct PasswordHash: Column<8, NScheme::NTypeIds::Utf8> {}; + struct Sid : Column<1, NScheme::NTypeIds::Utf8> {}; + struct IsEnabled : Column<2, NScheme::NTypeIds::Bool> {}; + struct IsLockedOut : Column<3, NScheme::NTypeIds::Bool> {}; + struct CreatedAt : Column<4, NScheme::NTypeIds::Timestamp> {}; + struct LastSuccessfulAttemptAt : Column<5, NScheme::NTypeIds::Timestamp> {}; + struct LastFailedAttemptAt : Column<6, NScheme::NTypeIds::Timestamp> {}; + struct FailedAttemptCount : Column<7, NScheme::NTypeIds::Uint32> {}; + struct PasswordHash : Column<8, NScheme::NTypeIds::Utf8> {}; using TKey = TableKey; using TColumns = TableColumns< @@ -685,7 +707,7 @@ struct Schema : NIceDb::Schema { }; struct AuthGroups : Table<16> { - struct Sid: Column<1, NScheme::NTypeIds::Utf8> {}; + struct Sid : Column<1, NScheme::NTypeIds::Utf8> {}; using TKey = TableKey; using TColumns = TableColumns< @@ -694,8 +716,8 @@ struct Schema : NIceDb::Schema { }; struct AuthGroupMembers : Table<17> { - struct GroupSid: Column<1, NScheme::NTypeIds::Utf8> {}; - struct MemberSid: Column<2, NScheme::NTypeIds::Utf8> {}; + struct GroupSid : Column<1, NScheme::NTypeIds::Utf8> {}; + struct MemberSid : Column<2, NScheme::NTypeIds::Utf8> {}; using TKey = TableKey; using TColumns = TableColumns< @@ -705,8 +727,8 @@ struct Schema : NIceDb::Schema { }; struct AuthOwners : Table<18> { - struct Path: Column<1, NScheme::NTypeIds::Utf8> {}; - struct Sid: Column<2, NScheme::NTypeIds::Utf8> {}; + struct Path : Column<1, NScheme::NTypeIds::Utf8> {}; + struct Sid : Column<2, NScheme::NTypeIds::Utf8> {}; using TKey = TableKey; using TColumns = TableColumns< @@ -716,9 +738,9 @@ struct Schema : NIceDb::Schema { }; struct AuthPermissions : Table<19> { - struct Path: Column<1, NScheme::NTypeIds::Utf8> {}; - struct Sid: Column<2, NScheme::NTypeIds::Utf8> {}; - struct Permission: Column<3, NScheme::NTypeIds::Utf8> {}; + struct Path : Column<1, NScheme::NTypeIds::Utf8> {}; + struct Sid : Column<2, NScheme::NTypeIds::Utf8> {}; + struct Permission : Column<3, NScheme::NTypeIds::Utf8> {}; using TKey = TableKey; using TColumns = TableColumns< @@ -744,10 +766,10 @@ struct Schema : NIceDb::Schema { }; struct ResourcePoolClassifiers : Table<20> { - struct Name: Column<1, NScheme::NTypeIds::Utf8> {}; - struct Rank: Column<2, NScheme::NTypeIds::Int64> {}; - struct MemberName: Column<4, NScheme::NTypeIds::Utf8> {}; - struct ResourcePool: Column<5, NScheme::NTypeIds::Utf8> {}; + struct Name : Column<1, NScheme::NTypeIds::Utf8> {}; + struct Rank : Column<2, NScheme::NTypeIds::Int64> {}; + struct MemberName : Column<4, NScheme::NTypeIds::Utf8> {}; + struct ResourcePool : Column<5, NScheme::NTypeIds::Utf8> {}; using TKey = TableKey; using TColumns = TableColumns< @@ -757,10 +779,10 @@ struct Schema : NIceDb::Schema { ResourcePool>; }; - struct ShowCreate: Table<21> { - struct Path: Column<1, NScheme::NTypeIds::Utf8> {}; - struct CreateQuery: Column<2, NScheme::NTypeIds::Utf8> {}; - struct PathType: Column<3, NScheme::NTypeIds::Utf8> {}; + struct ShowCreate : Table<21> { + struct Path : Column<1, NScheme::NTypeIds::Utf8> {}; + struct CreateQuery : Column<2, NScheme::NTypeIds::Utf8> {}; + struct PathType : Column<3, NScheme::NTypeIds::Utf8> {}; using TKey = TableKey; using TColumns = TableColumns< @@ -771,14 +793,14 @@ struct Schema : NIceDb::Schema { }; struct ResourcePools : Table<22> { - struct Name: Column<1, NScheme::NTypeIds::Utf8> {}; - struct ConcurrentQueryLimit: Column<2, NScheme::NTypeIds::Int32> {}; - struct QueueSize: Column<3, NScheme::NTypeIds::Int32> {}; - struct DatabaseLoadCpuThreshold: Column<4, NScheme::NTypeIds::Double> {}; - struct ResourceWeight: Column<5, NScheme::NTypeIds::Double> {}; - struct TotalCpuLimitPercentPerNode: Column<6, NScheme::NTypeIds::Double> {}; - struct QueryCpuLimitPercentPerNode: Column<7, NScheme::NTypeIds::Double> {}; - struct QueryMemoryLimitPercentPerNode: Column<8, NScheme::NTypeIds::Double> {}; + struct Name : Column<1, NScheme::NTypeIds::Utf8> {}; + struct ConcurrentQueryLimit : Column<2, NScheme::NTypeIds::Int32> {}; + struct QueueSize : Column<3, NScheme::NTypeIds::Int32> {}; + struct DatabaseLoadCpuThreshold : Column<4, NScheme::NTypeIds::Double> {}; + struct ResourceWeight : Column<5, NScheme::NTypeIds::Double> {}; + struct TotalCpuLimitPercentPerNode : Column<6, NScheme::NTypeIds::Double> {}; + struct QueryCpuLimitPercentPerNode : Column<7, NScheme::NTypeIds::Double> {}; + struct QueryMemoryLimitPercentPerNode : Column<8, NScheme::NTypeIds::Double> {}; using TKey = TableKey; using TColumns = TableColumns< @@ -860,46 +882,27 @@ struct Schema : NIceDb::Schema { }; }; -bool MaybeSystemViewPath(const TVector& path); -bool MaybeSystemViewFolderPath(const TVector& path); +struct SysViewsRegistryRecord { + using FillSchemaPointer = void(*)(ISystemViewResolver::TSchema&); -class ISystemViewResolver { -public: - virtual ~ISystemViewResolver() = default; - - enum class ETarget : ui8 { - Domain, - SubDomain, - OlapStore, - ColumnTable - }; - - struct TSystemViewPath { - TVector Parent; - TString ViewName; - }; - - struct TSchema { - THashMap Columns; - TVector KeyColumnTypes; - }; - - virtual bool IsSystemViewPath(const TVector& path, TSystemViewPath& sysViewPath) const = 0; - - virtual TMaybe GetSystemViewSchema(const TStringBuf viewName, ETarget target) const = 0; - - virtual TMaybe GetSystemViewSchema(NKikimrSysView::ESysViewType viewType) const = 0; + const TStringBuf Name; + const NKikimrSysView::ESysViewType Type; + const TSet SourceObjectTypes; + const FillSchemaPointer FillSchemaFunc; +}; - virtual bool IsSystemView(const TStringBuf viewName) const = 0; +struct SysViewsRegistry { + using ESysViewType = NKikimrSysView::ESysViewType; + using ESource = ISystemViewResolver::ESource; - virtual TVector GetSystemViewNames(ETarget target) const = 0; + SysViewsRegistry(); - virtual const THashMap& GetSystemViewsTypes(ETarget target) const = 0; -}; + static const TVector SysViews; + static const TVector RewrittenSysViews; -THolder CreateSystemViewResolver(); + THashMap SysViewTypesMap; -THolder CreateSystemViewRewrittenResolver(); +} const extern Registry; } // NSysView } // NKikimr diff --git a/ydb/core/sys_view/common/schema.cpp b/ydb/core/sys_view/common/resolver.cpp similarity index 51% rename from ydb/core/sys_view/common/schema.cpp rename to ydb/core/sys_view/common/resolver.cpp index e04730fc2174..84619c122616 100644 --- a/ydb/core/sys_view/common/schema.cpp +++ b/ydb/core/sys_view/common/resolver.cpp @@ -1,46 +1,13 @@ -#include "schema.h" +#include "resolver.h" -#include -#include +#include +#include -namespace { - using NKikimrSysView::ESysViewType; -} +using NKikimrSysView::ESysViewType; namespace NKikimr { namespace NSysView { -namespace { -TVector GetPgStaticTableColumns(const TString& schema, const TString& tableName) { - TVector res; - auto columns = NYql::NPg::GetStaticColumns().FindPtr(NYql::NPg::TTableInfoKey{schema, tableName}); - res.reserve(columns->size()); - for (size_t i = 0; i < columns->size(); i++) { - const auto& column = columns->at(i); - res.emplace_back(i, column.UdtType, column.Name); - } - return res; -} -} - -Schema::PgColumn::PgColumn(NIceDb::TColumnId columnId, TStringBuf columnTypeName, TStringBuf columnName) - : _ColumnId(columnId) - , _ColumnTypeInfo(NPg::TypeDescFromPgTypeId(NYql::NPg::LookupType(TString(columnTypeName)).TypeId)) - , _ColumnName(columnName) -{} - -const TVector& Schema::PgTablesSchemaProvider::GetColumns(TStringBuf tableName) const { - TString key(tableName); - Y_ENSURE(columnsStorage.contains(key)); - return columnsStorage.at(key); -} - -Schema::PgTablesSchemaProvider::PgTablesSchemaProvider() { - columnsStorage[TString(PgTablesName)] = GetPgStaticTableColumns("pg_catalog", "pg_tables"); - columnsStorage[TString(InformationSchemaTablesName)] = GetPgStaticTableColumns("information_schema", "tables"); - columnsStorage[TString(PgClassName)] = GetPgStaticTableColumns("pg_catalog", "pg_class"); -} - bool MaybeSystemViewPath(const TVector& path) { auto length = path.size(); // minimal system view path should be /Root/.sys/view @@ -58,73 +25,6 @@ bool MaybeSystemViewFolderPath(const TVector& path) { return true; } -template -struct TSchemaFiller { - - using TSchema = ISystemViewResolver::TSchema; - - template - struct TFiller; - - template - struct TFiller { - static void Fill(TSchema& schema) { - schema.Columns[Column::ColumnId] = TSysTables::TTableColumnInfo( - Table::template TableColumns::GetColumnName(), - Column::ColumnId, NScheme::TTypeInfo(Column::ColumnType), "", -1); - } - }; - - template - struct TFiller { - static void Fill(TSchema& schema) { - TFiller::Fill(schema); - TFiller::Fill(schema); - } - }; - - template - using TColumnsType = typename Table::template TableColumns; - - template - static void FillColumns(TSchema& schema, TColumnsType) { - TFiller::Fill(schema); - } - - template - struct TKeyFiller; - - template - struct TKeyFiller { - static void Fill(TSchema& schema, i32 index) { - auto& column = schema.Columns[Key::ColumnId]; - column.KeyOrder = index; - schema.KeyColumnTypes.push_back(column.PType); - } - }; - - template - struct TKeyFiller { - static void Fill(TSchema& schema, i32 index) { - TKeyFiller::Fill(schema, index); - TKeyFiller::Fill(schema, index + 1); - } - }; - - template - using TKeysType = typename Table::template TableKey; - - template - static void FillKeys(TSchema& schema, TKeysType) { - TKeyFiller::Fill(schema, 0); - } - - static void Fill(TSchema& schema) { - FillColumns(schema, typename Table::TColumns()); - FillKeys(schema, typename Table::TKey()); - } -}; - class TSystemViewResolver : public ISystemViewResolver { public: TSystemViewResolver() { @@ -155,19 +55,19 @@ class TSystemViewResolver : public ISystemViewResolver { return false; } - TMaybe GetSystemViewSchema(const TStringBuf viewName, ETarget target) const override final { + TMaybe GetSystemViewSchema(const TStringBuf viewName, ESource sourceObjectType) const override final { const TSchema* view = nullptr; - switch (target) { - case ETarget::Domain: + switch (sourceObjectType) { + case ESource::Domain: view = DomainSystemViews.FindPtr(viewName); break; - case ETarget::SubDomain: + case ESource::SubDomain: view = SubDomainSystemViews.FindPtr(viewName); break; - case ETarget::OlapStore: + case ESource::OlapStore: view = OlapStoreSystemViews.FindPtr(viewName); break; - case ETarget::ColumnTable: + case ESource::ColumnTable: view = ColumnTableSystemViews.FindPtr(viewName); break; } @@ -179,28 +79,28 @@ class TSystemViewResolver : public ISystemViewResolver { return view ? TMaybe(*view) : Nothing(); } - TVector GetSystemViewNames(ETarget target) const override { + TVector GetSystemViewNames(ESource sourceObjectType) const override { TVector result; - switch (target) { - case ETarget::Domain: + switch (sourceObjectType) { + case ESource::Domain: result.reserve(DomainSystemViews.size()); for (const auto& [name, _] : DomainSystemViews) { result.push_back(name); } break; - case ETarget::SubDomain: + case ESource::SubDomain: result.reserve(SubDomainSystemViews.size()); for (const auto& [name, _] : SubDomainSystemViews) { result.push_back(name); } break; - case ETarget::OlapStore: + case ESource::OlapStore: result.reserve(OlapStoreSystemViews.size()); for (const auto& [name, _] : OlapStoreSystemViews) { result.push_back(name); } break; - case ETarget::ColumnTable: + case ESource::ColumnTable: result.reserve(ColumnTableSystemViews.size()); for (const auto& [name, _] : ColumnTableSystemViews) { result.push_back(name); @@ -210,15 +110,15 @@ class TSystemViewResolver : public ISystemViewResolver { return result; } - const THashMap& GetSystemViewsTypes(ETarget target) const override { - switch (target) { - case ETarget::Domain: + const THashMap& GetSystemViewsTypes(ESource sourceObjectType) const override { + switch (sourceObjectType) { + case ESource::Domain: return DomainSystemViewTypes; - case ETarget::SubDomain: + case ESource::SubDomain: return SubDomainSystemViewTypes; - case ETarget::OlapStore: + case ESource::OlapStore: return OlapStoreSystemViewTypes; - case ETarget::ColumnTable: + case ESource::ColumnTable: return ColumnTableSystemViewTypes; } } @@ -267,22 +167,6 @@ class TSystemViewResolver : public ISystemViewResolver { ); } - template - void RegisterSystemView(const TStringBuf& name, ESysViewType type) { - TSchemaFiller::Fill(DomainSystemViews[name]); - DomainSystemViewTypes[name] = type; - TSchemaFiller
::Fill(SubDomainSystemViews[name]); - SubDomainSystemViewTypes[name] = type; - TSchemaFiller
::Fill(SystemViews[type]); - } - - template - void RegisterDomainSystemView(const TStringBuf& name, ESysViewType type) { - TSchemaFiller
::Fill(DomainSystemViews[name]); - DomainSystemViewTypes[name] = type; - TSchemaFiller
::Fill(SystemViews[type]); - } - template void RegisterOlapStoreSystemView(const TStringBuf& name) { TSchemaFiller
::Fill(OlapStoreSystemViews[name]); @@ -294,31 +178,34 @@ class TSystemViewResolver : public ISystemViewResolver { } void RegisterSystemViews() { - RegisterSystemView(PartitionStatsName, ESysViewType::EPartitionStats); - - RegisterSystemView(NodesName, ESysViewType::ENodes); - - RegisterSystemView(TopQueriesByDuration1MinuteName, ESysViewType::ETopQueriesByDurationOneMinute); - RegisterSystemView(TopQueriesByDuration1HourName, ESysViewType::ETopQueriesByDurationOneHour); - RegisterSystemView(TopQueriesByReadBytes1MinuteName, ESysViewType::ETopQueriesByReadBytesOneMinute); - RegisterSystemView(TopQueriesByReadBytes1HourName, ESysViewType::ETopQueriesByReadBytesOneHour); - RegisterSystemView(TopQueriesByCpuTime1MinuteName, ESysViewType::ETopQueriesByCpuTimeOneMinute); - RegisterSystemView(TopQueriesByCpuTime1HourName, ESysViewType::ETopQueriesByCpuTimeOneHour); - RegisterSystemView(TopQueriesByRequestUnits1MinuteName, ESysViewType::ETopQueriesByRequestUnitsOneMinute); - RegisterSystemView(TopQueriesByRequestUnits1HourName, ESysViewType::ETopQueriesByRequestUnitsOneHour); - RegisterSystemView(QuerySessions, ESysViewType::EQuerySessions); - // don't have approved schema yet - // RegisterSystemView(CompileCacheQueries, ESysViewType::ECompileCacheQueries); - - RegisterDomainSystemView(PDisksName, ESysViewType::EPDisks); - RegisterDomainSystemView(VSlotsName, ESysViewType::EVSlots); - RegisterDomainSystemView(GroupsName, ESysViewType::EGroups); - RegisterDomainSystemView(StoragePoolsName, ESysViewType::EStoragePools); - RegisterDomainSystemView(StorageStatsName, ESysViewType::EStorageStats); - - RegisterDomainSystemView(TabletsName, ESysViewType::ETablets); - - RegisterSystemView(QueryMetricsName, ESysViewType::EQueryMetricsOneMinute); + for (const auto& registryRecord : SysViewsRegistry::SysViews) { + TSchema schema; + registryRecord.FillSchemaFunc(schema); + SystemViews[registryRecord.Type] = schema; + + for (const auto sourceObjectType : registryRecord.SourceObjectTypes) { + switch (sourceObjectType) { + case ESource::Domain: { + DomainSystemViews[registryRecord.Name] = schema; + DomainSystemViewTypes[registryRecord.Name] = registryRecord.Type; + break; + }; + case ESource::SubDomain: { + SubDomainSystemViews[registryRecord.Name] = schema; + SubDomainSystemViewTypes[registryRecord.Name] = registryRecord.Type; + break; + }; + case ESource::OlapStore: { + OlapStoreSystemViews[registryRecord.Name] = schema; + break; + }; + case ESource::ColumnTable: { + ColumnTableSystemViews[registryRecord.Name] = schema; + break; + }; + } + } + } RegisterOlapStoreSystemView(StorePrimaryIndexStatsName); RegisterOlapStoreSystemView(StorePrimaryIndexSchemaStatsName); @@ -331,25 +218,7 @@ class TSystemViewResolver : public ISystemViewResolver { RegisterColumnTableSystemView(TablePrimaryIndexGranuleStatsName); RegisterColumnTableSystemView(TablePrimaryIndexOptimizerStatsName); - RegisterSystemView(TopPartitionsByCpu1MinuteName, ESysViewType::ETopPartitionsByCpuOneMinute); - RegisterSystemView(TopPartitionsByCpu1HourName, ESysViewType::ETopPartitionsByCpuOneHour); - RegisterSystemView(TopPartitionsByTli1MinuteName, ESysViewType::ETopPartitionsByTliOneMinute); - RegisterSystemView(TopPartitionsByTli1HourName, ESysViewType::ETopPartitionsByTliOneHour); - RegisterPgTablesSystemViews(); - - RegisterSystemView(ResourcePoolClassifiersName, ESysViewType::EResourcePoolClassifiers); - RegisterSystemView(ResourcePoolsName, ESysViewType::EResourcePools); - - { - using namespace NAuth; - RegisterSystemView(UsersName, ESysViewType::EAuthUsers); - RegisterSystemView(NAuth::GroupsName, ESysViewType::EAuthGroups); - RegisterSystemView(GroupMembersName, ESysViewType::EAuthGroupMembers); - RegisterSystemView(OwnersName, ESysViewType::EAuthOwners); - RegisterSystemView(PermissionsName, ESysViewType::EAuthPermissions); - RegisterSystemView(EffectivePermissionsName, ESysViewType::EAuthEffectivePermissions); - } } private: @@ -368,7 +237,11 @@ class TSystemViewRewrittenResolver : public ISystemViewResolver { public: TSystemViewRewrittenResolver() { - TSchemaFiller::Fill(SystemViews[ShowCreateName]); + for (const auto& registryRecord : SysViewsRegistry::RewrittenSysViews) { + TSchema schema; + registryRecord.FillSchemaFunc(schema); + SystemViews[registryRecord.Name] = std::move(schema); + } } bool IsSystemViewPath(const TVector& path, TSystemViewPath& sysViewPath) const override final { @@ -385,8 +258,8 @@ class TSystemViewRewrittenResolver : public ISystemViewResolver { return false; } - TMaybe GetSystemViewSchema(const TStringBuf viewName, ETarget target) const override final { - Y_UNUSED(target); + TMaybe GetSystemViewSchema(const TStringBuf viewName, ESource sourceObjectType) const override final { + Y_UNUSED(sourceObjectType); const TSchema* view = SystemViews.FindPtr(viewName); return view ? TMaybe(*view) : Nothing(); } @@ -396,13 +269,13 @@ class TSystemViewRewrittenResolver : public ISystemViewResolver { return Nothing(); } - TVector GetSystemViewNames(ETarget target) const override { - Y_UNUSED(target); + TVector GetSystemViewNames(ESource sourceObjectType) const override { + Y_UNUSED(sourceObjectType); return {}; } - const THashMap& GetSystemViewsTypes(ETarget target) const override { - Y_UNUSED(target); + const THashMap& GetSystemViewsTypes(ESource sourceObjectType) const override { + Y_UNUSED(sourceObjectType); return SystemViewTypes; } diff --git a/ydb/core/sys_view/common/resolver.h b/ydb/core/sys_view/common/resolver.h new file mode 100644 index 000000000000..3b0560f8001c --- /dev/null +++ b/ydb/core/sys_view/common/resolver.h @@ -0,0 +1,52 @@ +#pragma once + +#include +#include +#include + +namespace NKikimr { +namespace NSysView { + +class ISystemViewResolver { +public: + virtual ~ISystemViewResolver() = default; + + enum class ESource : ui8 { + Domain, + SubDomain, + OlapStore, + ColumnTable + }; + + struct TSystemViewPath { + TVector Parent; + TString ViewName; + }; + + struct TSchema { + THashMap Columns; + TVector KeyColumnTypes; + }; + + virtual bool IsSystemViewPath(const TVector& path, TSystemViewPath& sysViewPath) const = 0; + + virtual TMaybe GetSystemViewSchema(const TStringBuf viewName, ESource sourceObjectType) const = 0; + + virtual TMaybe GetSystemViewSchema(NKikimrSysView::ESysViewType viewType) const = 0; + + virtual bool IsSystemView(const TStringBuf viewName) const = 0; + + virtual TVector GetSystemViewNames(ESource target) const = 0; + + virtual const THashMap& GetSystemViewsTypes(ESource target) const = 0; +}; + +bool MaybeSystemViewPath(const TVector& path); +bool MaybeSystemViewFolderPath(const TVector& path); + +THolder CreateSystemViewResolver(); + +THolder CreateSystemViewRewrittenResolver(); + +} // NSysView +} // NKikimr diff --git a/ydb/core/sys_view/common/scan_actor_base_impl.h b/ydb/core/sys_view/common/scan_actor_base_impl.h index 92e5e8bd7da0..e57199c63dc4 100644 --- a/ydb/core/sys_view/common/scan_actor_base_impl.h +++ b/ydb/core/sys_view/common/scan_actor_base_impl.h @@ -1,11 +1,10 @@ #pragma once -#include "utils.h" - #include #include #include #include +#include #include #include diff --git a/ydb/core/sys_view/common/ya.make b/ydb/core/sys_view/common/ya.make index e139dec639b8..580b51b486e7 100644 --- a/ydb/core/sys_view/common/ya.make +++ b/ydb/core/sys_view/common/ya.make @@ -6,8 +6,8 @@ SRCS( keys.h path.h scan_actor_base_impl.h - schema.h - schema.cpp + registry.cpp + resolver.cpp utils.h processor_scan.h ) diff --git a/ydb/core/sys_view/compile_cache/compile_cache.cpp b/ydb/core/sys_view/compile_cache/compile_cache.cpp index bd7eb9e8af89..2327816ed22f 100644 --- a/ydb/core/sys_view/compile_cache/compile_cache.cpp +++ b/ydb/core/sys_view/compile_cache/compile_cache.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include #include diff --git a/ydb/core/sys_view/nodes/nodes.cpp b/ydb/core/sys_view/nodes/nodes.cpp index 5795836152b5..1c9599a7bf56 100644 --- a/ydb/core/sys_view/nodes/nodes.cpp +++ b/ydb/core/sys_view/nodes/nodes.cpp @@ -1,7 +1,7 @@ #include "nodes.h" #include -#include +#include #include #include diff --git a/ydb/core/sys_view/partition_stats/partition_stats.cpp b/ydb/core/sys_view/partition_stats/partition_stats.cpp index 9826fc2ca3e3..63e3afaefb62 100644 --- a/ydb/core/sys_view/partition_stats/partition_stats.cpp +++ b/ydb/core/sys_view/partition_stats/partition_stats.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include diff --git a/ydb/core/sys_view/partition_stats/top_partitions.cpp b/ydb/core/sys_view/partition_stats/top_partitions.cpp index 54abe33d1807..0c3d34ac309e 100644 --- a/ydb/core/sys_view/partition_stats/top_partitions.cpp +++ b/ydb/core/sys_view/partition_stats/top_partitions.cpp @@ -2,6 +2,7 @@ #include #include +#include namespace { using NKikimrSysView::ESysViewType; diff --git a/ydb/core/sys_view/pg_tables/pg_tables.cpp b/ydb/core/sys_view/pg_tables/pg_tables.cpp index 6c67c60386d2..f16f2f74d956 100644 --- a/ydb/core/sys_view/pg_tables/pg_tables.cpp +++ b/ydb/core/sys_view/pg_tables/pg_tables.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include #include diff --git a/ydb/core/sys_view/pg_tables/pg_tables.h b/ydb/core/sys_view/pg_tables/pg_tables.h index 84b320d8d76e..b7c0446c4df0 100644 --- a/ydb/core/sys_view/pg_tables/pg_tables.h +++ b/ydb/core/sys_view/pg_tables/pg_tables.h @@ -4,7 +4,7 @@ #include #include -#include +#include #include namespace NKikimr { diff --git a/ydb/core/sys_view/query_stats/query_metrics.cpp b/ydb/core/sys_view/query_stats/query_metrics.cpp index 6e377da42d11..a5cb54dfec24 100644 --- a/ydb/core/sys_view/query_stats/query_metrics.cpp +++ b/ydb/core/sys_view/query_stats/query_metrics.cpp @@ -2,6 +2,7 @@ #include #include +#include namespace NKikimr::NSysView { diff --git a/ydb/core/sys_view/query_stats/query_stats.cpp b/ydb/core/sys_view/query_stats/query_stats.cpp index 9253bad9726b..e0871a3d0fd4 100644 --- a/ydb/core/sys_view/query_stats/query_stats.cpp +++ b/ydb/core/sys_view/query_stats/query_stats.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/ydb/core/sys_view/resource_pool_classifiers/resource_pool_classifiers.cpp b/ydb/core/sys_view/resource_pool_classifiers/resource_pool_classifiers.cpp index fa2c9264f5b4..b61849d81bdb 100644 --- a/ydb/core/sys_view/resource_pool_classifiers/resource_pool_classifiers.cpp +++ b/ydb/core/sys_view/resource_pool_classifiers/resource_pool_classifiers.cpp @@ -5,8 +5,8 @@ #include #include #include +#include #include -#include #include #include diff --git a/ydb/core/sys_view/resource_pools/resource_pools.cpp b/ydb/core/sys_view/resource_pools/resource_pools.cpp index 783486a674eb..760e5dbc118a 100644 --- a/ydb/core/sys_view/resource_pools/resource_pools.cpp +++ b/ydb/core/sys_view/resource_pools/resource_pools.cpp @@ -2,8 +2,8 @@ #include #include +#include #include -#include #include #include diff --git a/ydb/core/sys_view/scan.cpp b/ydb/core/sys_view/scan.cpp index b6495047110b..feb7052d2557 100644 --- a/ydb/core/sys_view/scan.cpp +++ b/ydb/core/sys_view/scan.cpp @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include #include @@ -34,57 +34,7 @@ namespace NKikimr { namespace NSysView { -namespace { - using NKikimrSysView::ESysViewType; - - const THashMap SYS_VIEW_TYPES_MAP = { - {PartitionStatsName, ESysViewType::EPartitionStats}, - {NodesName, ESysViewType::ENodes}, - - {TopQueriesByDuration1MinuteName, ESysViewType::ETopQueriesByDurationOneMinute}, - {TopQueriesByDuration1HourName, ESysViewType::ETopQueriesByDurationOneHour}, - {TopQueriesByReadBytes1MinuteName, ESysViewType::ETopQueriesByReadBytesOneMinute}, - {TopQueriesByReadBytes1HourName, ESysViewType::ETopQueriesByReadBytesOneHour}, - {TopQueriesByCpuTime1MinuteName, ESysViewType::ETopQueriesByCpuTimeOneMinute}, - {TopQueriesByCpuTime1HourName, ESysViewType::ETopQueriesByCpuTimeOneHour}, - {TopQueriesByRequestUnits1MinuteName, ESysViewType::ETopQueriesByRequestUnitsOneMinute}, - {TopQueriesByRequestUnits1HourName, ESysViewType::ETopQueriesByRequestUnitsOneHour}, - - {QuerySessions, ESysViewType::EQuerySessions}, - {CompileCacheQueries, ESysViewType::ECompileCacheQueries}, - - {PDisksName, ESysViewType::EPDisks}, - {VSlotsName, ESysViewType::EVSlots}, - {GroupsName, ESysViewType::EGroups}, - {StoragePoolsName, ESysViewType::EStoragePools}, - {StorageStatsName, ESysViewType::EStorageStats}, - - {TabletsName, ESysViewType::ETablets}, - - {QueryMetricsName, ESysViewType::EQueryMetricsOneMinute}, - - {TopPartitionsByCpu1MinuteName, ESysViewType::ETopPartitionsByCpuOneMinute}, - {TopPartitionsByCpu1HourName, ESysViewType::ETopPartitionsByCpuOneHour}, - {TopPartitionsByTli1MinuteName, ESysViewType::ETopPartitionsByTliOneMinute}, - {TopPartitionsByTli1HourName, ESysViewType::ETopPartitionsByTliOneHour}, - - {PgTablesName, ESysViewType::EPgTables}, - {InformationSchemaTablesName, ESysViewType::EInformationSchemaTables}, - {PgClassName, ESysViewType::EPgClass}, - - {ResourcePoolClassifiersName, ESysViewType::EResourcePoolClassifiers}, - {ResourcePoolsName, ESysViewType::EResourcePools}, - - {NAuth::UsersName, ESysViewType::EAuthUsers}, - {NAuth::GroupsName, ESysViewType::EAuthGroups}, - {NAuth::GroupMembersName, ESysViewType::EAuthGroupMembers}, - {NAuth::OwnersName, ESysViewType::EAuthOwners}, - {NAuth::PermissionsName, ESysViewType::EAuthPermissions}, - {NAuth::EffectivePermissionsName, ESysViewType::EAuthEffectivePermissions}, - - {ShowCreateName, ESysViewType::EShowCreate} - }; -} +using NKikimrSysView::ESysViewType; class TSysViewRangesReader : public TActor { public: @@ -255,8 +205,8 @@ THolder CreateSystemViewScan( if (sysViewInfo) { sysViewDescription = *sysViewInfo; } else { - auto typesIt = SYS_VIEW_TYPES_MAP.find(tableId.SysViewInfo); - Y_ABORT_UNLESS(typesIt != SYS_VIEW_TYPES_MAP.end()); + auto typesIt = Registry.SysViewTypesMap.find(tableId.SysViewInfo); + Y_ABORT_UNLESS(typesIt != Registry.SysViewTypesMap.end()); sysViewDescription.SetType(typesIt->second); *sysViewDescription.MutableSourceObject() = tableId.PathId.ToProto(); } diff --git a/ydb/core/sys_view/sessions/sessions.cpp b/ydb/core/sys_view/sessions/sessions.cpp index 61142a58581f..99ee4023fe38 100644 --- a/ydb/core/sys_view/sessions/sessions.cpp +++ b/ydb/core/sys_view/sessions/sessions.cpp @@ -3,7 +3,7 @@ #include #include -#include +#include #include #include #include diff --git a/ydb/core/sys_view/show_create/show_create.cpp b/ydb/core/sys_view/show_create/show_create.cpp index 1b1d4c2556e6..d1a70f25a306 100644 --- a/ydb/core/sys_view/show_create/show_create.cpp +++ b/ydb/core/sys_view/show_create/show_create.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include #include @@ -113,7 +113,7 @@ class TShowCreate : public TScanActorBase { void StartScan() { if (!AppData()->FeatureFlags.GetEnableShowCreate()) { ReplyErrorAndDie(Ydb::StatusIds::SCHEME_ERROR, - TStringBuilder() << "Sys view is not supported: " << ShowCreateName); + TStringBuilder() << "Sys view 'show_create' is not supported"); return; } diff --git a/ydb/core/sys_view/storage/base.h b/ydb/core/sys_view/storage/base.h index 709be1ae64ef..e36940c8301b 100644 --- a/ydb/core/sys_view/storage/base.h +++ b/ydb/core/sys_view/storage/base.h @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include diff --git a/ydb/core/sys_view/tablets/tablets.cpp b/ydb/core/sys_view/tablets/tablets.cpp index 2b7b71ef2504..a337d58b31cd 100644 --- a/ydb/core/sys_view/tablets/tablets.cpp +++ b/ydb/core/sys_view/tablets/tablets.cpp @@ -1,7 +1,7 @@ #include "tablets.h" #include -#include +#include #include #include #include diff --git a/ydb/core/sys_view/ut/ya.make b/ydb/core/sys_view/ut/ya.make index bcab5546ca83..385f46b4b4f8 100644 --- a/ydb/core/sys_view/ut/ya.make +++ b/ydb/core/sys_view/ut/ya.make @@ -25,6 +25,7 @@ SRCS( ut_common.cpp ut_counters.cpp ut_labeled.cpp + ut_registry.cpp ) END() diff --git a/ydb/core/sys_view/ut_registry.cpp b/ydb/core/sys_view/ut_registry.cpp new file mode 100644 index 000000000000..fe6e8682af94 --- /dev/null +++ b/ydb/core/sys_view/ut_registry.cpp @@ -0,0 +1,864 @@ +#include + +#include +#include + +namespace NKikimr { +namespace NSysView { + +template +extern void FillSchema(ISystemViewResolver::TSchema& schema); + +namespace { + +using NKikimrSysView::ESysViewType; +using ESource = ISystemViewResolver::ESource; + +// SCHEMAS MODIFICATION POLICY: +// --------------------------- +// These schemas are CANONICAL and subject to strict modification rules: +// +// FORBIDDEN: +// - Changing column data types +// - Renaming existing columns +// - Removing columns from the primary key +// - Deleting existing columns +// +// ALLOWED: +// - Adding new columns (use the next available sequential ID) +// - Adding new schema definitions +// +// When adding columns, maintain sequential Column IDs and update the TColumns list. +// When adding tables, maintain sequential Table IDs . + +struct Schema : NIceDb::Schema { + struct PartitionStats : Table<1> { + struct OwnerId : Column<1, NScheme::NTypeIds::Uint64> {}; + struct PathId : Column<2, NScheme::NTypeIds::Uint64> {}; + struct PartIdx : Column<3, NScheme::NTypeIds::Uint64> {}; + struct DataSize : Column<4, NScheme::NTypeIds::Uint64> {}; + struct RowCount : Column<5, NScheme::NTypeIds::Uint64> {}; + struct IndexSize : Column<6, NScheme::NTypeIds::Uint64> {}; + struct CPUCores : Column<7, NScheme::NTypeIds::Double> {}; + struct TabletId : Column<8, NScheme::NTypeIds::Uint64> {}; + struct Path : Column<9, NScheme::NTypeIds::Utf8> {}; + struct NodeId : Column<10, NScheme::NTypeIds::Uint32> {}; + struct StartTime : Column<11, NScheme::NTypeIds::Timestamp> {}; + struct AccessTime : Column<12, NScheme::NTypeIds::Timestamp> {}; + struct UpdateTime : Column<13, NScheme::NTypeIds::Timestamp> {}; + struct InFlightTxCount : Column<14, NScheme::NTypeIds::Uint32> {}; + struct RowUpdates : Column<15, NScheme::NTypeIds::Uint64> {}; + struct RowDeletes : Column<16, NScheme::NTypeIds::Uint64> {}; + struct RowReads : Column<17, NScheme::NTypeIds::Uint64> {}; + struct RangeReads : Column<18, NScheme::NTypeIds::Uint64> {}; + struct RangeReadRows : Column<19, NScheme::NTypeIds::Uint64> {}; + struct ImmediateTxCompleted : Column<20, NScheme::NTypeIds::Uint64> {}; + struct CoordinatedTxCompleted : Column<21, NScheme::NTypeIds::Uint64> {}; + struct TxRejectedByOverload : Column<22, NScheme::NTypeIds::Uint64> {}; + struct TxRejectedByOutOfStorage : Column<23, NScheme::NTypeIds::Uint64> {}; + struct LastTtlRunTime : Column<24, NScheme::NTypeIds::Timestamp> {}; + struct LastTtlRowsProcessed : Column<25, NScheme::NTypeIds::Uint64> {}; + struct LastTtlRowsErased : Column<26, NScheme::NTypeIds::Uint64> {}; + struct FollowerId : Column<27, NScheme::NTypeIds::Uint32> {}; + struct LocksAcquired : Column<28, NScheme::NTypeIds::Uint64> {}; + struct LocksWholeShard : Column<29, NScheme::NTypeIds::Uint64> {}; + struct LocksBroken : Column<30, NScheme::NTypeIds::Uint64> {}; + + using TKey = TableKey; + using TColumns = TableColumns< + OwnerId, + PathId, + PartIdx, + DataSize, + RowCount, + IndexSize, + CPUCores, + TabletId, + Path, + NodeId, + AccessTime, + UpdateTime, + StartTime, + InFlightTxCount, + RowUpdates, + RowDeletes, + RowReads, + RangeReads, + RangeReadRows, + ImmediateTxCompleted, + CoordinatedTxCompleted, + TxRejectedByOverload, + TxRejectedByOutOfStorage, + LastTtlRunTime, + LastTtlRowsProcessed, + LastTtlRowsErased, + FollowerId, + LocksAcquired, + LocksWholeShard, + LocksBroken + >; + }; + + struct Nodes : Table<2> { + struct NodeId : Column<1, NScheme::NTypeIds::Uint32> {}; + struct Address : Column<2, NScheme::NTypeIds::Utf8> {}; + struct Host : Column<3, NScheme::NTypeIds::Utf8> {}; + struct Port : Column<4, NScheme::NTypeIds::Uint32> {}; + struct StartTime : Column<5, NScheme::NTypeIds::Timestamp> {}; + struct UpTime : Column<6, NScheme::NTypeIds::Interval> {}; + struct CpuThreads : Column<7, NScheme::NTypeIds::Uint32> {}; + struct CpuUsage : Column<8, NScheme::NTypeIds::Double> {}; + struct CpuIdle : Column<9, NScheme::NTypeIds::Double> {}; + + using TKey = TableKey; + using TColumns = TableColumns< + NodeId, + Host, + Address, + Port, + StartTime, + UpTime, + CpuThreads, + CpuUsage, + CpuIdle>; + }; + + struct QueryStats : Table<3> { + struct IntervalEnd : Column<1, NScheme::NTypeIds::Timestamp> {}; + struct Rank : Column<2, NScheme::NTypeIds::Uint32> {}; + struct QueryText : Column<3, NScheme::NTypeIds::Utf8> {}; + struct Duration : Column<4, NScheme::NTypeIds::Interval> {}; + struct EndTime : Column<5, NScheme::NTypeIds::Timestamp> {}; + struct ReadRows : Column<6, NScheme::NTypeIds::Uint64> {}; + struct ReadBytes : Column<7, NScheme::NTypeIds::Uint64> {}; + struct UpdateRows : Column<8, NScheme::NTypeIds::Uint64> {}; + struct UpdateBytes : Column<9, NScheme::NTypeIds::Uint64> {}; + struct DeleteRows : Column<10, NScheme::NTypeIds::Uint64> {}; + struct DeleteBytes : Column<11, NScheme::NTypeIds::Uint64> {}; // deprecated, always 0 + struct Partitions : Column<12, NScheme::NTypeIds::Uint64> {}; + struct UserSID : Column<13, NScheme::NTypeIds::Utf8> {}; + struct ParametersSize : Column<14, NScheme::NTypeIds::Uint64> {}; + struct CompileDuration : Column<15, NScheme::NTypeIds::Interval> {}; + struct FromQueryCache : Column<16, NScheme::NTypeIds::Bool> {}; + struct CPUTime : Column<17, NScheme::NTypeIds::Uint64> {}; + struct ShardCount : Column<18, NScheme::NTypeIds::Uint64> {}; + struct SumShardCPUTime : Column<19, NScheme::NTypeIds::Uint64> {}; + struct MinShardCPUTime : Column<20, NScheme::NTypeIds::Uint64> {}; + struct MaxShardCPUTime : Column<21, NScheme::NTypeIds::Uint64> {}; + struct ComputeNodesCount : Column<22, NScheme::NTypeIds::Uint64> {}; + struct SumComputeCPUTime : Column<23, NScheme::NTypeIds::Uint64> {}; + struct MinComputeCPUTime : Column<24, NScheme::NTypeIds::Uint64> {}; + struct MaxComputeCPUTime : Column<25, NScheme::NTypeIds::Uint64> {}; + struct CompileCPUTime : Column<26, NScheme::NTypeIds::Uint64> {}; + struct ProcessCPUTime : Column<27, NScheme::NTypeIds::Uint64> {}; + struct TypeCol : Column<28, NScheme::NTypeIds::Utf8> { static TString GetColumnName(const TString&) { return "Type"; } }; + struct RequestUnits : Column<29, NScheme::NTypeIds::Uint64> {}; + + using TKey = TableKey; + using TColumns = TableColumns< + IntervalEnd, + Rank, + QueryText, + Duration, + EndTime, + ReadRows, + ReadBytes, + UpdateRows, + UpdateBytes, + DeleteRows, + DeleteBytes, + Partitions, + UserSID, + ParametersSize, + CompileDuration, + FromQueryCache, + CPUTime, + ShardCount, + SumShardCPUTime, + MinShardCPUTime, + MaxShardCPUTime, + ComputeNodesCount, + SumComputeCPUTime, + MinComputeCPUTime, + MaxComputeCPUTime, + CompileCPUTime, + ProcessCPUTime, + TypeCol, + RequestUnits>; + }; + + struct PDisks : Table<4> { + struct NodeId : Column<1, NScheme::NTypeIds::Uint32> {}; + struct PDiskId : Column<2, NScheme::NTypeIds::Uint32> {}; + struct TypeCol : Column<3, NScheme::NTypeIds::Utf8> { static TString GetColumnName(const TString&) { return "Type"; } }; + struct Kind : Column<4, NScheme::NTypeIds::Uint64> {}; + struct Path : Column<5, NScheme::NTypeIds::Utf8> {}; + struct Guid : Column<6, NScheme::NTypeIds::Uint64> {}; + struct BoxId : Column<7, NScheme::NTypeIds::Uint32> {}; + struct SharedWithOS : Column<8, NScheme::NTypeIds::Bool> {}; + struct ReadCentric : Column<9, NScheme::NTypeIds::Bool> {}; + struct AvailableSize : Column<10, NScheme::NTypeIds::Uint64> {}; + struct TotalSize : Column<11, NScheme::NTypeIds::Uint64> {}; + struct Status : Column<12, NScheme::NTypeIds::Utf8> {}; + struct StatusChangeTimestamp : Column<14, NScheme::NTypeIds::Timestamp> {}; + struct ExpectedSlotCount : Column<15, NScheme::NTypeIds::Uint32> {}; + struct NumActiveSlots : Column<16, NScheme::NTypeIds::Uint32> {}; + struct DecommitStatus : Column<17, NScheme::NTypeIds::Utf8> {}; + struct State : Column<18, NScheme::NTypeIds::Utf8> {}; + struct SlotSizeInUnits : Column<19, NScheme::NTypeIds::Uint32> {}; + struct InferPDiskSlotCountFromUnitSize : Column<20, NScheme::NTypeIds::Uint64> {}; + + using TKey = TableKey; + using TColumns = TableColumns< + NodeId, + PDiskId, + TypeCol, + Kind, + Path, + Guid, + BoxId, + SharedWithOS, + ReadCentric, + AvailableSize, + TotalSize, + Status, + State, + StatusChangeTimestamp, + ExpectedSlotCount, + NumActiveSlots, + DecommitStatus, + SlotSizeInUnits, + InferPDiskSlotCountFromUnitSize>; + }; + + struct VSlots : Table<5> { + struct NodeId : Column<1, NScheme::NTypeIds::Uint32> {}; + struct PDiskId : Column<2, NScheme::NTypeIds::Uint32> {}; + struct VSlotId : Column<3, NScheme::NTypeIds::Uint32> {}; + struct GroupId : Column<5, NScheme::NTypeIds::Uint32> {}; + struct GroupGeneration : Column<6, NScheme::NTypeIds::Uint32> {}; + struct FailDomain : Column<8, NScheme::NTypeIds::Uint32> {}; + struct VDisk : Column<9, NScheme::NTypeIds::Uint32> {}; + struct AllocatedSize : Column<10, NScheme::NTypeIds::Uint64> {}; + struct AvailableSize : Column<11, NScheme::NTypeIds::Uint64> {}; + struct Status : Column<12, NScheme::NTypeIds::Utf8> {}; + struct Kind : Column<14, NScheme::NTypeIds::Utf8> {}; + struct FailRealm : Column<15, NScheme::NTypeIds::Uint32> {}; + struct Replicated : Column<16, NScheme::NTypeIds::Bool> {}; + struct DiskSpace : Column<17, NScheme::NTypeIds::Utf8> {}; + struct State : Column <18, NScheme::NTypeIds::Utf8> {}; + + using TKey = TableKey; + using TColumns = TableColumns< + NodeId, + PDiskId, + VSlotId, + GroupId, + GroupGeneration, + FailDomain, + VDisk, + AllocatedSize, + AvailableSize, + Status, + State, + Kind, + FailRealm, + Replicated, + DiskSpace>; + }; + + struct Groups : Table<6> { + struct GroupId : Column<1, NScheme::NTypeIds::Uint32> {}; + struct Generation : Column<2, NScheme::NTypeIds::Uint32> {}; + struct ErasureSpecies : Column<3, NScheme::NTypeIds::Utf8> {}; + struct BoxId : Column<4, NScheme::NTypeIds::Uint64> {}; + struct StoragePoolId : Column<5, NScheme::NTypeIds::Uint64> {}; + struct EncryptionMode : Column<6, NScheme::NTypeIds::Uint32> {}; + struct LifeCyclePhase : Column<7, NScheme::NTypeIds::Uint32> {}; + struct AllocatedSize : Column<8, NScheme::NTypeIds::Uint64> {}; + struct AvailableSize : Column<9, NScheme::NTypeIds::Uint64> {}; + struct SeenOperational : Column<12, NScheme::NTypeIds::Bool> {}; + struct PutTabletLogLatency : Column<13, NScheme::NTypeIds::Interval> {}; + struct PutUserDataLatency : Column<14, NScheme::NTypeIds::Interval> {}; + struct GetFastLatency : Column<15, NScheme::NTypeIds::Interval> {}; + struct LayoutCorrect : Column<16, NScheme::NTypeIds::Bool> {}; + struct OperatingStatus : Column<17, NScheme::NTypeIds::Utf8> {}; + struct ExpectedStatus : Column<18, NScheme::NTypeIds::Utf8> {}; + struct ProxyGroupId : Column<19, NScheme::NTypeIds::Uint32> {}; + struct BridgePileId : Column<20, NScheme::NTypeIds::Uint32> {}; + struct GroupSizeInUnits : Column<21, NScheme::NTypeIds::Uint32> {}; + + using TKey = TableKey; + using TColumns = TableColumns< + GroupId, + Generation, + ErasureSpecies, + BoxId, + StoragePoolId, + EncryptionMode, + LifeCyclePhase, + AllocatedSize, + AvailableSize, + SeenOperational, + PutTabletLogLatency, + PutUserDataLatency, + GetFastLatency, + LayoutCorrect, + OperatingStatus, + ExpectedStatus, + ProxyGroupId, + BridgePileId, + GroupSizeInUnits>; + }; + + struct StoragePools : Table<7> { + struct BoxId : Column<1, NScheme::NTypeIds::Uint64> {}; + struct StoragePoolId : Column<2, NScheme::NTypeIds::Uint64> {}; + struct Name : Column<3, NScheme::NTypeIds::Utf8> {}; + struct Generation : Column<4, NScheme::NTypeIds::Uint64> {}; + struct ErasureSpecies : Column<5, NScheme::NTypeIds::Utf8> {}; + struct VDiskKind : Column<6, NScheme::NTypeIds::Utf8> {}; + struct Kind : Column<7, NScheme::NTypeIds::Utf8> {}; + struct NumGroups : Column<8, NScheme::NTypeIds::Uint32> {}; + struct EncryptionMode : Column<9, NScheme::NTypeIds::Uint32> {}; + struct SchemeshardId : Column<10, NScheme::NTypeIds::Uint64> {}; + struct PathId : Column<11, NScheme::NTypeIds::Uint64> {}; + struct DefaultGroupSizeInUnits : Column<12, NScheme::NTypeIds::Uint32> {}; + + using TKey = TableKey; + using TColumns = TableColumns< + BoxId, + StoragePoolId, + Name, + Generation, + ErasureSpecies, + VDiskKind, + Kind, + NumGroups, + EncryptionMode, + SchemeshardId, + PathId, + DefaultGroupSizeInUnits>; + }; + + struct Tablets : Table<8> { + struct TabletId : Column<1, NScheme::NTypeIds::Uint64> {}; + struct FollowerId : Column<2, NScheme::NTypeIds::Uint32> {}; + struct TypeCol : Column<3, NScheme::NTypeIds::Utf8> { static TString GetColumnName(const TString&) { return "Type"; } }; + struct State : Column<4, NScheme::NTypeIds::Utf8> {}; + struct VolatileState : Column<5, NScheme::NTypeIds::Utf8> {}; + struct BootState : Column<6, NScheme::NTypeIds::Utf8> {}; + struct Generation : Column<7, NScheme::NTypeIds::Uint32> {}; + struct NodeId : Column<8, NScheme::NTypeIds::Uint32> {}; + struct CPU : Column<9, NScheme::NTypeIds::Double> {}; + struct Memory : Column<10, NScheme::NTypeIds::Uint64> {}; + struct Network : Column<11, NScheme::NTypeIds::Uint64> {}; + + using TKey = TableKey; + using TColumns = TableColumns< + TabletId, + FollowerId, + TypeCol, + State, + VolatileState, + BootState, + Generation, + NodeId, + CPU, + Memory, + Network>; + }; + + struct QueryMetrics : Table<9> { + struct IntervalEnd : Column<1, NScheme::NTypeIds::Timestamp> {}; + struct Rank : Column<2, NScheme::NTypeIds::Uint32> {}; + struct QueryText : Column<3, NScheme::NTypeIds::Utf8> {}; + struct Count : Column<4, NScheme::NTypeIds::Uint64> {}; + struct SumCPUTime : Column<5, NScheme::NTypeIds::Uint64> {}; + struct MinCPUTime : Column<6, NScheme::NTypeIds::Uint64> {}; + struct MaxCPUTime : Column<7, NScheme::NTypeIds::Uint64> {}; + struct SumDuration : Column<8, NScheme::NTypeIds::Interval> {}; + struct MinDuration : Column<9, NScheme::NTypeIds::Interval> {}; + struct MaxDuration : Column<10, NScheme::NTypeIds::Interval> {}; + struct MinReadRows : Column<11, NScheme::NTypeIds::Uint64> {}; + struct MaxReadRows : Column<12, NScheme::NTypeIds::Uint64> {}; + struct SumReadRows : Column<13, NScheme::NTypeIds::Uint64> {}; + struct MinReadBytes : Column<14, NScheme::NTypeIds::Uint64> {}; + struct MaxReadBytes : Column<15, NScheme::NTypeIds::Uint64> {}; + struct SumReadBytes : Column<16, NScheme::NTypeIds::Uint64> {}; + struct MinUpdateRows : Column<17, NScheme::NTypeIds::Uint64> {}; + struct MaxUpdateRows : Column<18, NScheme::NTypeIds::Uint64> {}; + struct SumUpdateRows : Column<19, NScheme::NTypeIds::Uint64> {}; + struct MinUpdateBytes : Column<20, NScheme::NTypeIds::Uint64> {}; + struct MaxUpdateBytes : Column<21, NScheme::NTypeIds::Uint64> {}; + struct SumUpdateBytes : Column<22, NScheme::NTypeIds::Uint64> {}; + struct MinDeleteRows : Column<23, NScheme::NTypeIds::Uint64> {}; + struct MaxDeleteRows : Column<24, NScheme::NTypeIds::Uint64> {}; + struct SumDeleteRows : Column<25, NScheme::NTypeIds::Uint64> {}; + struct MinRequestUnits : Column<26, NScheme::NTypeIds::Uint64> {}; + struct MaxRequestUnits : Column<27, NScheme::NTypeIds::Uint64> {}; + struct SumRequestUnits : Column<28, NScheme::NTypeIds::Uint64> {}; + + using TKey = TableKey; + using TColumns = TableColumns< + IntervalEnd, + Rank, + QueryText, + Count, + SumCPUTime, MinCPUTime, MaxCPUTime, + SumDuration, MinDuration, MaxDuration, + SumReadRows, MinReadRows, MaxReadRows, + SumReadBytes, MinReadBytes, MaxReadBytes, + SumUpdateRows, MinUpdateRows, MaxUpdateRows, + SumUpdateBytes, MinUpdateBytes, MaxUpdateBytes, + SumDeleteRows, MinDeleteRows, MaxDeleteRows, + SumRequestUnits, MinRequestUnits, MaxRequestUnits>; + }; + + struct StorageStats : Table<10> { + struct PDiskFilter : Column<2, NScheme::NTypeIds::Utf8> {}; + struct ErasureSpecies : Column<3, NScheme::NTypeIds::Utf8> {}; + struct CurrentGroupsCreated : Column<4, NScheme::NTypeIds::Uint32> {}; + struct CurrentAllocatedSize : Column<5, NScheme::NTypeIds::Uint64> {}; + struct CurrentAvailableSize : Column<6, NScheme::NTypeIds::Uint64> {}; + struct AvailableGroupsToCreate : Column<7, NScheme::NTypeIds::Uint32> {}; + struct AvailableSizeToCreate : Column<8, NScheme::NTypeIds::Uint64> {}; + + using TKey = TableKey; + using TColumns = TableColumns; + }; + + struct TopPartitions : Table<11> { + struct IntervalEnd : Column<1, NScheme::NTypeIds::Timestamp> {}; + struct Rank : Column<2, NScheme::NTypeIds::Uint32> {}; + struct TabletId : Column<3, NScheme::NTypeIds::Uint64> {}; + struct Path : Column<4, NScheme::NTypeIds::Utf8> {}; + struct PeakTime : Column<5, NScheme::NTypeIds::Timestamp> {}; + struct CPUCores : Column<6, NScheme::NTypeIds::Double> {}; + struct NodeId : Column<7, NScheme::NTypeIds::Uint32> {}; + struct DataSize : Column<8, NScheme::NTypeIds::Uint64> {}; + struct RowCount : Column<9, NScheme::NTypeIds::Uint64> {}; + struct IndexSize : Column<10, NScheme::NTypeIds::Uint64> {}; + struct InFlightTxCount : Column<11, NScheme::NTypeIds::Uint32> {}; + struct FollowerId : Column<12, NScheme::NTypeIds::Uint32> {}; + + using TKey = TableKey; + using TColumns = TableColumns< + IntervalEnd, + Rank, + TabletId, + Path, + PeakTime, + CPUCores, + NodeId, + DataSize, + RowCount, + IndexSize, + InFlightTxCount, + FollowerId>; + }; + + struct QuerySessions : Table<12> { + struct SessionId : Column<1, NScheme::NTypeIds::Utf8> {}; + struct NodeId : Column<2, NScheme::NTypeIds::Uint32> {}; + struct State : Column<3, NScheme::NTypeIds::Utf8> {}; + struct Query : Column<4, NScheme::NTypeIds::Utf8> {}; + struct QueryCount : Column<5, NScheme::NTypeIds::Uint32> {}; + struct ClientAddress : Column<6, NScheme::NTypeIds::Utf8> {}; + struct ClientPID : Column<7, NScheme::NTypeIds::Utf8> {}; + struct ClientUserAgent : Column<8, NScheme::NTypeIds::Utf8> {}; + struct ClientSdkBuildInfo : Column<9, NScheme::NTypeIds::Utf8> {}; + struct ApplicationName : Column<10, NScheme::NTypeIds::Utf8> {}; + struct SessionStartAt : Column<11, NScheme::NTypeIds::Timestamp> {}; + struct QueryStartAt : Column<12, NScheme::NTypeIds::Timestamp> {}; + struct StateChangeAt : Column<13, NScheme::NTypeIds::Timestamp> {}; + struct UserSID : Column<14, NScheme::NTypeIds::Utf8> {}; + + using TKey = TableKey; + using TColumns = TableColumns< + SessionId, + NodeId, + State, + Query, + QueryCount, + ClientAddress, + ClientPID, + ClientUserAgent, + ClientSdkBuildInfo, + ApplicationName, + SessionStartAt, + QueryStartAt, + StateChangeAt, + UserSID>; + }; + + struct AuthUsers : Table<13> { + struct Sid : Column<1, NScheme::NTypeIds::Utf8> {}; + struct IsEnabled : Column<2, NScheme::NTypeIds::Bool> {}; + struct IsLockedOut : Column<3, NScheme::NTypeIds::Bool> {}; + struct CreatedAt : Column<4, NScheme::NTypeIds::Timestamp> {}; + struct LastSuccessfulAttemptAt : Column<5, NScheme::NTypeIds::Timestamp> {}; + struct LastFailedAttemptAt : Column<6, NScheme::NTypeIds::Timestamp> {}; + struct FailedAttemptCount : Column<7, NScheme::NTypeIds::Uint32> {}; + struct PasswordHash : Column<8, NScheme::NTypeIds::Utf8> {}; + + using TKey = TableKey; + using TColumns = TableColumns< + Sid, + IsEnabled, + IsLockedOut, + CreatedAt, + LastSuccessfulAttemptAt, + LastFailedAttemptAt, + FailedAttemptCount, + PasswordHash + >; + }; + + struct AuthGroups : Table<14> { + struct Sid : Column<1, NScheme::NTypeIds::Utf8> {}; + + using TKey = TableKey; + using TColumns = TableColumns< + Sid + >; + }; + + struct AuthGroupMembers : Table<15> { + struct GroupSid : Column<1, NScheme::NTypeIds::Utf8> {}; + struct MemberSid : Column<2, NScheme::NTypeIds::Utf8> {}; + + using TKey = TableKey; + using TColumns = TableColumns< + GroupSid, + MemberSid + >; + }; + + struct AuthOwners : Table<16> { + struct Path : Column<1, NScheme::NTypeIds::Utf8> {}; + struct Sid : Column<2, NScheme::NTypeIds::Utf8> {}; + + using TKey = TableKey; + using TColumns = TableColumns< + Path, + Sid + >; + }; + + struct AuthPermissions : Table<17> { + struct Path : Column<1, NScheme::NTypeIds::Utf8> {}; + struct Sid : Column<2, NScheme::NTypeIds::Utf8> {}; + struct Permission : Column<3, NScheme::NTypeIds::Utf8> {}; + + using TKey = TableKey; + using TColumns = TableColumns< + Path, + Sid, + Permission + >; + }; + + struct ResourcePoolClassifiers : Table<18> { + struct Name : Column<1, NScheme::NTypeIds::Utf8> {}; + struct Rank : Column<2, NScheme::NTypeIds::Int64> {}; + struct MemberName : Column<4, NScheme::NTypeIds::Utf8> {}; + struct ResourcePool : Column<5, NScheme::NTypeIds::Utf8> {}; + + using TKey = TableKey; + using TColumns = TableColumns< + Name, + Rank, + MemberName, + ResourcePool>; + }; + + struct ShowCreate : Table<19> { + struct Path : Column<1, NScheme::NTypeIds::Utf8> {}; + struct CreateQuery : Column<2, NScheme::NTypeIds::Utf8> {}; + struct PathType : Column<3, NScheme::NTypeIds::Utf8> {}; + + using TKey = TableKey; + using TColumns = TableColumns< + Path, + CreateQuery, + PathType + >; + }; + + struct ResourcePools : Table<20> { + struct Name : Column<1, NScheme::NTypeIds::Utf8> {}; + struct ConcurrentQueryLimit : Column<2, NScheme::NTypeIds::Int32> {}; + struct QueueSize : Column<3, NScheme::NTypeIds::Int32> {}; + struct DatabaseLoadCpuThreshold : Column<4, NScheme::NTypeIds::Double> {}; + struct ResourceWeight : Column<5, NScheme::NTypeIds::Double> {}; + struct TotalCpuLimitPercentPerNode : Column<6, NScheme::NTypeIds::Double> {}; + struct QueryCpuLimitPercentPerNode : Column<7, NScheme::NTypeIds::Double> {}; + struct QueryMemoryLimitPercentPerNode : Column<8, NScheme::NTypeIds::Double> {}; + + using TKey = TableKey; + using TColumns = TableColumns< + Name, + ConcurrentQueryLimit, + QueueSize, + DatabaseLoadCpuThreshold, + ResourceWeight, + TotalCpuLimitPercentPerNode, + QueryCpuLimitPercentPerNode, + QueryMemoryLimitPercentPerNode>; + }; + + struct TopPartitionsTli : Table<21> { + struct IntervalEnd : Column<1, NScheme::NTypeIds::Timestamp> {}; + struct Rank : Column<2, NScheme::NTypeIds::Uint32> {}; + struct TabletId : Column<3, NScheme::NTypeIds::Uint64> {}; + struct Path : Column<4, NScheme::NTypeIds::Utf8> {}; + struct LocksAcquired : Column<5, NScheme::NTypeIds::Uint64> {}; + struct LocksWholeShard : Column<6, NScheme::NTypeIds::Uint64> {}; + struct LocksBroken : Column<7, NScheme::NTypeIds::Uint64> {}; + struct NodeId : Column<8, NScheme::NTypeIds::Uint32> {}; + struct DataSize : Column<9, NScheme::NTypeIds::Uint64> {}; + struct RowCount : Column<10, NScheme::NTypeIds::Uint64> {}; + struct IndexSize : Column<11, NScheme::NTypeIds::Uint64> {}; + struct FollowerId : Column<12, NScheme::NTypeIds::Uint32> {}; + + using TKey = TableKey; + using TColumns = TableColumns< + IntervalEnd, + Rank, + TabletId, + Path, + LocksAcquired, + LocksWholeShard, + LocksBroken, + NodeId, + DataSize, + RowCount, + IndexSize, + FollowerId>; + }; + +}; + +// SYSTEM VIEWS REGISTRY MODIFICATION POLICY: +// --------------------------- +// These system view lists are CANONICAL and subject to strict modification rules: +// +// FORBIDDEN: +// - Changing sysview types (implementation ids) +// - Renaming existing sysviews +// - Adding sysviews with existing names and same source object types +// - Changing sysview schemas +// - Deleting existing sysviews +// - Deleting related source objects of existing sysviews +// +// ALLOWED: +// - Adding new sysviews +// - Adding new source object types of existing sysviews + +const TVector SysViews = { + {"partition_stats", ESysViewType::EPartitionStats, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"nodes", ESysViewType::ENodes, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + + {"top_queries_by_duration_one_minute", ESysViewType::ETopQueriesByDurationOneMinute, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"top_queries_by_duration_one_hour", ESysViewType::ETopQueriesByDurationOneHour, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"top_queries_by_read_bytes_one_minute", ESysViewType::ETopQueriesByReadBytesOneMinute, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"top_queries_by_read_bytes_one_hour", ESysViewType::ETopQueriesByReadBytesOneHour, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"top_queries_by_cpu_time_one_minute", ESysViewType::ETopQueriesByCpuTimeOneMinute, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"top_queries_by_cpu_time_one_hour", ESysViewType::ETopQueriesByCpuTimeOneHour, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"top_queries_by_request_units_one_minute", ESysViewType::ETopQueriesByRequestUnitsOneMinute, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"top_queries_by_request_units_one_hour", ESysViewType::ETopQueriesByRequestUnitsOneHour, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + + {"query_sessions", ESysViewType::EQuerySessions, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"query_metrics_one_minute", ESysViewType::EQueryMetricsOneMinute, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + + {"ds_pdisks", ESysViewType::EPDisks, {ESource::Domain}, &FillSchema}, + {"ds_vslots", ESysViewType::EVSlots, {ESource::Domain}, &FillSchema}, + {"ds_groups", ESysViewType::EGroups, {ESource::Domain}, &FillSchema}, + {"ds_storage_pools", ESysViewType::EStoragePools, {ESource::Domain}, &FillSchema}, + {"ds_storage_stats", ESysViewType::EStorageStats, {ESource::Domain}, &FillSchema}, + {"hive_tablets", ESysViewType::ETablets, {ESource::Domain}, &FillSchema}, + + {"top_partitions_one_minute", ESysViewType::ETopPartitionsByCpuOneMinute, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"top_partitions_one_hour", ESysViewType::ETopPartitionsByCpuOneHour, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"top_partitions_by_tli_one_minute", ESysViewType::ETopPartitionsByTliOneMinute, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"top_partitions_by_tli_one_hour", ESysViewType::ETopPartitionsByTliOneHour, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + + {"resource_pool_classifiers", ESysViewType::EResourcePoolClassifiers, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"resource_pools", ESysViewType::EResourcePools, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + + {"auth_users", ESysViewType::EAuthUsers, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"auth_groups", ESysViewType::EAuthGroups, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"auth_group_members", ESysViewType::EAuthGroupMembers, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"auth_owners", ESysViewType::EAuthOwners, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"auth_permissions", ESysViewType::EAuthPermissions, {ESource::Domain, ESource::SubDomain}, &FillSchema}, + {"auth_effective_permissions", ESysViewType::EAuthEffectivePermissions, {ESource::Domain, ESource::SubDomain}, &FillSchema}, +}; + +const TVector RewrittenSysViews = { + {"show_create", ESysViewType::EShowCreate, {}, &FillSchema}, +}; + +const THashMap SourceObjectTypeNames = { + {ESource::Domain, "domain"}, + {ESource::SubDomain, "subdomain"}, + {ESource::OlapStore, "olap store"}, + {ESource::ColumnTable, "column table"} +}; + +struct TSysViewDescription { + ESysViewType Type; + ISystemViewResolver::TSchema Schema; +}; + +THashMap> GroupSysViewsBySource(const TVector& sysViews, + TStringBuf registryName) +{ + THashMap> groupedSysViews; + for (const auto& registryRecord : sysViews) { + ISystemViewResolver::TSchema schema; + registryRecord.FillSchemaFunc(schema); + + for (auto sourceObjectType : registryRecord.SourceObjectTypes) { + auto& sourceTypeSysViews = groupedSysViews[sourceObjectType]; + UNIT_ASSERT_C(!sourceTypeSysViews.contains(registryRecord.Name), + TStringBuilder() << "Dupliсate sysview name '" << registryRecord.Name + << "' among " << SourceObjectTypeNames.at(sourceObjectType) << " sysviews" + << " in " << registryName << " registry"); + + sourceTypeSysViews[registryRecord.Name] = {registryRecord.Type, schema}; + } + } + + return groupedSysViews; +} + +THashMap MakeRewrittenSysViewsMap(const TVector& rewrittenSysViews, + TStringBuf registryName) +{ + THashMap sysViewsMap; + for (const auto& registryRecord : rewrittenSysViews) { + ISystemViewResolver::TSchema schema; + registryRecord.FillSchemaFunc(schema); + + UNIT_ASSERT_C(!sysViewsMap.contains(registryRecord.Name), + TStringBuilder() << "Dupliсate sysview name '" << registryRecord.Name << "'" + << " among rewritten sysviews in " << registryName << " registry"); + + sysViewsMap[registryRecord.Name] = {registryRecord.Type, schema}; + } + + return sysViewsMap; +} + +TSet FindColumnsSymmetricDifference(const THashMap& lhs, + const THashMap& rhs) +{ + TSet difference; + for (const auto& [id, column] : lhs) { + if (!rhs.contains(id)) { + difference.emplace(id); + } + } + + return difference; +} + +void CheckSysViewDescription(const TSysViewDescription& canonicalSysViewDescription, + const TSysViewDescription& actualSysViewDescription, TStringBuf errorSuffix) +{ + UNIT_ASSERT_C(canonicalSysViewDescription.Type == actualSysViewDescription.Type, + TStringBuilder() << "Type (implementation id) changed in " << errorSuffix); + + const auto& canonicalSchema = canonicalSysViewDescription.Schema; + const auto& actualSchema = actualSysViewDescription.Schema; + const auto& canonicalColumns = canonicalSchema.Columns; + const auto& actualColumns = actualSchema.Columns; + + const auto deletedColumns = FindColumnsSymmetricDifference(canonicalColumns, actualColumns); + UNIT_ASSERT_C(deletedColumns.empty(), + TStringBuilder() << "Column '" << canonicalColumns.at(*deletedColumns.begin()).Name << "'" + << " was deleted or its column id was changed in " << errorSuffix); + + const auto addedColumns = FindColumnsSymmetricDifference(actualColumns, canonicalColumns); + UNIT_ASSERT_C(addedColumns.empty(), + TStringBuilder() << "Column '" << actualColumns.at(*addedColumns.begin()).Name << "'" + << " was added or its column id was changed in " << errorSuffix); + + for (const auto& [canonicalId, canonicalColumn] : canonicalColumns) { + const auto actualColumnsIter = actualColumns.find(canonicalId); + if (actualColumnsIter != actualColumns.end()) { + const auto& actualColumn = actualColumnsIter->second; + UNIT_ASSERT_C(canonicalColumn.Name == actualColumn.Name, + TStringBuilder() << "Column '" << canonicalColumn.Name << "' was renamed in " << errorSuffix); + + const auto canonicalTypeName = NScheme::TypeName(canonicalColumn.PType, canonicalColumn.PTypeMod); + const auto actualTypeName = NScheme::TypeName(actualColumn.PType, actualColumn.PTypeMod); + UNIT_ASSERT_C(canonicalTypeName == actualTypeName, + TStringBuilder() << "Column '" << canonicalColumn.Name << "' changed type in " << errorSuffix); + + UNIT_ASSERT_C(canonicalColumn.KeyOrder == actualColumn.KeyOrder, + TStringBuilder() << "Primary key differs in " << errorSuffix); + + UNIT_ASSERT_C(canonicalColumn.IsNotNullColumn == actualColumn.IsNotNullColumn, + TStringBuilder() << "IsNotNull attribute of '" << canonicalColumn.Name << "' column" + << " changed in " << errorSuffix); + } + } + + + UNIT_ASSERT_C(canonicalSchema.KeyColumnTypes.size() == actualSchema.KeyColumnTypes.size(), + TStringBuilder() << "Primary key differs in " << errorSuffix); + + for (size_t i = 0; i < canonicalSchema.KeyColumnTypes.size(); ++i) { + UNIT_ASSERT_C(canonicalSchema.KeyColumnTypes[i] == actualSchema.KeyColumnTypes[i], + TStringBuilder() << "Primary key differs in " << errorSuffix); + } +} + +void CheckSysViewsLists(const THashMap& canonicalSysViewsList, + THashMap& actualSysViewsList, TStringBuf sysViewListName) +{ + for (const auto& [sysViewName, sysViewDescription] : canonicalSysViewsList) { + const auto actualSysViewsIter = actualSysViewsList.find(sysViewName); + UNIT_ASSERT_C(actualSysViewsIter != actualSysViewsList.end(), + TStringBuilder() << "Sysview '" << sysViewName << "'" + << " was deleted from" << sysViewListName << " sysviews"); + + CheckSysViewDescription(sysViewDescription, actualSysViewsIter->second, + TStringBuilder() << "sysview '" << sysViewName << "' from " << sysViewListName << " sysviews"); + + actualSysViewsList.erase(actualSysViewsIter); + } + + UNIT_ASSERT_C(actualSysViewsList.empty(), + TStringBuilder() << "Sysview '" << actualSysViewsList.begin()->first << "'" + << " was added to " << sysViewListName << " sysviews"); +} + +} + +Y_UNIT_TEST_SUITE(SysViewsRegistry) { + Y_UNIT_TEST(SysViews) { + auto canonicalSysViews = GroupSysViewsBySource(SysViews, "canonical"); + auto actualSysViews = GroupSysViewsBySource(SysViewsRegistry::SysViews, "actual"); + + for (const auto& [sourceObjectType, sourceObjectName] : SourceObjectTypeNames) { + auto& canonicalSysViewsList = canonicalSysViews[sourceObjectType]; + auto& actualSysViewsList = actualSysViews[sourceObjectType]; + + CheckSysViewsLists(canonicalSysViewsList, actualSysViewsList, sourceObjectName); + } + } + + Y_UNIT_TEST(RewrittenSysViews) { + auto canonicalSysViews = MakeRewrittenSysViewsMap(RewrittenSysViews, "canonical"); + auto actualSysViews = MakeRewrittenSysViewsMap(SysViewsRegistry::RewrittenSysViews, "actual"); + + CheckSysViewsLists(canonicalSysViews, actualSysViews, "rewritten"); + } +} + +} // NSysView +} // NKikimr diff --git a/ydb/core/tx/columnshard/common/portion.cpp b/ydb/core/tx/columnshard/common/portion.cpp index bc4c2ea7a075..3fa855d3aa70 100644 --- a/ydb/core/tx/columnshard/common/portion.cpp +++ b/ydb/core/tx/columnshard/common/portion.cpp @@ -1,6 +1,5 @@ #include "portion.h" #include -#include namespace NKikimr::NOlap::NPortion { diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/chunks/schema.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/chunks/schema.cpp index 18a5b5841635..155c75dd3ef9 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/chunks/schema.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/chunks/schema.cpp @@ -1,6 +1,8 @@ #include "metadata.h" #include "schema.h" +#include + namespace NKikimr::NOlap::NReader::NSimple::NSysView::NChunks { std::shared_ptr TSchemaAdapter::BuildMetadataAccessor( diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/chunks/source.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/chunks/source.cpp index 601a0bb32866..b4778d5ac27f 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/chunks/source.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/chunks/source.cpp @@ -1,5 +1,6 @@ #include "source.h" +#include #include #include #include diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/granules/schema.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/granules/schema.cpp index c1bf0e137212..564ee6ae5b6a 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/granules/schema.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/granules/schema.cpp @@ -1,6 +1,8 @@ #include "metadata.h" #include "schema.h" +#include + namespace NKikimr::NOlap::NReader::NSimple::NSysView::NGranules { NArrow::TSimpleRow TSchemaAdapter::GetPKSimpleRow(const NColumnShard::TSchemeShardLocalPathId& pathId, const ui64 tabletId) { diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/granules/source.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/granules/source.cpp index 72a6a0a5fcb4..05a2f0a5cff7 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/granules/source.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/granules/source.cpp @@ -1,6 +1,7 @@ #include "source.h" #include +#include #include #include diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/optimizer/schema.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/optimizer/schema.cpp index 67b62c179390..a8b69947a1fc 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/optimizer/schema.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/optimizer/schema.cpp @@ -1,6 +1,8 @@ #include "metadata.h" #include "schema.h" +#include + namespace NKikimr::NOlap::NReader::NSimple::NSysView::NOptimizer { NArrow::TSimpleRow TSchemaAdapter::GetPKSimpleRow(const NColumnShard::TSchemeShardLocalPathId& pathId, const ui64 tabletId, const ui64 taskId) { diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/optimizer/source.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/optimizer/source.cpp index d16c737ef264..807fd0b63e7c 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/optimizer/source.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/optimizer/source.cpp @@ -1,6 +1,7 @@ #include "source.h" #include +#include #include #include diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/portions/schema.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/portions/schema.cpp index b6e4d4f94bb2..f5ffd32714c5 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/portions/schema.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/portions/schema.cpp @@ -1,6 +1,8 @@ #include "metadata.h" #include "schema.h" +#include + namespace NKikimr::NOlap::NReader::NSimple::NSysView::NPortions { NArrow::TSimpleRow TSchemaAdapter::GetPKSimpleRow(const NColumnShard::TUnifiedPathId pathId, const ui64 tabletId, const ui64 portionId) { diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/portions/source.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/portions/source.cpp index 6d7b6b0c9c08..fdafa8009f5d 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/portions/source.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/portions/source.cpp @@ -1,6 +1,7 @@ #include "source.h" #include +#include #include #include diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/schema.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/schema.cpp index 5805a073eb01..b7c1f0686412 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/schema.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/schema.cpp @@ -1,6 +1,8 @@ #include "metadata.h" #include "schema.h" +#include + namespace NKikimr::NOlap::NReader::NSimple::NSysView::NSchemas { NArrow::TSimpleRow TSchemaAdapter::GetPKSimpleRow(const ui64 tabletId, const ui64 presetId, const ui64 schemaVersion) { diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/source.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/source.cpp index bc32891f55ec..1fe200d70e50 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/source.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sys_view/schemas/source.cpp @@ -1,6 +1,7 @@ #include "source.h" #include +#include #include #include diff --git a/ydb/core/tx/columnshard/engines/reader/transaction/tx_internal_scan.cpp b/ydb/core/tx/columnshard/engines/reader/transaction/tx_internal_scan.cpp index 659a9c0962b9..d47e35dcf428 100644 --- a/ydb/core/tx/columnshard/engines/reader/transaction/tx_internal_scan.cpp +++ b/ydb/core/tx/columnshard/engines/reader/transaction/tx_internal_scan.cpp @@ -1,7 +1,6 @@ #include "tx_internal_scan.h" #include -#include #include #include #include diff --git a/ydb/core/tx/columnshard/engines/reader/transaction/tx_scan.cpp b/ydb/core/tx/columnshard/engines/reader/transaction/tx_scan.cpp index fbd8af927097..15fd3685f4d7 100644 --- a/ydb/core/tx/columnshard/engines/reader/transaction/tx_scan.cpp +++ b/ydb/core/tx/columnshard/engines/reader/transaction/tx_scan.cpp @@ -1,7 +1,6 @@ #include "tx_scan.h" #include -#include #include #include #include diff --git a/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.cpp b/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.cpp index c51d93e98c24..1ae49925224b 100644 --- a/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/abstract/index_info.cpp @@ -1,6 +1,5 @@ #include "index_info.h" #include -#include #include #include @@ -17,7 +16,7 @@ const std::shared_ptr& IIndexInfo::GetColumnLoaderVerified(const void IIndexInfo::AddDeleteFlagsColumn(NArrow::TGeneralContainer& batch, const bool isDelete) { const i64 numRows = batch.num_rows(); - batch.AddField(arrow::field(SPEC_COL_DELETE_FLAG, arrow::boolean()), + batch.AddField(arrow::field(SPEC_COL_DELETE_FLAG, arrow::boolean()), NArrow::TThreadSimpleArraysCache::GetConst(arrow::boolean(), std::make_shared(isDelete), numRows)).Validate(); } diff --git a/ydb/core/tx/columnshard/engines/scheme/index_info.cpp b/ydb/core/tx/columnshard/engines/scheme/index_info.cpp index 580c23804f9b..9ee9cb10bb75 100644 --- a/ydb/core/tx/columnshard/engines/scheme/index_info.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/index_info.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include diff --git a/ydb/core/tx/columnshard/engines/scheme/index_info.h b/ydb/core/tx/columnshard/engines/scheme/index_info.h index fdfcdc8aa7ed..e287cb236dd3 100644 --- a/ydb/core/tx/columnshard/engines/scheme/index_info.h +++ b/ydb/core/tx/columnshard/engines/scheme/index_info.h @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include diff --git a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp index 5e8c3a6078d9..bd9957f898f4 100644 --- a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp +++ b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include #include #include diff --git a/ydb/core/tx/scheme_board/cache.cpp b/ydb/core/tx/scheme_board/cache.cpp index af1f1e0a7182..5c739c6f8356 100644 --- a/ydb/core/tx/scheme_board/cache.cpp +++ b/ydb/core/tx/scheme_board/cache.cpp @@ -17,7 +17,8 @@ #include #include #include -#include +#include +#include #include #include #include @@ -1779,7 +1780,7 @@ class TSchemeCache: public TMonitorableActor { } void FillSystemViewEntry(TNavigateContext* context, TNavigate::TEntry& entry, - NSysView::ISystemViewResolver::ETarget target) const + NSysView::ISystemViewResolver::ESource target) const { auto sysViewInfo = entry.TableId.SysViewInfo; @@ -1810,8 +1811,8 @@ class TSchemeCache: public TMonitorableActor { } entry.Kind = TNavigate::KindTable; - if (target == NSysView::ISystemViewResolver::ETarget::OlapStore || - target == NSysView::ISystemViewResolver::ETarget::ColumnTable) + if (target == NSysView::ISystemViewResolver::ESource::OlapStore || + target == NSysView::ISystemViewResolver::ESource::ColumnTable) { // OLAP sys views are represented by OLAP tables entry.Kind =TNavigate::KindColumnTable; @@ -1857,16 +1858,16 @@ class TSchemeCache: public TMonitorableActor { if (Kind == TNavigate::KindPath) { auto split = SplitPath(Path); if (split.size() == 1) { - return FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::Domain); + return FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ESource::Domain); } } else if (Kind == TNavigate::KindSubdomain || Kind == TNavigate::KindExtSubdomain) { - return FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::SubDomain); + return FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ESource::SubDomain); } else if (Kind == TNavigate::KindOlapStore) { - FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::OlapStore); + FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ESource::OlapStore); entry.OlapStoreInfo = OlapStoreInfo; return; } else if (Kind == TNavigate::KindColumnTable) { - FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::ColumnTable); + FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ESource::ColumnTable); entry.OlapStoreInfo = OlapStoreInfo; entry.ColumnTableInfo = ColumnTableInfo; return; @@ -2028,7 +2029,7 @@ class TSchemeCache: public TMonitorableActor { } void FillSystemViewEntry(TResolveContext* context, TResolve::TEntry& entry, - NSysView::ISystemViewResolver::ETarget target) const + NSysView::ISystemViewResolver::ESource target) const { TKeyDesc& keyDesc = *entry.KeyDescription; auto sysViewInfo = keyDesc.TableId.SysViewInfo; @@ -2079,12 +2080,12 @@ class TSchemeCache: public TMonitorableActor { if (Kind == TNavigate::KindPath) { auto split = SplitPath(Path); if (split.size() == 1) { - return FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::Domain); + return FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ESource::Domain); } } else if (Kind == TNavigate::KindSubdomain || Kind == TNavigate::KindExtSubdomain) { - return FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::SubDomain); + return FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ESource::SubDomain); } else if (Kind == TNavigate::KindOlapStore) { - FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::OlapStore); + FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ESource::OlapStore); // Add all shards of the OLAP store auto partitions = std::make_shared>(); for (ui64 columnShard : OlapStoreInfo->Description.GetColumnShards()) { @@ -2094,7 +2095,7 @@ class TSchemeCache: public TMonitorableActor { keyDesc.Partitioning = std::move(partitions); return; } else if (Kind == TNavigate::KindColumnTable) { - FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ETarget::ColumnTable); + FillSystemViewEntry(context, entry, NSysView::ISystemViewResolver::ESource::ColumnTable); // Add all shards of the OLAP table auto shardingInfo = NSharding::IShardingBase::BuildFromProto(ColumnTableInfo->Description.GetSharding()); if (shardingInfo.IsFail()) { diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index 97399d8674a4..236ae61f7926 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include #include #include @@ -138,8 +138,8 @@ void TSchemeShard::CollectSysViewUpdates(const TActorContext& ctx) { } const auto sysViewDirType = IsDomainSchemeShard - ? NSysView::ISystemViewResolver::ETarget::Domain - : NSysView::ISystemViewResolver::ETarget::SubDomain; + ? NSysView::ISystemViewResolver::ESource::Domain + : NSysView::ISystemViewResolver::ESource::SubDomain; const auto sysViewsRegistry = NSysView::CreateSystemViewResolver()->GetSystemViewsTypes(sysViewDirType); // create absent system views only if there's no '.sys' entry or '.sys' is a directory diff --git a/ydb/core/tx/tx_proxy/describe.cpp b/ydb/core/tx/tx_proxy/describe.cpp index 03399096e98b..429b0ebea12e 100644 --- a/ydb/core/tx/tx_proxy/describe.cpp +++ b/ydb/core/tx/tx_proxy/describe.cpp @@ -7,7 +7,8 @@ #include #include #include -#include +#include +#include #include #include diff --git a/ydb/tests/compatibility/test_system_views.py b/ydb/tests/compatibility/test_system_views.py new file mode 100644 index 000000000000..cf41c79148e3 --- /dev/null +++ b/ydb/tests/compatibility/test_system_views.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- +import logging +import pytest +from ydb.tests.library.compatibility.fixtures import RestartToAnotherVersionFixture, RollingUpgradeAndDowngradeFixture +from ydb.tests.oss.ydb_sdk_import import ydb + +logger = logging.getLogger(__name__) + +pg_sysviews = {'pg_tables', 'tables', 'pg_class'} + + +class TestSystemViewsRegistry(RestartToAnotherVersionFixture): + @pytest.fixture(autouse=True, scope="function") + def setup(self): + if self.versions[0] >= self.versions[1]: + pytest.skip("Only check forward compatibility") + + yield from self.setup_cluster() + + def collect_sysviews(self): + sysviews = dict() + with ydb.SessionPool(self.driver, size=1) as pool: + with pool.checkout() as session: + for sysview in self.driver.scheme_client.list_directory("/Root/.sys").children: + if sysview.name not in pg_sysviews: + sysview_descr = dict() + + response = session.describe_table(f"/Root/.sys/{sysview.name}") + columns = dict() + for col in response.columns: + columns[col.name] = str(col.type) + + sysview_descr['columns'] = columns + sysview_descr['primary_key'] = [pk_col for pk_col in response.primary_key] + + sysviews[sysview.name] = sysview_descr + + return sysviews + + def compare_sysviews_dicts(self, dict_before, dict_after): + for sysview_name, sysview_descr in dict_before.items(): + if sysview_name not in dict_after: + logger.debug(f"sysview '{sysview_name}' was deleted") + return False + + columns_before = sysview_descr['columns'] + columns_after = dict_after[sysview_name]['columns'] + + for col_name, col_type in columns_before.items(): + if col_name not in columns_after: + logger.debug(f"column '{col_name}' was deleted from sysview '{sysview_name}'") + return False + + if col_type != columns_after[col_name]: + logger.debug(f"column '{col_name}' changed type in sysview '{sysview_name}'") + return False + + primary_key_before = sysview_descr['primary_key'] + primary_key_after = dict_after[sysview_name]['primary_key'] + + for i, primary_key_col in enumerate(primary_key_before): + if primary_key_col != primary_key_after[i]: + logger.debug(f"column '{primary_key_col}' was deleted from sysview '{sysview_name}' primary key") + return False + + return True + + def test_domain_sys_dir(self): + sysview_folder_content_before = self.collect_sysviews() + self.change_cluster_version() + sysview_folder_content_after = self.collect_sysviews() + + assert self.compare_sysviews_dicts(sysview_folder_content_before, sysview_folder_content_after) + + +class TestSystemViewsRollingUpgrade(RollingUpgradeAndDowngradeFixture): + @pytest.fixture(autouse=True, scope="function") + def setup(self): + yield from self.setup_cluster() + + def create_table(self, table_name): + with ydb.QuerySessionPool(self.driver) as session_pool: + query = f""" + CREATE TABLE `{table_name}` ( + key Int32 NOT NULL, + value String, + PRIMARY KEY (key) + ); + """ + session_pool.execute_with_retries(query) + + query = f"UPSERT INTO `{table_name}` (key, value) VALUES (1, 'test value');" + session_pool.execute_with_retries(query) + + def read_partition_stats(self, table_name): + with ydb.QuerySessionPool(self.driver) as session_pool: + query = f""" + SELECT PathId, PartIdx, Path + FROM `/Root/.sys/partition_stats` + WHERE Path = '/Root/{table_name}';""" + result_sets = session_pool.execute_with_retries(query) + + assert len(result_sets) == 1 + assert len(result_sets[0].rows) == 1 + assert result_sets[0].rows[0]["Path"] == f'/Root/{table_name}' + assert result_sets[0].rows[0]["PartIdx"] == 0 + assert result_sets[0].rows[0]["PathId"] > 1 + + def test_path_resolving(self): + table_name = "table" + self.create_table(table_name) + + for _ in self.roll(): + self.read_partition_stats(table_name) diff --git a/ydb/tests/compatibility/ya.make b/ydb/tests/compatibility/ya.make index ed2cd6e39e65..12c626a6070a 100644 --- a/ydb/tests/compatibility/ya.make +++ b/ydb/tests/compatibility/ya.make @@ -13,6 +13,7 @@ TEST_SRCS( test_compatibility.py test_stress.py test_statistics.py + test_system_views.py test_rolling.py test_data_type.py test_ctas.py diff --git a/ydb/tests/library/compatibility/fixtures.py b/ydb/tests/library/compatibility/fixtures.py index b76b409aae14..1ff93e4bce99 100644 --- a/ydb/tests/library/compatibility/fixtures.py +++ b/ydb/tests/library/compatibility/fixtures.py @@ -55,13 +55,13 @@ def string_version_to_tuple(s): } all_binary_combinations_restart = [ - [[inter_stable_binary_path], [current_binary_path]], - [[current_binary_path], [inter_stable_binary_path]], - [[current_binary_path], [current_binary_path]], + [inter_stable_binary_path, current_binary_path], + [current_binary_path, inter_stable_binary_path], + [current_binary_path, current_binary_path], - [[init_stable_binary_path], [inter_stable_binary_path]], - [[inter_stable_binary_path], [init_stable_binary_path]], - [[inter_stable_binary_path], [inter_stable_binary_path]], + [init_stable_binary_path, inter_stable_binary_path], + [inter_stable_binary_path, init_stable_binary_path], + [inter_stable_binary_path, inter_stable_binary_path], ] all_binary_combinations_ids_restart = [ "restart_{}_to_{}".format(inter_stable_name, current_name), @@ -79,7 +79,7 @@ class RestartToAnotherVersionFixture: def base_setup(self, request): self.current_binary_paths_index = 0 self.all_binary_paths = request.param - self.versions = list([path_to_version[path] for path_list in self.all_binary_paths for path in path_list]) + self.versions = [path_to_version[path] for path in self.all_binary_paths] def setup_cluster(self, **kwargs): extra_feature_flags = kwargs.pop("extra_feature_flags", {}) @@ -87,7 +87,7 @@ def setup_cluster(self, **kwargs): extra_feature_flags["suppress_compatibility_check"] = True self.config = KikimrConfigGenerator( erasure=Erasure.MIRROR_3_DC, - binary_paths=self.all_binary_paths[self.current_binary_paths_index], + binary_paths=[self.all_binary_paths[self.current_binary_paths_index]], use_in_memory_pdisks=False, extra_feature_flags=extra_feature_flags, **kwargs, @@ -110,7 +110,7 @@ def setup_cluster(self, **kwargs): def change_cluster_version(self): self.current_binary_paths_index = (self.current_binary_paths_index + 1) % len(self.all_binary_paths) new_binary_paths = self.all_binary_paths[self.current_binary_paths_index] - self.config.set_binary_paths(new_binary_paths) + self.config.set_binary_paths([new_binary_paths]) self.cluster.update_configurator_and_restart(self.config) self.driver = ydb.Driver( ydb.DriverConfig( From 5ff926d9e510eabab1d90d91229e585486c11acd Mon Sep 17 00:00:00 2001 From: Semyon Date: Fri, 8 Aug 2025 12:26:49 +0300 Subject: [PATCH 29/52] deduplication abortion handling (#22535) --- .../engines/reader/simple_reader/duplicates/context.h | 3 ++- .../reader/simple_reader/duplicates/manager.cpp | 4 ++-- .../reader/simple_reader/iterator/fetching.cpp | 3 --- .../engines/reader/simple_reader/iterator/fetching.h | 11 ----------- 4 files changed, 4 insertions(+), 17 deletions(-) diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/context.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/context.h index b2a27aa003bf..db03c8cb702d 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/context.h +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/context.h @@ -80,7 +80,8 @@ class TInternalFilterConstructor: TMoveOnly { TInternalFilterConstructor(const TEvRequestFilter::TPtr& request, TColumnDataSplitter&& splitter); ~TInternalFilterConstructor() { - AFL_VERIFY(IsDone())("state", DebugString()); + AFL_VERIFY(IsDone() || (OriginalRequest->Get()->GetAbortionFlag() && OriginalRequest->Get()->GetAbortionFlag()->Val()))( + "state", DebugString()); } TString DebugString() const { diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.cpp index 7a161831c342..602e921ecd56 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.cpp @@ -35,7 +35,7 @@ class TColumnFetchingCallback: public ::NKikimr::NGeneralCache::NPublic::ICallba } virtual bool DoIsAborted() const override { - return false; + return Context->GetRequest()->Get()->GetAbortionFlag() && Context->GetRequest()->Get()->GetAbortionFlag()->Val(); } public: @@ -117,7 +117,7 @@ class TColumnDataAccessorFetching: public IDataAccessorRequestsSubscriber { { std::make_shared(Callback, Portions, Columns, ColumnDataManager, mem) }, std::nullopt); } virtual const std::shared_ptr& DoGetAbortionFlag() const override { - return Default>(); + return Callback->GetContext()->GetRequest()->Get()->GetAbortionFlag(); } public: diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/fetching.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/fetching.cpp index 6a4255308063..ee6a7faacc40 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/fetching.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/fetching.cpp @@ -243,7 +243,6 @@ void TDuplicateFilter::TFilterSubscriber::OnFilterReady(NArrow::TColumnFilter&& AFL_TRACE(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "fetch_filter")("source", source->GetSourceId())( "filter", filter.DebugString())("aborted", source->GetContext()->IsAborted()); if (source->GetContext()->IsAborted()) { - OnDone(); return; } if (const std::shared_ptr appliedFilter = source->GetStageData().GetAppliedFilter()) { @@ -256,14 +255,12 @@ void TDuplicateFilter::TFilterSubscriber::OnFilterReady(NArrow::TColumnFilter&& auto task = std::make_shared(std::move(source), std::move(Step), scanActorId, false); NConveyorComposite::TScanServiceOperator::SendTaskToExecute(task, convActorId); } - OnDone(); } void TDuplicateFilter::TFilterSubscriber::OnFailure(const TString& reason) { if (auto source = Source.lock()) { source->GetContext()->GetCommonContext()->AbortWithError("cannot build duplicate filter: " + reason); } - OnDone(); } TDuplicateFilter::TFilterSubscriber::TFilterSubscriber(const std::shared_ptr& source, const TFetchingScriptCursor& step) diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/fetching.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/fetching.h index ae543e5b7774..2aece6c76509 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/fetching.h +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/fetching.h @@ -283,23 +283,12 @@ class TDuplicateFilter: public IFetchingStep { std::weak_ptr Source; TFetchingScriptCursor Step; NColumnShard::TCounterGuard TaskGuard; - bool IsDone = false; virtual void OnFilterReady(NArrow::TColumnFilter&& filter) override; virtual void OnFailure(const TString& reason) override; - private: - void OnDone() { - AFL_VERIFY(!IsDone); - IsDone = true; - } - public: TFilterSubscriber(const std::shared_ptr& source, const TFetchingScriptCursor& step); - - ~TFilterSubscriber() { - AFL_VERIFY(IsDone); - } }; public: From 1130071a5a7cb0884ae60e65c62930c5cf8fe57c Mon Sep 17 00:00:00 2001 From: Evgeniy Ivanov Date: Fri, 8 Aug 2025 11:48:17 +0200 Subject: [PATCH 30/52] Add more try-catch blocks to avoid unhandled exception (#17333) (#22470) --- ydb/library/workload/tpcc/import.cpp | 15 ++++++++++----- ydb/library/workload/tpcc/import_tui.cpp | 21 ++++++++++++++------- ydb/library/workload/tpcc/import_tui.h | 4 +++- ydb/library/workload/tpcc/runner.cpp | 11 ++++++++--- ydb/library/workload/tpcc/runner_tui.cpp | 21 ++++++++++++++------- ydb/library/workload/tpcc/runner_tui.h | 3 ++- ydb/library/workload/tpcc/terminal.cpp | 9 ++++++++- 7 files changed, 59 insertions(+), 25 deletions(-) diff --git a/ydb/library/workload/tpcc/import.cpp b/ydb/library/workload/tpcc/import.cpp index 6addc37197d7..acb5957d9329 100644 --- a/ydb/library/workload/tpcc/import.cpp +++ b/ydb/library/workload/tpcc/import.cpp @@ -826,7 +826,7 @@ class TPCCLoader { : ConnectionConfig(connectionConfig) , Config(runConfig) , LogBackend(new TLogBackendWithCapture("cerr", runConfig.LogPriority, TUI_LOG_LINES)) - , Log(std::make_unique(THolder(static_cast(LogBackend)))) + , Log(std::make_shared(THolder(static_cast(LogBackend)))) , PreviousDataSizeLoaded(0) , StartTime(Clock::now()) , LoadState(GetGlobalInterruptSource().get_token()) @@ -859,7 +859,7 @@ class TPCCLoader { if (Config.DisplayMode == TRunConfig::EDisplayMode::Tui) { LogBackend->StartCapture(); TImportDisplayData dataToDisplay(LoadState); - Tui = std::make_unique(Config, *LogBackend, dataToDisplay); + Tui = std::make_unique(Log, Config, *LogBackend, dataToDisplay); } // TODO: calculate optimal number of drivers (but per thread looks good) @@ -1142,7 +1142,7 @@ class TPCCLoader { // XXX Log instance owns LogBackend (unfortunately, it accepts THolder with LogBackend) TLogBackendWithCapture* LogBackend; - std::unique_ptr Log; + std::shared_ptr Log; Clock::time_point LastDisplayUpdate; size_t PreviousDataSizeLoaded; @@ -1157,8 +1157,13 @@ class TPCCLoader { //----------------------------------------------------------------------------- void ImportSync(const NConsoleClient::TClientCommand::TConfig& connectionConfig, const TRunConfig& runConfig) { - TPCCLoader loader(connectionConfig, runConfig); - loader.ImportSync(); + try { + TPCCLoader loader(connectionConfig, runConfig); + loader.ImportSync(); + } catch (const std::exception& ex) { + std::cerr << "Exception while execution: " << ex.what() << std::endl; + throw NConsoleClient::TNeedToExitWithCode(EXIT_FAILURE); + } } } // namespace NYdb::NTPCC diff --git a/ydb/library/workload/tpcc/import_tui.cpp b/ydb/library/workload/tpcc/import_tui.cpp index 9cbeb6483057..b647f1a9a7c4 100644 --- a/ydb/library/workload/tpcc/import_tui.cpp +++ b/ydb/library/workload/tpcc/import_tui.cpp @@ -15,8 +15,9 @@ using namespace ftxui; namespace NYdb::NTPCC { -TImportTui::TImportTui(const TRunConfig& runConfig, TLogBackendWithCapture& logBacked, const TImportDisplayData& data) - : Config(runConfig) +TImportTui::TImportTui(std::shared_ptr& log, const TRunConfig& runConfig, TLogBackendWithCapture& logBacked, const TImportDisplayData& data) + : Log(log) + , Config(runConfig) , LogBackend(logBacked) , DataToDisplay(data) , Screen(ScreenInteractive::Fullscreen()) @@ -137,11 +138,17 @@ Element TImportTui::BuildUpperPart() { } Component TImportTui::BuildComponent() { - // Main layout - return Container::Vertical({ - Renderer([=]{ return BuildUpperPart(); }), - LogsScroller(LogBackend), - }); + try { + // Main layout + return Container::Vertical({ + Renderer([=]{ return BuildUpperPart(); }), + LogsScroller(LogBackend), + }); + } catch (const std::exception& ex) { + LOG_E("Exception in TUI: " << ex.what()); + RequestStop(); + return Renderer([] { return filler(); }); + } } } // namespace NYdb::NTPCC diff --git a/ydb/library/workload/tpcc/import_tui.h b/ydb/library/workload/tpcc/import_tui.h index b12e3e000245..e0ff5c2dea2f 100644 --- a/ydb/library/workload/tpcc/import_tui.h +++ b/ydb/library/workload/tpcc/import_tui.h @@ -1,6 +1,7 @@ #pragma once #include "import_display_data.h" +#include "log.h" #include "runner.h" #include @@ -12,7 +13,7 @@ class TLogBackendWithCapture; class TImportTui { public: - TImportTui(const TRunConfig& runConfig, TLogBackendWithCapture& logBacked, const TImportDisplayData& data); + TImportTui(std::shared_ptr& log, const TRunConfig& runConfig, TLogBackendWithCapture& logBacked, const TImportDisplayData& data); ~TImportTui(); void Update(const TImportDisplayData& data); @@ -22,6 +23,7 @@ class TImportTui { ftxui::Component BuildComponent(); private: + std::shared_ptr Log; const TRunConfig Config; TLogBackendWithCapture& LogBackend; TImportDisplayData DataToDisplay; diff --git a/ydb/library/workload/tpcc/runner.cpp b/ydb/library/workload/tpcc/runner.cpp index e8007cadf03c..2bcd491bb296 100644 --- a/ydb/library/workload/tpcc/runner.cpp +++ b/ydb/library/workload/tpcc/runner.cpp @@ -312,7 +312,7 @@ void TPCCRunner::RunSync() { // produced after this point and before the first screen update if (Config.DisplayMode == TRunConfig::EDisplayMode::Tui) { LogBackend->StartCapture(); // start earlier? - Tui = std::make_unique(*LogBackend, DataToDisplay); + Tui = std::make_unique(Log, *LogBackend, DataToDisplay); } if (forcedWarmup) { @@ -754,8 +754,13 @@ void TRunConfig::SetDisplay() { //----------------------------------------------------------------------------- void RunSync(const NConsoleClient::TClientCommand::TConfig& connectionConfig, const TRunConfig& runConfig) { - TPCCRunner runner(connectionConfig, runConfig); - runner.RunSync(); + try { + TPCCRunner runner(connectionConfig, runConfig); + runner.RunSync(); + } catch (const std::exception& ex) { + std::cerr << "Exception while execution: " << ex.what() << std::endl; + throw NConsoleClient::TNeedToExitWithCode(EXIT_FAILURE); + } } } // namespace NYdb::NTPCC diff --git a/ydb/library/workload/tpcc/runner_tui.cpp b/ydb/library/workload/tpcc/runner_tui.cpp index f799a3d95941..4c714d9c4ca4 100644 --- a/ydb/library/workload/tpcc/runner_tui.cpp +++ b/ydb/library/workload/tpcc/runner_tui.cpp @@ -12,8 +12,9 @@ using namespace ftxui; namespace NYdb::NTPCC { -TRunnerTui::TRunnerTui(TLogBackendWithCapture& logBacked, std::shared_ptr data) - : LogBackend(logBacked) +TRunnerTui::TRunnerTui(std::shared_ptr& log, TLogBackendWithCapture& logBacked, std::shared_ptr data) + : Log(log) + , LogBackend(logBacked) , DataToDisplay(std::move(data)) , Screen(ScreenInteractive::Fullscreen()) { @@ -227,11 +228,17 @@ Element TRunnerTui::BuildUpperPart() { } Component TRunnerTui::BuildComponent() { - // Main layout - return Container::Vertical({ - Renderer([=]{ return BuildUpperPart(); }), - LogsScroller(LogBackend), - }); + try { + // Main layout + return Container::Vertical({ + Renderer([=]{ return BuildUpperPart(); }), + LogsScroller(LogBackend), + }); + } catch (const std::exception& ex) { + LOG_E("Exception in TUI: " << ex.what()); + RequestStop(); + return Renderer([] { return filler(); }); + } } } // namespace NYdb::NTPCC diff --git a/ydb/library/workload/tpcc/runner_tui.h b/ydb/library/workload/tpcc/runner_tui.h index 0349f7d856c8..997c4387d9ff 100644 --- a/ydb/library/workload/tpcc/runner_tui.h +++ b/ydb/library/workload/tpcc/runner_tui.h @@ -11,7 +11,7 @@ class TLogBackendWithCapture; class TRunnerTui { public: - TRunnerTui(TLogBackendWithCapture& logBacked, std::shared_ptr data); + TRunnerTui(std::shared_ptr& log, TLogBackendWithCapture& logBacked, std::shared_ptr data); ~TRunnerTui(); void Update(std::shared_ptr data); @@ -21,6 +21,7 @@ class TRunnerTui { ftxui::Component BuildComponent(); private: + std::shared_ptr Log; TLogBackendWithCapture& LogBackend; std::shared_ptr DataToDisplay; ftxui::ScreenInteractive Screen; diff --git a/ydb/library/workload/tpcc/terminal.cpp b/ydb/library/workload/tpcc/terminal.cpp index 6d64d2a3af53..c9a24fc84f65 100644 --- a/ydb/library/workload/tpcc/terminal.cpp +++ b/ydb/library/workload/tpcc/terminal.cpp @@ -190,9 +190,16 @@ TTerminalTask TTerminal::Run() { LOG_E(ss.Str()); RequestStop(); co_return; + } catch (const std::exception& ex) { + TStringStream ss; + ss << "Terminal " << Context.TerminalID << " got exception while " << transaction.Name << " execution: " + << ex.what(); + LOG_E(ss.Str()); + RequestStop(); + co_return; } - // only here if exception cought + // only here if exception caught TaskQueue.DecInflight(); } From a798aa2b8550497f2d69767e3c26e711802563e3 Mon Sep 17 00:00:00 2001 From: Semyon Date: Fri, 8 Aug 2025 13:30:35 +0300 Subject: [PATCH 31/52] deduplication memory fixes (#22532) --- .../run/kikimr_services_initializers.cpp | 23 +++++++++++++++++++ .../run/kikimr_services_initializers.h | 6 +++++ ydb/core/driver_lib/run/run.cpp | 1 + ydb/core/protos/config.proto | 1 + ydb/core/protos/console_config.proto | 1 + .../simple_reader/duplicates/context.cpp | 9 +------- .../reader/simple_reader/duplicates/context.h | 4 ++-- .../simple_reader/duplicates/manager.cpp | 2 +- ydb/tests/library/harness/kikimr_config.py | 3 +++ 9 files changed, 39 insertions(+), 11 deletions(-) diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index df4e34fecaae..5f44dd6a1e6c 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -2260,6 +2260,29 @@ void TCompGroupedMemoryLimiterInitializer::InitializeServices(NActors::TActorSys } } +TDeduplicationGroupedMemoryLimiterInitializer::TDeduplicationGroupedMemoryLimiterInitializer(const TKikimrRunConfig& runConfig) + : IKikimrServicesInitializer(runConfig) +{ +} + +void TDeduplicationGroupedMemoryLimiterInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { + NOlap::NGroupedMemoryManager::TConfig serviceConfig; + if (Config.GetDeduplicationGroupedMemoryLimiterConfig().GetCountBuckets() == 0) { + Config.MutableDeduplicationGroupedMemoryLimiterConfig()->SetCountBuckets(1); + } + Y_ABORT_UNLESS(serviceConfig.DeserializeFromProto(Config.GetDeduplicationGroupedMemoryLimiterConfig())); + + if (serviceConfig.IsEnabled()) { + TIntrusivePtr<::NMonitoring::TDynamicCounters> tabletGroup = GetServiceCounters(appData->Counters, "tablets"); + TIntrusivePtr<::NMonitoring::TDynamicCounters> countersGroup = tabletGroup->GetSubgroup("type", "TX_DEDU_GROUPED_MEMORY_LIMITER"); + + auto service = NOlap::NGroupedMemoryManager::TDeduplicationMemoryLimiterOperator::CreateService(serviceConfig, countersGroup); + + setup->LocalServices.push_back(std::make_pair(NOlap::NGroupedMemoryManager::TDeduplicationMemoryLimiterOperator::MakeServiceId(NodeId), + TActorSetupCmd(service, TMailboxType::HTSwap, appData->UserPoolId))); + } +} + TCompDiskLimiterInitializer::TCompDiskLimiterInitializer(const TKikimrRunConfig& runConfig) : IKikimrServicesInitializer(runConfig) { } diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h index ba70439e7701..8142b70424e5 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.h +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h @@ -415,6 +415,12 @@ class TCompGroupedMemoryLimiterInitializer: public IKikimrServicesInitializer { void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; +class TDeduplicationGroupedMemoryLimiterInitializer: public IKikimrServicesInitializer { +public: + TDeduplicationGroupedMemoryLimiterInitializer(const TKikimrRunConfig& runConfig); + void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; +}; + class TCompPrioritiesInitializer: public IKikimrServicesInitializer { public: TCompPrioritiesInitializer(const TKikimrRunConfig& runConfig); diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index 6a09de02277b..5625fcbca188 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -1741,6 +1741,7 @@ TIntrusivePtr TKikimrRunner::CreateServiceInitializers if (serviceMask.EnableGroupedMemoryLimiter) { sil->AddServiceInitializer(new TScanGroupedMemoryLimiterInitializer(runConfig)); sil->AddServiceInitializer(new TCompGroupedMemoryLimiterInitializer(runConfig)); + sil->AddServiceInitializer(new TDeduplicationGroupedMemoryLimiterInitializer(runConfig)); } if (serviceMask.EnableCompPriorities) { diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index dca22365598a..8fb4e2925e0f 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -2523,6 +2523,7 @@ message TAppConfig { optional TCompositeConveyorConfig CompositeConveyorConfig = 109; optional TGeneralCacheConfig PortionsMetadataCache = 110; optional TGeneralCacheConfig ColumnDataCache = 111; + optional TGroupedMemoryLimiterConfig DeduplicationGroupedMemoryLimiterConfig = 112; } message TYdbVersion { diff --git a/ydb/core/protos/console_config.proto b/ydb/core/protos/console_config.proto index 24f7a486e9b8..af548d552e8f 100644 --- a/ydb/core/protos/console_config.proto +++ b/ydb/core/protos/console_config.proto @@ -154,6 +154,7 @@ message TConfigItem { CompositeConveyorConfigItem = 109; PortionsMetadataCacheItem = 110; + DeduGroupedMemoryLimiterConfig = 112; // synthetic kinds for audit purposes only DatabaseYamlConfigChangeItem = 32767; diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/context.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/context.cpp index eeac3c4a6835..5d0774c2a0b6 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/context.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/context.cpp @@ -4,17 +4,10 @@ namespace NKikimr::NOlap::NReader::NSimple::NDuplicateFiltering { - namespace { - - static std::shared_ptr DeduplicationStageFeatures = - NOlap::NGroupedMemoryManager::TDeduplicationMemoryLimiterOperator::BuildStageFeatures("DEFAULT", 1000000000); - - } - TInternalFilterConstructor::TInternalFilterConstructor(const TEvRequestFilter::TPtr& request, TColumnDataSplitter&& splitter) : OriginalRequest(request) , Intervals(std::move(splitter)) - , ProcessGuard(NGroupedMemoryManager::TDeduplicationMemoryLimiterOperator::BuildProcessGuard({ DeduplicationStageFeatures })) + , ProcessGuard(NGroupedMemoryManager::TDeduplicationMemoryLimiterOperator::BuildProcessGuard({})) , ScopeGuard(ProcessGuard->BuildScopeGuard(1)) , GroupGuard(ScopeGuard->BuildGroupGuard()) { diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/context.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/context.h index db03c8cb702d..57852edf7b64 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/context.h +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/context.h @@ -101,10 +101,10 @@ class TInternalFilterConstructor: TMoveOnly { return ProcessGuard->GetProcessId(); } ui64 GetMemoryScopeId() const { - return ScopeGuard->GetProcessId(); + return ScopeGuard->GetScopeId(); } ui64 GetMemoryGroupId() const { - return GroupGuard->GetProcessId(); + return GroupGuard->GetGroupId(); } }; diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.cpp index 602e921ecd56..8882307cfcd1 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.cpp @@ -69,7 +69,7 @@ class TColumnDataAllocation: public NGroupedMemoryManager::IAllocation { private: virtual void DoOnAllocationImpossible(const TString& errorMessage) override { AFL_VERIFY(Callback); - Callback->OnError(errorMessage); + Callback->OnError(TStringBuilder() << "cannot allocate memory: " << errorMessage); } virtual bool DoOnAllocated(std::shared_ptr&& guard, const std::shared_ptr& /*allocation*/) override { diff --git a/ydb/tests/library/harness/kikimr_config.py b/ydb/tests/library/harness/kikimr_config.py index 0e9d45e3ab09..f6e25143b4d5 100644 --- a/ydb/tests/library/harness/kikimr_config.py +++ b/ydb/tests/library/harness/kikimr_config.py @@ -165,6 +165,7 @@ def __init__( enable_resource_pools=None, scan_grouped_memory_limiter_config=None, comp_grouped_memory_limiter_config=None, + deduplication_grouped_memory_limiter_config=None, query_service_config=None, domain_login_only=None, use_self_management=False, @@ -408,6 +409,8 @@ def __init__( self.yaml_config["scan_grouped_memory_limiter_config"] = scan_grouped_memory_limiter_config if comp_grouped_memory_limiter_config: self.yaml_config["comp_grouped_memory_limiter_config"] = comp_grouped_memory_limiter_config + if deduplication_grouped_memory_limiter_config: + self.yaml_config["deduplication_grouped_memory_limiter_config"] = deduplication_grouped_memory_limiter_config self.__build() From 6d9c938a847687b046f15bb461fafab0e820e951 Mon Sep 17 00:00:00 2001 From: Hor911 Date: Fri, 8 Aug 2025 13:44:20 +0300 Subject: [PATCH 32/52] Show Aggregation Pushdown in Explain Plan (#22495) --- ydb/core/kqp/opt/kqp_query_plan.cpp | 45 +++++++++++++++++-- ydb/core/kqp/ut/olap/aggregations_ut.cpp | 10 ++--- .../queries-original-plan-column-0 | 12 ++++- .../queries-original-plan-column-1 | 12 ++++- .../queries-original-plan-column-12 | 12 ++++- .../queries-original-plan-column-14 | 12 ++++- .../queries-original-plan-column-15 | 12 ++++- .../queries-original-plan-column-16 | 12 ++++- .../queries-original-plan-column-17 | 12 ++++- .../queries-original-plan-column-2 | 12 ++++- .../queries-original-plan-column-20 | 12 ++++- .../queries-original-plan-column-21 | 12 ++++- .../queries-original-plan-column-3 | 12 ++++- .../queries-original-plan-column-30 | 12 ++++- .../queries-original-plan-column-31 | 12 ++++- .../queries-original-plan-column-32 | 12 ++++- .../queries-original-plan-column-33 | 12 ++++- .../queries-original-plan-column-34 | 12 ++++- .../queries-original-plan-column-36 | 12 ++++- .../queries-original-plan-column-37 | 12 ++++- .../queries-original-plan-column-38 | 12 ++++- .../queries-original-plan-column-40 | 12 ++++- .../queries-original-plan-column-41 | 12 ++++- .../queries-original-plan-column-6 | 12 ++++- .../queries-original-plan-column-7 | 12 ++++- 25 files changed, 299 insertions(+), 32 deletions(-) diff --git a/ydb/core/kqp/opt/kqp_query_plan.cpp b/ydb/core/kqp/opt/kqp_query_plan.cpp index 6f95486edee2..6deb0a83e3ae 100644 --- a/ydb/core/kqp/opt/kqp_query_plan.cpp +++ b/ydb/core/kqp/opt/kqp_query_plan.cpp @@ -1014,6 +1014,8 @@ class TxPlanSerializer { TVector> Visit(TExprNode::TPtr node, TQueryPlanNode& planNode) { TMaybe> operatorId; + auto maybeCallable = TMaybeNode(node); + if (auto maybeRead = TMaybeNode(node)) { auto read = maybeRead.Cast(); TString table = TString(read.Table().Path()); TKqlKeyRange range = read.Range(); @@ -1066,6 +1068,11 @@ class TxPlanSerializer { operatorId = Visit(maybeCombiner.Cast(), planNode); } else if (auto maybeBlockCombine = TMaybeNode(node)) { operatorId = Visit(maybeBlockCombine.Cast(), planNode); + } else if (maybeCallable && (maybeCallable.Cast().CallableName() == "BlockMergeFinalizeHashed" || maybeCallable.Cast().CallableName() == "BlockMergeManyFinalizeHashed")) { + TOperator op; + op.Properties["Name"] = "Aggregate"; + op.Properties["Phase"] = "Final"; + operatorId = AddOperator(planNode, "Aggregate", std::move(op)); } else if (auto maybeCombiner = TMaybeNode(node)) { operatorId = Visit(maybeCombiner.Cast(), planNode); } else if (auto maybeSort = TMaybeNode(node)) { @@ -1124,6 +1131,25 @@ class TxPlanSerializer { } else if (TMaybeNode(node)) { auto olapTable = TExprBase(node).Cast(); + ui32 currentOperatorId = 0; + + auto aggr = [](const TExprNode::TPtr& n) -> bool { + if (auto maybeAggregation = TMaybeNode(n)) { return true; } return false; + }; + + if (auto maybeKqpOlapAggregation = FindNode(olapTable.Process().Body().Ptr(), aggr)) { + auto kqpOlapAggregation = TExprBase(maybeKqpOlapAggregation).Cast(); + + TOperator op; + op.Properties["Name"] = "Aggregate"; + op.Properties["Phase"] = "Intermediate"; + op.Properties["Pushdown"] = "True"; + + AddOptimizerEstimates(op, kqpOlapAggregation); + currentOperatorId = AddOperator(planNode, "Aggregate", std::move(op)); + operatorId = currentOperatorId; + } + auto pred = [](const TExprNode::TPtr& n) -> bool { if (auto maybeFilter = TMaybeNode(n)) { return true; } return false; }; @@ -1137,11 +1163,22 @@ class TxPlanSerializer { op.Properties["Pushdown"] = "True"; AddOptimizerEstimates(op, kqpOlapFilter); + auto filterOperatorId = AddOperator(planNode, "Filter", std::move(op)); + + if (operatorId) { + planNode.Operators[currentOperatorId].Inputs.push_back(filterOperatorId); + } else { + operatorId = filterOperatorId; + } + currentOperatorId = filterOperatorId; + } + + auto tableOperatorId = Visit(olapTable, planNode); - operatorId = AddOperator(planNode, "Filter", std::move(op)); - inputIds.push_back(Visit(olapTable, planNode)); + if (operatorId) { + planNode.Operators[currentOperatorId].Inputs.push_back(tableOperatorId); } else { - operatorId = Visit(olapTable, planNode); + operatorId = tableOperatorId; } } else if (TMaybeNode(node)) { // do nothing @@ -1768,7 +1805,7 @@ class TxPlanSerializer { return AddOperator(planNode, readName, std::move(op)); } - std::variant Visit(const TKqlReadTableRangesBase& read, TQueryPlanNode& planNode) { + ui32 Visit(const TKqlReadTableRangesBase& read, TQueryPlanNode& planNode) { const auto tablePath = TString(read.Table().Path()); const auto explainPrompt = TKqpReadTableExplainPrompt::Parse(read); const auto rangesDesc = NPlanUtils::PrettyExprStr(read.Ranges()); diff --git a/ydb/core/kqp/ut/olap/aggregations_ut.cpp b/ydb/core/kqp/ut/olap/aggregations_ut.cpp index 62678a3c0c8f..280fd5b41a7a 100644 --- a/ydb/core/kqp/ut/olap/aggregations_ut.cpp +++ b/ydb/core/kqp/ut/olap/aggregations_ut.cpp @@ -145,7 +145,7 @@ Y_UNIT_TEST_SUITE(KqpOlapAggregations) { CompareYson(result, R"([[23000u;]])"); // Check plan - CheckPlanForAggregatePushdown(query, tableClient, { "TKqpOlapAgg" }, "TableFullScan"); + CheckPlanForAggregatePushdown(query, tableClient, { "TKqpOlapAgg" }, "Aggregate-TableFullScan"); } } @@ -184,7 +184,7 @@ Y_UNIT_TEST_SUITE(KqpOlapAggregations) { CompareYson(result, R"([[[0];4600u];[[1];4600u];[[2];4600u];[[3];4600u];[[4];4600u]])"); // Check plan - CheckPlanForAggregatePushdown(query, tableClient, { "WideCombiner" }, "TableFullScan"); + CheckPlanForAggregatePushdown(query, tableClient, { "WideCombiner" }, "Aggregate-TableFullScan"); // CheckPlanForAggregatePushdown(query, tableClient, { "TKqpOlapAgg" }, "TableFullScan"); } } @@ -309,7 +309,7 @@ Y_UNIT_TEST_SUITE(KqpOlapAggregations) { CompareYson(result, R"([[23000u;]])"); // Check plan - CheckPlanForAggregatePushdown(query, tableClient, { "TKqpOlapAgg" }, "TableFullScan"); + CheckPlanForAggregatePushdown(query, tableClient, { "TKqpOlapAgg" }, "Aggregate-TableFullScan"); } } @@ -348,7 +348,7 @@ Y_UNIT_TEST_SUITE(KqpOlapAggregations) { CompareYson(result, R"([[23000u;]])"); // Check plan - CheckPlanForAggregatePushdown(query, tableClient, { "TKqpOlapAgg" }, "TableFullScan"); + CheckPlanForAggregatePushdown(query, tableClient, { "TKqpOlapAgg" }, "Aggregate-TableFullScan"); } } @@ -1033,7 +1033,7 @@ Y_UNIT_TEST_SUITE(KqpOlapAggregations) { ORDER BY c, resource_id DESC LIMIT 3 )") .SetExpectedReply("[[[\"40999\"];[4];1u];[[\"40998\"];[3];1u];[[\"40997\"];[2];1u]]") - .SetExpectedReadNodeType("TableFullScan"); + .SetExpectedReadNodeType("Aggregate-TableFullScan"); testCase.FillExpectedAggregationGroupByPlanOptions(); TestAggregations({ testCase }); } diff --git a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-0 b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-0 index 4dd058066df6..8613c362f499 100644 --- a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-0 +++ b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-0 @@ -73,8 +73,18 @@ "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "TableFullScan", + "Node Type": "Aggregate-TableFullScan", "Operators": [ + { + "Inputs": [ + { + "InternalOperatorId": 1 + } + ], + "Name": "Aggregate", + "Phase": "Intermediate", + "Pushdown": "True" + }, { "Inputs": [], "Name": "TableFullScan", diff --git a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-1 b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-1 index 093aee53b7c1..cfa4e725a849 100644 --- a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-1 +++ b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-1 @@ -73,7 +73,7 @@ "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "Filter-TableFullScan", + "Node Type": "Aggregate-Filter-TableFullScan", "Operators": [ { "Inputs": [ @@ -81,6 +81,16 @@ "InternalOperatorId": 1 } ], + "Name": "Aggregate", + "Phase": "Intermediate", + "Pushdown": "True" + }, + { + "Inputs": [ + { + "InternalOperatorId": 2 + } + ], "Name": "Filter", "Predicate": "AdvEngineID != 0", "Pushdown": "True" diff --git a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-12 b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-12 index d3b202aafc8c..df2900e93bf8 100644 --- a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-12 +++ b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-12 @@ -69,7 +69,7 @@ "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "Filter-TableFullScan", + "Node Type": "Aggregate-Filter-TableFullScan", "Operators": [ { "Inputs": [ @@ -77,6 +77,16 @@ "InternalOperatorId": 1 } ], + "Name": "Aggregate", + "Phase": "Intermediate", + "Pushdown": "True" + }, + { + "Inputs": [ + { + "InternalOperatorId": 2 + } + ], "Name": "Filter", "Predicate": "SearchPhrase != \"\"", "Pushdown": "True" diff --git a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-14 b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-14 index c339f15704fd..abd9ebd54579 100644 --- a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-14 +++ b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-14 @@ -70,7 +70,7 @@ "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "Filter-TableFullScan", + "Node Type": "Aggregate-Filter-TableFullScan", "Operators": [ { "Inputs": [ @@ -78,6 +78,16 @@ "InternalOperatorId": 1 } ], + "Name": "Aggregate", + "Phase": "Intermediate", + "Pushdown": "True" + }, + { + "Inputs": [ + { + "InternalOperatorId": 2 + } + ], "Name": "Filter", "Predicate": "SearchPhrase != \"\"", "Pushdown": "True" diff --git a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-15 b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-15 index da43d509d310..6d5fc35607db 100644 --- a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-15 +++ b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-15 @@ -69,8 +69,18 @@ "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "TableFullScan", + "Node Type": "Aggregate-TableFullScan", "Operators": [ + { + "Inputs": [ + { + "InternalOperatorId": 1 + } + ], + "Name": "Aggregate", + "Phase": "Intermediate", + "Pushdown": "True" + }, { "Inputs": [], "Name": "TableFullScan", diff --git a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-16 b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-16 index e1d3cb9ec128..f8d106c1aace 100644 --- a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-16 +++ b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-16 @@ -70,8 +70,18 @@ "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "TableFullScan", + "Node Type": "Aggregate-TableFullScan", "Operators": [ + { + "Inputs": [ + { + "InternalOperatorId": 1 + } + ], + "Name": "Aggregate", + "Phase": "Intermediate", + "Pushdown": "True" + }, { "Inputs": [], "Name": "TableFullScan", diff --git a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-17 b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-17 index df2e26d4a99e..e26834e0e919 100644 --- a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-17 +++ b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-17 @@ -69,8 +69,18 @@ "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "TableFullScan", + "Node Type": "Aggregate-TableFullScan", "Operators": [ + { + "Inputs": [ + { + "InternalOperatorId": 1 + } + ], + "Name": "Aggregate", + "Phase": "Intermediate", + "Pushdown": "True" + }, { "Inputs": [], "Name": "TableFullScan", diff --git a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-2 b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-2 index 119991fe409b..2226f6596edf 100644 --- a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-2 +++ b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-2 @@ -82,8 +82,18 @@ "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "TableFullScan", + "Node Type": "Aggregate-TableFullScan", "Operators": [ + { + "Inputs": [ + { + "InternalOperatorId": 1 + } + ], + "Name": "Aggregate", + "Phase": "Intermediate", + "Pushdown": "True" + }, { "Inputs": [], "Name": "TableFullScan", diff --git a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-20 b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-20 index b177fcdae327..91d162e63d5f 100644 --- a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-20 +++ b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-20 @@ -73,7 +73,7 @@ "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "Filter-TableFullScan", + "Node Type": "Aggregate-Filter-TableFullScan", "Operators": [ { "Inputs": [ @@ -81,6 +81,16 @@ "InternalOperatorId": 1 } ], + "Name": "Aggregate", + "Phase": "Intermediate", + "Pushdown": "True" + }, + { + "Inputs": [ + { + "InternalOperatorId": 2 + } + ], "Name": "Filter", "Predicate": "URL LIKE \"%google%\"", "Pushdown": "True" diff --git a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-21 b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-21 index 17174a5557d5..603a8a9d4f56 100644 --- a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-21 +++ b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-21 @@ -69,7 +69,7 @@ "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "Filter-TableFullScan", + "Node Type": "Aggregate-Filter-TableFullScan", "Operators": [ { "Inputs": [ @@ -77,6 +77,16 @@ "InternalOperatorId": 1 } ], + "Name": "Aggregate", + "Phase": "Intermediate", + "Pushdown": "True" + }, + { + "Inputs": [ + { + "InternalOperatorId": 2 + } + ], "Name": "Filter", "Predicate": "(SearchPhrase != \"\") AND (URL LIKE \"%google%\")", "Pushdown": "True" diff --git a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-3 b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-3 index ecf31167e577..e9085dbbbd64 100644 --- a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-3 +++ b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-3 @@ -73,8 +73,18 @@ "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "TableFullScan", + "Node Type": "Aggregate-TableFullScan", "Operators": [ + { + "Inputs": [ + { + "InternalOperatorId": 1 + } + ], + "Name": "Aggregate", + "Phase": "Intermediate", + "Pushdown": "True" + }, { "Inputs": [], "Name": "TableFullScan", diff --git a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-30 b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-30 index 75e81d7c2731..c1883e7a55b8 100644 --- a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-30 +++ b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-30 @@ -70,7 +70,7 @@ "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "Filter-TableFullScan", + "Node Type": "Aggregate-Filter-TableFullScan", "Operators": [ { "Inputs": [ @@ -78,6 +78,16 @@ "InternalOperatorId": 1 } ], + "Name": "Aggregate", + "Phase": "Intermediate", + "Pushdown": "True" + }, + { + "Inputs": [ + { + "InternalOperatorId": 2 + } + ], "Name": "Filter", "Predicate": "SearchPhrase != \"\"", "Pushdown": "True" diff --git a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-31 b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-31 index e7ab9dee9de4..ebb318010b68 100644 --- a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-31 +++ b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-31 @@ -70,7 +70,7 @@ "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "Filter-TableFullScan", + "Node Type": "Aggregate-Filter-TableFullScan", "Operators": [ { "Inputs": [ @@ -78,6 +78,16 @@ "InternalOperatorId": 1 } ], + "Name": "Aggregate", + "Phase": "Intermediate", + "Pushdown": "True" + }, + { + "Inputs": [ + { + "InternalOperatorId": 2 + } + ], "Name": "Filter", "Predicate": "SearchPhrase != \"\"", "Pushdown": "True" diff --git a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-32 b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-32 index 6340084d72e3..df951218332b 100644 --- a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-32 +++ b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-32 @@ -70,8 +70,18 @@ "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "TableFullScan", + "Node Type": "Aggregate-TableFullScan", "Operators": [ + { + "Inputs": [ + { + "InternalOperatorId": 1 + } + ], + "Name": "Aggregate", + "Phase": "Intermediate", + "Pushdown": "True" + }, { "Inputs": [], "Name": "TableFullScan", diff --git a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-33 b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-33 index a42d42b2ddc5..045d108d45e7 100644 --- a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-33 +++ b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-33 @@ -69,8 +69,18 @@ "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "TableFullScan", + "Node Type": "Aggregate-TableFullScan", "Operators": [ + { + "Inputs": [ + { + "InternalOperatorId": 1 + } + ], + "Name": "Aggregate", + "Phase": "Intermediate", + "Pushdown": "True" + }, { "Inputs": [], "Name": "TableFullScan", diff --git a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-34 b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-34 index 51ef05797732..df6d52a10ecd 100644 --- a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-34 +++ b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-34 @@ -70,8 +70,18 @@ "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "TableFullScan", + "Node Type": "Aggregate-TableFullScan", "Operators": [ + { + "Inputs": [ + { + "InternalOperatorId": 1 + } + ], + "Name": "Aggregate", + "Phase": "Intermediate", + "Pushdown": "True" + }, { "Inputs": [], "Name": "TableFullScan", diff --git a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-36 b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-36 index 8d83298cfe74..cf7c0abf53bb 100644 --- a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-36 +++ b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-36 @@ -69,7 +69,7 @@ "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "Filter-TableRangeScan", + "Node Type": "Aggregate-Filter-TableRangeScan", "Operators": [ { "Inputs": [ @@ -77,6 +77,16 @@ "InternalOperatorId": 1 } ], + "Name": "Aggregate", + "Phase": "Intermediate", + "Pushdown": "True" + }, + { + "Inputs": [ + { + "InternalOperatorId": 2 + } + ], "Name": "Filter", "Predicate": "DontCountHits == 0 AND IsRefresh == 0 AND URL != \"\"", "Pushdown": "True" diff --git a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-37 b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-37 index 4542b8caf9a5..41a151847f76 100644 --- a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-37 +++ b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-37 @@ -69,7 +69,7 @@ "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "Filter-TableRangeScan", + "Node Type": "Aggregate-Filter-TableRangeScan", "Operators": [ { "Inputs": [ @@ -77,6 +77,16 @@ "InternalOperatorId": 1 } ], + "Name": "Aggregate", + "Phase": "Intermediate", + "Pushdown": "True" + }, + { + "Inputs": [ + { + "InternalOperatorId": 2 + } + ], "Name": "Filter", "Predicate": "DontCountHits == 0 AND IsRefresh == 0 AND Title != \"\"", "Pushdown": "True" diff --git a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-38 b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-38 index 3ff02a76abaf..283b10f0baeb 100644 --- a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-38 +++ b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-38 @@ -98,7 +98,7 @@ "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "Filter-TableRangeScan", + "Node Type": "Aggregate-Filter-TableRangeScan", "Operators": [ { "Inputs": [ @@ -106,6 +106,16 @@ "InternalOperatorId": 1 } ], + "Name": "Aggregate", + "Phase": "Intermediate", + "Pushdown": "True" + }, + { + "Inputs": [ + { + "InternalOperatorId": 2 + } + ], "Name": "Filter", "Predicate": "IsRefresh == 0 AND IsLink != 0 AND IsDownload == 0", "Pushdown": "True" diff --git a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-40 b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-40 index 842665431ca6..acdbb7d8578b 100644 --- a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-40 +++ b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-40 @@ -99,7 +99,7 @@ "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "Filter-TableRangeScan", + "Node Type": "Aggregate-Filter-TableRangeScan", "Operators": [ { "Inputs": [ @@ -107,6 +107,16 @@ "InternalOperatorId": 1 } ], + "Name": "Aggregate", + "Phase": "Intermediate", + "Pushdown": "True" + }, + { + "Inputs": [ + { + "InternalOperatorId": 2 + } + ], "Name": "Filter", "Predicate": "IsRefresh == 0 AND RefererHash == 3594120000172545465 AND TraficSourceID == -1 OR TraficSourceID == 6", "Pushdown": "True" diff --git a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-41 b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-41 index 743d9f089af5..f3d4128c440f 100644 --- a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-41 +++ b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-41 @@ -99,7 +99,7 @@ "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "Filter-TableRangeScan", + "Node Type": "Aggregate-Filter-TableRangeScan", "Operators": [ { "Inputs": [ @@ -107,6 +107,16 @@ "InternalOperatorId": 1 } ], + "Name": "Aggregate", + "Phase": "Intermediate", + "Pushdown": "True" + }, + { + "Inputs": [ + { + "InternalOperatorId": 2 + } + ], "Name": "Filter", "Predicate": "IsRefresh == 0 AND DontCountHits == 0 AND URLHash == 2868770270353813622", "Pushdown": "True" diff --git a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-6 b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-6 index d6cb90bb8a9b..5ec00f549abd 100644 --- a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-6 +++ b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-6 @@ -82,8 +82,18 @@ "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "TableFullScan", + "Node Type": "Aggregate-TableFullScan", "Operators": [ + { + "Inputs": [ + { + "InternalOperatorId": 1 + } + ], + "Name": "Aggregate", + "Phase": "Intermediate", + "Pushdown": "True" + }, { "Inputs": [], "Name": "TableFullScan", diff --git a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-7 b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-7 index f860a26ab045..1216daef5e28 100644 --- a/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-7 +++ b/ydb/tests/functional/clickbench/canondata/test.test_plans_column_/queries-original-plan-column-7 @@ -69,7 +69,7 @@ "PlanNodeType": "Connection", "Plans": [ { - "Node Type": "Filter-TableFullScan", + "Node Type": "Aggregate-Filter-TableFullScan", "Operators": [ { "Inputs": [ @@ -77,6 +77,16 @@ "InternalOperatorId": 1 } ], + "Name": "Aggregate", + "Phase": "Intermediate", + "Pushdown": "True" + }, + { + "Inputs": [ + { + "InternalOperatorId": 2 + } + ], "Name": "Filter", "Predicate": "AdvEngineID != 0", "Pushdown": "True" From 84e06acf9f818dbe1d5169be7d24c2c19d62f1c4 Mon Sep 17 00:00:00 2001 From: Nikolay Shestakov Date: Fri, 8 Aug 2025 15:47:42 +0500 Subject: [PATCH 33/52] Fixed lost secrets after alter the transfer or the async replication (#22547) --- ydb/core/transfer/ut/common/utils.h | 24 +++++- .../transfer/ut/functional/transfer_ut.cpp | 79 ++++++++++++++++++- .../tx/replication/controller/replication.cpp | 23 +++++- .../tx/replication/controller/replication.h | 2 + .../controller/secret_resolver.cpp | 10 ++- .../replication/controller/secret_resolver.h | 2 +- .../controller/tx_alter_replication.cpp | 2 + .../controller/tx_resolve_secret_result.cpp | 6 ++ .../tx/replication/service/topic_reader.cpp | 2 +- ydb/core/tx/replication/service/worker.cpp | 6 +- 10 files changed, 141 insertions(+), 15 deletions(-) diff --git a/ydb/core/transfer/ut/common/utils.h b/ydb/core/transfer/ut/common/utils.h index a36974d73f17..6de08fba97de 100644 --- a/ydb/core/transfer/ut/common/utils.h +++ b/ydb/core/transfer/ut/common/utils.h @@ -408,6 +408,7 @@ struct MainTestCase { std::optional BatchSizeBytes = 8_MB; std::optional ExpectedError; std::optional Username; + std::optional UserSecretName; std::optional Directory; CreateTransferSettings() {}; @@ -455,6 +456,12 @@ struct MainTestCase { result.Directory = directory; return result; } + + static CreateTransferSettings WithSecretName(const TString& secret) { + CreateTransferSettings result; + result.UserSecretName = secret; + return result; + } }; void CreateTransfer(const std::string& lambda, const CreateTransferSettings& settings = CreateTransferSettings()) { @@ -475,6 +482,9 @@ struct MainTestCase { if (settings.BatchSizeBytes) { options.push_back(TStringBuilder() << "BATCH_SIZE_BYTES = " << *settings.BatchSizeBytes); } + if (settings.UserSecretName) { + options.push_back(TStringBuilder() << "TOKEN_SECRET_NAME = '" << *settings.UserSecretName << "'"); + } if (settings.Username) { options.push_back(TStringBuilder() << "TOKEN = '" << *settings.Username << "@builtin'"); } @@ -696,10 +706,16 @@ struct MainTestCase { return result; } - void CreateUser(const std::string& username) { - ExecuteDDL(Sprintf(R"( - CREATE USER %s - )", username.data())); + void CreateUser(const std::string& username, const std::optional password = std::nullopt) { + if (password) { + ExecuteDDL(Sprintf(R"( + CREATE USER %s PASSWORD '%s' + )", username.data(), password.value().data())); + } else { + ExecuteDDL(Sprintf(R"( + CREATE USER %s + )", username.data())); + } } void Write(const TMessage& message) { diff --git a/ydb/core/transfer/ut/functional/transfer_ut.cpp b/ydb/core/transfer/ut/functional/transfer_ut.cpp index 44a6443ee5c5..e107b241ab62 100644 --- a/ydb/core/transfer/ut/functional/transfer_ut.cpp +++ b/ydb/core/transfer/ut/functional/transfer_ut.cpp @@ -976,7 +976,7 @@ Y_UNIT_TEST_SUITE(Transfer) |> ]; }; - )", MainTestCase::CreateTransferSettings::WithBatching(TDuration::Seconds(1), 1)); + )", MainTestCase::CreateTransferSettings::WithBatching(TDuration::Seconds(2), 1_MB)); testCase.Write({"Message-1"}); @@ -1013,13 +1013,27 @@ Y_UNIT_TEST_SUITE(Transfer) _C("Message", TString("Message-2")), }}); + testCase.Write({"Message-3"}); + Sleep(TDuration::MilliSeconds(500)); + // More cycles for pause/resume testCase.PauseTransfer(); - testCase.CheckTransferState(TTransferDescription::EState::Paused); + + testCase.Write({"Message-4"}); testCase.ResumeTransfer(); testCase.CheckTransferState(TTransferDescription::EState::Running); + testCase.CheckResult({{ + _C("Message", TString("Message-1")) + }, { + _C("Message", TString("Message-2")), + }, { + _C("Message", TString("Message-3")), + }, { + _C("Message", TString("Message-4")), + }}); + testCase.DropTransfer(); testCase.DropTable(); testCase.DropTopic(); @@ -1342,5 +1356,66 @@ Y_UNIT_TEST_SUITE(Transfer) MainTestCase::AlterTransferSettings::WithBatching(TDuration::Seconds(1), 1)); } + Y_UNIT_TEST(Alter_WithSecret) + { + auto id = RandomNumber(); + auto username = TStringBuilder() << "u" << id; + auto secretName = TStringBuilder() << "s" << id; + + MainTestCase permissionSetup; + permissionSetup.CreateUser(username); + permissionSetup.Grant("", username, {"ydb.granular.create_table", "ydb.granular.create_queue", "ydb.granular.alter_schema"}); + + MainTestCase testCase(username); + testCase.ExecuteDDL(Sprintf(R"( + CREATE OBJECT %s (TYPE SECRET) WITH value="%s@builtin" + )", secretName.data(), username.data())); + + permissionSetup.ExecuteDDL(Sprintf(R"( + CREATE TABLE `%s` ( + Key Uint64 NOT NULL, + Message Utf8, + PRIMARY KEY (Key) + ) WITH ( + STORE = COLUMN + ); + )", testCase.TableName.data())); + permissionSetup.Grant(testCase.TableName, username, {"ydb.generic.write", "ydb.generic.read"}); + + testCase.CreateTopic(1); + permissionSetup.Grant(testCase.TopicName, username, {"ALL"}); + + testCase.CreateTransfer(R"( + $l = ($x) -> { + return [ + <| + Key:CAST($x._offset AS Uint64), + Message:CAST($x._data AS Utf8) + |> + ]; + }; + )", MainTestCase::CreateTransferSettings::WithSecretName(secretName)); + + testCase.Write({"Message-1"}); + + testCase.CheckResult({{ + _C("Message", TString("Message-1")) + }}); + + testCase.PauseTransfer(); + testCase.CheckTransferState(TTransferDescription::EState::Paused); + testCase.ResumeTransfer(); + + testCase.Write({"Message-2"}); + + testCase.CheckResult({{ + _C("Message", TString("Message-1")) + }, { + _C("Message", TString("Message-2")) + }}); + + testCase.DropTopic(); + testCase.DropTransfer(); + } } diff --git a/ydb/core/tx/replication/controller/replication.cpp b/ydb/core/tx/replication/controller/replication.cpp index e032ed5f4a3c..321a63970c78 100644 --- a/ydb/core/tx/replication/controller/replication.cpp +++ b/ydb/core/tx/replication/controller/replication.cpp @@ -41,7 +41,11 @@ class TReplication::TImpl: public TLagProvider { return; } - SecretResolver = ctx.Register(CreateSecretResolver(ctx.SelfID, ReplicationId, PathId, secretName)); + SecretResolver = ctx.Register(CreateSecretResolver(ctx.SelfID, ReplicationId, PathId, secretName, ++SecretResolverCookie)); + } + + ui64 GetExpectedSecretResolverCookie() const { + return SecretResolverCookie; } template @@ -215,6 +219,14 @@ class TReplication::TImpl: public TLagProvider { Config = config; } + void ResetCredentials(const TActorContext& ctx) { + for (auto* x : TVector{&SecretResolver, &YdbProxy}) { + if (auto actorId = std::exchange(*x, {})) { + ctx.Send(actorId, new TEvents::TEvPoison()); + } + } + } + void ErrorState(TString issue) { SetState(EState::Error, issue); } @@ -243,6 +255,7 @@ class TReplication::TImpl: public TLagProvider { THashSet PendingAlterTargets; mutable TVector TargetTablePaths; TActorId SecretResolver; + ui64 SecretResolverCookie = 0; TActorId YdbProxy; TActorId TenantResolver; TActorId TargetDiscoverer; @@ -323,6 +336,10 @@ void TReplication::SetConfig(NKikimrReplication::TReplicationConfig&& config) { Impl->SetConfig(std::move(config)); } +void TReplication::ResetCredentials(const TActorContext& ctx) { + Impl->ResetCredentials(ctx); +} + const NKikimrReplication::TReplicationConfig& TReplication::GetConfig() const { return Impl->Config; } @@ -373,6 +390,10 @@ void TReplication::UpdateSecret(const TString& secretValue) { } } +ui64 TReplication::GetExpectedSecretResolverCookie() const { + return Impl->GetExpectedSecretResolverCookie(); +} + void TReplication::SetTenant(const TString& value) { Impl->Tenant = value; Impl->TenantResolver = {}; diff --git a/ydb/core/tx/replication/controller/replication.h b/ydb/core/tx/replication/controller/replication.h index f593abbf65aa..f3cbfc73249f 100644 --- a/ydb/core/tx/replication/controller/replication.h +++ b/ydb/core/tx/replication/controller/replication.h @@ -139,6 +139,7 @@ class TReplication: public TSimpleRefCount { const TActorId& GetYdbProxy() const; ui64 GetSchemeShardId() const; void SetConfig(NKikimrReplication::TReplicationConfig&& config); + void ResetCredentials(const TActorContext& ctx); const NKikimrReplication::TReplicationConfig& GetConfig() const; const TString& GetDatabase() const; void SetState(EState state, TString issue = {}); @@ -152,6 +153,7 @@ class TReplication: public TSimpleRefCount { ui64 GetNextTargetId() const; void UpdateSecret(const TString& secretValue); + ui64 GetExpectedSecretResolverCookie() const; void SetTenant(const TString& value); const TString& GetTenant() const; diff --git a/ydb/core/tx/replication/controller/secret_resolver.cpp b/ydb/core/tx/replication/controller/secret_resolver.cpp index 03c2d603772b..b04b1d152605 100644 --- a/ydb/core/tx/replication/controller/secret_resolver.cpp +++ b/ydb/core/tx/replication/controller/secret_resolver.cpp @@ -57,7 +57,7 @@ class TSecretResolver: public TActorBootstrapped { template void Reply(Args&&... args) { - Send(Parent, new TEvPrivate::TEvResolveSecretResult(ReplicationId, std::forward(args)...)); + Send(Parent, new TEvPrivate::TEvResolveSecretResult(ReplicationId, std::forward(args)...), 0, Cookie); PassAway(); } @@ -66,11 +66,12 @@ class TSecretResolver: public TActorBootstrapped { return NKikimrServices::TActivity::REPLICATION_CONTROLLER_SECRET_RESOLVER; } - explicit TSecretResolver(const TActorId& parent, ui64 rid, const TPathId& pathId, const TString& secretName) + explicit TSecretResolver(const TActorId& parent, ui64 rid, const TPathId& pathId, const TString& secretName, const ui64 cookie) : Parent(parent) , ReplicationId(rid) , PathId(pathId) , SecretName(secretName) + , Cookie(cookie) , LogPrefix("SecretResolver", ReplicationId) { } @@ -106,6 +107,7 @@ class TSecretResolver: public TActorBootstrapped { const ui64 ReplicationId; const TPathId PathId; const TString SecretName; + const ui64 Cookie; const TActorLogPrefix LogPrefix; static constexpr auto RetryInterval = TDuration::Seconds(1); @@ -113,8 +115,8 @@ class TSecretResolver: public TActorBootstrapped { }; // TSecretResolver -IActor* CreateSecretResolver(const TActorId& parent, ui64 rid, const TPathId& pathId, const TString& secretName) { - return new TSecretResolver(parent, rid, pathId, secretName); +IActor* CreateSecretResolver(const TActorId& parent, ui64 rid, const TPathId& pathId, const TString& secretName, const ui64 cookie) { + return new TSecretResolver(parent, rid, pathId, secretName, cookie); } } diff --git a/ydb/core/tx/replication/controller/secret_resolver.h b/ydb/core/tx/replication/controller/secret_resolver.h index 6408469c7633..dd3b0af55c43 100644 --- a/ydb/core/tx/replication/controller/secret_resolver.h +++ b/ydb/core/tx/replication/controller/secret_resolver.h @@ -4,6 +4,6 @@ namespace NKikimr::NReplication::NController { -IActor* CreateSecretResolver(const TActorId& parent, ui64 rid, const TPathId& pathId, const TString& secretName); +IActor* CreateSecretResolver(const TActorId& parent, ui64 rid, const TPathId& pathId, const TString& secretName, const ui64 cookie); } diff --git a/ydb/core/tx/replication/controller/tx_alter_replication.cpp b/ydb/core/tx/replication/controller/tx_alter_replication.cpp index 5fc1b94fe46d..838c38c1782f 100644 --- a/ydb/core/tx/replication/controller/tx_alter_replication.cpp +++ b/ydb/core/tx/replication/controller/tx_alter_replication.cpp @@ -88,6 +88,8 @@ class TController::TTxAlterReplication: public TTxBase { } Replication->SetConfig(std::move(*record.MutableConfig())); + Replication->ResetCredentials(ctx); + NIceDb::TNiceDb db(txc.DB); db.Table().Key(Replication->GetId()).Update( NIceDb::TUpdate(record.GetConfig().SerializeAsString()), diff --git a/ydb/core/tx/replication/controller/tx_resolve_secret_result.cpp b/ydb/core/tx/replication/controller/tx_resolve_secret_result.cpp index 52ba9d835aeb..9b121e6d29eb 100644 --- a/ydb/core/tx/replication/controller/tx_resolve_secret_result.cpp +++ b/ydb/core/tx/replication/controller/tx_resolve_secret_result.cpp @@ -29,6 +29,12 @@ class TController::TTxResolveSecretResult: public TTxBase { return true; } + if (Ev->Cookie != Replication->GetExpectedSecretResolverCookie()) { + CLOG_E(ctx, "Unexpected cookie" + << ": cookie# " << Ev->Cookie); + return true; + } + if (Ev->Get()->IsSuccess()) { CLOG_N(ctx, "Secret resolved" << ": rid# " << rid); diff --git a/ydb/core/tx/replication/service/topic_reader.cpp b/ydb/core/tx/replication/service/topic_reader.cpp index fc7a867eb376..602b2e90f47b 100644 --- a/ydb/core/tx/replication/service/topic_reader.cpp +++ b/ydb/core/tx/replication/service/topic_reader.cpp @@ -122,7 +122,7 @@ class TRemoteTopicReader: public TActor { case NYdb::EStatus::SCHEME_ERROR: return Leave(TEvWorker::TEvGone::SCHEME_ERROR, ev->Get()->Result.GetIssues().ToOneLineString()); default: - return Leave(TEvWorker::TEvGone::UNAVAILABLE); + return Leave(TEvWorker::TEvGone::UNAVAILABLE, ev->Get()->Result.GetIssues().ToOneLineString()); } } diff --git a/ydb/core/tx/replication/service/worker.cpp b/ydb/core/tx/replication/service/worker.cpp index c9f57daba3e7..923df075d186 100644 --- a/ydb/core/tx/replication/service/worker.cpp +++ b/ydb/core/tx/replication/service/worker.cpp @@ -259,11 +259,13 @@ class TWorker: public TActorBootstrapped { void Handle(TEvWorker::TEvGone::TPtr& ev) { if (ev->Sender == Reader) { LOG_I("Reader has gone" - << ": sender# " << ev->Sender); + << ": sender# " << ev->Sender + << ": " << ev->Get()->ToString()); MaybeRecreateActor(ev, Reader); } else if (ev->Sender == Writer) { LOG_I("Writer has gone" - << ": sender# " << ev->Sender); + << ": sender# " << ev->Sender + << ": " << ev->Get()->ToString()); MaybeRecreateActor(ev, Writer); } else { LOG_W("Unknown actor has gone" From 9706794d463f7aa437fd32c1aec58f051f80245e Mon Sep 17 00:00:00 2001 From: Vasily Gerasimov Date: Fri, 8 Aug 2025 14:44:28 +0300 Subject: [PATCH 34/52] Add changelog for PR #18484 (#22560) Co-authored-by: Nikolay Perfilov --- ydb/apps/ydb/CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ydb/apps/ydb/CHANGELOG.md b/ydb/apps/ydb/CHANGELOG.md index eabff09fd0dc..142447cfba01 100644 --- a/ydb/apps/ydb/CHANGELOG.md +++ b/ydb/apps/ydb/CHANGELOG.md @@ -1,3 +1,5 @@ +* Added a new paths approach in the `ydb export s3` and `ydb import s3` commands with the new `--include` option instead of the `--item` option. +* Added support for encryption features in the `ydb export s3` and `ydb import s3` commands. ## 2.24.1 ## From 7d9a9463a1a3e655cbfcb1945021aceb5dcbbff9 Mon Sep 17 00:00:00 2001 From: Yaroslav Dynnikov Date: Fri, 8 Aug 2025 15:01:37 +0300 Subject: [PATCH 35/52] Docs for SysView ds_pdisks, ds_groups, ds_storage_pools (#22276) --- .../core/devops/observability/system-views.md | 25 +++++++++++++------ .../core/devops/observability/system-views.md | 23 ++++++++++++----- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/ydb/docs/en/core/devops/observability/system-views.md b/ydb/docs/en/core/devops/observability/system-views.md index a44f32ce17d9..f5f7d4284e82 100644 --- a/ydb/docs/en/core/devops/observability/system-views.md +++ b/ydb/docs/en/core/devops/observability/system-views.md @@ -16,10 +16,10 @@ Similar system views exist for what happens inside a specific database, they are Information about distributed storage operation is contained in several interconnected views, each responsible for describing its own entity: -* PDisk -* VSlot -* Group -* Storage Pool +* [PDisk](../../concepts/glossary.md#pdisk) +* [VSlot](../../concepts/glossary.md#slot) +* [Group](../../concepts/glossary.md#storage-group) +* [Storage Pool](../../concepts/glossary.md#storage-pool) Additionally, there is a separate view that shows statistics on the usage of group numbers in different storage pools and the growth capabilities of these pools. @@ -40,8 +40,13 @@ Additionally, there is a separate view that shows statistics on the usage of gro | TotalSize | Uint64 | | Total number of bytes on PDisk | | Status | String | | PDisk operating mode that affects its participation in group allocation (ACTIVE, INACTIVE, BROKEN, FAULTY, TO_BE_REMOVED) | | StatusChangeTimestamp | Timestamp | | Time when Status last changed; if NULL, Status has not changed since PDisk creation | -| ExpectedSlotCount | Uint32 | | Maximum number of slots (VSlot) that can be created on this PDisk | -| NumActiveSlots | Uint32 | | Number of currently running slots | +| ExpectedSlotCount | Uint32 | | Maximum number of VSlots that can be created on this PDisk. Either user-defined or inferred from InferPDiskSlotCountFromUnitSize. | +| NumActiveSlots | Uint32 | | Number of currently occupied VSlots with respect to GroupSizeInUnits of VDisks | +| SlotSizeInUnits | Uint32 | | Size of VSlot in abstract units. Either user-defined or inferred from InferPDiskSlotCountFromUnitSize. | +| DecommitStatus | String | | Status of PDisk [decommissioning](../deployment-options/manual/decommissioning.md) (DECOMMIT_NONE, DECOMMIT_PENDING, DECOMMIT_IMMINENT, DECOMMIT_REJECTED) | +| InferPDiskSlotCountFromUnitSize | Uint64 | | Size of VSlot in bytes from which ExpectedSlotCount and SlotSizeInUnits values are inferred unless user-defined. | + +The inferred values of ExpectedSlotCount and SlotSizeInUnits are defined by the formula `ExpectedSlotCount * SlotSizeInUnits = TotalSize / InferPDiskSlotCountFromUnitSize`, where `SlotSizeInUnits = 2^N` is chosen to meet `ExpectedSlotCount <= 16`. ### ds_vslots @@ -79,6 +84,11 @@ Note that the tuple (NodeId, PDiskId) forms a foreign key to the `ds_pdisks` vie | PutTabletLogLatency | Interval | | 90th percentile of PutTabletLog request execution time | | PutUserDataLatency | Interval | | 90th percentile of PutUserData request execution time | | GetFastLatency | Interval | | 90th percentile of GetFast request execution time | +| OperatingStatus | String | | Group status based on latest VDisk reports only (UNKNOWN, FULL, PARTIAL, DEGRADED, DISINTEGRATED) | +| ExpectedStatus | String | | Status based not only on operational report, but on PDisk status and plans too (UNKNOWN, FULL, PARTIAL, DEGRADED, DISINTEGRATED) | +| GroupSizeInUnits | Uint32 | | Size of the group in abstract units. In proportion to it VDisks receive storage quota. | + +The number of VSlots occupied by VDisk is defined as `ceil(VDisk.GroupSizeInUnits / PDisk.SlotSizeInUnits)`. In this view, the tuple (BoxId, StoragePoolId) forms a foreign key to the `ds_storage_pools` view. @@ -97,6 +107,7 @@ In this view, the tuple (BoxId, StoragePoolId) forms a foreign key to the `ds_st | EncryptionMode | Uint32 | | Data encryption setting for all groups (similar to ds_groups.EncryptionMode) | | SchemeshardId | Uint64 | | SchemeShard identifier of the schema object to which this storage pool belongs (currently always NULL) | | PathId | Uint64 | | Schema object node identifier within the specified SchemeShard to which this storage pool belongs | +| DefaultGroupSizeInUnits | Uint32 | | The value of GroupSizeInUnits inherited by groups when new groups are added to the pool | ### ds_storage_stats @@ -119,4 +130,4 @@ Note that AvailableGroupsToCreate shows the maximum number of groups that can be Accessing system views is more of an analytical workload. Frequent access to them in large databases will significantly consume system resources. The recommended load is no more than 1-2 RPS. -{% endnote %} \ No newline at end of file +{% endnote %} diff --git a/ydb/docs/ru/core/devops/observability/system-views.md b/ydb/docs/ru/core/devops/observability/system-views.md index d331004ece7f..0ca725c7e42d 100644 --- a/ydb/docs/ru/core/devops/observability/system-views.md +++ b/ydb/docs/ru/core/devops/observability/system-views.md @@ -16,10 +16,10 @@ Информация о работе распределённого хранилища содержится в нескольких взаимосвязанных представлениях, каждое из которых отвечает за описание своей сущности, а именно: -* PDisk -* VSlot -* Group -* Storage Pool +* [PDisk](../../concepts/glossary.md#pdisk) +* [VSlot](../../concepts/glossary.md#slot) +* [Group](../../concepts/glossary.md#storage-group) +* [Storage Pool](../../concepts/glossary.md#storage-pool) Кроме этого, есть отдельное представление, которое показывает статистику использования количества групп в разных пулах хранилища и возможности роста этих пулов. @@ -40,8 +40,13 @@ | TotalSize | Uint64 | | Общее число байт на PDisk | | Status | String | | Режим работы PDisk, который влияет на его участие в выделении групп (ACTIVE, INACTIVE, BROKEN, FAULTY, TO_BE_REMOVED) | | StatusChangeTimestamp | Timestamp | | Время, когда последний раз поменялся Status; если NULL, то Status не менялся с момента создания PDisk | -| ExpectedSlotCount | Uint32 | | Максимальное число слотов (VSlot), которое может быть создано на этом PDisk | -| NumActiveSlots | Uint32 | | Количество работающих слотов в настоящий момент | +| ExpectedSlotCount | Uint32 | | Максимальное число VSlot, которое может быть создано на этом PDisk. Либо заданное пользователем, либо вычисленное из параметра InferPDiskSlotCountFromUnitSize. | +| NumActiveSlots | Uint32 | | Количество занятых VSlot с учетом значений GroupSizeInUnits у VDisk | +| SlotSizeInUnits | Uint32 | | Размер VSlot в условных единицах. Либо заданный пользователем, либо вычисленный из параметра InferPDiskSlotCountFromUnitSize. | +| DecommitStatus | String | | Статус вывода из эксплуатации ([декоммиссии](../deployment-options/manual/decommissioning.md)) PDisk (DECOMMIT_NONE, DECOMMIT_PENDING, DECOMMIT_IMMINENT, DECOMMIT_REJECTED) | +| InferPDiskSlotCountFromUnitSize | Uint64 | | Размер VSlot в байтах, исходя из которого вычисляются ExpectedSlotCount и SlotSizeInUnits если не заданы пользователем. | + +Вычисленные значения ExpectedSlotCount и SlotSizeInUnits определяются по формуле `ExpectedSlotCount * SlotSizeInUnits = TotalSize / InferPDiskSlotCountFromUnitSize`, где `SlotSizeInUnits = 2^N` выбирается так, чтобы выполнялось условие `ExpectedSlotCount <= 16`. ### ds_vslots @@ -79,6 +84,11 @@ | PutTabletLogLatency | Interval | | 90 процентиль времени выполнения запроса PutTabletLog | | PutUserDataLatency | Interval | | 90 процентиль времени выполнения запроса PutUserData | | GetFastLatency | Interval | | 90 процентиль времени выполнения запроса GetFast | +| OperatingStatus | String | | Статус группы по последним отчетам VDisk (UNKNOWN, FULL, PARTIAL, DEGRADED, DISINTEGRATED) | +| ExpectedStatus | String | | Статус, основанный не только на операционном отчете, но и на статусе PDisk и планах (UNKNOWN, FULL, PARTIAL, DEGRADED, DISINTEGRATED) | +| GroupSizeInUnits | Uint32 | | Размер группы в абстрактных единицах. Пропорционально ему VDisk получают квоту на хранение. | + +Количество VSlot, занимаемых VDisk, определяется как `ceil(VDisk.GroupSizeInUnits / PDisk.SlotSizeInUnits)`. В данном представлении кортеж (BoxId, StoragePoolId) формирует внешний ключ к представлению `ds_storage_pools`. @@ -97,6 +107,7 @@ | EncryptionMode | Uint32 | | Настройка шифрования данных для всех групп (аналогично ds_groups.EncryptionMode) | | SchemeshardId | Uint64 | | Идентификатор SchemeShard объекта схемы, к которому относится данный пул хранения (сейчас всегда NULL) | | PathId | Uint64 | | Идентификатор узла объекта схемы внутри указанного SchemeShard, к которому относится данный пул хранения | +| DefaultGroupSizeInUnits | Uint32 | | Значение GroupSizeInUnits, наследуемое группами при добавлении новых групп в пул | ### ds_storage_stats From 900c8676ac8493afb57bf0adfc6fdde5e8110ba7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9E=D0=BB=D0=B5=D0=B3?= <150132506+iddqdex@users.noreply.github.com> Date: Fri, 8 Aug 2025 15:05:39 +0300 Subject: [PATCH 36/52] Make 'unknown' branches instead NULL (#22569) --- .../analytics/data_mart_queries/perfomance_olap_mart.sql | 6 +++--- .../data_mart_queries/perfomance_olap_suites_mart.sql | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/scripts/analytics/data_mart_queries/perfomance_olap_mart.sql b/.github/scripts/analytics/data_mart_queries/perfomance_olap_mart.sql index a48b0ef237d3..91d8f5e4cb35 100644 --- a/.github/scripts/analytics/data_mart_queries/perfomance_olap_mart.sql +++ b/.github/scripts/analytics/data_mart_queries/perfomance_olap_mart.sql @@ -18,9 +18,9 @@ $launch_times = ( SELECT launch_times_raw.*, all_suites.*, - SubString(CAST(launch_times_raw.Version AS String), 0U, FIND(CAST(launch_times_raw.Version AS String), '.')) As Branch, - SubString(CAST(launch_times_raw.CiVersion AS String), 0U, FIND(CAST(launch_times_raw.CiVersion AS String), '.')) As CiBranch, - SubString(CAST(launch_times_raw.TestToolsVersion AS String), 0U, FIND(CAST(launch_times_raw.TestToolsVersion AS String), '.')) As TestToolsBranch, + COALESCE(SubString(CAST(launch_times_raw.Version AS String), 0U, FIND(CAST(launch_times_raw.Version AS String), '.')), 'unknown') As Branch, + COALESCE(SubString(CAST(launch_times_raw.CiVersion AS String), 0U, FIND(CAST(launch_times_raw.CiVersion AS String), '.')), 'unknown') As CiBranch, + COALESCE(SubString(CAST(launch_times_raw.TestToolsVersion AS String), 0U, FIND(CAST(launch_times_raw.TestToolsVersion AS String), '.')), 'unknown') As TestToolsBranch, FROM ( SELECT Db, diff --git a/.github/scripts/analytics/data_mart_queries/perfomance_olap_suites_mart.sql b/.github/scripts/analytics/data_mart_queries/perfomance_olap_suites_mart.sql index 99f4e07fda9c..9164a6ad76dd 100644 --- a/.github/scripts/analytics/data_mart_queries/perfomance_olap_suites_mart.sql +++ b/.github/scripts/analytics/data_mart_queries/perfomance_olap_suites_mart.sql @@ -126,9 +126,9 @@ SELECT WHEN s.Db LIKE '%/row%' THEN 'row' ELSE 'other' END AS DbAlias, - SubString(CAST(s.Version AS String), 0U, FIND(CAST(s.Version AS String), '.')) As Branch, - SubString(CAST(s.CiVersion AS String), 0U, FIND(CAST(s.CiVersion AS String), '.')) As CiBranch, - SubString(CAST(s.TestToolsVersion AS String), 0U, FIND(CAST(s.TestToolsVersion AS String), '.')) As TestToolsBranch + COALESCE(SubString(CAST(s.Version AS String), 0U, FIND(CAST(s.Version AS String), '.')), 'unknown') As Branch, + COALESCE(SubString(CAST(s.CiVersion AS String), 0U, FIND(CAST(s.CiVersion AS String), '.')), 'unknown') As CiBranch, + COALESCE(SubString(CAST(s.TestToolsVersion AS String), 0U, FIND(CAST(s.TestToolsVersion AS String), '.')), 'unknown') As TestToolsBranch FROM $suites AS s LEFT JOIN $diff_tests AS d ON s.RunId = d.RunId AND s.Db = d.Db AND s.Suite = d.Suite LEFT JOIN $fail_tests AS f ON s.RunId = f.RunId AND s.Db = f.Db AND s.Suite = f.Suite From 10f796c9205608850c680e7a33c72e94e537740e Mon Sep 17 00:00:00 2001 From: vporyadke Date: Fri, 8 Aug 2025 14:07:52 +0200 Subject: [PATCH 37/52] healthcheck: fix handling of response from a disconnected node (#22556) --- ydb/core/health_check/health_check.cpp | 119 ++++++++++++++++------ ydb/core/health_check/health_check_ut.cpp | 33 ++++++ 2 files changed, 119 insertions(+), 33 deletions(-) diff --git a/ydb/core/health_check/health_check.cpp b/ydb/core/health_check/health_check.cpp index 16b70e68b6e4..486aeaf63759 100644 --- a/ydb/core/health_check/health_check.cpp +++ b/ydb/core/health_check/health_check.cpp @@ -59,6 +59,7 @@ struct std::hash { #define BLOG_CRIT(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::HEALTH, stream) #define BLOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::HEALTH, stream) +#define BLOG_TRACE(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::HEALTH, stream) namespace NKikimr::NHealthCheck { @@ -490,22 +491,23 @@ class TSelfCheckRequest : public TActorBootstrapped { TRequestResponse& operator =(const TRequestResponse&) = delete; TRequestResponse& operator =(TRequestResponse&&) = default; - void Set(std::unique_ptr&& response) { + bool Set(std::unique_ptr&& response) { + if (IsDone()) { + return false; + } constexpr bool hasErrorCheck = requires(const std::unique_ptr& r) {TSelfCheckRequest::IsSuccess(r);}; if constexpr (hasErrorCheck) { if (!TSelfCheckRequest::IsSuccess(response)) { - Error(TSelfCheckRequest::GetError(response)); - return; + return Error(TSelfCheckRequest::GetError(response)); } } - if (!IsDone()) { - Span.EndOk(); - } + Span.EndOk(); Response = std::move(response); + return true; } - void Set(TAutoPtr>&& response) { - Set(std::unique_ptr(response->Release().Release())); + bool Set(TAutoPtr>&& response) { + return Set(std::unique_ptr(response->Release().Release())); } bool Error(const TString& error) { @@ -847,7 +849,9 @@ class TSelfCheckRequest : public TActorBootstrapped { } void Handle(TEvNodeWardenStorageConfig::TPtr ev) { - NodeWardenStorageConfig->Set(std::move(ev)); + if (!NodeWardenStorageConfig->Set(std::move(ev))) { + return; + } const NKikimrBlobStorage::TStorageConfig& config = *NodeWardenStorageConfig->Get()->Config; RequestDone("TEvNodeWardenStorageConfig"); if (config.GetSelfManagementConfig().GetEnabled() && config.GetGeneration() == 0) { @@ -940,6 +944,7 @@ class TSelfCheckRequest : public TActorBootstrapped { void RequestDone(const char* name) { --Requests; + BLOG_TRACE("RequestDone(" << name << "): remaining " << Requests); if (Requests == 0) { ReplyAndPassAway(); } @@ -1159,7 +1164,9 @@ class TSelfCheckRequest : public TActorBootstrapped { } void Handle(TEvStateStorage::TEvBoardInfo::TPtr& ev) { - DatabaseBoardInfo->Set(std::move(ev)); + if (!DatabaseBoardInfo->Set(std::move(ev))) { + return; + } if (DatabaseBoardInfo->IsOk()) { TDatabaseState& database = DatabaseState[FilterDatabase]; for (const auto& entry : DatabaseBoardInfo->Get()->InfoEntries) { @@ -1177,18 +1184,34 @@ class TSelfCheckRequest : public TActorBootstrapped { auto eventId = ev->Get()->EventId; auto nodeId = ev->Get()->NodeId; switch (eventId) { - case TEvWhiteboard::EvSystemStateRequest: - NodeSystemState[nodeId] = RequestNodeWhiteboard(nodeId, {-1}); + case TEvWhiteboard::EvSystemStateRequest: { + auto& request = NodeSystemState[nodeId]; + if (!request.IsOk()) { + request = RequestNodeWhiteboard(nodeId, {-1}); + } break; - case TEvWhiteboard::EvVDiskStateRequest: - NodeVDiskState[nodeId] = RequestNodeWhiteboard(nodeId); + } + case TEvWhiteboard::EvVDiskStateRequest: { + auto& request = NodeVDiskState[nodeId]; + if (!request.IsOk()) { + request = RequestNodeWhiteboard(nodeId); + } break; - case TEvWhiteboard::EvPDiskStateRequest: - NodePDiskState[nodeId] = RequestNodeWhiteboard(nodeId); + } + case TEvWhiteboard::EvPDiskStateRequest: { + auto& request = NodePDiskState[nodeId]; + if (!request.IsOk()) { + request = RequestNodeWhiteboard(nodeId); + } break; - case TEvWhiteboard::EvBSGroupStateRequest: - NodeBSGroupState[nodeId] = RequestNodeWhiteboard(nodeId); + } + case TEvWhiteboard::EvBSGroupStateRequest: { + auto& request = NodeBSGroupState[nodeId]; + if (!request.IsOk()) { + request = RequestNodeWhiteboard(nodeId); + } break; + } default: RequestDone("unsupported event scheduled"); break; @@ -1383,7 +1406,9 @@ class TSelfCheckRequest : public TActorBootstrapped { void Handle(TEvInterconnect::TEvNodesInfo::TPtr& ev) { bool needComputeFromStaticNodes = !IsSpecificDatabaseFilter(); - NodesInfo->Set(std::move(ev)); + if (!NodesInfo->Set(std::move(ev))) { + return; + } for (const auto& ni : NodesInfo->Get()->Nodes) { MergedNodeInfo[ni.NodeId] = ∋ if (IsStaticNode(ni.NodeId) && needComputeFromStaticNodes) { @@ -1404,28 +1429,36 @@ class TSelfCheckRequest : public TActorBootstrapped { void Handle(TEvSysView::TEvGetStoragePoolsResponse::TPtr& ev) { TabletRequests.CompleteRequest(TTabletRequestsState::RequestStoragePools); - StoragePools->Set(std::move(ev)); + if (!StoragePools->Set(std::move(ev))) { + return; + } AggregateBSControllerState(); RequestDone("TEvGetStoragePoolsRequest"); } void Handle(TEvSysView::TEvGetGroupsResponse::TPtr& ev) { TabletRequests.CompleteRequest(TTabletRequestsState::RequestGroups); - Groups->Set(std::move(ev)); + if (!Groups->Set(std::move(ev))) { + return; + } AggregateBSControllerState(); RequestDone("TEvGetGroupsRequest"); } void Handle(TEvSysView::TEvGetVSlotsResponse::TPtr& ev) { TabletRequests.CompleteRequest(TTabletRequestsState::RequestVSlots); - VSlots->Set(std::move(ev)); + if (!VSlots->Set(std::move(ev))) { + return; + } AggregateBSControllerState(); RequestDone("TEvGetVSlotsRequest"); } void Handle(TEvSysView::TEvGetPDisksResponse::TPtr& ev) { TabletRequests.CompleteRequest(TTabletRequestsState::RequestPDisks); - PDisks->Set(std::move(ev)); + if (!PDisks->Set(std::move(ev))) { + return; + } AggregateBSControllerState(); RequestDone("TEvGetPDisksRequest"); } @@ -1435,7 +1468,9 @@ class TSelfCheckRequest : public TActorBootstrapped { TTabletId schemeShardId = TabletRequests.CompleteRequest(TTabletRequestsState::RequestGetPartitionStats); if (schemeShardId) { auto& partitionStatsResult(GetPartitionStatsResult[schemeShardId]); - partitionStatsResult.Set(std::move(ev)); + if (!partitionStatsResult.Set(std::move(ev))) { + return; + } if (partitionStatsResult.IsOk()) { std::map> overloadedPaths; for (const NKikimrSysView::TPartitionStatsResult& statsPath : partitionStatsResult.Get()->Record.GetStats()) { @@ -1502,7 +1537,9 @@ class TSelfCheckRequest : public TActorBootstrapped { TabletRequests.CompleteRequest(ev->Cookie); TString path = ev->Get()->GetRecord().path(); auto& response = DescribeByPath[path]; - response.Set(std::move(ev)); + if (!response.Set(std::move(ev))) { + return; + } if (response.IsOk()) { auto itOverloadedShardHint = OverloadedShardHints.find(path); if (itOverloadedShardHint != OverloadedShardHints.end()) { @@ -1547,7 +1584,9 @@ class TSelfCheckRequest : public TActorBootstrapped { void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { TRequestResponse& response = NavigateKeySet[ev->Get()->Request->Cookie]; - response.Set(std::move(ev)); + if (!response.Set(std::move(ev))) { + return; + } if (response.IsOk()) { auto domainInfo = response.Get()->Request->ResultSet.begin()->DomainInfo; TString path = CanonizePath(response.Get()->Request->ResultSet.begin()->Path); @@ -1612,7 +1651,9 @@ class TSelfCheckRequest : public TActorBootstrapped { TInstant aliveBarrier = TInstant::Now() - TDuration::Minutes(5); { auto& response = HiveNodeStats[hiveId]; - response.Set(std::move(ev)); + if (!response.Set(std::move(ev))) { + return; + } if (hiveId != RootHiveId) { for (const NKikimrHive::THiveNodeStats& hiveStat : response.Get()->Record.GetNodeStats()) { if (hiveStat.HasNodeDomain()) { @@ -1649,13 +1690,17 @@ class TSelfCheckRequest : public TActorBootstrapped { void Handle(TEvHive::TEvResponseHiveInfo::TPtr& ev) { TTabletId hiveId = TabletRequests.CompleteRequest(ev->Cookie); - HiveInfo[hiveId].Set(std::move(ev)); + if (!HiveInfo[hiveId].Set(std::move(ev))) { + return; + } RequestDone("TEvResponseHiveInfo"); } void Handle(TEvConsole::TEvListTenantsResponse::TPtr& ev) { TabletRequests.CompleteRequest(ev->Cookie); - ListTenants->Set(std::move(ev)); + if (!ListTenants->Set(std::move(ev))) { + return; + } RequestSchemeCacheNavigate(DomainPath); Ydb::Cms::ListDatabasesResult listTenantsResult; ListTenants->Get()->Record.GetResponse().operation().result().UnpackTo(&listTenantsResult); @@ -1669,7 +1714,9 @@ class TSelfCheckRequest : public TActorBootstrapped { void Handle(TEvWhiteboard::TEvSystemStateResponse::TPtr& ev) { TNodeId nodeId = ev.Get()->Cookie; auto& nodeSystemState(NodeSystemState[nodeId]); - nodeSystemState.Set(std::move(ev)); + if (!nodeSystemState.Set(std::move(ev))) { + return; + } RequestDone("TEvSystemStateResponse"); } @@ -2470,21 +2517,27 @@ class TSelfCheckRequest : public TActorBootstrapped { void Handle(TEvWhiteboard::TEvVDiskStateResponse::TPtr& ev) { TNodeId nodeId = ev.Get()->Cookie; auto& nodeVDiskState(NodeVDiskState[nodeId]); - nodeVDiskState.Set(std::move(ev)); + if (!nodeVDiskState.Set(std::move(ev))) { + return; + } RequestDone("TEvVDiskStateResponse"); } void Handle(TEvWhiteboard::TEvPDiskStateResponse::TPtr& ev) { TNodeId nodeId = ev.Get()->Cookie; auto& nodePDiskState(NodePDiskState[nodeId]); - nodePDiskState.Set(std::move(ev)); + if (!nodePDiskState.Set(std::move(ev))) { + return; + } RequestDone("TEvPDiskStateResponse"); } void Handle(TEvWhiteboard::TEvBSGroupStateResponse::TPtr& ev) { ui64 nodeId = ev.Get()->Cookie; auto& nodeBSGroupState(NodeBSGroupState[nodeId]); - nodeBSGroupState.Set(std::move(ev)); + if (!nodeBSGroupState.Set(std::move(ev))) { + return; + } RequestDone("TEvBSGroupStateResponse"); } diff --git a/ydb/core/health_check/health_check_ut.cpp b/ydb/core/health_check/health_check_ut.cpp index 3f6283657cfa..ec3b0250bf41 100644 --- a/ydb/core/health_check/health_check_ut.cpp +++ b/ydb/core/health_check/health_check_ut.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include "health_check.cpp" @@ -2722,6 +2723,38 @@ Y_UNIT_TEST_SUITE(THealthCheckTest) { UNIT_ASSERT_VALUES_EQUAL(result.self_check_result(), Ydb::Monitoring::SelfCheck::GOOD); } + Y_UNIT_TEST(TestNodeDisconnected) { + TPortManager tp; + ui16 port = tp.GetPort(2134); + ui16 grpcPort = tp.GetPort(2135); + auto settings = TServerSettings(port) + .SetNodeCount(1) + .SetUseRealThreads(false) + .SetDomainName("Root"); + TServer server(settings); + server.EnableGRpc(grpcPort); + TClient client(settings); + TTestActorRuntime& runtime = *server.GetRuntime(); + + TActorId sender = runtime.AllocateEdgeActor(); + TAutoPtr handle; + + auto observer = runtime.AddObserver([&](auto&& ev) { + auto actor = ev->Recipient; + auto nodeId = ev->Sender.NodeId(); + Cerr << "Observing " << ev->ToString() << Endl; + runtime.Send(ev.Release()); + runtime.Send(new IEventHandle(actor, sender, new TEvInterconnect::TEvNodeDisconnected(nodeId))); + }); + + TBlockEvents block(runtime); // just make it arrive later + runtime.Send(new IEventHandle(NHealthCheck::MakeHealthCheckID(), sender, new NHealthCheck::TEvSelfCheckRequest(), 0)); + runtime.DispatchEvents({}, TDuration::MilliSeconds(500)); + block.Stop().Unblock(); + auto result = runtime.GrabEdgeEvent(handle)->Result; + UNIT_ASSERT_VALUES_EQUAL(result.self_check_result(), Ydb::Monitoring::SelfCheck::GOOD); + } + Y_UNIT_TEST(CLusterNotBootstrapped) { TPortManager tp; ui16 port = tp.GetPort(2134); From a642f4082ad4c7ed33c4c28e52c867d596498da0 Mon Sep 17 00:00:00 2001 From: Nikita Vasilev Date: Fri, 8 Aug 2025 15:22:08 +0300 Subject: [PATCH 38/52] CTAS: fix make (#22563) --- ydb/tests/stress/ctas/workload/__init__.py | 17 +++++++---------- ydb/tests/stress/ya.make | 1 + 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/ydb/tests/stress/ctas/workload/__init__.py b/ydb/tests/stress/ctas/workload/__init__.py index 48158dd1c1b9..74e616674a8e 100644 --- a/ydb/tests/stress/ctas/workload/__init__.py +++ b/ydb/tests/stress/ctas/workload/__init__.py @@ -1,9 +1,7 @@ # -*- coding: utf-8 -*- import ydb import time -import random import threading -from enum import Enum from ydb.tests.stress.common.common import WorkloadBase @@ -23,7 +21,7 @@ def __init__(self, client, prefix, stop, is_olap): def get_stat(self): with self.lock: return f"Tables: {self.tables_count}, TotalRows: {self.total_rows}, LastTableRows: {self.prev_rows[0]}" - + def get_create_query(self): table_path = self.get_table_path(self.table_name + str(self.tables_count)) @@ -32,7 +30,7 @@ def get_create_query(self): params = "STORE = COLUMN" else: params = "STORE = ROW" - + if self.tables_count == 0: return f""" CREATE TABLE `{table_path}` ( @@ -51,9 +49,9 @@ def get_create_query(self): WITH ( {params} ) AS SELECT - Unwrap(CAST(1 AS Uint64)) AS Key, "{ ('0' * self.value_bytes) }" AS Value; + Unwrap(CAST(1 AS Uint64)) AS Key, "{('0' * self.value_bytes)}" AS Value; """ - + def get_ctas_query(self): table_path = self.get_table_path(self.table_name + str(self.tables_count)) prev_table_path1 = self.get_table_path(self.table_name + str(self.tables_count - 1)) @@ -66,10 +64,10 @@ def get_ctas_query(self): FROM `{prev_table_path1}` UNION ALL SELECT - Key + Unwrap(CAST({self.prev_rows[0]} AS Uint64)) AS Key, Value + Key + Unwrap(CAST({self.prev_rows[0]} AS Uint64)) AS Key, Value FROM `{prev_table_path2}` """ - + def prepare(self): create_query = self.get_create_query() self.client.query( @@ -84,7 +82,6 @@ def prepare(self): ) self.tables_count += 1 - def _loop(self): self.prepare() @@ -105,7 +102,7 @@ def _loop(self): expected = self.prev_rows[0] + self.prev_rows[1] if actual != expected: raise Exception(f"Incorrect result: expected:{expected}, actual:{actual}") - + self.prev_rows = (actual, self.prev_rows[0]) self.total_rows += actual self.tables_count += 1 diff --git a/ydb/tests/stress/ya.make b/ydb/tests/stress/ya.make index b1678e56f268..7c6b39ec63b7 100644 --- a/ydb/tests/stress/ya.make +++ b/ydb/tests/stress/ya.make @@ -1,6 +1,7 @@ RECURSE( cdc common + ctas kv log mixedpy From fe092f204b68da16ac422868cece86c57a0025ff Mon Sep 17 00:00:00 2001 From: Nikita Vasilev Date: Fri, 8 Aug 2025 15:41:07 +0300 Subject: [PATCH 39/52] EnableTempTables flag (#22475) --- .../kqp/compile_service/kqp_compile_actor.cpp | 1 + .../compile_service/kqp_compile_service.cpp | 3 ++ ydb/core/kqp/host/kqp_statement_rewrite.cpp | 2 ++ ydb/core/kqp/provider/yql_kikimr_datasink.cpp | 23 ++++++++++++++ ydb/core/kqp/provider/yql_kikimr_settings.h | 1 + .../kqp/session_actor/kqp_session_actor.cpp | 3 +- ydb/core/kqp/ut/pg/kqp_pg_ut.cpp | 5 +++ ydb/core/kqp/ut/query/kqp_query_ut.cpp | 1 + ydb/core/kqp/ut/scheme/kqp_acl_ut.cpp | 1 + ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp | 31 +++++++++++++++++++ ydb/core/protos/table_service_config.proto | 2 ++ ydb/core/sys_view/ut_common.cpp | 1 + 12 files changed, 73 insertions(+), 1 deletion(-) diff --git a/ydb/core/kqp/compile_service/kqp_compile_actor.cpp b/ydb/core/kqp/compile_service/kqp_compile_actor.cpp index 076f29c6b0e1..b595bc3355f7 100644 --- a/ydb/core/kqp/compile_service/kqp_compile_actor.cpp +++ b/ydb/core/kqp/compile_service/kqp_compile_actor.cpp @@ -661,6 +661,7 @@ void ApplyServiceConfig(TKikimrConfiguration& kqpConfig, const TTableServiceConf kqpConfig.EnableOlapPushdownProjections = serviceConfig.GetEnableOlapPushdownProjections(); kqpConfig.LangVer = serviceConfig.GetDefaultLangVer(); kqpConfig.EnableParallelUnionAllConnectionsForExtend = serviceConfig.GetEnableParallelUnionAllConnectionsForExtend(); + kqpConfig.EnableTempTablesForUser = serviceConfig.GetEnableTempTablesForUser(); if (const auto limit = serviceConfig.GetResourceManager().GetMkqlHeavyProgramMemoryLimit()) { kqpConfig._KqpYqlCombinerMemoryLimit = std::max(1_GB, limit - (limit >> 2U)); diff --git a/ydb/core/kqp/compile_service/kqp_compile_service.cpp b/ydb/core/kqp/compile_service/kqp_compile_service.cpp index 0c081b451e60..f4c8779b6f5e 100644 --- a/ydb/core/kqp/compile_service/kqp_compile_service.cpp +++ b/ydb/core/kqp/compile_service/kqp_compile_service.cpp @@ -345,6 +345,8 @@ class TKqpCompileService : public TActorBootstrapped { bool enableOlapPushdownProjections = TableServiceConfig.GetEnableOlapPushdownProjections(); + bool enableTempTablesForUser = TableServiceConfig.GetEnableTempTablesForUser(); + ui32 defaultLangVer = TableServiceConfig.GetDefaultLangVer(); TableServiceConfig.Swap(event.MutableConfig()->MutableTableServiceConfig()); @@ -385,6 +387,7 @@ class TKqpCompileService : public TActorBootstrapped { TableServiceConfig.GetEnableOlapSubstringPushdown() != enableOlapSubstringPushdown || TableServiceConfig.GetEnableIndexStreamWrite() != enableIndexStreamWrite || TableServiceConfig.GetEnableOlapPushdownProjections() != enableOlapPushdownProjections || + TableServiceConfig.GetEnableTempTablesForUser() != enableTempTablesForUser || TableServiceConfig.GetDefaultLangVer() != defaultLangVer) { diff --git a/ydb/core/kqp/host/kqp_statement_rewrite.cpp b/ydb/core/kqp/host/kqp_statement_rewrite.cpp index 01c4d3539624..1a2a6c79c0b9 100644 --- a/ydb/core/kqp/host/kqp_statement_rewrite.cpp +++ b/ydb/core/kqp/host/kqp_statement_rewrite.cpp @@ -245,6 +245,8 @@ namespace { } settingsNodes.push_back( exprCtx.NewList(pos, {exprCtx.NewAtom(pos, "temporary")})); + settingsNodes.push_back( + exprCtx.NewList(pos, {exprCtx.NewAtom(pos, "ctas")})); create = exprCtx.ReplaceNode(std::move(create), *create->Child(4), exprCtx.NewList(pos, std::move(settingsNodes))); create = exprCtx.ReplaceNode(std::move(create), *tableNameNode, exprCtx.NewAtom(pos, tmpTableName)); diff --git a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp index 2e9b52fba4bd..9ab769186bfc 100644 --- a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp @@ -890,6 +890,11 @@ class TKikimrDataSink : public TDataProviderBase ? settings.Temporary.Cast() : Build(ctx, node->Pos()).Value("false").Done(); + if (temporary.Value() == "true") { + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Creating temporary sequence data is not supported.")); + return nullptr; + } + auto existringOk = (settings.Mode.Cast().Value() == "create_if_not_exists"); return Build(ctx, node->Pos()) @@ -1297,6 +1302,24 @@ class TKikimrDataSink : public TDataProviderBase auto temporary = settings.Temporary.IsValid() ? settings.Temporary.Cast() : Build(ctx, node->Pos()).Value("false").Done(); + + const bool isCreateTableAs = std::any_of( + settings.Other.Ptr()->Children().begin(), + settings.Other.Ptr()->Children().end(), + [&](const TExprNode::TPtr& child) { + NYql::NNodes::TExprBase expr(child); + if (auto maybeTuple = expr.Maybe()) { + const auto tuple = maybeTuple.Cast(); + const auto name = tuple.Name().Value(); + return name == "ctas"; + } + return false; + }); + + if (temporary.Value() == "true" && !SessionCtx->Config().EnableTempTablesForUser && !isCreateTableAs) { + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Creating temporary table is not supported.")); + return nullptr; + } auto replaceIfExists = (settings.Mode.Cast().Value() == "create_or_replace"); auto existringOk = (settings.Mode.Cast().Value() == "create_if_not_exists"); diff --git a/ydb/core/kqp/provider/yql_kikimr_settings.h b/ydb/core/kqp/provider/yql_kikimr_settings.h index 2d1cede6b390..e27392f6e414 100644 --- a/ydb/core/kqp/provider/yql_kikimr_settings.h +++ b/ydb/core/kqp/provider/yql_kikimr_settings.h @@ -212,6 +212,7 @@ struct TKikimrConfiguration : public TKikimrSettings, public NCommon::TSettingDi bool EnableIndexStreamWrite = false; bool EnableOlapPushdownProjections = false; bool EnableParallelUnionAllConnectionsForExtend = false; + bool EnableTempTablesForUser = false; ui32 LangVer = NYql::MinLangVersion; NYql::EBackportCompatibleFeaturesMode BackportMode = NYql::EBackportCompatibleFeaturesMode::Released; diff --git a/ydb/core/kqp/session_actor/kqp_session_actor.cpp b/ydb/core/kqp/session_actor/kqp_session_actor.cpp index dcefd337362d..93fd3fdd4774 100644 --- a/ydb/core/kqp/session_actor/kqp_session_actor.cpp +++ b/ydb/core/kqp/session_actor/kqp_session_actor.cpp @@ -1440,7 +1440,6 @@ class TKqpSessionActor : public TActorBootstrapped { } SendToSchemeExecuter(tx); - ++QueryState->CurrentTx; return false; case NKqpProto::TKqpPhyTx::TYPE_DATA: @@ -1607,6 +1606,8 @@ class TKqpSessionActor : public TActorBootstrapped { temporary, TempTablesState.SessionId, QueryState->UserRequestContext, KqpTempTablesAgentActor); ExecuterId = RegisterWithSameMailbox(executerActor); + + ++QueryState->CurrentTx; } static ui32 GetResultsCount(const IKqpGateway::TExecPhysicalRequest& req) { diff --git a/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp b/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp index 1bf2d6b567cc..e7c8afd15a82 100644 --- a/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp +++ b/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp @@ -2258,6 +2258,7 @@ Y_UNIT_TEST_SUITE(KqpPg) { Y_UNIT_TEST(CreateTempTable) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings().SetKqpSettings({setting}); + serverSettings.AppConfig.MutableTableServiceConfig()->SetEnableTempTablesForUser(true); TKikimrRunner kikimr( serverSettings.SetWithSampleTables(false).SetEnableTempTables(true)); auto clientConfig = NGRpcProxy::TGRpcClientConfig(kikimr.GetEndpoint()); @@ -2305,6 +2306,7 @@ Y_UNIT_TEST_SUITE(KqpPg) { Y_UNIT_TEST(CreateTempTableSerial) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings().SetKqpSettings({setting}); + serverSettings.AppConfig.MutableTableServiceConfig()->SetEnableTempTablesForUser(true); TKikimrRunner kikimr( serverSettings.SetWithSampleTables(false).SetEnableTempTables(true)); auto clientConfig = NGRpcProxy::TGRpcClientConfig(kikimr.GetEndpoint()); @@ -2959,6 +2961,7 @@ Y_UNIT_TEST_SUITE(KqpPg) { Y_UNIT_TEST(TempTablesSessionsIsolation) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings().SetKqpSettings({setting}); + serverSettings.AppConfig.MutableTableServiceConfig()->SetEnableTempTablesForUser(true); TKikimrRunner kikimr( serverSettings.SetWithSampleTables(false).SetEnableTempTables(true)); auto clientConfig = NGRpcProxy::TGRpcClientConfig(kikimr.GetEndpoint()); @@ -3007,6 +3010,7 @@ Y_UNIT_TEST_SUITE(KqpPg) { Y_UNIT_TEST(TempTablesDrop) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings().SetKqpSettings({setting}); + serverSettings.AppConfig.MutableTableServiceConfig()->SetEnableTempTablesForUser(true); TKikimrRunner kikimr( serverSettings.SetWithSampleTables(false).SetEnableTempTables(true)); auto clientConfig = NGRpcProxy::TGRpcClientConfig(kikimr.GetEndpoint()); @@ -3079,6 +3083,7 @@ Y_UNIT_TEST_SUITE(KqpPg) { Y_UNIT_TEST(TempTablesWithCache) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings().SetKqpSettings({setting}); + serverSettings.AppConfig.MutableTableServiceConfig()->SetEnableTempTablesForUser(true); TKikimrRunner kikimr( serverSettings.SetWithSampleTables(false).SetEnableTempTables(true)); auto clientConfig = NGRpcProxy::TGRpcClientConfig(kikimr.GetEndpoint()); diff --git a/ydb/core/kqp/ut/query/kqp_query_ut.cpp b/ydb/core/kqp/ut/query/kqp_query_ut.cpp index 6b529ce9d888..61c8b5ab4d1b 100644 --- a/ydb/core/kqp/ut/query/kqp_query_ut.cpp +++ b/ydb/core/kqp/ut/query/kqp_query_ut.cpp @@ -1433,6 +1433,7 @@ Y_UNIT_TEST_SUITE(KqpQuery) { Y_UNIT_TEST(OlapTemporary) { auto settings = TKikimrSettings().SetEnableTempTables(true).SetWithSampleTables(false); + settings.AppConfig.MutableTableServiceConfig()->SetEnableTempTablesForUser(true); settings.AppConfig.MutableTableServiceConfig()->SetEnableOlapSink(true); settings.AppConfig.MutableTableServiceConfig()->SetEnableCreateTableAs(true); settings.AppConfig.MutableTableServiceConfig()->SetEnablePerStatementQueryExecution(true); diff --git a/ydb/core/kqp/ut/scheme/kqp_acl_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_acl_ut.cpp index 38ffd8c7b7e9..451216300b04 100644 --- a/ydb/core/kqp/ut/scheme/kqp_acl_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_acl_ut.cpp @@ -776,6 +776,7 @@ Y_UNIT_TEST_SUITE(KqpAcl) { auto settings = NKqp::TKikimrSettings().SetWithSampleTables(false).SetEnableTempTables(true); settings.AppConfig.MutableTableServiceConfig()->SetEnableOltpSink(true); settings.AppConfig.MutableTableServiceConfig()->SetEnableOlapSink(true); + settings.AppConfig.MutableTableServiceConfig()->SetEnableTempTablesForUser(true); TKikimrRunner kikimr(settings); if (UseAdmin) { kikimr.GetTestClient().GrantConnect("user_write@builtin"); diff --git a/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp b/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp index daa5136370f3..79e4583d474b 100644 --- a/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp +++ b/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp @@ -1885,6 +1885,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { Y_UNIT_TEST(CreateTempTable) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings().SetKqpSettings({setting}).SetAuthToken("user0@builtin"); + serverSettings.AppConfig.MutableTableServiceConfig()->SetEnableTempTablesForUser(true); TKikimrRunner kikimr( serverSettings.SetWithSampleTables(false).SetEnableTempTables(true)); auto clientConfig = NGRpcProxy::TGRpcClientConfig(kikimr.GetEndpoint()); @@ -1959,9 +1960,38 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { } } + Y_UNIT_TEST(CreateTempTableDisabled) { + auto setting = NKikimrKqp::TKqpSetting(); + auto serverSettings = TKikimrSettings().SetKqpSettings({setting}).SetAuthToken("user0@builtin"); + serverSettings.AppConfig.MutableTableServiceConfig()->SetEnableTempTablesForUser(false); + TKikimrRunner kikimr( + serverSettings.SetWithSampleTables(false).SetEnableTempTables(true)); + auto clientConfig = NGRpcProxy::TGRpcClientConfig(kikimr.GetEndpoint()); + auto client = kikimr.GetQueryClient(); + + TString SessionId; + { + auto session = client.GetSession().GetValueSync().GetSession(); + auto id = session.GetId(); + + const auto queryCreate = Q_(R"( + --!syntax_v1 + CREATE TEMP TABLE Temp ( + Key Uint64 NOT NULL, + Value String, + PRIMARY KEY (Key) + );)"); + + auto resultCreate = session.ExecuteQuery(queryCreate, NYdb::NQuery::TTxControl::NoTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(resultCreate.GetStatus(), NYdb::EStatus::GENERIC_ERROR, resultCreate.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS_C(resultCreate.GetIssues().ToString(), "Creating temporary table is not supported", resultCreate.GetIssues().ToString()); + } + } + Y_UNIT_TEST(AlterTempTable) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings().SetKqpSettings({setting}); + serverSettings.AppConfig.MutableTableServiceConfig()->SetEnableTempTablesForUser(true); TKikimrRunner kikimr( serverSettings.SetWithSampleTables(false).SetEnableTempTables(true)); auto clientConfig = NGRpcProxy::TGRpcClientConfig(kikimr.GetEndpoint()); @@ -2121,6 +2151,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { Y_UNIT_TEST(TempTablesDrop) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings().SetKqpSettings({setting}); + serverSettings.AppConfig.MutableTableServiceConfig()->SetEnableTempTablesForUser(true); TKikimrRunner kikimr( serverSettings.SetWithSampleTables(false).SetEnableTempTables(true)); auto clientConfig = NGRpcProxy::TGRpcClientConfig(kikimr.GetEndpoint()); diff --git a/ydb/core/protos/table_service_config.proto b/ydb/core/protos/table_service_config.proto index 4fa06729e26f..2a4979129043 100644 --- a/ydb/core/protos/table_service_config.proto +++ b/ydb/core/protos/table_service_config.proto @@ -411,4 +411,6 @@ message TTableServiceConfig { optional EHashKind DefaultHashShuffleFuncType = 93 [ default = HASH_V2 ]; optional bool EnableParallelUnionAllConnectionsForExtend = 94 [ default = false ]; + + optional bool EnableTempTablesForUser = 95 [default = false]; }; diff --git a/ydb/core/sys_view/ut_common.cpp b/ydb/core/sys_view/ut_common.cpp index 88072815d14c..5da04698877c 100644 --- a/ydb/core/sys_view/ut_common.cpp +++ b/ydb/core/sys_view/ut_common.cpp @@ -67,6 +67,7 @@ TTestEnv::TTestEnv(ui32 staticNodes, ui32 dynamicNodes, const TTestEnvSettings& *appConfig.MutableFeatureFlags() = Settings->FeatureFlags; appConfig.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage"); appConfig.MutableColumnShardConfig()->SetAlterObjectEnabled(settings.AlterObjectEnabled); + appConfig.MutableTableServiceConfig()->SetEnableTempTablesForUser(true); Settings->SetAppConfig(appConfig); for (ui32 i : xrange(settings.StoragePools)) { From f233f13c298deab787612642874eeb99d960a448 Mon Sep 17 00:00:00 2001 From: Andrey Zaspa Date: Fri, 8 Aug 2025 16:49:58 +0400 Subject: [PATCH 40/52] Assign '+inf' to current version value in compatibility tests (#22575) --- ydb/tests/library/compatibility/fixtures.py | 23 ++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/ydb/tests/library/compatibility/fixtures.py b/ydb/tests/library/compatibility/fixtures.py index 1ff93e4bce99..b46e1e5db9e2 100644 --- a/ydb/tests/library/compatibility/fixtures.py +++ b/ydb/tests/library/compatibility/fixtures.py @@ -13,14 +13,23 @@ def string_version_to_tuple(s): result = [] s = s.replace('.', '-') - for idx, elem in enumerate(s.split("-")): - # skipping 'stable' in stable-25-1-1 version - if idx == 0 and elem == 'stable': - continue + version_components = s.split("-") + for idx, elem in enumerate(version_components): + if idx == 0: + # skipping 'stable' in stable-25-1-1 version + if elem == 'stable': + continue + elif elem == 'current': + result.append(float('+inf')) + continue + elif idx == len(version_components) - 1: + # skipping 'hotfix' in stable-24-4-4-hotfix version + if elem == 'hotfix': + continue try: result.append(int(elem)) except ValueError: - result.append(float("NaN")) + result.append(float('NaN')) return tuple(result) @@ -37,12 +46,12 @@ def string_version_to_tuple(s): inter_stable_version = None init_stable_version = None -inter_stable_name = "intermediate" +inter_stable_name = 'intermediate' if inter_stable_binary_path is not None: # in import_test yatest.common.binary_path returns None with open(yatest.common.binary_path("ydb/tests/library/compatibility/binaries/ydbd-inter-name")) as f: inter_stable_name = f.read().strip() inter_stable_version = string_version_to_tuple(inter_stable_name) -init_stable_name = "initial" +init_stable_name = 'initial' if init_stable_binary_path: # in import_test yatest.common.binary_path returns None with open(yatest.common.binary_path("ydb/tests/library/compatibility/binaries/ydbd-init-name")) as f: init_stable_name = f.read().strip() From 879cdf42e8e4b8eb5574376d877dc9373780470e Mon Sep 17 00:00:00 2001 From: YDBot Date: Fri, 8 Aug 2025 14:55:00 +0200 Subject: [PATCH 41/52] Update muted_ya.txt in main (#22465) --- .github/config/muted_ya.txt | 54 +++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt index adc52120c2dd..9b9729118bf4 100644 --- a/.github/config/muted_ya.txt +++ b/.github/config/muted_ya.txt @@ -1,10 +1,13 @@ ydb/apps/ydb/ut YdbWorkloadTransferTopicToTable.Default_Run ydb/core/blobstorage/dsproxy/ut TBlobStorageProxySequenceTest.TestBlock42PutWithChangingSlowDisk +ydb/core/blobstorage/dsproxy/ut TDSProxyPutTest.TestBlock42PutStatusOkWith_1_1_VdiskErrors +ydb/core/blobstorage/dsproxy/ut TDSProxyPutTest.TestBlock42PutStatusOkWith_2_0_VdiskErrors ydb/core/blobstorage/pdisk/ut TPDiskTest.FailedToFormatDiskInfoUpdate ydb/core/blobstorage/pdisk/ut TPDiskTest.PDiskSlotSizeInUnits ydb/core/blobstorage/pdisk/ut TPDiskTest.TestStartEncryptedOrPlainAndRestart ydb/core/blobstorage/ut_blobstorage BlobPatching.PatchBlock42 ydb/core/blobstorage/ut_blobstorage Defragmentation.DoesItWork +ydb/core/blobstorage/ut_blobstorage GroupReconfiguration.BsControllerDoesNotDisableGroup ydb/core/blobstorage/ut_blobstorage GroupReconfiguration.BsControllerDoesNotDisableGroupNoRequestsToNodesWVDisks ydb/core/blobstorage/ut_blobstorage unittest.[*/*] chunk ydb/core/blobstorage/ut_blobstorage/ut_group_reconfiguration GroupReconfigurationRace.Test_block42 @@ -22,6 +25,7 @@ ydb/core/blobstorage/ut_blobstorage/ut_scrub unittest.[*/*] chunk ydb/core/blobstorage/ut_blobstorage/ut_statestorage TStateStorageRingGroupState.TestProxyConfigMismatch ydb/core/blobstorage/ut_vdisk TBsLocalRecovery.WriteRestartReadHugeDecreased ydb/core/blobstorage/ut_vdisk TBsVDiskGC.GCPutKeepBarrierSync +ydb/core/blobstorage/ut_vdisk TBsVDiskGC.TGCManyVPutsCompactGCAllTest ydb/core/blobstorage/ut_vdisk TBsVDiskManyPutGet.ManyPutRangeGetCompactionIndexOnly ydb/core/blobstorage/ut_vdisk unittest.[*/*] chunk ydb/core/client/ut TObjectStorageListingTest.TestSkipShards @@ -34,9 +38,11 @@ ydb/core/keyvalue/ut_trace TKeyValueTracingTest.ReadSmall ydb/core/keyvalue/ut_trace TKeyValueTracingTest.WriteHuge ydb/core/keyvalue/ut_trace TKeyValueTracingTest.WriteSmall ydb/core/kqp/proxy_service/ut TestScriptExecutionsUtils.TestRetryLimiter +ydb/core/kqp/ut/data_integrity KqpDataIntegrityTrails.UpsertEvWriteQueryService+isOlap+useOltpSink ydb/core/kqp/ut/data_integrity KqpDataIntegrityTrails.UpsertEvWriteQueryService+isOlap-useOltpSink ydb/core/kqp/ut/data_integrity KqpDataIntegrityTrails.UpsertEvWriteQueryService-isOlap+useOltpSink ydb/core/kqp/ut/data_integrity KqpDataIntegrityTrails.UpsertEvWriteQueryService-isOlap-useOltpSink +ydb/core/kqp/ut/federated_query/datastreams KqpFederatedQueryDatastreams.RestoreScriptPhysicalGraphOnRetry ydb/core/kqp/ut/federated_query/large_results KqpScriptExecResults.ExecuteScriptWithLargeFile ydb/core/kqp/ut/federated_query/large_results KqpScriptExecResults.ExecuteScriptWithThinFile ydb/core/kqp/ut/federated_query/s3 KqpFederatedQuery.ExecuteScriptWithExternalTableResolve @@ -47,17 +53,21 @@ ydb/core/kqp/ut/olap KqpOlap.ManyColumnShardsWithRestartsWithResolving ydb/core/kqp/ut/olap KqpOlapOptimizer.OptimizationAfterDeletion ydb/core/kqp/ut/opt KqpKv.ReadRows_TimeoutCancelsReads ydb/core/kqp/ut/query KqpAnalyze.AnalyzeTable+ColumnStore -ydb/core/kqp/ut/query KqpAnalyze.AnalyzeTable-ColumnStore +ydb/core/kqp/ut/query KqpStats.CreateTableAsStats+IsOlap ydb/core/kqp/ut/query KqpStats.DeferredEffects+UseSink ydb/core/kqp/ut/query KqpStats.SysViewClientLost -ydb/core/kqp/ut/query KqpStats.CreateTableAsStats+IsOlap ydb/core/kqp/ut/scan KqpScan.StreamExecuteScanQueryClientTimeoutBruteForce +ydb/core/kqp/ut/scheme KqpAcl.AclCreateTableAs+IsOlap+UseAdmin +ydb/core/kqp/ut/scheme KqpAcl.AclCreateTableAs+IsOlap-UseAdmin +ydb/core/kqp/ut/scheme KqpAcl.AclCreateTableAs-IsOlap+UseAdmin ydb/core/kqp/ut/scheme KqpAcl.AlterDatabasePrivilegesRequiredToChangeSchemeLimits-AsClusterAdmin ydb/core/kqp/ut/scheme KqpOlapScheme.AddPgColumnWithStore ydb/core/kqp/ut/scheme KqpScheme.CreateDropTableMultipleTime +ydb/core/kqp/ut/scheme KqpScheme.ModifySysViewDirPermissions ydb/core/kqp/ut/service KqpService.CloseSessionsWithLoad ydb/core/kqp/ut/service unittest.[*/*] chunk ydb/core/kqp/ut/yql KqpScripting.StreamExecuteYqlScriptScanOperationTmeoutBruteForce +ydb/core/kqp/workload_service/ut KqpWorkloadServiceDistributed.TestDistributedLessConcurrentQueryLimit ydb/core/quoter/ut QuoterWithKesusTest.PrefetchCoefficient ydb/core/statistics/aggregator/ut AnalyzeColumnshard.AnalyzeRebootColumnShard ydb/core/sys_view/ut SystemView.AuthGroupMembers_Access @@ -73,20 +83,22 @@ ydb/core/transfer/ut/large TransferLarge.Transfer1KM_1KP_RowTable_TopicAutoParti ydb/core/tx/conveyor_composite/ut CompositeConveyorTests.TestUniformDistribution ydb/core/tx/schemeshard/ut_export_reboots_s3 TExportToS3WithRebootsTests.ShouldSucceedOnMultiShardTable ydb/core/tx/schemeshard/ut_export_reboots_s3 unittest.[*/*] chunk +ydb/core/tx/schemeshard/ut_index_build_reboots IndexBuildTestReboots.BaseCaseWithDataColumns +ydb/core/tx/schemeshard/ut_index_build_reboots unittest.[*/*] chunk ydb/core/tx/schemeshard/ut_login_large TSchemeShardLoginLargeTest.RemoveLogin_Many ydb/core/tx/schemeshard/ut_restore TImportTests.ShouldSucceedOnManyTables ydb/core/tx/schemeshard/ut_shred_reboots ShredReboots.SimpleShredTest ydb/core/tx/schemeshard/ut_shred_reboots unittest.[*/*] chunk ydb/core/tx/schemeshard/ut_split_merge_reboots TSchemeShardSplitTestReboots.SplitWithTxInFlightWithReboots ydb/core/tx/schemeshard/ut_system_names unittest.[*/*] chunk -ydb/core/tx/schemeshard/ut_vector_index_build_reboots VectorIndexBuildTestReboots.BaseCase+Prefixed[PipeResets] +ydb/core/tx/schemeshard/ut_vector_index_build_reboots VectorIndexBuildTestReboots.BaseCase+Prefixed[TabletReboots] ydb/core/tx/schemeshard/ut_vector_index_build_reboots VectorIndexBuildTestReboots.BaseCase-Prefixed[TabletReboots] ydb/core/tx/schemeshard/ut_vector_index_build_reboots unittest.[*/*] chunk ydb/core/tx/tiering/ut ColumnShardTiers.TTLUsage -ydb/core/tx/tx_proxy/ut_ext_tenant TExtSubDomainTest.CreateTableInsideThenStopTenantAndForceDeleteSubDomain-AlterDatabaseCreateHiveFirst-false ydb/core/tx/tx_proxy/ut_schemereq SchemeReqAccess.AlterLoginProtect-RootDB-Auth-LocalUser-CreateUser-23 ydb/core/viewer/tests test.py.test_topic_data ydb/core/viewer/tests test.py.test_transfer_describe +ydb/library/actors/interconnect/ut DynamicProxy.RaceCheck10 ydb/library/actors/interconnect/ut_fat InterconnectUnstableConnection.InterconnectTestWithProxyTlsReestablishWithXdc ydb/library/actors/interconnect/ut_fat InterconnectZcLocalOp.ZcDisabledAfterHiddenCopy ydb/library/actors/interconnect/ut_fat InterconnectZcLocalOp.ZcIsDisabledByDefault @@ -102,22 +114,24 @@ ydb/library/yql/tests/sql/dq_file/part3 pytest.[*/*] chunk ydb/library/yql/tests/sql/dq_file/part3 test.py.test[pg-tpch-q20-default.txt-ForceBlocks] ydb/library/yql/tests/sql/dq_file/part9 pytest.[*/*] chunk ydb/library/yql/tests/sql/dq_file/part9 test.py.test[pg-tpch-q08-default.txt-ForceBlocks] -ydb/mvp/oidc_proxy/ut Mvp.OidcWhoamiForward307 ydb/mvp/oidc_proxy/ut Mvp.OidcYandexIgnoresWhoamiExtention ydb/mvp/oidc_proxy/ut unittest.sole chunk ydb/public/sdk/cpp/src/client/federated_topic/ut BasicUsage.PropagateSessionClosed ydb/public/sdk/cpp/src/client/federated_topic/ut unittest.[*/*] chunk +ydb/public/sdk/cpp/src/client/topic/ut TxUsage.Sinks_Oltp_WriteToTopicAndTable_6_Query ydb/public/sdk/cpp/src/client/topic/ut TxUsage.Sinks_Oltp_WriteToTopicAndTable_6_Table ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut TxUsage.Sinks_Olap_WriteToTopicAndTable_4_Query ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut TxUsage.Sinks_Olap_WriteToTopicAndTable_4_Table ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut TxUsage.Sinks_Oltp_WriteToTopicAndTable_6_Query ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut TxUsage.Sinks_Oltp_WriteToTopicAndTable_6_Table ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut TxUsage.Transactions_Conflict_On_SeqNo_Query +ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut TxUsage.Transactions_Conflict_On_SeqNo_Table ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut TxUsage.WriteToTopic_Demo_23_RestartAfterCommit_Table ydb/public/sdk/cpp/tests/integration/sessions_pool YdbSdkSessionsPool.StressTestSync/0 ydb/public/sdk/cpp/tests/integration/sessions_pool YdbSdkSessionsPool.StressTestSync/1 ydb/public/sdk/cpp/tests/integration/topic/with_direct_read DirectReadWithClient.ManyMessages ydb/public/sdk/cpp/tests/integration/topic/with_direct_read gtest.[*/*] chunk +ydb/services/keyvalue/ut KeyValueGRPCService.SimpleWriteReadWithGetChannelStatus ydb/services/persqueue_v1/ut TPersQueueTest.Cache ydb/services/persqueue_v1/ut TPersQueueTest.CacheHead ydb/services/persqueue_v1/ut TPersQueueTest.LOGBROKER_7820 @@ -129,33 +143,21 @@ ydb/services/ydb/backup_ut CommonEncryptionRequirementsTest.CommonEncryptionRequ ydb/services/ydb/ut TAuthenticationWithSqlExecution.CreateAlterUserWithHash ydb/services/ydb/ut TGRpcAuthentication.NoConnectRights ydb/services/ydb/ut TGRpcAuthentication.NoDescribeRights -ydb/services/ydb/ut TGRpcAuthentication.ValidCredentials ydb/services/ydb/ut YdbLogStore.AlterLogTable ydb/tests/compatibility py3test.[test_transfer.py */*] chunk -ydb/tests/compatibility test_compatibility.py.TestCompatibility.test_simple[restart_current_to_stable-25-1-3-row] ydb/tests/compatibility test_compatibility.py.TestCompatibility.test_simple[restart_prestable-25-2_to_prestable-25-2-row] ydb/tests/compatibility test_compatibility.py.TestCompatibility.test_simple[restart_stable-24-4-4-hotfix_to_stable-25-1-3-column] -ydb/tests/compatibility test_compatibility.py.TestCompatibility.test_simple[restart_stable-25-1-3_to_current-column] ydb/tests/compatibility test_compatibility.py.TestCompatibility.test_simple[restart_stable-25-1-3_to_stable-25-1-3-row] -ydb/tests/compatibility test_compatibility.py.TestCompatibility.test_tpch1[restart_current_to_current-column-date64] -ydb/tests/compatibility test_compatibility.py.TestCompatibility.test_tpch1[restart_current_to_current-column] -ydb/tests/compatibility test_compatibility.py.TestCompatibility.test_tpch1[restart_current_to_stable-25-1-3-row] ydb/tests/compatibility test_compatibility.py.TestCompatibility.test_tpch1[restart_stable-24-4-4-hotfix_to_stable-25-1-3-column-date64] ydb/tests/compatibility test_compatibility.py.TestCompatibility.test_tpch1[restart_stable-24-4-4-hotfix_to_stable-25-1-3-column] ydb/tests/compatibility test_compatibility.py.TestCompatibility.test_tpch1[restart_stable-25-1-3_to_stable-24-4-4-hotfix-column-date64] ydb/tests/compatibility test_compatibility.py.TestCompatibility.test_tpch1[restart_stable-25-1-3_to_stable-24-4-4-hotfix-column] -ydb/tests/compatibility test_followers.py.TestFollowersCompatibility.test_followers_compatability[mixed_current] -ydb/tests/compatibility test_followers.py.TestFollowersCompatibility.test_followers_compatability[mixed_current_and_stable-25-1-3] -ydb/tests/compatibility test_followers.py.TestSecondaryIndexFollowers.test_secondary_index_followers[rolling_stable-25-1-3_to_current-True] -ydb/tests/compatibility test_node_broker_delta_protocol.py.TestNodeBrokerDeltaProtocolRestartToAnotherVersion.test[restart_stable-25-1-3_to_current] -ydb/tests/compatibility test_node_broker_delta_protocol.py.TestNodeBrokerDeltaProtocolRestartToAnotherVersion.test[restart_stable-25-1-3_to_stable-25-1-3] +ydb/tests/compatibility test_ctas.py.TestCTASOperations.test_ctas_olap[rolling_stable-25-1-3_to_current] +ydb/tests/compatibility test_ctas.py.TestCTASOperations.test_ctas_oltp[rolling_stable-25-1-3_to_current] ydb/tests/compatibility test_rolling.py.TestRolling.test_kv[rolling_prestable-25-2_to_main-column] ydb/tests/compatibility test_rolling.py.TestRolling.test_kv[rolling_stable-25-1-3_to_current-column] -ydb/tests/compatibility test_rolling.py.TestRolling.test_kv[rolling_stable-25-1-3_to_current-row] ydb/tests/compatibility test_stress.py.TestStress.test_kv[mixed_current-column] ydb/tests/compatibility test_stress.py.TestStress.test_kv[mixed_main-column] -ydb/tests/compatibility test_stress.py.TestStress.test_kv[mixed_stable-25-1-3_and_stable-24-4-4-hotfix-column] -ydb/tests/compatibility test_stress.py.TestStress.test_log[mixed_current-column] ydb/tests/compatibility test_stress.py.TestStress.test_log[mixed_current_and_stable-25-1-3-column] ydb/tests/compatibility test_stress.py.TestStress.test_log[mixed_prestable-25-2-column] ydb/tests/compatibility test_stress.py.TestStress.test_log[mixed_stable-25-1-3_and_stable-24-4-4-hotfix-column] @@ -185,8 +187,10 @@ ydb/tests/compatibility/s3_backups test_export_import_s3.py.TestExportImportS3.t ydb/tests/compatibility/s3_backups test_export_import_s3.py.TestExportImportS3.test_topics[mixed_prestable-25-2] ydb/tests/datashard/vector_index/large test_vector_index.py.TestVectorIndex.test_vector_index ydb/tests/datashard/vector_index/large test_vector_index_large_levels_and_clusters.py.TestVectorIndexLargeLevelsAndClusters.test_vecot_index_large_levels_and_clusters -ydb/tests/fq/mem_alloc test_alloc_default.py.TestAlloc.test_alloc_and_free[kikimr0] ydb/tests/fq/mem_alloc test_scheduling.py.TestSchedule.test_skip_busy[kikimr0] +ydb/tests/fq/multi_plane py3test.[test_dispatch.py] chunk +ydb/tests/fq/multi_plane test_dispatch.py.TestMapping.test_mapping +ydb/tests/fq/pq_async_io/ut TDqPqReadActorTest.TestSaveLoadPqRead ydb/tests/fq/yds test_2_selects_limit.py.TestSelectLimit.test_select_same[v1] ydb/tests/fq/yds test_2_selects_limit.py.TestSelectLimit.test_select_sequence[v1] ydb/tests/fq/yds test_mem_alloc.py.TestMemAlloc.test_hop_alloc[v1] @@ -212,10 +216,9 @@ ydb/tests/functional/restarts test_restarts.py.TestRestartSingleBlock42.test_res ydb/tests/functional/serverless test_serverless.py.test_database_with_disk_quotas[enable_alter_database_create_hive_first--false] ydb/tests/functional/serverless test_serverless.py.test_database_with_disk_quotas[enable_alter_database_create_hive_first--true] ydb/tests/functional/sqs/cloud test_yandex_cloud_mode.py.TestSqsYandexCloudMode.test_dlq_mechanics_in_cloud[tables_format_v0-tables_format_v0-fifo] -ydb/tests/functional/sqs/cloud test_yandex_cloud_mode.py.TestSqsYandexCloudMode.test_dlq_mechanics_in_cloud[tables_format_v0-tables_format_v0-std] ydb/tests/functional/sqs/cloud test_yandex_cloud_mode.py.TestSqsYandexCloudMode.test_dlq_mechanics_in_cloud[tables_format_v0-tables_format_v1-fifo] ydb/tests/functional/sqs/cloud test_yandex_cloud_mode.py.TestSqsYandexCloudMode.test_dlq_mechanics_in_cloud[tables_format_v0-tables_format_v1-std] -ydb/tests/functional/sqs/cloud test_yandex_cloud_mode.py.TestSqsYandexCloudMode.test_dlq_mechanics_in_cloud[tables_format_v1-tables_format_v1-fifo] +ydb/tests/functional/sqs/cloud test_yandex_cloud_mode.py.TestSqsYandexCloudMode.test_fifo_groups_with_dlq_in_cloud[tables_format_v0] ydb/tests/functional/tenants test_dynamic_tenants.py.test_create_and_drop_tenants[enable_alter_database_create_hive_first--false] ydb/tests/functional/tenants test_dynamic_tenants.py.test_create_and_drop_tenants[enable_alter_database_create_hive_first--true] ydb/tests/functional/tenants test_dynamic_tenants.py.test_create_and_drop_the_same_tenant2[enable_alter_database_create_hive_first--false] @@ -252,10 +255,13 @@ ydb/tests/functional/tpc/large test_tpch_spilling.py.TestTpchSpillingS10.test_tp ydb/tests/functional/tpc/large test_tpch_spilling.py.TestTpchSpillingS10.test_tpch[8] ydb/tests/functional/tpc/large test_tpch_spilling.py.TestTpchSpillingS10.test_tpch[9] ydb/tests/functional/tpc/medium py3test.[test_upload.py] chunk +ydb/tests/functional/tpc/medium test_upload.py.TestUploadTpchS1.test_tpch[20] ydb/tests/functional/tpc/medium test_upload.py.TestUploadTpchS1.test_tpch[21] +ydb/tests/functional/tpc/medium test_upload.py.TestUploadTpchS1.test_tpch[22] +ydb/tests/functional/ydb_cli test_ydb_backup.py.TestDatabaseBackup.test_database_backup ydb/tests/functional/ydb_cli test_ydb_backup.py.TestDatabaseBackupRestore.test_database_backup_restore -ydb/tests/olap/s3_import test_tpch_import.py.TestS3TpchImport.test_import_and_export ydb/tests/olap/scenario test_alter_tiering.py.TestAlterTiering.test[many_tables] +ydb/tests/olap/scenario test_alter_tiering.py.TestAlterTiering.test_multi[many_tables] ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[Test64BitErrorChecking] ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestArrayScanBackend] ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestArrayValueBackend] @@ -333,5 +339,7 @@ ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generate ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestTimestampWithTimeZone] ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestTxOptions] ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py.test_pg_generated[TestXactMultiStmt] +ydb/tests/postgres_integrations/go-libpq py3test.[docker_wrapper_test.py] chunk ydb/tests/stress/mixedpy py3test.sole chunk +ydb/tests/stress/topic/tests test_workload_topic.py.TestYdbTopicWorkload.test ydb/tools/stress_tool/ut TDeviceTestTool.PDiskTestLogWrite From 3623f275c0b664ce4a81517eb571a49708d2e8c4 Mon Sep 17 00:00:00 2001 From: Nikolay Shestakov Date: Fri, 8 Aug 2025 18:04:29 +0500 Subject: [PATCH 42/52] Fixed describe of the paused state of the transfer (#22553) --- .../transfer/ut/functional/transfer_ut.cpp | 35 +++++++++++++++++++ .../controller/tx_describe_replication.cpp | 14 +++++--- .../ydb_cli/commands/ydb_service_scheme.cpp | 1 + 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/ydb/core/transfer/ut/functional/transfer_ut.cpp b/ydb/core/transfer/ut/functional/transfer_ut.cpp index e107b241ab62..4a3d11b4fdfa 100644 --- a/ydb/core/transfer/ut/functional/transfer_ut.cpp +++ b/ydb/core/transfer/ut/functional/transfer_ut.cpp @@ -570,6 +570,41 @@ Y_UNIT_TEST_SUITE(Transfer) testCase.DropTopic(); } + Y_UNIT_TEST(DescribeTransferWithErrorTopicNotFound) + { + MainTestCase testCase; + testCase.CreateTable(R"( + CREATE TABLE `%s` ( + Key Uint64 NOT NULL, + Message Utf8, + PRIMARY KEY (Key) + ) WITH ( + STORE = %s + ); + )"); + + testCase.CreateTransfer(R"( + $l = ($x) -> { + return [ + <| + Key:CAST($x._offset AS Uint64), + Message:CAST($x._data AS Utf8) + |> + ]; + }; + )", MainTestCase::CreateTransferSettings::WithLocalTopic(false)); + + testCase.CheckTransferStateError("Path not found"); + + auto d = testCase.DescribeTransfer(); + UNIT_ASSERT_VALUES_EQUAL(d.GetTransferDescription().GetState(), TTransferDescription::EState::Error); + UNIT_ASSERT_VALUES_EQUAL(d.GetTransferDescription().GetSrcPath(), TStringBuilder() << "local/" << testCase.TopicName); + UNIT_ASSERT_VALUES_EQUAL(d.GetTransferDescription().GetDstPath(), TStringBuilder() << "/local/" << testCase.TableName); + + testCase.DropTransfer(); + testCase.DropTable(); + } + Y_UNIT_TEST(CustomConsumer) { MainTestCase testCase; diff --git a/ydb/core/tx/replication/controller/tx_describe_replication.cpp b/ydb/core/tx/replication/controller/tx_describe_replication.cpp index 79ebedaa330d..94b4342b4a3c 100644 --- a/ydb/core/tx/replication/controller/tx_describe_replication.cpp +++ b/ydb/core/tx/replication/controller/tx_describe_replication.cpp @@ -198,6 +198,16 @@ class TController::TTxDescribeReplication: public TTxBase { } bool isTransfer = replication->GetConfig().HasTransferSpecific(); + if (isTransfer) { + auto& specific = replication->GetConfig().GetTransferSpecific(); + + auto& transferSpecific = *Result->Record.MutableTransferSpecific(); + transferSpecific.MutableTarget()->SetSrcPath(specific.GetTarget().GetSrcPath()); + transferSpecific.MutableTarget()->SetDstPath(specific.GetTarget().GetDstPath()); + transferSpecific.MutableTarget()->SetTransformLambda(specific.GetTarget().GetTransformLambda()); + transferSpecific.MutableBatching()->CopyFrom(specific.GetBatching()); + } + for (ui64 tid = 0; tid < replication->GetNextTargetId(); ++tid) { auto* target = replication->FindTarget(tid); if (!target) { @@ -209,11 +219,7 @@ class TController::TTxDescribeReplication: public TTxBase { auto& specific = replication->GetConfig().GetTransferSpecific(); auto& transferSpecific = *Result->Record.MutableTransferSpecific(); - transferSpecific.MutableTarget()->SetSrcPath(target->GetSrcPath()); - transferSpecific.MutableTarget()->SetDstPath(target->GetDstPath()); transferSpecific.MutableTarget()->SetConsumerName(target->GetStreamConsumerName() ? target->GetStreamConsumerName() : specific.GetTarget().GetConsumerName()); - transferSpecific.MutableTarget()->SetTransformLambda(specific.GetTarget().GetTransformLambda()); - transferSpecific.MutableBatching()->CopyFrom(specific.GetBatching()); } auto& item = *Result->Record.AddTargets(); diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp index b4cd0c06957b..6c476aa1d294 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp @@ -613,6 +613,7 @@ int TCommandDescribe::PrintTransferResponsePretty(const NYdb::NReplication::TDes Cout << Endl << "State: "; switch (desc.GetState()) { case NReplication::TTransferDescription::EState::Running: + case NReplication::TTransferDescription::EState::Paused: Cout << desc.GetState(); break; case NReplication::TTransferDescription::EState::Error: From 604a3ecfc4f424dfe1d362b641c2ed50174f5350 Mon Sep 17 00:00:00 2001 From: qyryq Date: Fri, 8 Aug 2025 16:07:39 +0300 Subject: [PATCH 43/52] Topic SDK: fix in direct reader (#22539) --- .../sdk/cpp/src/client/topic/impl/direct_reader.cpp | 13 +++++++------ .../sdk/cpp/tests/integration/topic/direct_read.cpp | 7 +++++++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/direct_reader.cpp b/ydb/public/sdk/cpp/src/client/topic/impl/direct_reader.cpp index 623535296d47..42cabfa2303b 100644 --- a/ydb/public/sdk/cpp/src/client/topic/impl/direct_reader.cpp +++ b/ydb/public/sdk/cpp/src/client/topic/impl/direct_reader.cpp @@ -731,12 +731,6 @@ void TDirectReadSession::OnReadDoneImpl(Ydb::Topic::StreamDirectReadMessage::Sta auto partitionSessionId = response.partition_session_id(); auto it = PartitionSessions.find(partitionSessionId); - if (it->second.Location.GetGeneration() != response.generation()) { - LOG_LAZY(Log, TLOG_DEBUG, GetLogPrefix() << "Got StartDirectReadPartitionSessionResponse for wrong generation " - << "(expected " << it->second.Location.GetGeneration() - << ", got " << response.generation() << ") partition_session_id=" << partitionSessionId); - return; - } if (it == PartitionSessions.end()) { // We could get a StopPartitionSessionRequest from server before processing this response. @@ -744,6 +738,13 @@ void TDirectReadSession::OnReadDoneImpl(Ydb::Topic::StreamDirectReadMessage::Sta return; } + if (it->second.Location.GetGeneration() != response.generation()) { + LOG_LAZY(Log, TLOG_DEBUG, GetLogPrefix() << "Got StartDirectReadPartitionSessionResponse for wrong generation " + << "(expected " << it->second.Location.GetGeneration() + << ", got " << response.generation() << ") partition_session_id=" << partitionSessionId); + return; + } + auto& partitionSession = it->second; auto transitioned = partitionSession.TransitionTo(TDirectReadPartitionSession::EState::WORKING); diff --git a/ydb/public/sdk/cpp/tests/integration/topic/direct_read.cpp b/ydb/public/sdk/cpp/tests/integration/topic/direct_read.cpp index d152365e0ad5..40846ca6a6d4 100644 --- a/ydb/public/sdk/cpp/tests/integration/topic/direct_read.cpp +++ b/ydb/public/sdk/cpp/tests/integration/topic/direct_read.cpp @@ -1292,6 +1292,13 @@ TEST_F(DirectReadWithControlSession, StopPartitionSession) { } } + { + // Send a StartDirectReadPartitionSessionResponse with a random partition session id, + // the session should ignore it. + auto r = TMockDirectReadSessionProcessor::TServerReadInfo(); + setup.AddDirectReadResponse(r.StartDirectReadPartitionSessionResponse(333)); + } + setup.GetControlSession()->Start(); { auto r = TMockReadSessionProcessor::TServerReadInfo(); From f2c19c8d67b5cd1c6709021f47f4d661de7f07c6 Mon Sep 17 00:00:00 2001 From: Sergey Belyakov Date: Fri, 8 Aug 2025 16:30:26 +0300 Subject: [PATCH 44/52] Add UT for group cache, improve SentToWarden for local messages (#22541) --- .../blobstorage/ut_blobstorage/bsc_cache.cpp | 122 ++++++++++++++++++ .../ut_blobstorage/lib/ut_helpers.h | 46 ++++--- ydb/core/blobstorage/ut_blobstorage/ya.make | 1 + ydb/core/mind/bscontroller/config.cpp | 10 +- ydb/core/mind/bscontroller/config.h | 17 ++- ydb/core/mind/bscontroller/register_node.cpp | 2 +- 6 files changed, 177 insertions(+), 21 deletions(-) create mode 100644 ydb/core/blobstorage/ut_blobstorage/bsc_cache.cpp diff --git a/ydb/core/blobstorage/ut_blobstorage/bsc_cache.cpp b/ydb/core/blobstorage/ut_blobstorage/bsc_cache.cpp new file mode 100644 index 000000000000..9bff92b2747a --- /dev/null +++ b/ydb/core/blobstorage/ut_blobstorage/bsc_cache.cpp @@ -0,0 +1,122 @@ +#include +#include + +Y_UNIT_TEST_SUITE(GroupConfigurationPropagation) { + +struct TTestCtx : public TTestCtxBase { + TTestCtx(const TBlobStorageGroupType& erasure) + : TTestCtxBase(TEnvironmentSetup::TSettings{ + .NodeCount = erasure.BlobSubgroupSize() * 2, + .Erasure = erasure, + .ControllerNodeId = erasure.BlobSubgroupSize() * 2, + }) + {} + + static bool BlockBscResponses(ui32, std::unique_ptr& ev) { + if (ev->GetTypeRewrite() == TEvBlobStorage::TEvControllerNodeServiceSetUpdate::EventType) { + return false; + } + + return true; + } + + void AllocateEdgeActorOnNodeWOConfig() { + std::set nodesWOConfig = GetComplementNodeSet(NodesWithConfig); + UNIT_ASSERT(!NodesWithConfig.empty()); + AllocateEdgeActorOnSpecificNode(*nodesWOConfig.begin()); + NodeWithProxy = Edge.NodeId(); + } + + void CheckStatus() { + AllocateEdgeActorOnNodeWOConfig(); + auto res = GetGroupStatus(GroupId, WaitTime); + UNIT_ASSERT(res); + UNIT_ASSERT_VALUES_EQUAL_C(res->Get()->Status, NKikimrProto::OK, res->Get()->ErrorReason); + } + + void Setup() { + Initialize(); + NodesWithConfig = GetNodesWithVDisks(); + NodesWithConfig.insert(Env->Settings.ControllerNodeId); + AllocateEdgeActorOnNodeWOConfig(); + UNIT_ASSERT(NodeWithProxy != Env->Settings.ControllerNodeId); + Env->Sim(TDuration::Minutes(10)); + Env->Runtime->FilterFunction = BlockBscResponses; + } + + void StopNode() { + Env->StopNode(NodeWithProxy); + } + + void StartNode() { + Env->StartNode(NodeWithProxy); + AllocateEdgeActorOnNodeWOConfig(); + } + + void RestartNode() { + StopNode(); + StartNode(); + } + + void UpdateGroupConfiguration() { + NKikimrBlobStorage::TConfigRequest request; + auto* reassign = request.AddCommand()->MutableReassignGroupDisk(); + for (const auto& slot : BaseConfig.GetVSlot()) { + if (slot.GetGroupId() == GroupId) { + reassign->SetGroupId(slot.GetGroupId()); + reassign->SetGroupGeneration(slot.GetGroupGeneration()); + reassign->SetFailRealmIdx(slot.GetFailRealmIdx()); + reassign->SetFailDomainIdx(slot.GetFailDomainIdx()); + reassign->SetVDiskIdx(slot.GetVDiskIdx()); + break; + } + } + auto response = Env->Invoke(request); + BaseConfig = Env->FetchBaseConfig(); + + std::set nodesWithVDisks = GetNodesWithVDisks(); + NodesWithConfig.insert(nodesWithVDisks.begin(), nodesWithVDisks.end()); + } + + const TDuration WaitTime = TDuration::Seconds(1); + ui32 NodeWithProxy; + std::set NodesWithConfig; +}; + +Y_UNIT_TEST(Simple) { + TTestCtx ctx(TBlobStorageGroupType::Erasure4Plus2Block); + ctx.Setup(); + ctx.CheckStatus(); +} + +Y_UNIT_TEST(Reassign) { + TTestCtx ctx(TBlobStorageGroupType::Erasure4Plus2Block); + ctx.Setup(); + ctx.UpdateGroupConfiguration(); + ctx.CheckStatus(); +} + +Y_UNIT_TEST(NodeRestart) { + TTestCtx ctx(TBlobStorageGroupType::Erasure4Plus2Block); + ctx.Setup(); + ctx.RestartNode(); + ctx.CheckStatus(); +} + +Y_UNIT_TEST(NodeRestartAndUpdate) { + TTestCtx ctx(TBlobStorageGroupType::Erasure4Plus2Block); + ctx.Setup(); + ctx.StopNode(); + ctx.UpdateGroupConfiguration(); + ctx.StartNode(); + ctx.CheckStatus(); +} + +Y_UNIT_TEST(BscRestart) { + TTestCtx ctx(TBlobStorageGroupType::Erasure4Plus2Block); + ctx.Setup(); + ctx.Env->RestartNode(ctx.Env->Settings.ControllerNodeId); + ctx.CheckStatus(); +} + +} diff --git a/ydb/core/blobstorage/ut_blobstorage/lib/ut_helpers.h b/ydb/core/blobstorage/ut_blobstorage/lib/ut_helpers.h index 6d76e832f691..67696937191e 100644 --- a/ydb/core/blobstorage/ut_blobstorage/lib/ut_helpers.h +++ b/ydb/core/blobstorage/ut_blobstorage/lib/ut_helpers.h @@ -287,33 +287,29 @@ struct TTestCtxBase { if (!findNodeWithoutVDisks) { chosenNodeId = NodeCount; } else { - std::set nodesWithoutVDisks; - for (ui32 nodeId = 1; nodeId <= NodeCount; ++nodeId) { - nodesWithoutVDisks.insert(nodeId); - } - - for (const auto& vslot : BaseConfig.GetVSlot()) { - nodesWithoutVDisks.erase(vslot.GetVSlotId().GetNodeId()); - } - - if (!nodesWithoutVDisks.empty()) { - chosenNodeId = *nodesWithoutVDisks.begin(); - } + std::set nodesWOVDisks = GetComplementNodeSet(GetNodesWithVDisks()); + chosenNodeId = *nodesWOVDisks.begin(); } Y_VERIFY_S(chosenNodeId != 0, "No available nodes to allocate"); - Edge = Env->Runtime->AllocateEdgeActor(NodeCount); + Edge = Env->Runtime->AllocateEdgeActor(chosenNodeId); + } + + void AllocateEdgeActorOnSpecificNode(ui32 nodeId) { + Edge = Env->Runtime->AllocateEdgeActor(nodeId); } void FetchBaseConfig() { BaseConfig = Env->FetchBaseConfig(); } - TAutoPtr> GetGroupStatus(ui32 groupId) { + TAutoPtr> GetGroupStatus(ui32 groupId, + TDuration waitTime = TDuration::Max()) { + TInstant ts = (waitTime == TDuration::Max()) ? TInstant::Max() : Env->Now() + waitTime; Env->Runtime->WrapInActorContext(Edge, [&] { - SendToBSProxy(Edge, groupId, new TEvBlobStorage::TEvStatus(TInstant::Max())); + SendToBSProxy(Edge, groupId, new TEvBlobStorage::TEvStatus(ts)); }); - return Env->WaitForEdgeActorEvent(Edge, false, TInstant::Max()); + return Env->WaitForEdgeActorEvent(Edge, false, ts); } virtual void Initialize() { @@ -431,6 +427,24 @@ struct TTestCtxBase { return blobs; } + std::set GetNodesWithVDisks() { + std::set res; + for (const auto& vslot : BaseConfig.GetVSlot()) { + res.insert(vslot.GetVSlotId().GetNodeId()); + } + return res; + } + + std::set GetComplementNodeSet(std::set nodes) { + std::set res; + for (ui32 nodeId = 1; nodeId <= NodeCount; ++nodeId) { + if (!nodes.contains(nodeId)) { + res.insert(nodeId); + } + } + return res; + } + public: ui32 NodeCount; TBlobStorageGroupType Erasure; diff --git a/ydb/core/blobstorage/ut_blobstorage/ya.make b/ydb/core/blobstorage/ut_blobstorage/ya.make index 64122d2d7209..894bea7c1073 100644 --- a/ydb/core/blobstorage/ut_blobstorage/ya.make +++ b/ydb/core/blobstorage/ut_blobstorage/ya.make @@ -18,6 +18,7 @@ SRCS( assimilation.cpp backpressure.cpp block_race.cpp + bsc_cache.cpp counting_events.cpp deadlines.cpp decommit_3dc.cpp diff --git a/ydb/core/mind/bscontroller/config.cpp b/ydb/core/mind/bscontroller/config.cpp index 78b3ac002d84..62e9fd9a85a2 100644 --- a/ydb/core/mind/bscontroller/config.cpp +++ b/ydb/core/mind/bscontroller/config.cpp @@ -40,7 +40,7 @@ namespace NKikimr::NBsController { if (CacheUpdate.KeyValuePairsSize()) { State.Outbox.emplace_back(Self->SelfId().NodeId(), std::make_unique( - std::move(CacheUpdate)), 0); + std::move(CacheUpdate)), 0, true); } } @@ -722,8 +722,12 @@ namespace NKikimr::NBsController { } ui64 TBlobStorageController::TConfigState::ApplyConfigUpdates() { - for (auto& [nodeId, ev, cookie] : Outbox) { - Self.SendToWarden(nodeId, std::move(ev), cookie); + for (TOutgoingMessage& msg : Outbox) { + if (msg.ToLocalWarden) { + Self.Send(MakeBlobStorageNodeWardenID(Self.SelfId().NodeId()), std::move(msg.Event), 0, msg.Cookie); + } else { + Self.SendToWarden(msg.NodeId, std::move(msg.Event), msg.Cookie); + } } for (auto& ev : StatProcessorOutbox) { Self.SelfId().Send(Self.StatProcessorActorId, ev.release()); diff --git a/ydb/core/mind/bscontroller/config.h b/ydb/core/mind/bscontroller/config.h index 720a6bc521ef..902a1147dfd3 100644 --- a/ydb/core/mind/bscontroller/config.h +++ b/ydb/core/mind/bscontroller/config.h @@ -92,7 +92,22 @@ namespace NKikimr { THashSet PDisksToRemove; // outgoing messages - std::deque, ui64>> Outbox; + struct TOutgoingMessage { + TNodeId NodeId; + std::unique_ptr Event; + ui64 Cookie; + bool ToLocalWarden; + + TOutgoingMessage(TNodeId nodeId, std::unique_ptr&& event, + ui64 cookie, bool toLocalWarden = false) + : NodeId(nodeId) + , Event(std::forward>(event)) + , Cookie(cookie) + , ToLocalWarden(toLocalWarden) + {} + }; + + std::deque Outbox; std::deque> StatProcessorOutbox; std::deque> NodeWhiteboardOutbox; THolder UpdateSelfHealInfoMsg; diff --git a/ydb/core/mind/bscontroller/register_node.cpp b/ydb/core/mind/bscontroller/register_node.cpp index effefe91ea19..ad5f93dc8a8d 100644 --- a/ydb/core/mind/bscontroller/register_node.cpp +++ b/ydb/core/mind/bscontroller/register_node.cpp @@ -667,7 +667,7 @@ void TBlobStorageController::EraseKnownDrivesOnDisconnected(TNodeInfo *nodeInfo) void TBlobStorageController::SendToWarden(TNodeId nodeId, std::unique_ptr ev, ui64 cookie) { Y_ABORT_UNLESS(nodeId); - if (auto *node = FindNode(nodeId); node && node->ConnectedServerId) { + if (TNodeInfo* node = FindNode(nodeId); node && node->ConnectedServerId) { auto h = std::make_unique(MakeBlobStorageNodeWardenID(nodeId), SelfId(), ev.release(), 0, cookie); if (node->InterconnectSessionId) { h->Rewrite(TEvInterconnect::EvForward, node->InterconnectSessionId); From efff3f287ffb356a066925318a58c766db79d029 Mon Sep 17 00:00:00 2001 From: Semyon Date: Fri, 8 Aug 2025 17:13:08 +0300 Subject: [PATCH 45/52] enable simple reader in cs (#22468) --- .github/config/muted_ya.txt | 1 + ydb/core/kqp/ut/olap/delete_ut.cpp | 2 +- ydb/core/kqp/ut/olap/kqp_olap_ut.cpp | 4 ++-- ydb/core/kqp/ut/olap/locks_ut.cpp | 6 +++--- ydb/core/kqp/ut/olap/optimizer_ut.cpp | 2 +- ydb/core/kqp/ut/olap/sparsed_ut.cpp | 2 +- ydb/core/kqp/ut/olap/tiering_ut.cpp | 2 +- ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp | 8 ++++---- ydb/core/kqp/ut/tx/kqp_sink_common.h | 2 +- ydb/core/protos/config.proto | 2 +- .../engines/reader/transaction/tx_scan.cpp | 2 +- .../ut_schema/ut_columnshard_move_table.cpp | 2 ++ ydb/tests/olap/oom/overlapping_portions.py | 14 +++++++------- 13 files changed, 26 insertions(+), 23 deletions(-) diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt index 9b9729118bf4..e46468d317c1 100644 --- a/.github/config/muted_ya.txt +++ b/.github/config/muted_ya.txt @@ -66,6 +66,7 @@ ydb/core/kqp/ut/scheme KqpScheme.CreateDropTableMultipleTime ydb/core/kqp/ut/scheme KqpScheme.ModifySysViewDirPermissions ydb/core/kqp/ut/service KqpService.CloseSessionsWithLoad ydb/core/kqp/ut/service unittest.[*/*] chunk +ydb/core/kqp/ut/tx KqpSinkMvcc.OlapNamedStatement ydb/core/kqp/ut/yql KqpScripting.StreamExecuteYqlScriptScanOperationTmeoutBruteForce ydb/core/kqp/workload_service/ut KqpWorkloadServiceDistributed.TestDistributedLessConcurrentQueryLimit ydb/core/quoter/ut QuoterWithKesusTest.PrefetchCoefficient diff --git a/ydb/core/kqp/ut/olap/delete_ut.cpp b/ydb/core/kqp/ut/olap/delete_ut.cpp index 522a9571dc38..42cb972ad2bb 100644 --- a/ydb/core/kqp/ut/olap/delete_ut.cpp +++ b/ydb/core/kqp/ut/olap/delete_ut.cpp @@ -5,7 +5,7 @@ namespace NKikimr::NKqp { Y_UNIT_TEST_SUITE(KqpOlapDelete) { Y_UNIT_TEST_TWIN(DeleteWithDiffrentTypesPKColumns, isStream) { - auto runnerSettings = TKikimrSettings().SetWithSampleTables(true).SetColumnShardReaderClassName("PLAIN"); + auto runnerSettings = TKikimrSettings().SetWithSampleTables(true); runnerSettings.AppConfig.MutableTableServiceConfig()->SetEnableOlapSink(true); TTestHelper testHelper(runnerSettings); diff --git a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp index 032fb0535322..41239042cf6d 100644 --- a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp +++ b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp @@ -3405,7 +3405,7 @@ Y_UNIT_TEST_SUITE(KqpOlap) { Y_UNIT_TEST(BulkUpsertUpdate) { TKikimrSettings runnerSettings; runnerSettings.WithSampleTables = false; - runnerSettings.SetColumnShardAlterObjectEnabled(true).SetColumnShardReaderClassName("PLAIN"); + runnerSettings.SetColumnShardAlterObjectEnabled(true); auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); TTestHelper testHelper(runnerSettings); @@ -4333,7 +4333,7 @@ Y_UNIT_TEST_SUITE(KqpOlap) { Y_UNIT_TEST(ReverseMerge) { auto runnerSettings = - TKikimrSettings().SetWithSampleTables(true).SetColumnShardAlterObjectEnabled(true).SetColumnShardReaderClassName("PLAIN"); + TKikimrSettings().SetWithSampleTables(true).SetColumnShardAlterObjectEnabled(true); runnerSettings.AppConfig.MutableTableServiceConfig()->SetEnableOlapSink(true); auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); diff --git a/ydb/core/kqp/ut/olap/locks_ut.cpp b/ydb/core/kqp/ut/olap/locks_ut.cpp index 9c26adc39d53..37536e1123db 100644 --- a/ydb/core/kqp/ut/olap/locks_ut.cpp +++ b/ydb/core/kqp/ut/olap/locks_ut.cpp @@ -16,7 +16,7 @@ namespace NKikimr::NKqp { Y_UNIT_TEST_SUITE(KqpOlapLocks) { Y_UNIT_TEST(TwoQueriesWithRestartTablet) { - auto settings = TKikimrSettings().SetWithSampleTables(false).SetColumnShardReaderClassName("PLAIN"); + auto settings = TKikimrSettings().SetWithSampleTables(false); settings.AppConfig.MutableTableServiceConfig()->SetEnableOlapSink(true); TKikimrRunner kikimr(settings); @@ -160,7 +160,7 @@ Y_UNIT_TEST_SUITE(KqpOlapLocks) { } Y_UNIT_TEST(TableSinkWithOlapStore) { - auto settings = TKikimrSettings().SetWithSampleTables(false).SetColumnShardReaderClassName("PLAIN"); + auto settings = TKikimrSettings().SetWithSampleTables(false); settings.AppConfig.MutableTableServiceConfig()->SetEnableOlapSink(true); TKikimrRunner kikimr(settings); @@ -193,7 +193,7 @@ Y_UNIT_TEST_SUITE(KqpOlapLocks) { //It corresponds to a SCAN, then NO write then COMMIT on that shard auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); - auto settings = TKikimrSettings().SetWithSampleTables(false).SetColumnShardReaderClassName("PLAIN"); + auto settings = TKikimrSettings().SetWithSampleTables(false); settings.AppConfig.MutableTableServiceConfig()->SetEnableOlapSink(true); TTestHelper testHelper(settings); diff --git a/ydb/core/kqp/ut/olap/optimizer_ut.cpp b/ydb/core/kqp/ut/olap/optimizer_ut.cpp index 6af8d40c61aa..7b096ca4c64a 100644 --- a/ydb/core/kqp/ut/olap/optimizer_ut.cpp +++ b/ydb/core/kqp/ut/olap/optimizer_ut.cpp @@ -17,7 +17,7 @@ namespace NKikimr::NKqp { Y_UNIT_TEST_SUITE(KqpOlapOptimizer) { Y_UNIT_TEST(SpecialSliceToOneLayer) { - auto settings = TKikimrSettings().SetWithSampleTables(false).SetColumnShardReaderClassName("PLAIN"); + auto settings = TKikimrSettings().SetWithSampleTables(false); TKikimrRunner kikimr(settings); auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); diff --git a/ydb/core/kqp/ut/olap/sparsed_ut.cpp b/ydb/core/kqp/ut/olap/sparsed_ut.cpp index c030bed70beb..063ed45dbcc2 100644 --- a/ydb/core/kqp/ut/olap/sparsed_ut.cpp +++ b/ydb/core/kqp/ut/olap/sparsed_ut.cpp @@ -17,7 +17,7 @@ Y_UNIT_TEST_SUITE(KqpOlapSparsed) { class TSparsedDataTest { private: const TKikimrSettings Settings = - TKikimrSettings().SetColumnShardAlterObjectEnabled(true).SetWithSampleTables(false).SetColumnShardReaderClassName("PLAIN"); + TKikimrSettings().SetColumnShardAlterObjectEnabled(true).SetWithSampleTables(false); TKikimrRunner Kikimr; NKikimr::NYDBTest::TControllers::TGuard CSController; const TString StoreName; diff --git a/ydb/core/kqp/ut/olap/tiering_ut.cpp b/ydb/core/kqp/ut/olap/tiering_ut.cpp index 5ea497165073..88212ff2333b 100644 --- a/ydb/core/kqp/ut/olap/tiering_ut.cpp +++ b/ydb/core/kqp/ut/olap/tiering_ut.cpp @@ -256,7 +256,7 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) { auto it = tableClient.StreamExecuteScanQuery(selectQuery, NYdb::NTable::TStreamExecScanQuerySettings()).GetValueSync(); auto streamPart = it.ReadNext().GetValueSync(); UNIT_ASSERT(!streamPart.IsSuccess()); - UNIT_ASSERT_STRING_CONTAINS(streamPart.GetIssues().ToString(), "Error reading blob range"); + UNIT_ASSERT_STRING_CONTAINS(streamPart.GetIssues().ToString(), "/Root/tier1"); } testHelper.CreateTier("tier1"); diff --git a/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp b/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp index 79e4583d474b..d60669d3636d 100644 --- a/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp +++ b/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp @@ -3930,7 +3930,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { } Y_UNIT_TEST_TWIN(TableSink_Htap, withOltpSink) { - auto settings = TKikimrSettings().SetWithSampleTables(false).SetColumnShardReaderClassName("PLAIN"); + auto settings = TKikimrSettings().SetWithSampleTables(false); settings.AppConfig.MutableTableServiceConfig()->SetEnableOlapSink(true); settings.AppConfig.MutableTableServiceConfig()->SetEnableOltpSink(withOltpSink); settings.AppConfig.MutableTableServiceConfig()->SetEnableHtapTx(true); @@ -4117,7 +4117,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { Y_UNIT_TEST_TWIN(TableSink_HtapComplex, withOltpSink) { auto settings = TKikimrSettings() - .SetWithSampleTables(false).SetColumnShardReaderClassName("PLAIN"); + .SetWithSampleTables(false); settings.AppConfig.MutableTableServiceConfig()->SetEnableOlapSink(true); settings.AppConfig.MutableTableServiceConfig()->SetEnableOltpSink(withOltpSink); settings.AppConfig.MutableTableServiceConfig()->SetEnableHtapTx(true); @@ -4665,7 +4665,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { virtual void DoExecute() = 0; public: void Execute() { - auto settings = TKikimrSettings().SetWithSampleTables(false).SetColumnShardReaderClassName("PLAIN"); + auto settings = TKikimrSettings().SetWithSampleTables(false); settings.AppConfig.MutableTableServiceConfig()->SetEnableOlapSink(IsOlap); settings.AppConfig.MutableTableServiceConfig()->SetEnableOltpSink(!IsOlap); @@ -5007,7 +5007,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { } Y_UNIT_TEST(TableSink_ReplaceDuplicatesOlap) { - auto settings = TKikimrSettings().SetWithSampleTables(false).SetColumnShardReaderClassName("PLAIN"); + auto settings = TKikimrSettings().SetWithSampleTables(false); settings.AppConfig.MutableTableServiceConfig()->SetEnableOlapSink(true); TKikimrRunner kikimr(settings); diff --git a/ydb/core/kqp/ut/tx/kqp_sink_common.h b/ydb/core/kqp/ut/tx/kqp_sink_common.h index 68086472faed..775b9efd5fe9 100644 --- a/ydb/core/kqp/ut/tx/kqp_sink_common.h +++ b/ydb/core/kqp/ut/tx/kqp_sink_common.h @@ -22,7 +22,7 @@ class TTableDataModificationTester { virtual void DoExecute() = 0; public: void Execute() { - auto settings = TKikimrSettings().SetWithSampleTables(false).SetColumnShardReaderClassName("PLAIN"); + auto settings = TKikimrSettings().SetWithSampleTables(false); settings.AppConfig.MutableTableServiceConfig()->SetEnableOlapSink(!DisableSinks); settings.AppConfig.MutableTableServiceConfig()->SetEnableOltpSink(!DisableSinks); settings.AppConfig.MutableTableServiceConfig()->SetEnableSnapshotIsolationRW(true); diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index 8fb4e2925e0f..761dddd17f52 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -2065,7 +2065,7 @@ message TColumnShardConfig { optional bool GenerateInternalPathId = 43 [default = false]; optional bool ProxyWritingEnabled = 44 [default = true]; optional bool AllowExtraSymbolsForColumnTableColumns = 45 [default = false]; - optional bool DeduplicationEnabled = 46 [default = false]; + optional bool DeduplicationEnabled = 46 [default = true]; } message TSchemeShardConfig { diff --git a/ydb/core/tx/columnshard/engines/reader/transaction/tx_scan.cpp b/ydb/core/tx/columnshard/engines/reader/transaction/tx_scan.cpp index 15fd3685f4d7..fc3015359885 100644 --- a/ydb/core/tx/columnshard/engines/reader/transaction/tx_scan.cpp +++ b/ydb/core/tx/columnshard/engines/reader/transaction/tx_scan.cpp @@ -87,7 +87,7 @@ void TTxScan::Complete(const TActorContext& ctx) { const TString defaultReader = [&]() { const TString defGlobal = - AppDataVerified().ColumnShardConfig.GetReaderClassName() ? AppDataVerified().ColumnShardConfig.GetReaderClassName() : "PLAIN"; + AppDataVerified().ColumnShardConfig.GetReaderClassName() ? AppDataVerified().ColumnShardConfig.GetReaderClassName() : "SIMPLE"; if (Self->HasIndex()) { return Self->GetIndexAs() .GetVersionedIndex() diff --git a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_move_table.cpp b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_move_table.cpp index e1af3c0393a5..f048a18f0287 100644 --- a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_move_table.cpp +++ b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_move_table.cpp @@ -101,6 +101,7 @@ Y_UNIT_TEST_SUITE(MoveTable) { { TShardReader reader(runtime, TTestTxConfig::TxTablet0, dstPathId, NOlap::TSnapshot(planStep, txId)); + reader.SetReplyColumnIds(TTestSchema::ExtractIds(testTabe.Schema)); auto rb = reader.ReadAll(); UNIT_ASSERT(rb); UNIT_ASSERT_EQUAL(rb->num_rows(), 100); @@ -108,6 +109,7 @@ Y_UNIT_TEST_SUITE(MoveTable) { { TShardReader reader(runtime, TTestTxConfig::TxTablet0, srcPathId, NOlap::TSnapshot(planStep, txId)); + reader.SetReplyColumnIds(TTestSchema::ExtractIds(testTabe.Schema)); auto rb = reader.ReadAll(); UNIT_ASSERT(!rb); } diff --git a/ydb/tests/olap/oom/overlapping_portions.py b/ydb/tests/olap/oom/overlapping_portions.py index 2e8b42387b7b..595703f6d85c 100644 --- a/ydb/tests/olap/oom/overlapping_portions.py +++ b/ydb/tests/olap/oom/overlapping_portions.py @@ -23,13 +23,13 @@ def setup_class(cls): logger.info(yatest.common.execute([ydb_path, "-V"], wait=True).stdout.decode("utf-8")) config = KikimrConfigGenerator( column_shard_config={"compaction_enabled": False}, - scan_grouped_memory_limiter_config={ + deduplication_grouped_memory_limiter_config={ "enabled": True, - "memory_limit": 300 * 1024 * 1024, - "hard_memory_limit": 300 * 1024 * 1024, + "memory_limit": 1024 * 1024, + "hard_memory_limit": 1024 * 1024, }, memory_controller_config={ - "column_tables_read_execution_limit_bytes": 300 * 1024 * 1024 + "column_tables_read_execution_limit_bytes": 1024 * 1024 }, ) cls.cluster = KiKiMR(config) @@ -90,9 +90,9 @@ def test(self): f""" CREATE TABLE `{table_path}` ( ts Timestamp NOT NULL, - s String, - val Uint64, - PRIMARY KEY(ts), + s String NOT NULL, + val Uint64 NOT NULL, + PRIMARY KEY(ts, s, val), ) WITH ( STORE = COLUMN, From b9d5713d2157043a50253974a829b009a751e63e Mon Sep 17 00:00:00 2001 From: Semyon Date: Fri, 8 Aug 2025 17:16:44 +0300 Subject: [PATCH 46/52] reduce memory usage by splitter in deduplication (#22555) --- .../simple_reader/duplicates/context.cpp | 3 +- .../reader/simple_reader/duplicates/context.h | 7 +-- .../simple_reader/duplicates/manager.cpp | 54 +++++++++++-------- .../reader/simple_reader/duplicates/manager.h | 3 ++ 4 files changed, 37 insertions(+), 30 deletions(-) diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/context.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/context.cpp index 5d0774c2a0b6..b0a255174bf7 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/context.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/context.cpp @@ -4,9 +4,8 @@ namespace NKikimr::NOlap::NReader::NSimple::NDuplicateFiltering { -TInternalFilterConstructor::TInternalFilterConstructor(const TEvRequestFilter::TPtr& request, TColumnDataSplitter&& splitter) +TInternalFilterConstructor::TInternalFilterConstructor(const TEvRequestFilter::TPtr& request) : OriginalRequest(request) - , Intervals(std::move(splitter)) , ProcessGuard(NGroupedMemoryManager::TDeduplicationMemoryLimiterOperator::BuildProcessGuard({})) , ScopeGuard(ProcessGuard->BuildScopeGuard(1)) , GroupGuard(ScopeGuard->BuildGroupGuard()) diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/context.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/context.h index 57852edf7b64..980381f99d0f 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/context.h +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/context.h @@ -10,7 +10,6 @@ namespace NKikimr::NOlap::NReader::NSimple::NDuplicateFiltering { class TInternalFilterConstructor: TMoveOnly { private: const TEvRequestFilter::TPtr OriginalRequest; - const TColumnDataSplitter Intervals; const std::shared_ptr ProcessGuard; const std::shared_ptr ScopeGuard; const std::shared_ptr GroupGuard; @@ -73,11 +72,7 @@ class TInternalFilterConstructor: TMoveOnly { return OriginalRequest; } - const TColumnDataSplitter& GetIntervals() const { - return Intervals; - } - - TInternalFilterConstructor(const TEvRequestFilter::TPtr& request, TColumnDataSplitter&& splitter); + TInternalFilterConstructor(const TEvRequestFilter::TPtr& request); ~TInternalFilterConstructor() { AFL_VERIFY(IsDone() || (OriginalRequest->Get()->GetAbortionFlag() && OriginalRequest->Get()->GetAbortionFlag()->Val()))( diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.cpp index 8882307cfcd1..97ec37723c06 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.cpp @@ -162,6 +162,33 @@ class TDuplicateManager::TPortionsSlice { } }; +std::vector TDuplicateManager::FindIntervalBorders( + const THashMap>& dataByPortion, + const std::shared_ptr& context) const { + THashMap borders; + for (const auto& [portionId, _] : dataByPortion) { + const auto& portion = GetPortionVerified(portionId); + borders.emplace( + portionId, NArrow::TFirstLastSpecialKeys(portion->IndexKeyStart(), portion->IndexKeyEnd(), portion->IndexKeyStart().GetSchema())); + } + const auto& mainSource = GetPortionVerified(context->GetRequest()->Get()->GetSourceId()); + TColumnDataSplitter splitter( + borders, NArrow::TFirstLastSpecialKeys(mainSource->IndexKeyStart(), mainSource->IndexKeyEnd(), mainSource->IndexKeyStart().GetSchema())); + + std::vector slices; + for (ui64 i = 0; i < splitter.NumIntervals(); ++i) { + slices.emplace_back(TPortionsSlice(splitter.GetIntervalFinish(i))); + } + for (const auto& [id, data] : dataByPortion) { + auto intervals = splitter.SplitPortion(data); + AFL_VERIFY(intervals.size() == splitter.NumIntervals()); + for (ui64 i = 0; i < splitter.NumIntervals(); ++i) { + slices[i].AddRange(id, intervals[i]); + } + } + return slices; +} + #define LOCAL_LOG_TRACE \ AFL_TRACE(NKikimrServices::TX_COLUMNSHARD_SCAN)("component", "duplicates_manager")("self", TActivationContext::AsActorContext().SelfID) @@ -180,14 +207,11 @@ TDuplicateManager::TDuplicateManager(const TSpecialReadContext& context, const s void TDuplicateManager::Handle(const TEvRequestFilter::TPtr& ev) { std::vector sourcesToFetch; - THashMap borders; const std::shared_ptr& source = GetPortionVerified(ev->Get()->GetSourceId()); { - const auto collector = [&sourcesToFetch, &borders]( + const auto collector = [&sourcesToFetch]( const TPortionIntervalTree::TRange& /*interval*/, const std::shared_ptr& portion) { sourcesToFetch.emplace_back(portion); - borders.emplace(portion->GetPortionId(), - NArrow::TFirstLastSpecialKeys(portion->IndexKeyStart(), portion->IndexKeyEnd(), portion->IndexKeyStart().GetSchema())); }; Intervals.EachIntersection(TPortionIntervalTree::TRange(source->IndexKeyStart(), true, source->IndexKeyEnd(), true), collector); } @@ -202,9 +226,7 @@ void TDuplicateManager::Handle(const TEvRequestFilter::TPtr& ev) { return; } - TColumnDataSplitter splitter( - borders, NArrow::TFirstLastSpecialKeys(source->IndexKeyStart(), source->IndexKeyEnd(), source->IndexKeyStart().GetSchema())); - auto constructor = std::make_shared(ev, std::move(splitter)); + auto constructor = std::make_shared(ev); { THashSet portionAddresses; @@ -239,24 +261,12 @@ void TDuplicateManager::Handle(const NPrivate::TEvDuplicateSourceCacheResult::TP THashMap> dataByPortion = ev->Get()->ExtractResult().ExtractDataByPortion(GetFetchingColumns()); const std::shared_ptr& context = ev->Get()->GetContext(); - const TColumnDataSplitter& splitter = context->GetIntervals(); auto allocationGuard = ev->Get()->ExtractAllocationGuard(); - std::vector slices; - for (ui64 i = 0; i < splitter.NumIntervals(); ++i) { - slices.emplace_back(TPortionsSlice(splitter.GetIntervalFinish(i))); - } - for (const auto& [id, data] : dataByPortion) { - auto intervals = splitter.SplitPortion(data); - AFL_VERIFY(intervals.size() == splitter.NumIntervals()); - for (ui64 i = 0; i < splitter.NumIntervals(); ++i) { - slices[i].AddRange(id, intervals[i]); - } - } - LOCAL_LOG_TRACE("event", "construct_filters")("context", context->DebugString())("splitter", splitter.DebugString()); + LOCAL_LOG_TRACE("event", "construct_filters")("context", context->DebugString()); - for (ui64 i = 0; i < splitter.NumIntervals(); ++i) { - const auto& slice = slices[i]; + auto slices = FindIntervalBorders(dataByPortion, context); + for (const auto& slice : slices) { BuildFilterForSlice(slice, context, allocationGuard, dataByPortion); } } diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.h index 15f697825cfe..a414614aa8e4 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.h +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.h @@ -67,6 +67,9 @@ class TDuplicateManager: public NActors::TActor { const std::shared_ptr& allocationGuard, const THashMap>& dataByPortion); + std::vector FindIntervalBorders(const THashMap>& dataByPortion, + const std::shared_ptr& context) const; + private: STATEFN(StateMain) { switch (ev->GetTypeRewrite()) { From 07b141baab39533f572ea3d64b1108f6abc61191 Mon Sep 17 00:00:00 2001 From: Alexey Pozdniakov Date: Fri, 8 Aug 2025 18:16:00 +0300 Subject: [PATCH 47/52] [YQ-4427] Watermarks: RD: pushdown watermark expr to RD (#22411) --- .../yql_clickhouse_dq_integration.cpp | 7 +- .../providers/common/pushdown/collection.cpp | 98 +++++- .../providers/common/pushdown/collection.h | 19 +- .../yql/providers/common/pushdown/settings.h | 7 +- .../api/service/protos/connector.proto | 28 +- .../provider/yql_generic_dq_integration.cpp | 12 +- .../yql_generic_predicate_pushdown.cpp | 209 +++++++++++-- .../provider/yql_generic_predicate_pushdown.h | 10 +- .../pushdown/yql_generic_match_predicate.cpp | 12 + .../pq/expr_nodes/yql_pq_expr_nodes.json | 6 +- .../yql/providers/pq/proto/dq_io.proto | 1 + .../pq/provider/yql_pq_datasource.cpp | 4 + .../provider/yql_pq_datasource_type_ann.cpp | 290 +++++++++++++----- .../pq/provider/yql_pq_dq_integration.cpp | 41 ++- .../pq/provider/yql_pq_logical_opt.cpp | 10 +- .../pq/provider/yql_pq_topic_key_parser.cpp | 4 + .../pq/provider/yql_pq_topic_key_parser.h | 13 +- .../s3/provider/yql_s3_dq_integration.cpp | 8 +- .../provider/yql_solomon_dq_integration.cpp | 14 +- .../ydb/provider/yql_ydb_dq_integration.cpp | 8 +- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 4 +- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 2 +- .../ast.txt | 4 +- 41 files changed, 683 insertions(+), 164 deletions(-) diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_dq_integration.cpp b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_dq_integration.cpp index 8481141a3088..c97c082e92b7 100644 --- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_dq_integration.cpp +++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_dq_integration.cpp @@ -33,12 +33,17 @@ class TClickHouseDqIntegration: public TDqIntegrationBase { return Nothing(); } - TExprNode::TPtr WrapRead(const TExprNode::TPtr& read, TExprContext& ctx, const TWrapReadSettings& ) override { + TExprNode::TPtr WrapRead(const TExprNode::TPtr& read, TExprContext& ctx, const TWrapReadSettings& wrSettings) override { if (const auto maybeClReadTable = TMaybeNode(read)) { const auto clReadTable = maybeClReadTable.Cast(); const auto token = TString("cluster:default_") += clReadTable.DataSource().Cluster().StringValue(); YQL_CLOG(INFO, ProviderClickHouse) << "Wrap " << read->Content() << " with token: " << token; + if (wrSettings.WatermarksMode.GetOrElse("") == "default") { + ctx.AddError(TIssue(ctx.GetPosition(clReadTable.Pos()), "Cannot use watermarks in ClickHouse")); + return {}; + } + const auto rowType = clReadTable.Ref().GetTypeAnn()->Cast()->GetItems().back()->Cast()->GetItemType(); auto columns = clReadTable.Columns().Ptr(); if (!columns->IsList()) { diff --git a/ydb/library/yql/providers/common/pushdown/collection.cpp b/ydb/library/yql/providers/common/pushdown/collection.cpp index c0225e666739..27f6dd9e810d 100644 --- a/ydb/library/yql/providers/common/pushdown/collection.cpp +++ b/ydb/library/yql/providers/common/pushdown/collection.cpp @@ -16,7 +16,7 @@ namespace { // | | // C AND D COALESCE(X, Y, Z) // | | | | | -// Member Comparation ..... +// Member Comparision ..... // // Each node has flag if it can be pushed entirely, // Next some tree nodes will be split @@ -187,7 +187,7 @@ class TPredicateMarkup { } private: - // Genric expression checking + // Generic expression checking bool IsSupportedDataType(const TCoDataCtor& node) const { if (node.Maybe() || node.Maybe() || @@ -205,6 +205,9 @@ class TPredicateMarkup { if (Settings.IsEnabled(EFlag::TimestampCtor) && node.Maybe()) { return true; } + if (Settings.IsEnabled(EFlag::IntervalCtor) && node.Maybe()) { + return true; + } if (Settings.IsEnabled(EFlag::StringTypes) && (node.Maybe() || node.Maybe())) { return true; } @@ -212,7 +215,7 @@ class TPredicateMarkup { } bool IsMemberColumn(const TCoMember& member) const { - // We allow member acces only for top level predicate argument + // We allow member access only for top level predicate argument return member.Struct().Raw() == LambdaArg.Raw(); } @@ -229,7 +232,10 @@ class TPredicateMarkup { } const auto targetType = DataSlotFromOptionalDataType(UnwrapExprType(cast.Type().Ref().GetTypeAnn())); - if (targetType == EDataSlot::Bool || IsNumericType(targetType) || IsStringType(targetType) && Settings.IsEnabled(EFlag::StringTypes)) { + if (targetType == EDataSlot::Bool || + IsNumericType(targetType) || + IsStringType(targetType) && Settings.IsEnabled(EFlag::StringTypes) || + IsDateTimeType(targetType) && Settings.IsEnabled(EFlag::DateTimeTypes)) { return CheckExpressionNodeForPushdown(cast.Value()); } return false; @@ -309,6 +315,7 @@ class TPredicateMarkup { return LambdaArguments.contains(expr.Raw()); } +public: bool CheckExpressionNodeForPushdown(const TExprBase& node) { if (auto maybeSafeCast = node.Maybe()) { return IsSupportedSafeCast(maybeSafeCast.Cast()); @@ -373,11 +380,26 @@ class TPredicateMarkup { if (auto flatMap = node.Maybe()) { return IsSupportedFlatMap(flatMap.Cast()); } + if (auto maybeDependsOn = node.Maybe()) { + return DependsOnCanBePushed(maybeDependsOn.Cast()); + } + if (auto maybeUnwrap = node.Maybe()) { + return UnwrapCanBePushed(maybeUnwrap.Cast()); + } + if (auto maybeMin = node.Maybe()) { + return MinCanBePushed(maybeMin.Cast()); + } + if (auto maybeMax = node.Maybe()) { + return MaxCanBePushed(maybeMax.Cast()); + } + if (auto maybeNonDeterministic = node.Maybe()) { + return NonDeterministicCanBePushed(maybeNonDeterministic.Cast()); + } return IsLambdaArgument(node); } private: - // Comprasion checking + // Comparision checking bool IsSupportedLikeOperator(const TCoCompare& compare) const { if (!IsSimpleLikeOperator(compare)) { return false; @@ -657,6 +679,62 @@ class TPredicateMarkup { return true; } + bool DependsOnCanBePushed(const TCoDependsOn& dependsOn) { + return CheckExpressionNodeForPushdown(dependsOn.Input()); + } + + bool UnwrapCanBePushed(const TCoUnwrap& unwrap) { + if (!Settings.IsEnabled(EFlag::JustPassthroughOperators)) { + return false; + } + + return CheckExpressionNodeForPushdown(unwrap.Optional()); + } + + bool MaxCanBePushed(const TCoMax& sqlMax) { + if (!Settings.IsEnabled(EFlag::MinMax)) { + return false; + } + + for (const auto& childNodePtr : sqlMax.Args()) { + if (!CheckExpressionNodeForPushdown(TExprBase(childNodePtr))) { + return false; + } + } + return true; + } + + bool MinCanBePushed(const TCoMin& sqlMin) { + if (!Settings.IsEnabled(EFlag::MinMax)) { + return false; + } + + for (const auto& childNodePtr : sqlMin.Args()) { + if (!CheckExpressionNodeForPushdown(TExprBase(childNodePtr))) { + return false; + } + } + return true; + } + + bool NonDeterministicCanBePushed(const TCoNonDeterministicBase& nonDeterministic) { + if (!Settings.IsEnabled(EFlag::NonDeterministic)) { + return false; + } + + if (const auto maybeCurrentUtcTimestamp = nonDeterministic.Maybe()) { + const auto currentUtcTimestamp = maybeCurrentUtcTimestamp.Cast(); + for (const auto& childNodePtr : currentUtcTimestamp.Args()) { + if (!CheckExpressionNodeForPushdown(TExprBase(childNodePtr))) { + return false; + } + } + return true; + } else { + return false; + } + } + private: const TExprBase& LambdaArg; // Predicate input item, has struct type const TSettings& Settings; @@ -679,4 +757,14 @@ void CollectPredicates( markup.MarkupPredicates(predicate, predicateTree); } +[[nodiscard]] bool TestExprForPushdown( + TExprContext& ctx, + const TExprBase& lambdaArg, + const TExprBase& lambdaBody, + const TSettings& settings +) { + TPredicateMarkup markup(lambdaArg, settings, ctx); + return markup.CheckExpressionNodeForPushdown(lambdaBody); +} + } // namespace NYql::NPushdown diff --git a/ydb/library/yql/providers/common/pushdown/collection.h b/ydb/library/yql/providers/common/pushdown/collection.h index 03e0ee5cc2fc..3ef4d8133fd7 100644 --- a/ydb/library/yql/providers/common/pushdown/collection.h +++ b/ydb/library/yql/providers/common/pushdown/collection.h @@ -8,9 +8,20 @@ namespace NYql::NPushdown { // Collects subpredicate that we can then push down -void CollectPredicates(TExprContext& ctx, - const NNodes::TExprBase& predicate, TPredicateNode& predicateTree, - const NNodes::TExprBase& lambdaArg, const NNodes::TExprBase& lambdaBody, - const TSettings& settings); +void CollectPredicates( + TExprContext& ctx, + const NNodes::TExprBase& predicate, + TPredicateNode& predicateTree, + const NNodes::TExprBase& lambdaArg, + const NNodes::TExprBase& lambdaBody, + const TSettings& settings +); + +[[nodiscard]] bool TestExprForPushdown( + TExprContext& ctx, + const NNodes::TExprBase& lambdaArg, + const NNodes::TExprBase& lambdaBody, + const TSettings& settings +); } // namespace NYql::NPushdown diff --git a/ydb/library/yql/providers/common/pushdown/settings.h b/ydb/library/yql/providers/common/pushdown/settings.h index a8ba0807371f..2a2027fd8685 100644 --- a/ydb/library/yql/providers/common/pushdown/settings.h +++ b/ydb/library/yql/providers/common/pushdown/settings.h @@ -29,7 +29,7 @@ struct TSettings { TimestampCtor = 1 << 17, JustPassthroughOperators = 1 << 18, // if + coalesce + just InOperator = 1 << 19, // IN() - IsDistinctOperator = 1 << 20, // IS NOT DISTINCT FROM / IS DISTINCT FROM + IsDistinctOperator = 1 << 20, // IS NOT DISTINCT FROM / IS DISTINCT FROM DivisionExpressions = 1 << 21, // %, / -- NOTE: division by zero is not handled and also pushdown // Option which enables partial pushdown for sequence of OR @@ -40,8 +40,11 @@ struct TSettings { // In case of unsupported / complicated expressions $B and $D SplitOrOperator = 1 << 22, ToBytesFromStringExpressions = 1 << 23, // ToBytes(string like) - FlatMapOverOptionals = 1 << 24, // FlatMap(Optional, Lmabda (T) -> Optional) + FlatMapOverOptionals = 1 << 24, // FlatMap(Optional, Lambda (T) -> Optional) ToStringFromStringExpressions = 1 << 25, // ToString(string like) + IntervalCtor = 1 << 26, + MinMax = 1 << 27, + NonDeterministic = 1 << 28, }; explicit TSettings(NLog::EComponent logComponent) diff --git a/ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto b/ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto index c6ef31fb95a9..62295b843f66 100644 --- a/ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto +++ b/ydb/library/yql/providers/generic/connector/api/service/protos/connector.proto @@ -139,7 +139,7 @@ message TSelect { // NOTE: this API intentionally makes it not possible to request 'SELECT *'. // YQ must provide all the column names explicitly. // - // Еmpty list means that YQ wants to get empty tuples in the response. + // Empty list means that YQ wants to get empty tuples in the response. // On the connector's side this request will be transformed into something like // SELECT 1 FROM $table (...) repeated TItem items = 1; @@ -367,6 +367,22 @@ message TExpression { message TNull { } + message TUnwrap { + TExpression operand = 1; + } + + message TMinOf { + repeated TExpression operands = 1; + } + + message TMaxOf { + repeated TExpression operands = 1; + } + + message TCurrentUtcTimestamp { + repeated TExpression operands = 1; + } + oneof payload { // A scalar value Ydb.TypedValue typed_value = 1; @@ -382,6 +398,14 @@ message TExpression { TIf if = 6; TCast cast = 7; + + TUnwrap unwrap = 8; + + TMinOf min_of = 9; + + TMaxOf max_of = 10; + + TCurrentUtcTimestamp current_utc_timestamp = 11; } } @@ -440,7 +464,7 @@ message TPredicate { TExpression value = 1; } - // Expression wich has bool type + // Expression which has bool type // For example, bool column message TBoolExpression { TExpression value = 1; diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_dq_integration.cpp b/ydb/library/yql/providers/generic/provider/yql_generic_dq_integration.cpp index b04e2b93ba9d..6927b691663d 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_dq_integration.cpp +++ b/ydb/library/yql/providers/generic/provider/yql_generic_dq_integration.cpp @@ -47,7 +47,7 @@ namespace NYql { case NYql::EGenericDataSourceKind::MONGO_DB: return "MongoDBGeneric"; case NYql::EGenericDataSourceKind::OPENSEARCH: - return "OpenSearchGeneric"; + return "OpenSearchGeneric"; default: throw yexception() << "Data source kind is unknown or not specified"; } @@ -72,9 +72,15 @@ namespace NYql { return Nothing(); } - TExprNode::TPtr WrapRead(const TExprNode::TPtr& read, TExprContext& ctx, const TWrapReadSettings&) override { + TExprNode::TPtr WrapRead(const TExprNode::TPtr& read, TExprContext& ctx, const TWrapReadSettings& wrSettings) override { if (const auto maybeGenReadTable = TMaybeNode(read)) { const auto genReadTable = maybeGenReadTable.Cast(); + + if (wrSettings.WatermarksMode.GetOrElse("") == "default") { + ctx.AddError(TIssue(ctx.GetPosition(genReadTable.Pos()), "Cannot use watermarks")); + return {}; + } + YQL_ENSURE(genReadTable.Ref().GetTypeAnn(), "No type annotation for node " << genReadTable.Ref().Content()); const auto token = TString("cluster:default_") += genReadTable.DataSource().Cluster().StringValue(); const auto rowType = genReadTable.Ref() @@ -301,7 +307,7 @@ namespace NYql { break; case NYql::EGenericDataSourceKind::OPENSEARCH: properties["SourceType"] = "OpenSearch"; - break; + break; case NYql::EGenericDataSourceKind::DATA_SOURCE_KIND_UNSPECIFIED: break; default: diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_predicate_pushdown.cpp b/ydb/library/yql/providers/generic/provider/yql_generic_predicate_pushdown.cpp index cf2ec2258df6..79edd7ba33b7 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_predicate_pushdown.cpp +++ b/ydb/library/yql/providers/generic/provider/yql_generic_predicate_pushdown.cpp @@ -11,7 +11,9 @@ namespace NYql { using namespace NConnector::NApi; TString FormatColumn(const TString& value); - TString FormatValue(const Ydb::TypedValue& value); + TString FormatType(const Ydb::Type& type); + TString FormatTypedValue(const Ydb::TypedValue& typedValue); + TString FormatValue(const Ydb::Value& value); TString FormatNull(const TExpression_TNull&); TString FormatExpression(const TExpression& expression); TString FormatArithmeticalExpression(const TExpression_TArithmeticalExpression& expression); @@ -25,8 +27,27 @@ namespace NYql { TString FormatIn(const TPredicate_TIn& in); TString FormatCoalesce(const TExpression::TCoalesce& coalesce); TString FormatIfExpression(const TExpression::TIf& sqlIf); + TString FormatUnwrap(const TExpression::TUnwrap& unwrap); + TString FormatMinOf(const TExpression::TMinOf& minOf); + TString FormatMaxOf(const TExpression::TMaxOf& maxOf); + TString FormatCurrentUtcTimestamp(const TExpression::TCurrentUtcTimestamp& currentUtcTimestamp); namespace { + TString ToIso8601(TDuration duration) { + if (duration == TDuration::Zero()) { + return "PT0S"; + } + TStringBuilder result; + result << "PT" << duration.Seconds(); + auto fraction = duration.MicroSecondsOfSecond(); + if (fraction != 0) { + result << "."; + result << Sprintf("%06d", fraction); + } + result << "S"; + return result; + } + struct TSerializationContext { const TCoArgument& Arg; TStringBuilder& Err; @@ -57,7 +78,7 @@ namespace NYql { return FromString(from); } - // Special convertation from TStringBuf to TString + // Special conversion from TStringBuf to TString template <> TString Cast(const TStringBuf& from) { return TString(from); @@ -114,6 +135,7 @@ namespace NYql { MATCH_TYPE(Utf8, UTF8); MATCH_TYPE(Json, JSON); MATCH_TYPE(Timestamp, TIMESTAMP); + MATCH_TYPE(Interval, INTERVAL); ctx.Err << "unknown data slot " << static_cast(dataSlot) << " for safe cast"; return false; @@ -127,8 +149,8 @@ namespace NYql { return false; } - const auto toBytexExpr = TExprBase(toBytes.Ref().Child(0)); - auto typeAnnotation = toBytexExpr.Ref().GetTypeAnn(); + const auto toBytesExpr = TExprBase(toBytes.Ref().Child(0)); + auto typeAnnotation = toBytesExpr.Ref().GetTypeAnn(); if (!typeAnnotation) { ctx.Err << "expected non empty type annotation for ToBytes"; return false; @@ -149,7 +171,7 @@ namespace NYql { auto* dstProto = proto->mutable_cast(); dstProto->mutable_type()->set_type_id(Ydb::Type::STRING); - return SerializeExpression(toBytexExpr, dstProto->mutable_value(), ctx, depth + 1); + return SerializeExpression(toBytesExpr, dstProto->mutable_value(), ctx, depth + 1); } bool SerializeToStringExpression(const TExprBase& toString, TExpression* proto, TSerializationContext& ctx, ui64 depth) { @@ -223,6 +245,29 @@ namespace NYql { return SerializeExpression(expr.Left(), exprProto->mutable_left_value(), ctx, depth + 1) && SerializeExpression(expr.Right(), exprProto->mutable_right_value(), ctx, depth + 1); \ } +#define MATCH_UNARY_OP(OpType, op_name) \ + if (auto maybeExpr = expression.Maybe()) { \ + auto expr = maybeExpr.Cast(); \ + auto* exprProto = proto->Y_CAT(mutable_, op_name)(); \ + const auto child = expression.Ptr()->Child(0); \ + if (!SerializeExpression(TExprBase(child), exprProto->mutable_operand(), ctx, depth + 1)) { \ + return false; \ + } \ + return true; \ + } + +#define MATCH_VAR_ARG_OP(OpType, op_name) \ + if (auto maybeExpr = expression.Maybe()) { \ + auto expr = maybeExpr.Cast(); \ + auto* exprProto = proto->Y_CAT(mutable_, op_name)(); \ + for (const auto& child : expr.Ref().Children()) { \ + if (!SerializeExpression(TExprBase(child), exprProto->add_operands(), ctx, depth + 1)) { \ + return false; \ + } \ + } \ + return true; \ + } + bool SerializeSqlIfExpression(const TCoIf& sqlIf, TExpression* proto, TSerializationContext& ctx, ui64 depth); bool SerializeCoalesceExpression(const TCoCoalesce& coalesce, TExpression* proto, TSerializationContext& ctx, ui64 depth); @@ -252,6 +297,9 @@ namespace NYql { if (auto flatMap = expression.Maybe()) { return SerializeFlatMap(flatMap.Cast(), proto, ctx, depth); } + if (auto dependsOn = expression.Maybe()) { + return SerializeExpression(dependsOn.Cast().Input(), proto, ctx, depth + 1); + } // data MATCH_ATOM(Bool, BOOL, bool, bool); @@ -268,11 +316,16 @@ namespace NYql { MATCH_ATOM(String, STRING, bytes, TString); MATCH_ATOM(Utf8, UTF8, text, TString); MATCH_ATOM(Timestamp, TIMESTAMP, int64, i64); + MATCH_ATOM(Interval, INTERVAL, int64, i64); MATCH_ARITHMETICAL(Sub, SUB); MATCH_ARITHMETICAL(Add, ADD); MATCH_ARITHMETICAL(Mul, MUL); MATCH_ARITHMETICAL(Div, DIV); MATCH_ARITHMETICAL(Mod, MOD); + MATCH_UNARY_OP(Unwrap, unwrap); + MATCH_VAR_ARG_OP(Min, min_of); + MATCH_VAR_ARG_OP(Max, max_of); + MATCH_VAR_ARG_OP(CurrentUtcTimestamp, current_utc_timestamp); if (auto maybeNull = expression.Maybe()) { proto->mutable_null(); @@ -290,6 +343,8 @@ namespace NYql { #undef MATCH_ATOM #undef MATCH_ARITHMETICAL +#undef MATCH_UNARY_OP +#undef MATCH_VAR_ARG_OP #define EXPR_NODE_TO_COMPARE_TYPE(TExprNodeType, COMPARE_TYPE) \ if (!opMatched && compare.Maybe()) { \ @@ -341,7 +396,7 @@ namespace NYql { template void UnwrapNestedCoalesce(TProto* proto) { // We can unwrap nested COALESCE: - // COALESCE(..., COALESCE(Predicat_1, Predicat_2), ...) -> COALESCE(..., Predicat_1, Predicat_2, ...) + // COALESCE(..., COALESCE(Predicate_1, Predicate_2), ...) -> COALESCE(..., Predicate_1, Predicate_2, ...) if (proto->operands().rbegin()->has_coalesce()) { auto coalesceOperands = std::move(*proto->mutable_operands()->rbegin()->mutable_coalesce()->mutable_operands()); proto->mutable_operands()->RemoveLast(); @@ -361,7 +416,7 @@ namespace NYql { } bool SerializeCoalescePredicate(const TCoCoalesce& coalesce, TPredicate* proto, TSerializationContext& ctx, ui64 depth) { - // Special case for top level COALESCE: COALESCE(Predicat, FALSE) + // Special case for top level COALESCE: COALESCE(Predicate, FALSE) // We can assume NULL as FALSE and skip COALESCE if (depth == 0) { auto value = coalesce.Value().Maybe(); @@ -545,28 +600,28 @@ namespace NYql { return NFq::EncloseAndEscapeString(value, '`'); } - TString FormatValue(const Ydb::TypedValue& value) { - switch (value.value().value_case()) { + TString FormatValue(const Ydb::Value& value) { + switch (value.value_case()) { case Ydb::Value::kBoolValue: - return value.value().bool_value() ? "TRUE" : "FALSE"; + return value.bool_value() ? "TRUE" : "FALSE"; case Ydb::Value::kInt32Value: - return ToString(value.value().int32_value()); + return ToString(value.int32_value()); case Ydb::Value::kUint32Value: - return ToString(value.value().uint32_value()); + return ToString(value.uint32_value()); case Ydb::Value::kInt64Value: - return ToString(value.value().int64_value()); + return ToString(value.int64_value()); case Ydb::Value::kUint64Value: - return ToString(value.value().uint64_value()); + return ToString(value.uint64_value()); case Ydb::Value::kFloatValue: - return ToString(value.value().float_value()); + return ToString(value.float_value()); case Ydb::Value::kDoubleValue: - return ToString(value.value().double_value()); + return ToString(value.double_value()); case Ydb::Value::kBytesValue: - return NFq::EncloseAndEscapeString(value.value().bytes_value(), '"'); + return NFq::EncloseAndEscapeString(value.bytes_value(), '"'); case Ydb::Value::kTextValue: - return NFq::EncloseAndEscapeString(value.value().text_value(), '"'); + return NFq::EncloseAndEscapeString(value.text_value(), '"'); default: - throw yexception() << "Failed to format ydb typed vlaue, value case " << static_cast(value.value().value_case()) << " is not supported"; + throw yexception() << "Failed to format ydb value, value case " << static_cast(value.value_case()) << " is not supported"; } } @@ -594,6 +649,10 @@ namespace NYql { return "Float"; case Ydb::Type::DOUBLE: return "Double"; + case Ydb::Type::TIMESTAMP: + return "Timestamp"; + case Ydb::Type::INTERVAL: + return "Interval"; case Ydb::Type::STRING: return "String"; case Ydb::Type::UTF8: @@ -616,6 +675,33 @@ namespace NYql { } } + TString FormatTypedValue(const Ydb::TypedValue& typedValue) { + const auto& type = typedValue.type(); + switch (type.type_case()) { + case Ydb::Type::kTypeId: { + const auto& typeId = type.type_id(); + switch (typeId) { + case Ydb::Type::INTERVAL: { + const auto& value = typedValue.value(); + switch (value.value_case()) { + case Ydb::Value::kInt64Value: { + const auto duration = TDuration::MicroSeconds(value.int64_value()); + return TStringBuilder() << FormatType(typedValue.type()) << "(\"" << ToIso8601(duration) << "\")"; + } + default: + [[fallthrough]]; + } + } + default: + [[fallthrough]]; + } + } + default: + // return TStringBuilder() << FormatType(typedValue.type()) << "(\"" << FormatValue(typedValue.value()) << "\")"; + return FormatValue(typedValue.value()); + } + } + TString FormatNull(const TExpression_TNull&) { return "NULL"; } @@ -626,12 +712,69 @@ namespace NYql { return TStringBuilder() << "CAST(" << value << " AS " << type << ")"; } + TString FormatUnwrap(const TExpression_TUnwrap& unwrap) { + return TStringBuilder() << "Unwrap(" << FormatExpression(unwrap.operand()) << ")"; + } + + TString FormatMinOf(const TExpression_TMinOf& minOf) { + TStringBuilder sb; + sb << "MIN_OF("; + + bool isFirst = true; + for (const auto& operand : minOf.operands()) { + if (!isFirst) { + sb << ", "; + } + sb << FormatExpression(operand); + isFirst = false; + } + sb << ")"; + + return sb; + } + + TString FormatMaxOf(const TExpression_TMaxOf& maxOf) { + TStringBuilder sb; + sb << "MAX_OF("; + + bool isFirst = true; + for (const auto& operand : maxOf.operands()) { + if (!isFirst) { + sb << ", "; + } + sb << FormatExpression(operand); + isFirst = false; + } + sb << ")"; + + return sb; + } + + TString FormatCurrentUtcTimestamp(const TExpression_TCurrentUtcTimestamp& currentUtcTimestamp) { + TStringBuilder sb; + sb << "CurrentUtcTimestamp("; + + bool isFirst = true; + for (const auto& operand : currentUtcTimestamp.operands()) { + if (!isFirst) { + sb << ", "; + } + sb << FormatExpression(operand); + isFirst = false; + } + sb << ")"; + + return sb; + } + TString FormatExpression(const TExpression& expression) { switch (expression.payload_case()) { + case TExpression::PAYLOAD_NOT_SET: + return {}; case TExpression::kColumn: return FormatColumn(expression.column()); case TExpression::kTypedValue: - return FormatValue(expression.typed_value()); + return FormatTypedValue(expression.typed_value()); case TExpression::kArithmeticalExpression: return FormatArithmeticalExpression(expression.arithmetical_expression()); case TExpression::kNull: @@ -642,6 +785,14 @@ namespace NYql { return FormatIfExpression(expression.if_()); case TExpression::kCast: return FormatCast(expression.cast()); + case TExpression::kUnwrap: + return FormatUnwrap(expression.unwrap()); + case TExpression::kMinOf: + return FormatMinOf(expression.min_of()); + case TExpression::kMaxOf: + return FormatMaxOf(expression.max_of()); + case TExpression::kCurrentUtcTimestamp: + return FormatCurrentUtcTimestamp(expression.current_utc_timestamp()); default: throw yexception() << "Failed to format expression, unimplemented payload_case " << static_cast(expression.payload_case()); } @@ -945,7 +1096,7 @@ namespace NYql { bool SerializeFilterPredicate( TExprContext& ctx, const TExprBase& predicateBody, - const TCoArgument& predicateArgument, + const TCoArgument& predicateArgument, NConnector::NApi::TPredicate* proto, TStringBuilder& err ) { @@ -955,13 +1106,23 @@ namespace NYql { bool SerializeFilterPredicate( TExprContext& ctx, - const TCoLambda& predicate, - TPredicate* proto, - TStringBuilder& err + const TCoLambda& predicate, + TPredicate* proto, + TStringBuilder& err ) { return SerializeFilterPredicate(ctx, predicate.Body(), predicate.Args().Arg(0), proto, err); } + bool SerializeWatermarkExpr( + TExprContext& ctx, + const TCoLambda& predicate, + TExpression* proto, + TStringBuilder& err + ) { + TSerializationContext serializationContext = {.Arg = predicate.Args().Arg(0), .Err = err, .Ctx = ctx}; + return SerializeExpression(predicate.Body(), proto, serializationContext, 0); + } + TString FormatWhere(const TPredicate& predicate) { auto stream = FormatPredicate(predicate); if (stream.empty()) { diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_predicate_pushdown.h b/ydb/library/yql/providers/generic/provider/yql_generic_predicate_pushdown.h index 0a8a15692f8a..298d2d485ce0 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_predicate_pushdown.h +++ b/ydb/library/yql/providers/generic/provider/yql_generic_predicate_pushdown.h @@ -3,6 +3,7 @@ #include namespace NYql::NConnector::NApi { + class TExpression; class TPredicate; } // namespace NYql::NConnector::NApi @@ -18,9 +19,16 @@ namespace NYql { ); bool SerializeFilterPredicate( TExprContext& ctx, - const NNodes::TCoLambda& predicate, + const NNodes::TCoLambda& predicate, NConnector::NApi::TPredicate* proto, TStringBuilder& err ); + bool SerializeWatermarkExpr( + TExprContext& ctx, + const NNodes::TCoLambda& watermarkExpr, + NConnector::NApi::TExpression* proto, + TStringBuilder& err + ); + TString FormatExpression(const NConnector::NApi::TExpression& expression); TString FormatWhere(const NConnector::NApi::TPredicate& predicate); } // namespace NYql diff --git a/ydb/library/yql/providers/generic/pushdown/yql_generic_match_predicate.cpp b/ydb/library/yql/providers/generic/pushdown/yql_generic_match_predicate.cpp index d93fe69d3288..47a6e57387cb 100644 --- a/ydb/library/yql/providers/generic/pushdown/yql_generic_match_predicate.cpp +++ b/ydb/library/yql/providers/generic/pushdown/yql_generic_match_predicate.cpp @@ -56,6 +56,10 @@ namespace NYql::NGenericPushDown { case NYql::NConnector::NApi::TExpression::kCoalesce: case NYql::NConnector::NApi::TExpression::kIf: case NYql::NConnector::NApi::TExpression::kCast: + case NYql::NConnector::NApi::TExpression::kUnwrap: + case NYql::NConnector::NApi::TExpression::kMinOf: + case NYql::NConnector::NApi::TExpression::kMaxOf: + case NYql::NConnector::NApi::TExpression::kCurrentUtcTimestamp: case NYql::NConnector::NApi::TExpression::PAYLOAD_NOT_SET: return false; } @@ -72,6 +76,10 @@ namespace NYql::NGenericPushDown { case NYql::NConnector::NApi::TExpression::kCoalesce: case NYql::NConnector::NApi::TExpression::kIf: case NYql::NConnector::NApi::TExpression::kCast: + case NYql::NConnector::NApi::TExpression::kUnwrap: + case NYql::NConnector::NApi::TExpression::kMinOf: + case NYql::NConnector::NApi::TExpression::kMaxOf: + case NYql::NConnector::NApi::TExpression::kCurrentUtcTimestamp: case NYql::NConnector::NApi::TExpression::PAYLOAD_NOT_SET: return false; } @@ -290,6 +298,10 @@ namespace NYql::NGenericPushDown { case NYql::NConnector::NApi::TExpression::kCoalesce: case NYql::NConnector::NApi::TExpression::kIf: case NYql::NConnector::NApi::TExpression::kCast: + case NYql::NConnector::NApi::TExpression::kUnwrap: + case NYql::NConnector::NApi::TExpression::kMinOf: + case NYql::NConnector::NApi::TExpression::kMaxOf: + case NYql::NConnector::NApi::TExpression::kCurrentUtcTimestamp: case NYql::NConnector::NApi::TExpression::PAYLOAD_NOT_SET: return Triple::Unknown; } diff --git a/ydb/library/yql/providers/pq/expr_nodes/yql_pq_expr_nodes.json b/ydb/library/yql/providers/pq/expr_nodes/yql_pq_expr_nodes.json index e029bf045c82..d878cbb5d9b8 100644 --- a/ydb/library/yql/providers/pq/expr_nodes/yql_pq_expr_nodes.json +++ b/ydb/library/yql/providers/pq/expr_nodes/yql_pq_expr_nodes.json @@ -59,7 +59,8 @@ {"Index": 4, "Name": "Format", "Type": "TCoAtom"}, {"Index": 5, "Name": "Compression", "Type": "TCoAtom"}, {"Index": 6, "Name": "LimitHint", "Type": "TExprBase", "Optional": true}, - {"Index": 7, "Name": "Settings", "Type": "TExprList", "Optional": true} + {"Index": 7, "Name": "Settings", "Type": "TExprList", "Optional": true}, + {"Index": 8, "Name": "Watermark", "Type": "TCoLambda", "Optional": true} ] }, { @@ -73,7 +74,8 @@ {"Index": 3, "Name": "Settings", "Type": "TCoNameValueTupleList"}, {"Index": 4, "Name": "Token", "Type": "TCoSecureParam"}, {"Index": 5, "Name": "FilterPredicate", "Type": "TCoAtom"}, - {"Index": 6, "Name": "RowType", "Type": "TExprBase"} + {"Index": 6, "Name": "RowType", "Type": "TExprBase"}, + {"Index": 7, "Name": "Watermark", "Type": "TCoAtom"} ] }, { diff --git a/ydb/library/yql/providers/pq/proto/dq_io.proto b/ydb/library/yql/providers/pq/proto/dq_io.proto index f17b2211fbda..3ba1e4f3e8f6 100644 --- a/ydb/library/yql/providers/pq/proto/dq_io.proto +++ b/ydb/library/yql/providers/pq/proto/dq_io.proto @@ -88,6 +88,7 @@ message TDqPqTopicSource { repeated TDqPqFederatedCluster FederatedClusters = 21; StreamingDisposition Disposition = 22; repeated TTaskSensorLabel TaskSensorLabel = 23; + string WatermarkExpr = 24; } message TDqPqTopicSink { diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_datasource.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_datasource.cpp index 23a8ef000360..b5a7b5e0c498 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_datasource.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_datasource.cpp @@ -199,6 +199,10 @@ class TPqDataSourceProvider : public TDataProviderBase { builder.Columns().Build(); } + if (topicKeyParser.GetWatermark()) { + builder.Watermark(topicKeyParser.GetWatermark()); + } + return Build(ctx, read.Pos()) .Input(builder.Done()) .Done().Ptr(); diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_datasource_type_ann.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_datasource_type_ann.cpp index 5e5ef2b0fc71..95bb467de88c 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_datasource_type_ann.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_datasource_type_ann.cpp @@ -6,7 +6,8 @@ #include #include -#include +#include +#include #include #include #include @@ -19,6 +20,33 @@ using namespace NNodes; namespace { +struct TWatermarkPushdownSettings: public NPushdown::TSettings { + TWatermarkPushdownSettings() + : NPushdown::TSettings(NLog::EComponent::ProviderGeneric) + { + using EFlag = NPushdown::TSettings::EFeatureFlag; + Enable( + // Type features + EFlag::DateTimeTypes | + EFlag::DecimalType | + EFlag::StringTypes | + EFlag::TimestampCtor | + EFlag::IntervalCtor | + EFlag::ImplicitConversionToInt64 | + EFlag::DoNotCheckCompareArgumentsTypes | + + // Expr features + EFlag::ArithmeticalExpressions | + EFlag::CastExpression | + EFlag::DivisionExpressions | + EFlag::JustPassthroughOperators | + EFlag::UnaryOperators | + EFlag::MinMax | + EFlag::NonDeterministic + ); + } +}; + class TPqDataSourceTypeAnnotationTransformer : public TVisitorTransformerBase { public: explicit TPqDataSourceTypeAnnotationTransformer(TPqState::TPtr state) @@ -34,34 +62,40 @@ class TPqDataSourceTypeAnnotationTransformer : public TVisitorTransformerBase { AddHandler({TDqPqFederatedCluster::CallableName()}, Hndl(&TSelf::HandleFederatedCluster)); } - TStatus HandleFederatedCluster(TExprBase input, TExprContext& ctx) { - const auto cluster = input.Cast(); - if (!EnsureMinMaxArgsCount(input.Ref(), 3, 4, ctx)) { + TStatus HandleFederatedCluster(const TExprNode::TPtr& input, TExprContext& ctx) { + if (!EnsureMinMaxArgsCount(*input, 3, 4, ctx)) { return TStatus::Error; } - if (!EnsureAtom(cluster.Name().Ref(), ctx)) { + const auto name = input->Child(TDqPqFederatedCluster::idx_Name); + const auto endpoint = input->Child(TDqPqFederatedCluster::idx_Endpoint); + const auto database = input->Child(TDqPqFederatedCluster::idx_Database); + + if (!EnsureAtom(*name, ctx)) { return TStatus::Error; } - if (!EnsureAtom(cluster.Endpoint().Ref(), ctx)) { + if (!EnsureAtom(*endpoint, ctx)) { return TStatus::Error; } - if (!EnsureAtom(cluster.Database().Ref(), ctx)) { + if (!EnsureAtom(*database, ctx)) { return TStatus::Error; } - if (TDqPqFederatedCluster::idx_PartitionsCount < input.Ref().ChildrenSize()) { - if (!EnsureAtom(cluster.PartitionsCount().Ref(), ctx)) { + if (TDqPqFederatedCluster::idx_PartitionsCount < input->ChildrenSize()) { + const auto partitionsCount = input->Child(TDqPqFederatedCluster::idx_PartitionsCount); + if (!EnsureAtom(*partitionsCount, ctx)) { return TStatus::Error; } - if (!TryFromString(cluster.PartitionsCount().Cast().StringValue())) { - ctx.AddError(TIssue(ctx.GetPosition(cluster.PartitionsCount().Cast().Pos()), TStringBuilder() << "Expected integer, but got: " << cluster.PartitionsCount().Cast().StringValue())); + if (!TryFromString(partitionsCount->Content())) { + ctx.AddError(TIssue(ctx.GetPosition(partitionsCount->Pos()), TStringBuilder() + << "Expected integer, but got: " << partitionsCount->Content())); return TStatus::Error; } } - input.Ptr()->SetTypeAnn(ctx.MakeType()); + + input->SetTypeAnn(ctx.MakeType()); return TStatus::Ok; } @@ -82,24 +116,26 @@ class TPqDataSourceTypeAnnotationTransformer : public TVisitorTransformerBase { return TStatus::Ok; } - const TTypeAnnotationNode* GetReadTopicSchema(TPqTopic topic, TMaybeNode columns, TExprContext& ctx, TColumnOrder& columnOrder) { + const TTypeAnnotationNode* GetReadTopicSchema(const TExprNode* topic, const TExprNode* columns, TExprContext& ctx, TColumnOrder& columnOrder) { + const auto metadata = topic->Child(TPqTopic::idx_Metadata); TVector items; - items.reserve((columns ? columns.Cast().Ref().ChildrenSize() : 0) + topic.Metadata().Size()); + items.reserve((columns ? columns->ChildrenSize() : 0) + metadata->ChildrenSize()); - const auto* itemSchema = topic.Ref().GetTypeAnn()->Cast() + const auto* itemSchema = topic->GetTypeAnn()->Cast() ->GetItemType()->Cast(); std::unordered_set addedFields; if (columns) { columnOrder.Reserve(items.capacity()); - for (auto c : columns.Cast().Ref().ChildrenList()) { + for (auto c : columns->Children()) { if (!EnsureAtom(*c, ctx)) { return nullptr; } auto index = itemSchema->FindItem(c->Content()); if (!index) { - ctx.AddError(TIssue(ctx.GetPosition(topic.Pos()), TStringBuilder() << "Unable to find column: " << c->Content())); + ctx.AddError(TIssue(ctx.GetPosition(topic->Pos()), TStringBuilder() + << "Unable to find column: " << c->Content())); return nullptr; } columnOrder.AddColumn(TString(c->Content())); @@ -119,102 +155,184 @@ class TPqDataSourceTypeAnnotationTransformer : public TVisitorTransformerBase { return ctx.MakeType(ctx.MakeType(items)); } - TStatus HandleReadTopic(TExprBase input, TExprContext& ctx) { - if (!EnsureMinMaxArgsCount(input.Ref(), 6, 9, ctx)) { + TStatus HandleReadTopic(const TExprNode::TPtr& input, TExprContext& ctx) { + if (!EnsureMinMaxArgsCount(*input, 6, 9, ctx)) { return TStatus::Error; } - TPqReadTopic read = input.Cast(); + const auto world = input->Child(TPqReadTopic::idx_World); + const auto dataSource = input->Child(TPqReadTopic::idx_DataSource); + const auto topic = input->Child(TPqReadTopic::idx_Topic); + const auto columns = input->Child(TPqReadTopic::idx_Columns); + const auto format = input->Child(TPqReadTopic::idx_Format); + const auto compression = input->Child(TPqReadTopic::idx_Compression); - if (!EnsureWorldType(read.World().Ref(), ctx)) { + if (!EnsureWorldType(*world, ctx)) { return TStatus::Error; } - if (!EnsureSpecificDataSource(read.DataSource().Ref(), PqProviderName, ctx)) { + if (!EnsureSpecificDataSource(*dataSource, PqProviderName, ctx)) { return TStatus::Error; } - TPqTopic topic = read.Topic(); - if (!EnsureCallable(topic.Ref(), ctx)) { + if (!topic->IsCallable(TPqTopic::CallableName())) { + ctx.AddError(TIssue(ctx.GetPosition(topic->Pos()), TStringBuilder() + << "Expected PqTopic, but got: " << topic->Content())); return TStatus::Error; } TColumnOrder columnOrder; - auto schema = GetReadTopicSchema(topic, read.Columns().Maybe(), ctx, columnOrder); + auto schema = GetReadTopicSchema(topic, columns, ctx, columnOrder); if (!schema) { return TStatus::Error; } - auto format = read.Format().Ref().Content(); + if (!EnsureAtom(*format, ctx)) { + return TStatus::Error; + } + + if (!EnsureAtom(*compression, ctx)) { + return TStatus::Error; + } + if (!State_->IsRtmrMode() && !NCommon::ValidateFormatForInput( // Rtmr has 3 field (key/subkey/value). - format, + format->Content(), schema->Cast()->GetItemType()->Cast(), [](TStringBuf fieldName) {return FindPqMetaFieldDescriptorBySysColumn(TString(fieldName)); }, ctx)) { return TStatus::Error; } - if (!NCommon::ValidateCompressionForInput(format, read.Compression().Ref().Content(), ctx)) { + if (!NCommon::ValidateCompressionForInput(format->Content(), compression->Content(), ctx)) { return TStatus::Error; } - input.Ptr()->SetTypeAnn(ctx.MakeType(TTypeAnnotationNode::TListType{ - read.World().Ref().GetTypeAnn(), + if (TPqReadTopic::idx_Watermark < input->ChildrenSize()) { + auto& watermark = input->ChildRef(TPqReadTopic::idx_Watermark); + const auto status = ConvertToLambda(watermark, ctx, 1, 1); + if (status != TStatus::Ok) { + return status; + } + if (!UpdateLambdaAllArgumentsTypes(watermark, {schema->Cast()->GetItemType()}, ctx)) { + return TStatus::Error; + } + if (!watermark->GetTypeAnn()) { + return TStatus::Repeat; + } + if (!EnsureSpecificDataType(*watermark, EDataSlot::Timestamp, ctx, false)) { + return TStatus::Error; + } + + const TCoLambda lambda(watermark); + const auto lambdaArg = TExprBase(lambda.Args().Arg(0).Ptr()); + const auto lambdaBody = lambda.Body(); + if (!TestExprForPushdown(ctx, lambdaArg, lambdaBody, TWatermarkPushdownSettings())) { + ctx.AddError(TIssue(ctx.GetPosition(watermark->Pos()), TStringBuilder() + << "Bad watermark expression")); + return TStatus::Error; + } + } + + input->SetTypeAnn(ctx.MakeType(TTypeAnnotationNode::TListType{ + world->GetTypeAnn(), schema })); - return State_->Types->SetColumnOrder(input.Ref(), columnOrder, ctx); + return State_->Types->SetColumnOrder(*input, columnOrder, ctx); } - TStatus HandleDqTopicSource(TExprBase input, TExprContext& ctx) { - if (!EnsureArgsCount(input.Ref(), 7, ctx)) { + TStatus HandleDqTopicSource(const TExprNode::TPtr& input, TExprContext& ctx) { + if (!EnsureMinMaxArgsCount(*input, 7, 8, ctx)) { return TStatus::Error; } - TDqPqTopicSource topicSource = input.Cast(); + const auto world = input->Child(TDqPqTopicSource::idx_World); + const auto topic = input->Child(TDqPqTopicSource::idx_Topic); + const auto settings = input->Child(TDqPqTopicSource::idx_Settings); + const auto rowType = input->Child(TDqPqTopicSource::idx_RowType); - if (!EnsureWorldType(topicSource.World().Ref(), ctx)) { + if (!EnsureWorldType(*world, ctx)) { return TStatus::Error; } - TPqTopic topic = topicSource.Topic(); + if (!EnsureCallable(*topic, ctx)) { + return TStatus::Error; + } - if (!EnsureCallable(topic.Ref(), ctx)) { + const auto cluster = topic->Child(TPqTopic::idx_Cluster); + const auto path = topic->Child(TPqTopic::idx_Path); + const auto metadata = topic->Child(TPqTopic::idx_Metadata); + + if (!EnsureAtom(*cluster, ctx)) { return TStatus::Error; } - const auto cluster = TString(topic.Cluster().Value()); - const auto topicPath = TString(topic.Path().Value()); - const auto* meta = State_->FindTopicMeta(cluster, topicPath); + if (!EnsureAtom(*path, ctx)) { + return TStatus::Error; + } + + const TString topicCluster(cluster->Content()); + const TString topicPath(path->Content()); + const auto* meta = State_->FindTopicMeta(topicCluster, topicPath); if (!meta) { - ctx.AddError(TIssue(ctx.GetPosition(input.Pos()), TStringBuilder() << "Unknown topic `" << cluster << "`.`" << topicPath << "`")); + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() + << "Unknown topic `" << topicCluster << "`.`" << topicPath << "`")); return TStatus::Error; } - if (const auto maybeSharedReadingSetting = FindSetting(topicSource.Settings().Ptr(), SharedReading)) { - const TExprNode& value = maybeSharedReadingSetting.Cast().Ref(); - if (value.IsAtom() && FromString(value.Content())) { - input.Ptr()->SetTypeAnn(ctx.MakeType(topicSource.RowType().Ref().GetTypeAnn())); + if (TDqPqTopicSource::idx_Watermark < input->ChildrenSize()) { + const auto watermark = input->Child(TDqPqTopicSource::idx_Watermark); + if (!EnsureAtom(*watermark, ctx)) { + return TStatus::Error; + } + } + + if (const auto maybeSharedReadingSetting = FindSetting(settings, SharedReading)) { + const auto value = maybeSharedReadingSetting.Cast().Ptr(); + if (!EnsureAtom(*value, ctx)) { + return TStatus::Error; + } + bool sharedReadingSetting; + if (!TryFromString(value->Content(), sharedReadingSetting)) { + ctx.AddError(TIssue(ctx.GetPosition(settings->Pos()), TStringBuilder() + << "Expected bool, but got: " << value->Content())); + return TStatus::Error; + } + if (sharedReadingSetting) { + input->SetTypeAnn(ctx.MakeType(rowType->GetTypeAnn())); return TStatus::Ok; } } - if (topic.Metadata().Empty()) { - input.Ptr()->SetTypeAnn(ctx.MakeType(ctx.MakeType(EDataSlot::String))); + if (metadata->ChildrenSize() == 0) { + input->SetTypeAnn(ctx.MakeType(ctx.MakeType(EDataSlot::String))); return TStatus::Ok; } - TTypeAnnotationNode::TListType tupleItems; - tupleItems.reserve(topic.Metadata().Size() + 1); + TTypeAnnotationNode::TListType items; + items.reserve(metadata->ChildrenSize() + 1); - tupleItems.emplace_back(ctx.MakeType(EDataSlot::String)); - for (const auto metadataField : topic.Metadata()) { - const auto metadataSysColumn = metadataField.Value().Maybe().Cast().StringValue(); - const auto* descriptor = FindPqMetaFieldDescriptorBySysColumn(metadataSysColumn); - Y_ENSURE(descriptor); - tupleItems.emplace_back(ctx.MakeType(descriptor->Type)); + items.emplace_back(ctx.MakeType(EDataSlot::String)); + for (const auto& metadataField : metadata->Children()) { + if (TCoNameValueTuple::idx_Value >= metadataField->ChildrenSize()) { + ctx.AddError(TIssue(ctx.GetPosition(metadataField->Pos()), TStringBuilder() + << "index out of range")); + return TStatus::Error; + } + const auto metadataSysColumn = metadataField->Child(TCoNameValueTuple::idx_Value); + if (!EnsureAtom(*metadataSysColumn, ctx)) { + return TStatus::Error; + } + const TString metadataSysColumnName(metadataSysColumn->Content()); + const auto descriptor = FindPqMetaFieldDescriptorBySysColumn(metadataSysColumnName); + if (!descriptor) { + ctx.AddError(TIssue(ctx.GetPosition(metadataField->Pos()), TStringBuilder() + << "Pq Meta Field Descriptor was not found")); + return TStatus::Error; + } + items.emplace_back(ctx.MakeType(descriptor->Type)); } - input.Ptr()->SetTypeAnn(ctx.MakeType(ctx.MakeType(tupleItems))); + input->SetTypeAnn(ctx.MakeType(ctx.MakeType(items))); return TStatus::Ok; } @@ -223,47 +341,62 @@ class TPqDataSourceTypeAnnotationTransformer : public TVisitorTransformerBase { return HandleTopicInRtmrMode(input, ctx); } - TPqTopic topic(input); - TVector outputItems; + const auto metadata = input->Child(TPqTopic::idx_Metadata); + const auto rowSpec = input->Child(TPqTopic::idx_RowSpec); + + TVector items; - auto rowSchema = topic.RowSpec().Ref().GetTypeAnn()->Cast()->GetType()->Cast(); + auto rowSchema = rowSpec->GetTypeAnn()->Cast()->GetType()->Cast(); for (const auto& rowSchemaItem : rowSchema->GetItems()) { - outputItems.push_back(rowSchemaItem); + items.push_back(rowSchemaItem); } - for (auto nameValue : topic.Metadata()) { - const auto metadataSysColumn = nameValue.Value().Maybe().Cast().StringValue(); - const auto* descriptor = FindPqMetaFieldDescriptorBySysColumn(metadataSysColumn); - Y_ENSURE(descriptor); - outputItems.emplace_back(ctx.MakeType(descriptor->SysColumn, ctx.MakeType(descriptor->Type))); + for (const auto& metadataField : metadata->Children()) { + if (TCoNameValueTuple::idx_Value >= metadataField->ChildrenSize()) { + ctx.AddError(TIssue(ctx.GetPosition(metadataField->Pos()), TStringBuilder() + << "index out of range")); + return TStatus::Error; + } + const auto metadataSysColumn = metadataField->Child(TCoNameValueTuple::idx_Value); + if (!EnsureAtom(*metadataSysColumn, ctx)) { + return TStatus::Error; + } + const TString metadataSysColumnName(metadataSysColumn->Content()); + const auto descriptor = FindPqMetaFieldDescriptorBySysColumn(metadataSysColumnName); + if (!descriptor) { + ctx.AddError(TIssue(ctx.GetPosition(metadataField->Pos()), TStringBuilder() + << "Pq Meta Field Descriptor was not found")); + return TStatus::Error; + } + items.emplace_back(ctx.MakeType(descriptor->SysColumn, ctx.MakeType(descriptor->Type))); } - const auto* itemType = ctx.MakeType(outputItems); - input->SetTypeAnn(ctx.MakeType(itemType)); + input->SetTypeAnn(ctx.MakeType(ctx.MakeType(items))); return TStatus::Ok; } TStatus HandleMetadata(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { const auto key = input->ChildPtr(0); if (!EnsureCallable(*key, ctx)) { - return IGraphTransformer::TStatus::Error; + return TStatus::Error; } const auto metadataKey = TString(key->TailPtr()->Content()); - const auto* descriptor = FindPqMetaFieldDescriptorByKey(metadataKey); + const auto descriptor = FindPqMetaFieldDescriptorByKey(metadataKey); if (!descriptor) { - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Metadata key " << metadataKey << " wasn't found")); - return IGraphTransformer::TStatus::Error; + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() + << "Metadata key " << metadataKey << " wasn't found")); + return TStatus::Error; } const auto dependsOn = input->Child(1); if (!EnsureDependsOn(*dependsOn, ctx)) { - return IGraphTransformer::TStatus::Error; + return TStatus::Error; } const auto row = dependsOn->TailPtr(); if (!EnsureStructType(*row, ctx)) { - return IGraphTransformer::TStatus::Error; + return TStatus::Error; } if (row->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Struct) { @@ -276,18 +409,19 @@ class TPqDataSourceTypeAnnotationTransformer : public TVisitorTransformerBase { } if (!pos) { - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Wrong place to use SystemMetadata")); - return IGraphTransformer::TStatus::Error; + ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() + << "Wrong place to use SystemMetadata")); + return TStatus::Error; } bool isOptional = false; const TDataExprType* dataType = nullptr; if (!EnsureDataOrOptionalOfData(row->Pos(), structType->GetItems()[*pos]->GetItemType(), isOptional, dataType, ctx)) { - return IGraphTransformer::TStatus::Error; + return TStatus::Error; } if (!EnsureSpecificDataType(row->Pos(), *dataType, descriptor->Type, ctx)) { - return IGraphTransformer::TStatus::Error; + return TStatus::Error; } output = ctx.Builder(input->Pos()) @@ -297,7 +431,7 @@ class TPqDataSourceTypeAnnotationTransformer : public TVisitorTransformerBase { .Seal() .Build(); - return IGraphTransformer::TStatus::Repeat; + return TStatus::Repeat; } input->SetTypeAnn(ctx.MakeType(descriptor->Type)); diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp index 1ec740230116..012475e6f965 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp @@ -46,10 +46,10 @@ class TPqDqIntegration: public TDqIntegrationBase { partitions.reserve(tasks); for (size_t i = 0; i < tasks; ++i) { NPq::NProto::TDqReadTaskParams params; - auto* partitioninigParams = params.MutablePartitioningParams(); - partitioninigParams->SetTopicPartitionsCount(topicPartitionsCount); - partitioninigParams->SetEachTopicPartitionGroupId(i); - partitioninigParams->SetDqPartitionsCount(tasks); + auto* partitioningParams = params.MutablePartitioningParams(); + partitioningParams->SetTopicPartitionsCount(topicPartitionsCount); + partitioningParams->SetEachTopicPartitionGroupId(i); + partitioningParams->SetDqPartitionsCount(tasks); YQL_CLOG(DEBUG, ProviderPq) << "Create DQ reading partition " << params; TString serializedParams; @@ -95,6 +95,28 @@ class TPqDqIntegration: public TDqIntegrationBase { }); auto columnNames = ctx.NewList(pos, std::move(colNames)); + TString serializedWatermarkExpr; + if (const auto maybeWatermark = pqReadTopic.Watermark()) { + const auto watermark = maybeWatermark.Cast(); + + if (wrSettings.WatermarksMode.GetOrElse("") != "default") { + ctx.AddError(TIssue(ctx.GetPosition(pqReadTopic.Pos()), R"(Enable watermarks using "PRAGMA dq.WatermarksMode="default";")")); + return {}; + } + + TStringBuilder err; + NYql::NConnector::NApi::TExpression watermarkExprProto; + if (!NYql::SerializeWatermarkExpr(ctx, watermark, &watermarkExprProto, err)) { + ctx.AddError(TIssue(ctx.GetPosition(pqReadTopic.Pos()), "Failed to serialize Watermark Expr to proto: " + err)); + return {}; + } + if (!watermarkExprProto.SerializeToString(&serializedWatermarkExpr)) { + ctx.AddError(TIssue(ctx.GetPosition(pqReadTopic.Pos()), "Failed to serialize Watermark Expr to string")); + return {}; + } + } + + return Build(ctx, pos) .Input() .World(pqReadTopic.World()) @@ -106,6 +128,7 @@ class TPqDqIntegration: public TDqIntegrationBase { .Build() .FilterPredicate().Value(TString()).Build() // Empty predicate by default <=> WHERE TRUE .RowType(ExpandType(pqReadTopic.Pos(), *rowType, ctx)) + .Watermark().Value(serializedWatermarkExpr).Build() .Build() .RowType(ExpandType(pqReadTopic.Pos(), *rowType, ctx)) .DataSource(pqReadTopic.DataSource().Cast()) @@ -279,10 +302,20 @@ class TPqDqIntegration: public TDqIntegrationBase { taskSensorLabel->SetLabel(label); taskSensorLabel->SetValue(value); } + + NYql::NConnector::NApi::TExpression watermarkExprProto; + auto serializedWatermarkExpr = topicSource.Watermark().Ref().Content(); + YQL_ENSURE(watermarkExprProto.ParseFromString(serializedWatermarkExpr)); + TString watermarkExprSql = NYql::FormatExpression(watermarkExprProto); + srcDesc.SetWatermarkExpr(watermarkExprSql); + protoSettings.PackFrom(srcDesc); if (sharedReading && !predicateSql.empty()) { ctx.AddWarning(TIssue(ctx.GetPosition(node.Pos()), "Row dispatcher will use the predicate: " + predicateSql)); } + if (sharedReading && !watermarkExprSql.empty()) { + ctx.AddWarning(TIssue(ctx.GetPosition(node.Pos()), "Row dispatcher will use watermark expr: " + watermarkExprSql)); + } sourceType = "PqSource"; } } diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_logical_opt.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_logical_opt.cpp index 1f8b6bcc1c89..b5e0e5cb9bac 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_logical_opt.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_logical_opt.cpp @@ -34,7 +34,7 @@ namespace { // Operator features EFlag::ExpressionAsPredicate | EFlag::ArithmeticalExpressions | EFlag::ImplicitConversionToInt64 | EFlag::StringTypes | EFlag::LikeOperator | EFlag::DoNotCheckCompareArgumentsTypes | EFlag::InOperator | - EFlag::IsDistinctOperator | EFlag::JustPassthroughOperators | DivisionExpressions | EFlag::CastExpression | + EFlag::IsDistinctOperator | EFlag::JustPassthroughOperators | EFlag::DivisionExpressions | EFlag::CastExpression | EFlag::ToBytesFromStringExpressions | EFlag::FlatMapOverOptionals | // Split features @@ -237,7 +237,7 @@ class TPqLogicalOptProposalTransformer : public TOptimizeTransformerBase { .Input(ctx.ReplaceNode(extractMembersInput.Ptr(), dqSourceWrap.Ref(), newDqSourceWrap)) .Done(); } - + bool IsEmptyFilterPredicate(const TCoLambda& lambda) const { auto maybeBool = lambda.Body().Maybe(); if (!maybeBool) { @@ -250,7 +250,7 @@ class TPqLogicalOptProposalTransformer : public TOptimizeTransformerBase { auto flatmap = node.Cast(); auto maybeExtractMembers = flatmap.Input().Maybe(); - auto maybeDqSourceWrap = + auto maybeDqSourceWrap = maybeExtractMembers ? maybeExtractMembers.Cast().Input().Maybe() : flatmap.Input().Maybe(); @@ -280,9 +280,9 @@ class TPqLogicalOptProposalTransformer : public TOptimizeTransformerBase { ctx.AddWarning(TIssue(ctx.GetPosition(node.Pos()), "Failed to serialize filter predicate for source: " + err)); return node; } - + TString serializedProto; - YQL_ENSURE(predicateProto.SerializeToString(&serializedProto)); + YQL_ENSURE(predicateProto.SerializeToString(&serializedProto)); YQL_CLOG(INFO, ProviderPq) << "Build new TCoFlatMap with predicate"; if (maybeExtractMembers) { diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_topic_key_parser.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_topic_key_parser.cpp index 9fd71db6c8b7..a6f1cae105ba 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_topic_key_parser.cpp +++ b/ydb/library/yql/providers/pq/provider/yql_pq_topic_key_parser.cpp @@ -71,6 +71,10 @@ bool TTopicKeyParser::Parse(const TExprNode& expr, TExprNode::TPtr readSettings, DateFormat = readSettings->Child(i); continue; } + if (readSettings->Child(i)->Head().IsAtom("watermark")) { + Watermark = readSettings->Child(i)->ChildPtr(1); + continue; + } } } diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_topic_key_parser.h b/ydb/library/yql/providers/pq/provider/yql_pq_topic_key_parser.h index f2f6ebdf04b6..2496e86a12a1 100644 --- a/ydb/library/yql/providers/pq/provider/yql_pq_topic_key_parser.h +++ b/ydb/library/yql/providers/pq/provider/yql_pq_topic_key_parser.h @@ -34,22 +34,26 @@ class TTopicKeyParser { return DateTimeFormatName; } - TExprNode::TPtr GetDateTimeFormat() { + TExprNode::TPtr GetDateTimeFormat() const { return DateTimeFormat; } - TExprNode::TPtr GetTimestampFormatName() { + TExprNode::TPtr GetTimestampFormatName() const { return TimestampFormatName; } - TExprNode::TPtr GetTimestampFormat() { + TExprNode::TPtr GetTimestampFormat() const { return TimestampFormat; } - TExprNode::TPtr GetDateFormat() { + TExprNode::TPtr GetDateFormat() const { return DateFormat; } + TExprNode::TPtr GetWatermark() const { + return Watermark; + } + bool Parse(const TExprNode& expr, TExprNode::TPtr readSettings, TExprContext& ctx); private: @@ -67,6 +71,7 @@ class TTopicKeyParser { TExprNode::TPtr DateFormat; TExprNode::TPtr UserSchema; TExprNode::TPtr ColumnOrder; + TExprNode::TPtr Watermark; }; } // namespace NYql diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp index b06b0e98c983..e7e00f98ddfe 100644 --- a/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp +++ b/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp @@ -239,9 +239,15 @@ class TS3DqIntegration: public TDqIntegrationBase { } } - TExprNode::TPtr WrapRead(const TExprNode::TPtr& read, TExprContext& ctx, const TWrapReadSettings& ) override { + TExprNode::TPtr WrapRead(const TExprNode::TPtr& read, TExprContext& ctx, const TWrapReadSettings& wrSettings) override { if (const auto& maybeS3ReadObject = TMaybeNode(read)) { const auto& s3ReadObject = maybeS3ReadObject.Cast(); + + if (wrSettings.WatermarksMode.GetOrElse("") == "default") { + ctx.AddError(TIssue(ctx.GetPosition(s3ReadObject.Pos()), "Cannot use watermarks in S3")); + return {}; + } + YQL_ENSURE(s3ReadObject.Ref().GetTypeAnn(), "No type annotation for node " << s3ReadObject.Ref().Content()); const auto rowType = s3ReadObject.Ref().GetTypeAnn()->Cast()->GetItems().back()->Cast()->GetItemType(); diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp index 7a688f285e2b..57dc3bcb3055 100644 --- a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp +++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp @@ -104,9 +104,15 @@ class TSolomonDqIntegration: public TDqIntegrationBase { return Nothing(); } - TExprNode::TPtr WrapRead(const TExprNode::TPtr& read, TExprContext& ctx, const TWrapReadSettings&) override { + TExprNode::TPtr WrapRead(const TExprNode::TPtr& read, TExprContext& ctx, const TWrapReadSettings& wrSettings) override { if (const auto& maybeSoReadObject = TMaybeNode(read)) { const auto& soReadObject = maybeSoReadObject.Cast(); + + if (wrSettings.WatermarksMode.GetOrElse("") == "default") { + ctx.AddError(TIssue(ctx.GetPosition(soReadObject.Pos()), "Cannot use watermarks in Solomon")); + return {}; + } + YQL_ENSURE(soReadObject.Ref().GetTypeAnn(), "No type annotation for node " << soReadObject.Ref().Content()); const auto& clusterName = soReadObject.DataSource().Cluster().StringValue(); @@ -286,10 +292,10 @@ class TSolomonDqIntegration: public TDqIntegrationBase { YQL_ENSURE(clusterDesc, "Unknown cluster " << cluster); NSo::NProto::TDqSolomonSource source = NSo::FillSolomonSource(clusterDesc, settings.Project().StringValue()); - + source.SetFrom(TInstant::ParseIso8601(settings.From().StringValue()).Seconds()); source.SetTo(TInstant::ParseIso8601(settings.To().StringValue()).Seconds()); - + auto selectors = settings.Selectors().StringValue(); if (!selectors.empty()) { std::map selectorValues; @@ -366,7 +372,7 @@ class TSolomonDqIntegration: public TDqIntegrationBase { auto providerFactory = CreateCredentialsProviderFactoryForStructuredToken(State_->CredentialsFactory, State_->Configuration->Tokens.at(cluster)); auto credentialsProvider = providerFactory->CreateProvider(); - + NDq::TDqSolomonReadParams readParams{ .Source = source }; YQL_ENSURE(NActors::TlsActivationContext); diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_dq_integration.cpp b/ydb/library/yql/providers/ydb/provider/yql_ydb_dq_integration.cpp index 0c9be9b9393b..bf45485e2c54 100644 --- a/ydb/library/yql/providers/ydb/provider/yql_ydb_dq_integration.cpp +++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_dq_integration.cpp @@ -80,9 +80,15 @@ class TYdbDqIntegration: public TDqIntegrationBase { return Nothing(); } - TExprNode::TPtr WrapRead(const TExprNode::TPtr& read, TExprContext& ctx, const TWrapReadSettings&) override { + TExprNode::TPtr WrapRead(const TExprNode::TPtr& read, TExprContext& ctx, const TWrapReadSettings& wrSettings) override { if (const auto& maybeYdbReadTable = TMaybeNode(read)) { const auto& ydbReadTable = maybeYdbReadTable.Cast(); + + if (wrSettings.WatermarksMode.GetOrElse("") == "default") { + ctx.AddError(TIssue(ctx.GetPosition(ydbReadTable.Pos()), "Cannot use watermarks in YDB")); + return {}; + } + YQL_ENSURE(ydbReadTable.Ref().GetTypeAnn(), "No type annotation for node " << ydbReadTable.Ref().Content()); const auto& clusterName = ydbReadTable.DataSource().Cluster().Value(); const auto token = "cluster:default_" + TString(clusterName); diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHop-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHop-default.txt_/ast.txt index 6f253d12644a..0f7c993101a4 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHop-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHop-default.txt_/ast.txt @@ -12,7 +12,7 @@ (let $11 '('"UseSsl" '"1")) (let $12 '('('"Consumer" '"test_client") $9 $10 '('"ReconnectPeriod" '"") '('"Format" '"raw") '('"ReadGroup" '"fqrun") $11)) (let $13 (SecureParam '"cluster:default_pq")) -(let $14 (DqPqTopicSource $3 $8 '('"Data") $12 $13 '"" $7)) +(let $14 (DqPqTopicSource $3 $8 '('"Data") $12 $13 '"" $7 '"")) (let $15 (ResourceType '"Yson2.Node")) (let $16 '('"strict")) (let $17 '($16)) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopByStringKey-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopByStringKey-default.txt_/ast.txt index 0777dfd237ce..17bf4fb989be 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopByStringKey-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopByStringKey-default.txt_/ast.txt @@ -12,7 +12,7 @@ (let $11 '('"UseSsl" '"1")) (let $12 '('('"Consumer" '"test_client") $9 $10 '('"ReconnectPeriod" '"") '('"Format" '"raw") '('"ReadGroup" '"fqrun") $11)) (let $13 (SecureParam '"cluster:default_pq")) -(let $14 (DqPqTopicSource $3 $8 '('"Data") $12 $13 '"" $7)) +(let $14 (DqPqTopicSource $3 $8 '('"Data") $12 $13 '"" $7 '"")) (let $15 (ResourceType '"Yson2.Node")) (let $16 '('"strict")) (let $17 '($16)) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopExprKey-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopExprKey-default.txt_/ast.txt index dd4ef6f82809..0883ec3e9fb9 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopExprKey-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopExprKey-default.txt_/ast.txt @@ -12,7 +12,7 @@ (let $11 '('"UseSsl" '"1")) (let $12 '('('"Consumer" '"test_client") $9 $10 '('"ReconnectPeriod" '"") '('"Format" '"raw") '('"ReadGroup" '"fqrun") $11)) (let $13 (SecureParam '"cluster:default_pq")) -(let $14 (DqPqTopicSource $3 $8 '('"Data") $12 $13 '"" $7)) +(let $14 (DqPqTopicSource $3 $8 '('"Data") $12 $13 '"" $7 '"")) (let $15 (ResourceType '"Yson2.Node")) (let $16 '('"strict")) (let $17 '($16)) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopListKey-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopListKey-default.txt_/ast.txt index 1aa87f821097..5d34171a7405 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopListKey-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopListKey-default.txt_/ast.txt @@ -12,7 +12,7 @@ (let $11 '('"UseSsl" '1)) (let $12 '('('"Consumer" '"test_client") $9 $10 '('"ReconnectPeriod" '"") '('"Format" '"raw") '('"ReadGroup" '"fqrun") $11)) (let $13 (SecureParam '"cluster:default_pq")) -(let $14 (DqPqTopicSource $3 $8 '('"Data") $12 $13 '"" $7)) +(let $14 (DqPqTopicSource $3 $8 '('"Data") $12 $13 '"" $7 '"")) (let $15 (ResourceType '"Yson2.Node")) (let $16 '('"strict")) (let $17 '($16)) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopNoKey-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopNoKey-default.txt_/ast.txt index f6924eada742..159a81f00aef 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopNoKey-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopNoKey-default.txt_/ast.txt @@ -12,7 +12,7 @@ (let $11 '('"UseSsl" '"1")) (let $12 '('('"Consumer" '"test_client") $9 $10 '('"ReconnectPeriod" '"") '('"Format" '"raw") '('"ReadGroup" '"fqrun") $11)) (let $13 (SecureParam '"cluster:default_pq")) -(let $14 (DqPqTopicSource $3 $8 '('"Data") $12 $13 '"" $7)) +(let $14 (DqPqTopicSource $3 $8 '('"Data") $12 $13 '"" $7 '"")) (let $15 (ResourceType '"Yson2.Node")) (let $16 '('"strict")) (let $17 '($16)) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopPercentile-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopPercentile-default.txt_/ast.txt index 2e271937fec7..e3c50d40c323 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopPercentile-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopPercentile-default.txt_/ast.txt @@ -12,7 +12,7 @@ (let $11 '('"UseSsl" '1)) (let $12 '('('"Consumer" '"test_client") $9 $10 '('"ReconnectPeriod" '"") '('"Format" '"raw") '('"ReadGroup" '"fqrun") $11)) (let $13 (SecureParam '"cluster:default_pq")) -(let $14 (DqPqTopicSource $3 $8 '('"Data") $12 $13 '"" $7)) +(let $14 (DqPqTopicSource $3 $8 '('"Data") $12 $13 '"" $7 '"")) (let $15 (ResourceType '"Yson2.Node")) (let $16 '('"2")) (let $17 '('"strict")) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopTimeExtractorUnusedColumns-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopTimeExtractorUnusedColumns-default.txt_/ast.txt index 2e271937fec7..e3c50d40c323 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopTimeExtractorUnusedColumns-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopTimeExtractorUnusedColumns-default.txt_/ast.txt @@ -12,7 +12,7 @@ (let $11 '('"UseSsl" '1)) (let $12 '('('"Consumer" '"test_client") $9 $10 '('"ReconnectPeriod" '"") '('"Format" '"raw") '('"ReadGroup" '"fqrun") $11)) (let $13 (SecureParam '"cluster:default_pq")) -(let $14 (DqPqTopicSource $3 $8 '('"Data") $12 $13 '"" $7)) +(let $14 (DqPqTopicSource $3 $8 '('"Data") $12 $13 '"" $7 '"")) (let $15 (ResourceType '"Yson2.Node")) (let $16 '('"2")) (let $17 '('"strict")) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopWithDataWatermarks-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopWithDataWatermarks-default.txt_/ast.txt index 6f253d12644a..0f7c993101a4 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopWithDataWatermarks-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHopWithDataWatermarks-default.txt_/ast.txt @@ -12,7 +12,7 @@ (let $11 '('"UseSsl" '"1")) (let $12 '('('"Consumer" '"test_client") $9 $10 '('"ReconnectPeriod" '"") '('"Format" '"raw") '('"ReadGroup" '"fqrun") $11)) (let $13 (SecureParam '"cluster:default_pq")) -(let $14 (DqPqTopicSource $3 $8 '('"Data") $12 $13 '"" $7)) +(let $14 (DqPqTopicSource $3 $8 '('"Data") $12 $13 '"" $7 '"")) (let $15 (ResourceType '"Yson2.Node")) (let $16 '('"strict")) (let $17 '($16)) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHoppingWithDataWatermarks-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHoppingWithDataWatermarks-default.txt_/ast.txt index 6f253d12644a..0f7c993101a4 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHoppingWithDataWatermarks-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-GroupByHoppingWithDataWatermarks-default.txt_/ast.txt @@ -12,7 +12,7 @@ (let $11 '('"UseSsl" '"1")) (let $12 '('('"Consumer" '"test_client") $9 $10 '('"ReconnectPeriod" '"") '('"Format" '"raw") '('"ReadGroup" '"fqrun") $11)) (let $13 (SecureParam '"cluster:default_pq")) -(let $14 (DqPqTopicSource $3 $8 '('"Data") $12 $13 '"" $7)) +(let $14 (DqPqTopicSource $3 $8 '('"Data") $12 $13 '"" $7 '"")) (let $15 (ResourceType '"Yson2.Node")) (let $16 '('"strict")) (let $17 '($16)) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopic-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopic-default.txt_/ast.txt index eecaa281ede8..bd535b7a77a5 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopic-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopic-default.txt_/ast.txt @@ -9,7 +9,7 @@ (let $8 '('"Data")) (let $9 '('"SharedReading" '"1")) (let $10 '('('"Consumer" '"test_client") '('"Endpoint" '"") $9 '('"ReconnectPeriod" '"") '('"Format" '"raw") '('"ReadGroup" '"fqrun") '('"UseSsl" '"1"))) -(let $11 (DqPqTopicSource world $7 $8 $10 (SecureParam '"cluster:default_pq") '"" $6)) +(let $11 (DqPqTopicSource world $7 $8 $10 (SecureParam '"cluster:default_pq") '"" $6 '"")) (let $12 (DqStage '((DqSource $5 $11)) (lambda '($16) (block '( (let $17 '('('"format" '"raw") '('"formatSettings" '('('"data.datetime.formatname" '"POSIX") '('"data.timestamp.formatname" '"POSIX"))) '('"settings" '($9)))) (let $18 (DqSourceWideWrap $16 $5 $6 $17)) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicGroupWriteToSolomon-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicGroupWriteToSolomon-default.txt_/ast.txt index 0647889aea67..adacffa1b577 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicGroupWriteToSolomon-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicGroupWriteToSolomon-default.txt_/ast.txt @@ -8,7 +8,7 @@ (let $7 (PqTopic '"pq" '"local" '"test_topic_input" '('('"PartitionsCount" '1)) '() $6)) (let $8 '('"SharedReading" '1)) (let $9 '('('"Consumer" '"test_client") '('"Endpoint" '"") $8 '('"ReconnectPeriod" '"") '('"Format" '"raw") '('"ReadGroup" '"fqrun") '('"UseSsl" '1))) -(let $10 (DqPqTopicSource $3 $7 '('"Data") $9 (SecureParam '"cluster:default_pq") '"" $6)) +(let $10 (DqPqTopicSource $3 $7 '('"Data") $9 (SecureParam '"cluster:default_pq") '"" $6 '"")) (let $11 (DqStage '((DqSource $4 $10)) (lambda '($17) (block '( (let $18 '('('"format" '"raw") '('"formatSettings" '('('"data.datetime.formatname" '"POSIX") '('"data.timestamp.formatname" '"POSIX"))) '('"settings" '($8)))) (let $19 (DqSourceWideWrap $17 $4 $6 $18)) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadata-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadata-default.txt_/ast.txt index 99756c957138..ae1bd17eddb1 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadata-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadata-default.txt_/ast.txt @@ -14,7 +14,7 @@ (let $13 (DataType 'Timestamp)) (let $14 (DataType 'Uint64)) (let $15 (StructType '('_yql_sys_create_time $13) '('_yql_sys_message_group_id $6) '('_yql_sys_offset $14) '('_yql_sys_partition_id $14) '('_yql_sys_seq_no $14) '('_yql_sys_tsp_write_time $13) $8 $9)) -(let $16 (DqPqTopicSource world $10 '('"color" '"value") $12 (SecureParam '"cluster:default_pq") '"" $15)) +(let $16 (DqPqTopicSource world $10 '('"color" '"value") $12 (SecureParam '"cluster:default_pq") '"" $15 '"")) (let $17 (DqStage '((DqSource $4 $16)) (lambda '($22) (block '( (let $23 '('_yql_sys_create_time '_yql_sys_tsp_write_time '_yql_sys_partition_id '_yql_sys_offset '_yql_sys_message_group_id '_yql_sys_seq_no)) (let $24 '('('"format" '"json_each_row") '('"metadataColumns" $23) '('"formatSettings" '('('"data.datetime.formatname" '"POSIX") '('"data.timestamp.formatname" '"POSIX"))) '('"settings" '($11)))) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadataInsideFilter-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadataInsideFilter-default.txt_/ast.txt index ac1812d02b4f..e464555c1fe0 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadataInsideFilter-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadataInsideFilter-default.txt_/ast.txt @@ -7,7 +7,7 @@ (let $6 '('"SharedReading" '"1")) (let $7 '('('"Consumer" '"test_client") '('"Endpoint" '"") $6 '('"ReconnectPeriod" '"") '('"Format" '"json_each_row") '('"ReadGroup" '"fqrun") '('"UseSsl" '"1"))) (let $8 (StructType '('_yql_sys_offset (DataType 'Uint64)))) -(let $9 (DqPqTopicSource world $5 '() $7 (SecureParam '"cluster:default_pq") '"" $8)) +(let $9 (DqPqTopicSource world $5 '() $7 (SecureParam '"cluster:default_pq") '"" $8 '"")) (let $10 (DqStage '((DqSource $4 $9)) (lambda '($14) (block '( (let $15 '('('"format" '"json_each_row") '('"metadataColumns" '('_yql_sys_offset)) '('"formatSettings" '('('"data.datetime.formatname" '"POSIX") '('"data.timestamp.formatname" '"POSIX"))) '('"settings" '($6)))) (let $16 (DqSourceWideWrap $14 $4 $8 $15)) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadataNestedDeep-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadataNestedDeep-default.txt_/ast.txt index f56abb841867..863facabfad0 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadataNestedDeep-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadataNestedDeep-default.txt_/ast.txt @@ -11,7 +11,7 @@ (let $10 '('"SharedReading" '"1")) (let $11 '('('"Consumer" '"test_client") '('"Endpoint" '"") $10 '('"ReconnectPeriod" '"") '('"Format" '"json_each_row") '('"ReadGroup" '"fqrun") '('"UseSsl" '"1"))) (let $12 (StructType '('_yql_sys_offset (DataType 'Uint64)) $7 $8)) -(let $13 (DqPqTopicSource world $9 '('"color" '"value") $11 (SecureParam '"cluster:default_pq") '"" $12)) +(let $13 (DqPqTopicSource world $9 '('"color" '"value") $11 (SecureParam '"cluster:default_pq") '"" $12 '"")) (let $14 (DqStage '((DqSource $4 $13)) (lambda '($19) (block '( (let $20 '('('"format" '"json_each_row") '('"metadataColumns" '('_yql_sys_offset)) '('"formatSettings" '('('"data.datetime.formatname" '"POSIX") '('"data.timestamp.formatname" '"POSIX"))) '('"settings" '($10)))) (let $21 (DqSourceWideWrap $19 $4 $12 $20)) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadataWithFilter-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadataWithFilter-default.txt_/ast.txt index 889dca624064..a9656addb51d 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadataWithFilter-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithMetadataWithFilter-default.txt_/ast.txt @@ -8,7 +8,7 @@ (let $7 '('"SharedReading" '"1")) (let $8 '('('"Consumer" '"test_client") '('"Endpoint" '"") $7 '('"ReconnectPeriod" '"") '('"Format" '"json_each_row") '('"ReadGroup" '"fqrun") '('"UseSsl" '"1"))) (let $9 (StructType '('_yql_sys_offset (DataType 'Uint64)) $5)) -(let $10 (DqPqTopicSource world $6 '('"value") $8 (SecureParam '"cluster:default_pq") '"B\x1B\b\x03\x12\a\x12\x05value\x1A\x0E\n\f\n\x03\b\x81 \x12\x05B\x03123" $9)) +(let $10 (DqPqTopicSource world $6 '('"value") $8 (SecureParam '"cluster:default_pq") '"B\x1B\b\x03\x12\a\x12\x05value\x1A\x0E\n\f\n\x03\b\x81 \x12\x05B\x03123" $9 '"")) (let $11 (DqStage '((DqSource $4 $10)) (lambda '($15) (block '( (let $16 '('('"format" '"json_each_row") '('"metadataColumns" '('_yql_sys_offset)) '('"formatSettings" '('('"data.datetime.formatname" '"POSIX") '('"data.timestamp.formatname" '"POSIX"))) '('"settings" '($7)))) (let $17 (DqSourceWideWrap $15 $4 $9 $16)) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithSchema-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithSchema-default.txt_/ast.txt index 6cf72308edd8..746165e85ff5 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithSchema-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTopicWithSchema-default.txt_/ast.txt @@ -9,7 +9,7 @@ (let $8 (PqTopic '"pq" '"local" '"test_topic_input" '('('"PartitionsCount" '"1")) '() $7)) (let $9 '('"SharedReading" '"1")) (let $10 '('('"Consumer" '"test_client") '('"Endpoint" '"") $9 '('"ReconnectPeriod" '"") '('"Format" '"json_each_row") '('"ReadGroup" '"fqrun") '('"UseSsl" '"1"))) -(let $11 (DqPqTopicSource world $8 '('"color" '"value") $10 (SecureParam '"cluster:default_pq") '"" $7)) +(let $11 (DqPqTopicSource world $8 '('"color" '"value") $10 (SecureParam '"cluster:default_pq") '"" $7 '"")) (let $12 (DqStage '((DqSource $5 $11)) (lambda '($15) (block '( (let $16 '('('"format" '"json_each_row") '('"formatSettings" '('('"data.datetime.formatname" '"POSIX") '('"data.timestamp.formatname" '"POSIX"))) '('"settings" '($9)))) (let $17 (DqSourceWideWrap $15 $5 $7 $16)) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTwoTopics-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTwoTopics-default.txt_/ast.txt index 2ce3e991be9a..c47a7fad5c34 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTwoTopics-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadTwoTopics-default.txt_/ast.txt @@ -12,7 +12,7 @@ (let $11 '('"UseSsl" '"1")) (let $12 '('('"Consumer" '"test_client") $9 $10 '('"ReconnectPeriod" '"") '('"Format" '"raw") '('"ReadGroup" '"fqrun") $11)) (let $13 (SecureParam '"cluster:default_pq")) -(let $14 (DqPqTopicSource $3 $7 $8 $12 $13 '"" $6)) +(let $14 (DqPqTopicSource $3 $7 $8 $12 $13 '"" $6 '"")) (let $15 (lambda '($23) (block '( (let $24 '('('"format" '"raw") '('"formatSettings" '('('"data.datetime.formatname" '"POSIX") '('"data.timestamp.formatname" '"POSIX"))) '('"settings" '($10)))) (let $25 (DqSourceWideWrap $23 $4 $6 $24)) @@ -20,7 +20,7 @@ )))) (let $16 (DqStage '((DqSource $4 $14)) $15 '('('"_logical_id" '0)))) (let $17 (PqTopic '"pq" '"local" '"test_topic_input2" $5 '() $6)) -(let $18 (DqPqTopicSource $3 $17 $8 $12 $13 '"" $6)) +(let $18 (DqPqTopicSource $3 $17 $8 $12 $13 '"" $6 '"")) (let $19 (DqStage '((DqSource $4 $18)) $15 '('('"_logical_id" '0)))) (let $20 (DataSink '"pq" '"pq")) (let $21 (PqTopic '"pq" '"local" '"test_topic_output" $5 '() $6)) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadWriteSameTopic-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadWriteSameTopic-default.txt_/ast.txt index c1af23d8499a..acd2a80ca030 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadWriteSameTopic-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadWriteSameTopic-default.txt_/ast.txt @@ -11,7 +11,7 @@ (let $10 '('"UseSsl" '"1")) (let $11 '('('"Consumer" '"test_client") $8 $9 '('"ReconnectPeriod" '"") '('"Format" '"raw") '('"ReadGroup" '"fqrun") $10)) (let $12 (SecureParam '"cluster:default_pq")) -(let $13 (DqPqTopicSource $3 $7 '($5) $11 $12 '"" $6)) +(let $13 (DqPqTopicSource $3 $7 '($5) $11 $12 '"" $6 '"")) (let $14 (DataSink '"pq" '"pq")) (let $15 (DqPqTopicSink $7 '($8 $10) $12)) (return (Commit! (DqQuery! $3 '((DqStage '((DqSource $4 $13)) (lambda '($16) (block '( diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadWriteTopic-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadWriteTopic-default.txt_/ast.txt index 2c0388d0f6ce..2233c6c93dc9 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadWriteTopic-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadWriteTopic-default.txt_/ast.txt @@ -11,7 +11,7 @@ (let $10 '('"UseSsl" '"1")) (let $11 '('('"Consumer" '"test_client") $8 $9 '('"ReconnectPeriod" '"") '('"Format" '"raw") '('"ReadGroup" '"fqrun") $10)) (let $12 (SecureParam '"cluster:default_pq")) -(let $13 (DqPqTopicSource $3 $7 '('"Data") $11 $12 '"" $6)) +(let $13 (DqPqTopicSource $3 $7 '('"Data") $11 $12 '"" $6 '"")) (let $14 (DataSink '"pq" '"pq")) (let $15 (PqTopic '"pq" '"local" '"test_topic_output" $5 '() $6)) (let $16 (DqPqTopicSink $15 '($8 $10) $12)) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadWriteTopicWithSchema-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadWriteTopicWithSchema-default.txt_/ast.txt index 826bc62150d9..7b54831a60c7 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadWriteTopicWithSchema-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-ReadWriteTopicWithSchema-default.txt_/ast.txt @@ -12,7 +12,7 @@ (let $11 '('"UseSsl" '"1")) (let $12 '('('"Consumer" '"test_client") $9 $10 '('"ReconnectPeriod" '"") '('"Format" '"json_each_row") '('"ReadGroup" '"fqrun") $11)) (let $13 (SecureParam '"cluster:default_pq")) -(let $14 (DqPqTopicSource $3 $8 '('"value") $12 $13 '"" $7)) +(let $14 (DqPqTopicSource $3 $8 '('"value") $12 $13 '"" $7 '"")) (let $15 (DataSink '"pq" '"pq")) (let $16 (PqTopic '"pq" '"local" '"test_topic_output" $5 '() (StructType '('"Data" $6)))) (let $17 (DqPqTopicSink $16 '($9 $11) $13)) diff --git a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-WriteTwoTopics-default.txt_/ast.txt b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-WriteTwoTopics-default.txt_/ast.txt index 5a6881b330b3..3f7248623c0f 100644 --- a/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-WriteTwoTopics-default.txt_/ast.txt +++ b/ydb/tests/fq/streaming_optimize/canondata/test_sql_streaming.test_suites-WriteTwoTopics-default.txt_/ast.txt @@ -12,7 +12,7 @@ (let $11 '('"UseSsl" '"1")) (let $12 '('('"Consumer" '"test_client") $9 $10 '('"ReconnectPeriod" '"") '('"Format" '"raw") '('"ReadGroup" '"fqrun") $11)) (let $13 (SecureParam '"cluster:default_pq")) -(let $14 (DqPqTopicSource $3 $7 $8 $12 $13 '"" $6)) +(let $14 (DqPqTopicSource $3 $7 $8 $12 $13 '"" $6 '"")) (let $15 (lambda '($25) (block '( (let $26 '('('"format" '"raw") '('"formatSettings" '('('"data.datetime.formatname" '"POSIX") '('"data.timestamp.formatname" '"POSIX"))) '('"settings" '($10)))) (let $27 (DqSourceWideWrap $25 $4 $6 $26)) @@ -24,7 +24,7 @@ (let $19 (DqPqTopicSink $17 $18 $13)) (let $20 (Commit! (DqQuery! $3 '((DqStage '((DqSource $4 $14)) $15 '('('"_logical_id" '0)) '((DqSink '"0" $16 $19))))) $16)) (let $21 (PqTopic '"pq" '"local" '"test_topic_input2" $5 '() $6)) -(let $22 (DqPqTopicSource $20 $21 $8 $12 $13 '"" $6)) +(let $22 (DqPqTopicSource $20 $21 $8 $12 $13 '"" $6 '"")) (let $23 (PqTopic '"pq" '"local" '"test_topic_output2" $5 '() $6)) (let $24 (DqPqTopicSink $23 $18 $13)) (return (Commit! (DqQuery! $20 '((DqStage '((DqSource $4 $22)) $15 '('('"_logical_id" '0)) '((DqSink '"0" $16 $24))))) $16)) From 896dd78dfc1ccdc5444666b638573ebe3b89b757 Mon Sep 17 00:00:00 2001 From: Daniil Demin Date: Fri, 8 Aug 2025 18:20:08 +0300 Subject: [PATCH 48/52] Speed up Scheme Board unit tests (#22606) --- ydb/core/tx/scheme_board/populator_ut.cpp | 5 +--- ydb/core/tx/scheme_board/subscriber_ut.cpp | 26 +++--------------- ydb/core/tx/scheme_board/ut_helpers.h | 31 ++++++++++++---------- 3 files changed, 22 insertions(+), 40 deletions(-) diff --git a/ydb/core/tx/scheme_board/populator_ut.cpp b/ydb/core/tx/scheme_board/populator_ut.cpp index 064b9b1a8de9..040463048ac7 100644 --- a/ydb/core/tx/scheme_board/populator_ut.cpp +++ b/ydb/core/tx/scheme_board/populator_ut.cpp @@ -256,10 +256,7 @@ Y_UNIT_TEST_SUITE(TPopulatorQuorumTest) { for (int i = 0; i + 1 < ssize(requiredAcks); ++i) { runtime.Send(requiredAcks[i].Release()); } - UNIT_CHECK_GENERATED_EXCEPTION( - runtime.GrabEdgeEvent(edge, TDuration::Seconds(10)), - TEmptyEventQueueException - ); + UNIT_ASSERT_VALUES_EQUAL(CountEvents(runtime, false, edge), 0); runtime.Send(requiredAcks.back().Release()); auto mainAck = runtime.GrabEdgeEvent(edge, TDuration::Seconds(10)); diff --git a/ydb/core/tx/scheme_board/subscriber_ut.cpp b/ydb/core/tx/scheme_board/subscriber_ut.cpp index ada31c2b78f1..b7800e56424b 100644 --- a/ydb/core/tx/scheme_board/subscriber_ut.cpp +++ b/ydb/core/tx/scheme_board/subscriber_ut.cpp @@ -635,12 +635,7 @@ Y_UNIT_TEST_SUITE(TSubscriberSinglePathUpdateTest) { Cerr << "Sending path update to replica: " << replica << '\n'; runtime.Send(replica, edge, GenerateUpdate(GenerateDescribe(Path, PathId, ++pathVersion))); if (shouldIgnore) { - // Every such check takes at least a minute! - // Make sure that there are not many ignored replicas. - UNIT_CHECK_GENERATED_EXCEPTION( - runtime.GrabEdgeEvent(edge, TDuration::Seconds(10)), - TEmptyEventQueueException - ); + UNIT_ASSERT_VALUES_EQUAL(CountEvents(runtime, false, edge), 0); } else { const auto ev = runtime.GrabEdgeEvent(edge, TDuration::Seconds(10)); UNIT_ASSERT_VALUES_EQUAL_C(ev->Sender, subscriber, ev->ToString()); @@ -663,10 +658,6 @@ Y_UNIT_TEST_SUITE(TSubscriberSinglePathUpdateTest) { TestSinglePathUpdate({ {.State = PRIMARY}, {.State = DISCONNECTED} }); } - Y_UNIT_TEST(OneSynchronizedRingGroup) { - TestSinglePathUpdate({ {.State = PRIMARY}, {.State = SYNCHRONIZED} }); - } - Y_UNIT_TEST(OneWriteOnlyRingGroup) { TestSinglePathUpdate({ {.State = PRIMARY}, {.State = PRIMARY, .WriteOnly = true} }); } @@ -711,10 +702,7 @@ Y_UNIT_TEST_SUITE(TSubscriberSinglePathUpdateTest) { ++pathVersion; runtime.Send(replica, edge, GenerateUpdate(GenerateDescribe(Path, PathId, pathVersion))); - UNIT_CHECK_GENERATED_EXCEPTION( - runtime.GrabEdgeEvent(edge, TDuration::Seconds(10)), - TEmptyEventQueueException - ); + UNIT_ASSERT_VALUES_EQUAL(CountEvents(runtime, false, edge), 0); } } @@ -888,10 +876,7 @@ Y_UNIT_TEST_SUITE(TSubscriberSyncQuorumTest) { UNIT_ASSERT_VALUES_EQUAL_C(syncResponse->Get()->Partial, false, syncResponse->ToString()); // No additional sync responses. - UNIT_CHECK_GENERATED_EXCEPTION( - runtime.GrabEdgeEvent(edge, TDuration::Seconds(10)), - TEmptyEventQueueException - ); + UNIT_ASSERT_VALUES_EQUAL(CountEvents(runtime, false, edge), 0); } Y_UNIT_TEST(ReconfigurationWithCurrentSyncRequest) { @@ -936,10 +921,7 @@ Y_UNIT_TEST_SUITE(TSubscriberSyncQuorumTest) { UNIT_ASSERT_VALUES_EQUAL_C(syncResponse->Get()->Partial, false, syncResponse->ToString()); // No additional sync responses. - UNIT_CHECK_GENERATED_EXCEPTION( - runtime.GrabEdgeEvent(edge, TDuration::Seconds(10)), - TEmptyEventQueueException - ); + UNIT_ASSERT_VALUES_EQUAL(CountEvents(runtime, false, edge), 0); } } diff --git a/ydb/core/tx/scheme_board/ut_helpers.h b/ydb/core/tx/scheme_board/ut_helpers.h index ecdbeb98bd06..03ce9cb64276 100644 --- a/ydb/core/tx/scheme_board/ut_helpers.h +++ b/ydb/core/tx/scheme_board/ut_helpers.h @@ -30,6 +30,22 @@ void SetupMinimalRuntime(TTestActorRuntime& runtime, const TStateStorageSetupper TIntrusiveConstPtr GetStateStorageInfo(TTestActorRuntime& runtime); TEvStateStorage::TEvResolveReplicasList::TPtr ResolveReplicas(TTestActorRuntime& runtime, const TString& path); +template +size_t CountEvents(TTestBasicRuntime& runtime, bool dispatch = true, TActorId recipient = {}) { + if (dispatch) { + if (!runtime.DispatchEvents()) { + return 0; + } + } else { + // is necessary to accumulate sent events + runtime.SimulateSleep(TDuration::Seconds(1)); + } + + return CountIf(runtime.CaptureEvents(), [=](const TAutoPtr ev) { + return ev->GetTypeRewrite() == TEvent::EventType && (recipient ? ev->Recipient == recipient : true); + }); +} + class TTestContext: public TTestBasicRuntime { public: using TTestBasicRuntime::TTestBasicRuntime; @@ -66,22 +82,9 @@ class TTestContext: public TTestBasicRuntime { WaitForEvent(TEvInterconnect::EvNodeDisconnected); } - template - size_t CountEvents(bool dispatch = true) { - if (dispatch) { - if (!DispatchEvents()) { - return 0; - } - } - - return CountIf(CaptureEvents(), [](const TAutoPtr ev) { - return ev->GetTypeRewrite() == TEvent::EventType; - }); - } - template size_t CountEdgeEvents() { - return CountEvents(false); + return CountEvents(*this, false); } NInternalEvents::TEvHandshakeResponse::TPtr HandshakeReplica( From 7695c8b29fdadc5ebb540ded6d7ccf24563cd175 Mon Sep 17 00:00:00 2001 From: Mark Ziganshin Date: Fri, 8 Aug 2025 20:03:08 +0300 Subject: [PATCH 49/52] Handle DependsOn with tuple arg in optimizers - YDB part (#22485) --- ydb/core/kqp/opt/logical/kqp_opt_log_helpers.cpp | 2 +- ydb/library/yql/dq/opt/dq_opt_log.cpp | 2 +- ydb/library/yql/dq/opt/dq_opt_phy.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log_helpers.cpp b/ydb/core/kqp/opt/logical/kqp_opt_log_helpers.cpp index d7a5e3da0ee2..52bc3d7b673a 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_log_helpers.cpp +++ b/ydb/core/kqp/opt/logical/kqp_opt_log_helpers.cpp @@ -384,7 +384,7 @@ bool ExtractUsedFields(const TExprNode::TPtr& start, const TExprNode& arg, TSet< if (parent->IsCallable("Member")) { usedFields.emplace(parent->Tail().Content()); - } else if (allowDependsOn && parent->IsCallable("DependsOn")) { + } else if (allowDependsOn && IsDependsOnUsage(*parent, parentsMap)) { continue; } else { // unknown node diff --git a/ydb/library/yql/dq/opt/dq_opt_log.cpp b/ydb/library/yql/dq/opt/dq_opt_log.cpp index 6c9a4abbf811..cd0790632dc6 100644 --- a/ydb/library/yql/dq/opt/dq_opt_log.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_log.cpp @@ -293,7 +293,7 @@ NNodes::TExprBase DqReplicateFieldSubset(NNodes::TExprBase node, TExprContext& c usedFields.insert(member.Value()); } } - else if (!TCoDependsOn::Match(parent)) { + else if (!IsDependsOnUsage(*parent, parentsMap)) { return node; } diff --git a/ydb/library/yql/dq/opt/dq_opt_phy.cpp b/ydb/library/yql/dq/opt/dq_opt_phy.cpp index 860f585f4985..eddd9b265120 100644 --- a/ydb/library/yql/dq/opt/dq_opt_phy.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_phy.cpp @@ -3394,7 +3394,7 @@ TMaybeNode DqUnorderedOverStageInput(TExprBase node, TExprContext& ct for (auto n: it->second) { if (TCoUnordered::Match(n)) { unordereds.push_back(n); - } else if (!TCoDependsOn::Match(n)) { + } else if (!IsDependsOnUsage(*n, parentsMap)) { unordereds.clear(); break; } From e033988dd1cc7a230ee04912893078ce5acac97e Mon Sep 17 00:00:00 2001 From: Pavel Zuev Date: Fri, 8 Aug 2025 22:04:24 +0500 Subject: [PATCH 50/52] Create counters instance for the memory profiles (#22617) --- ydb/tests/tools/kqprun/kqprun.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ydb/tests/tools/kqprun/kqprun.cpp b/ydb/tests/tools/kqprun/kqprun.cpp index 022f719e1244..2009e7f70c4f 100644 --- a/ydb/tests/tools/kqprun/kqprun.cpp +++ b/ydb/tests/tools/kqprun/kqprun.cpp @@ -21,6 +21,8 @@ #ifdef PROFILE_MEMORY_ALLOCATIONS #include +#include +#include #endif using namespace NKikimrRun; @@ -978,6 +980,11 @@ class TMain : public TMainBase { int main(int argc, const char* argv[]) { SetupSignalActions(); +#ifdef PROFILE_MEMORY_ALLOCATIONS + NMonitoring::TDynamicCounterPtr memoryProfilingCounters = MakeIntrusive(); + NKikimr::NMiniKQL::InitializeGlobalPagedBufferCounters(memoryProfilingCounters); +#endif + try { NKqpRun::TMain().Run(argc, argv); } catch (...) { From f98be89ee2d084a9da055a5985ac1b83e8125059 Mon Sep 17 00:00:00 2001 From: kungurtsev Date: Fri, 8 Aug 2025 19:06:05 +0200 Subject: [PATCH 51/52] Do not modify page states in TPrivateCache.PageMap (#22542) --- ydb/core/tablet_flat/flat_executor.cpp | 107 +++--- .../tablet_flat/flat_executor_counters.cpp | 16 +- ydb/core/tablet_flat/flat_ops_compact.h | 6 +- ydb/core/tablet_flat/flat_part_loader.cpp | 2 +- ydb/core/tablet_flat/flat_part_loader.h | 8 +- ydb/core/tablet_flat/flat_sausagecache.cpp | 324 ++++++------------ ydb/core/tablet_flat/flat_sausagecache.h | 157 ++++----- .../tablet_flat/ut/ut_shared_sausagecache.cpp | 42 +++ 8 files changed, 273 insertions(+), 389 deletions(-) diff --git a/ydb/core/tablet_flat/flat_executor.cpp b/ydb/core/tablet_flat/flat_executor.cpp index f83d7247d6fb..f94f6668efe5 100644 --- a/ydb/core/tablet_flat/flat_executor.cpp +++ b/ydb/core/tablet_flat/flat_executor.cpp @@ -659,9 +659,8 @@ void TExecutor::TryActivateWaitingTransaction(TIntrusivePtrPinned[collectionInfo->Id]; for (auto& loaded : loadedPages) { - auto inserted = pinnedCollection.insert(std::make_pair(loaded.PageId, loaded.Page)); + auto inserted = pinnedCollection.insert(std::make_pair(loaded.PageId, TPrivatePageCache::TPinnedPage(std::move(loaded.Page)))); Y_ENSURE(inserted.second); - Y_ENSURE(inserted.first->second.IsUsed()); } } @@ -732,7 +731,7 @@ void TExecutor::AddCachesOfBundle(const NTable::TPartView &partView, const THash void TExecutor::AddSingleCache(const TIntrusivePtr &info) { - PrivatePageCache->RegisterPageCollection(info); + PrivatePageCache->AddPageCollection(info); Send(MakeSharedPageCacheId(), new NSharedCache::TEvAttach(info->PageCollection, info->GetCacheMode())); StickyPagesMemory += info->GetStickySize(); @@ -772,7 +771,7 @@ void TExecutor::DropSingleCache(const TLogoBlobID &label) Y_ENSURE(TryKeepInMemoryMemory >= tryKeepInMemorySize); TryKeepInMemoryMemory -= tryKeepInMemorySize; - PrivatePageCache->ForgetPageCollection(pageCollection); + PrivatePageCache->DropPageCollection(pageCollection); // Note: Shared Cache will send TEvResult with NKikimrProto::RACE status // it activates all transactions that are waiting for being dropped page collection @@ -829,7 +828,7 @@ void TExecutor::StickInMemPages(NSharedCache::TEvResult *msg) { if (pageCollection->PageCollection == msg->PageCollection) { ui64 stickySizeBefore = pageCollection->GetStickySize(); for (auto& loaded : msg->Pages) { - pageCollection->AddSticky(loaded.PageId, loaded.Page); + pageCollection->AddStickyPage(loaded.PageId, loaded.Page); } StickyPagesMemory += pageCollection->GetStickySize() - stickySizeBefore; } @@ -2059,7 +2058,7 @@ void TExecutor::ExecuteTransaction(TSeat* seat) { THPTimer cpuTimer; - PrivatePageCache->ResetTouchesAndToLoad(true); + PrivatePageCache->BeginTransaction(&seat->Pinned); TPageCollectionTxEnv env(*Database, *PrivatePageCache); TTransactionContext txc(Owner->TabletID(), Generation(), Step(), *Database, env, seat->CurrentTxDataLimit, seat->TaskId, seat->Self->TxSpan); @@ -2144,13 +2143,13 @@ void TExecutor::ExecuteTransaction(TSeat* seat) { CommitTransactionLog(RemoveTransaction(seat->UniqID), env, prod.Change, cpuTimer); } else { Y_ENSURE(!seat->CapturedMemory); - if (!PrivatePageCache->GetStats().CurrentCacheMisses && !seat->RequestedMemory && !txc.IsRescheduled()) { + if (!PrivatePageCache->GetStats().ToLoadCount && !seat->RequestedMemory && !txc.IsRescheduled()) { Y_TABLET_ERROR(NFmt::Do(*this) << " " << NFmt::Do(*seat) << " type " << NFmt::Do(*seat->Self) << " postponed w/o demands"); } PostponeTransaction(seat, env, prod.Change, cpuTimer); } - PrivatePageCache->ResetTouchesAndToLoad(false); + PrivatePageCache->EndTransaction(); activeTransaction.Done(); PlanTransactionActivation(); @@ -2160,7 +2159,7 @@ void TExecutor::UnpinTransactionPages(TSeat &seat) { Y_ENSURE(TransactionPagesMemory >= seat.MemoryTouched); TransactionPagesMemory -= seat.MemoryTouched; - PrivatePageCache->TouchSharedCache(seat.Pinned); + PrivatePageCache->TranslatePinnedToSharedCacheTouches(); seat.Pinned.clear(); seat.MemoryTouched = 0; @@ -2190,20 +2189,11 @@ void TExecutor::PostponeTransaction(TSeat* seat, TPageCollectionTxEnv &env, { TTxType txType = seat->TxType; - ui32 touchedPages = 0; - ui32 newPinnedPages = 0; - ui64 prevTouched = seat->MemoryTouched; + seat->MemoryTouched += PrivatePageCache->GetStats().NewlyPinnedSize; + TransactionPagesMemory += PrivatePageCache->GetStats().NewlyPinnedSize; - PrivatePageCache->PinTouches(seat->Pinned, touchedPages, newPinnedPages, seat->MemoryTouched); - TransactionPagesMemory += seat->MemoryTouched - prevTouched; - - ui32 newTouchedPages = newPinnedPages; - ui64 newTouchedBytes = seat->MemoryTouched - prevTouched; - prevTouched = seat->MemoryTouched; - - PrivatePageCache->CountToLoad(seat->Pinned, newPinnedPages, seat->MemoryTouched); - ui64 loadBytes = seat->MemoryTouched - prevTouched; - TransactionPagesMemory += seat->MemoryTouched - prevTouched; + seat->MemoryTouched += PrivatePageCache->GetStats().ToLoadSize; + TransactionPagesMemory += PrivatePageCache->GetStats().ToLoadSize; if (seat->AttachedMemory) Memory->AttachMemory(*seat); @@ -2214,11 +2204,10 @@ void TExecutor::PostponeTransaction(TSeat* seat, TPageCollectionTxEnv &env, if (auto logl = Logger->Log(ELnLev::Debug)) { logl << NFmt::Do(*this) << " " << NFmt::Do(*seat) - << " touch new " << newTouchedBytes << "b" - << ", " << (seat->MemoryTouched - prevTouched) << "b lo load" - << " (" << seat->MemoryTouched << "b in total)" - << ", " << requestedMemory << "b requested for data" - << " (" << seat->CurrentTxDataLimit << "b in total)"; + << " pin " << PrivatePageCache->GetStats().NewlyPinnedCount + << " (" << PrivatePageCache->GetStats().NewlyPinnedSize << " b)" + << " load " << PrivatePageCache->GetStats().ToLoadCount + << " (" << PrivatePageCache->GetStats().ToLoadSize << " b)"; } // Check if additional resources should be requested. @@ -2271,7 +2260,7 @@ void TExecutor::PostponeTransaction(TSeat* seat, TPageCollectionTxEnv &env, // If memory was allocated and there is nothing to load // then tx may be re-activated. - if (!PrivatePageCache->GetStats().CurrentCacheMisses) { + if (!PrivatePageCache->GetStats().ToLoadCount) { EnqueueActivation(seat, CanExecuteTransaction()); return; } @@ -2281,15 +2270,13 @@ void TExecutor::PostponeTransaction(TSeat* seat, TPageCollectionTxEnv &env, auto waitPad = MakeIntrusive(seat); TransactionWaitPads[waitPad.Get()] = waitPad; - ui32 loadPages = 0; auto toLoad = PrivatePageCache->GetToLoad(); - for (auto &[pageCollectionInfo, pages] : toLoad) { + for (auto &[pageCollectionId, pages] : toLoad) { Y_DEBUG_ABORT_UNLESS(pages); - loadPages += pages.size(); if (auto logl = Logger->Log(ELnLev::Dbg03)) { logl - << NFmt::Do(*this) << " " << NFmt::Do(*seat) << " request page collection " << pageCollectionInfo->PageCollection->Label() + << NFmt::Do(*this) << " " << NFmt::Do(*seat) << " request page collection " << pageCollectionId << " pages [ "; for (auto pageId : pages) { logl << pageId << " "; @@ -2297,7 +2284,7 @@ void TExecutor::PostponeTransaction(TSeat* seat, TPageCollectionTxEnv &env, logl << "]"; } - auto request = new NSharedCache::TEvRequest(NBlockIO::EPriority::Fast, pageCollectionInfo->PageCollection, std::move(pages)); + auto request = new NSharedCache::TEvRequest(NBlockIO::EPriority::Fast, PrivatePageCache->GetPageCollection(pageCollectionId)->PageCollection, std::move(pages)); request->TraceId = waitPad->GetWaitingTraceId(); request->WaitPad = waitPad; ++waitPad->PendingRequests; @@ -2307,8 +2294,8 @@ void TExecutor::PostponeTransaction(TSeat* seat, TPageCollectionTxEnv &env, if (auto logl = Logger->Log(ELnLev::Debug)) { logl << NFmt::Do(*this) << " " << NFmt::Do(*seat) << " postponed" - << ", loading " << loadPages << " pages, " << loadBytes << " bytes" - << ", freshly touched " << newPinnedPages << " pages"; + << ", loading " << PrivatePageCache->GetStats().ToLoadCount << " pages, " << PrivatePageCache->GetStats().ToLoadSize << " bytes" + << ", newly pinned " << PrivatePageCache->GetStats().NewlyPinnedCount << " pages, " << PrivatePageCache->GetStats().NewlyPinnedSize << " bytes"; } seat->CPUBookkeepingTime += bookkeepingTimer.PassedReset(); @@ -2317,18 +2304,18 @@ void TExecutor::PostponeTransaction(TSeat* seat, TPageCollectionTxEnv &env, if (AppTxCounters && txType != UnknownTxType) AppTxCounters->TxCumulative(txType, COUNTER_TT_POSTPONED).Increment(1); - // Note: count all new touched pages (were obtained from cache), even not on the first attempt - Counters->Cumulative()[TExecutorCounters::TX_CACHE_HITS].Increment(newTouchedPages); - Counters->Cumulative()[TExecutorCounters::TX_BYTES_CACHED].Increment(newTouchedBytes); + // Note: previously counted non-unique cache hits for all retries + Counters->Cumulative()[TExecutorCounters::TX_CACHE_HITS].Increment(PrivatePageCache->GetStats().NewlyPinnedCount); + Counters->Cumulative()[TExecutorCounters::TX_BYTES_CACHED].Increment(PrivatePageCache->GetStats().NewlyPinnedSize); if (seat->Retries == 1) { Counters->Cumulative()[TExecutorCounters::TX_RETRIED].Increment(1); } - Counters->Cumulative()[TExecutorCounters::TX_CACHE_MISSES].Increment(loadPages); - Counters->Cumulative()[TExecutorCounters::TX_BYTES_READ].Increment(loadBytes); + Counters->Cumulative()[TExecutorCounters::TX_CACHE_MISSES].Increment(PrivatePageCache->GetStats().ToLoadCount); + Counters->Cumulative()[TExecutorCounters::TX_BYTES_READ].Increment(PrivatePageCache->GetStats().ToLoadSize); if (AppTxCounters && txType != UnknownTxType) { - AppTxCounters->TxCumulative(txType, COUNTER_TT_LOADED_BLOCKS).Increment(loadPages); - AppTxCounters->TxCumulative(txType, COUNTER_TT_BYTES_READ).Increment(loadBytes); + AppTxCounters->TxCumulative(txType, COUNTER_TT_LOADED_BLOCKS).Increment(PrivatePageCache->GetStats().ToLoadCount); + AppTxCounters->TxCumulative(txType, COUNTER_TT_BYTES_READ).Increment(PrivatePageCache->GetStats().ToLoadSize); } Counters->Simple()[TExecutorCounters::CACHE_TOTAL_USED] = TransactionPagesMemory; @@ -2339,23 +2326,15 @@ void TExecutor::CommitTransactionLog(std::unique_ptr seat, TPageCollectio const bool isReadOnly = !(change->HasAny() || env.HasChanges()); const TTxType txType = seat->TxType; - size_t touchedBlocks = PrivatePageCache->GetStats().CurrentCacheHits; + // Note: previously counted count (not size), but it requires an additional counter + size_t touchedBlocks = PrivatePageCache->GetStats().NewlyPinnedSize + seat->MemoryTouched; Counters->Percentile()[TExecutorCounters::TX_PERCENTILE_TOUCHED_BLOCKS].IncrementFor(touchedBlocks); if (AppTxCounters && txType != UnknownTxType) AppTxCounters->TxCumulative(txType, COUNTER_TT_TOUCHED_BLOCKS).Increment(touchedBlocks); - // Note: count all new touched pages (were obtained from cache), even not on the first attempt - ui32 newTouchedPages = 0; - ui64 newTouchedBytes = 0, pinnedTouchedBytes = 0; - PrivatePageCache->CountTouches(seat->Pinned, newTouchedPages, newTouchedBytes, pinnedTouchedBytes); - Counters->Cumulative()[TExecutorCounters::TX_CACHE_HITS].Increment(newTouchedPages); - Counters->Cumulative()[TExecutorCounters::TX_BYTES_CACHED].Increment(newTouchedBytes); - if (seat->MemoryTouched >= pinnedTouchedBytes) { - // memory that was pinned (for instance by Precharge) but wasn't used during the last successful execution - Counters->Cumulative()[TExecutorCounters::TX_BYTES_WASTED].Increment(seat->MemoryTouched - pinnedTouchedBytes); - } else { - Y_DEBUG_ABORT("Cache counters are out of sync"); - } + // Note: previously counted non-unique cache hits for all retries + Counters->Cumulative()[TExecutorCounters::TX_CACHE_HITS].Increment(PrivatePageCache->GetStats().NewlyPinnedCount); + Counters->Cumulative()[TExecutorCounters::TX_BYTES_CACHED].Increment(PrivatePageCache->GetStats().NewlyPinnedSize); UnpinTransactionPages(*seat); @@ -3096,10 +3075,10 @@ void TExecutor::Handle(NSharedCache::TEvResult::TPtr &ev) { case ESharedCacheRequestType::Transaction: case ESharedCacheRequestType::InMemPages: { - TPrivatePageCache::TInfo *collectionInfo = PrivatePageCache->Info(msg->PageCollection->Label()); - if (!collectionInfo) { + TPrivatePageCache::TInfo *pageCollection = PrivatePageCache->FindPageCollection(msg->PageCollection->Label()); + if (!pageCollection) { if (requestType == ESharedCacheRequestType::Transaction) { - TryActivateWaitingTransaction(std::move(msg->WaitPad), std::move(msg->Pages), collectionInfo); + TryActivateWaitingTransaction(std::move(msg->WaitPad), std::move(msg->Pages), pageCollection); } return; } @@ -3120,10 +3099,10 @@ void TExecutor::Handle(NSharedCache::TEvResult::TPtr &ev) { StickInMemPages(msg); } for (auto& loaded : msg->Pages) { - PrivatePageCache->ProvideBlock(loaded.PageId, loaded.Page, collectionInfo); + PrivatePageCache->AddPage(loaded.PageId, loaded.Page, pageCollection); } if (requestType == ESharedCacheRequestType::Transaction) { - TryActivateWaitingTransaction(std::move(msg->WaitPad), std::move(msg->Pages), collectionInfo); + TryActivateWaitingTransaction(std::move(msg->WaitPad), std::move(msg->Pages), pageCollection); } } return; @@ -3190,9 +3169,9 @@ void TExecutor::Handle(NSharedCache::TEvUpdated::TPtr &ev) { const auto *msg = ev->Get(); for (auto &kv : msg->DroppedPages) { - if (auto *info = PrivatePageCache->Info(kv.first)) { + if (auto *pageCollection = PrivatePageCache->FindPageCollection(kv.first)) { for (ui32 pageId : kv.second) { - PrivatePageCache->DropSharedBody(pageId, info); + PrivatePageCache->DropPage(pageId, pageCollection); } } } @@ -4051,8 +4030,6 @@ void TExecutor::UpdateCounters(const TActorContext &ctx) { const auto &stats = PrivatePageCache->GetStats(); Counters->Simple()[TExecutorCounters::CACHE_TOTAL_COLLECTIONS].Set(stats.TotalCollections); Counters->Simple()[TExecutorCounters::CACHE_TOTAL_SHARED_BODY].Set(stats.TotalSharedBody); - Counters->Simple()[TExecutorCounters::CACHE_TOTAL_PINNED_BODY].Set(stats.TotalPinnedBody); - Counters->Simple()[TExecutorCounters::CACHE_TOTAL_EXCLUSIVE].Set(stats.TotalExclusive); } Counters->Simple()[TExecutorCounters::CACHE_TOTAL_STICKY].Set(StickyPagesMemory); Counters->Simple()[TExecutorCounters::CACHE_TOTAL_TRY_KEEP_IN_MEMORY].Set(TryKeepInMemoryMemory); @@ -4589,8 +4566,6 @@ void TExecutor::RenderHtmlPage(NMon::TEvRemoteHttpInfo::TPtr &ev) const { TAG(TH3) {str << "Page collection cache:";} DIV_CLASS("row") {str << "Total collections: " << PrivatePageCache->GetStats().TotalCollections; } DIV_CLASS("row") {str << "Total bytes in shared cache: " << PrivatePageCache->GetStats().TotalSharedBody; } - DIV_CLASS("row") {str << "Total bytes in local cache: " << PrivatePageCache->GetStats().TotalPinnedBody; } - DIV_CLASS("row") {str << "Total bytes exclusive to local cache: " << PrivatePageCache->GetStats().TotalExclusive; } DIV_CLASS("row") {str << "Total bytes marked as sticky: " << StickyPagesMemory; } DIV_CLASS("row") {str << "Total bytes for try-keep-in-memory Mode: " << TryKeepInMemoryMemory; } DIV_CLASS("row") {str << "Total bytes currently in use: " << TransactionPagesMemory; } diff --git a/ydb/core/tablet_flat/flat_executor_counters.cpp b/ydb/core/tablet_flat/flat_executor_counters.cpp index 96e5dbc45d2d..e10b61bdb4b7 100644 --- a/ydb/core/tablet_flat/flat_executor_counters.cpp +++ b/ydb/core/tablet_flat/flat_executor_counters.cpp @@ -17,12 +17,16 @@ namespace NTabletFlatExecutor { XX(300000000, "10-30 s") #define FLAT_EXECUTOR_TOUCHED_BLOCKS(XX) \ - XX(0, "0") \ - XX(1, "1") \ - XX(10, "2-10") \ - XX(50, "10-50") \ - XX(200, "50-200") \ - XX(1000, "200-1000") + XX(0ULL, "0") \ + XX(10*1024ULL, "10240") \ + XX(100*1024ULL, "102400") \ + XX(1024*1024ULL, "1048576") \ + XX(10*1024*1024ULL, "10485760") \ + XX(100*1024*1024ULL, "104857600") \ + XX(1024*1024*1024ULL, "1073741824") \ + XX(10*1024*1024*1024ULL, "10737418240") \ + XX(100*1024*1024*1024ULL, "107374182400") \ + XX(1024*1024*1024*1024ULL, "1099511627776") #define FLAT_EXECUTOR_DATA_SIZE(XX) \ XX(0ULL, "0") \ diff --git a/ydb/core/tablet_flat/flat_ops_compact.h b/ydb/core/tablet_flat/flat_ops_compact.h index f9fadb39128f..c9a1aa54fc1e 100644 --- a/ydb/core/tablet_flat/flat_ops_compact.h +++ b/ydb/core/tablet_flat/flat_ops_compact.h @@ -353,7 +353,11 @@ namespace NTabletFlatExecutor { auto sharedPage = MakeIntrusive(pageId, pageSize, nullptr); sharedPage->Initialize(std::move(loadedPage.Data)); saveCompactedPages->Pages.push_back(sharedPage); - cache->Fill(pageId, TSharedPageRef::MakeUsed(std::move(sharedPage), gcList), sticky); + if (sticky) { + cache->AddStickyPage(pageId, TSharedPageRef::MakeUsed(std::move(sharedPage), gcList)); + } else { + cache->AddPage(pageId, TSharedPageRef::MakeUsed(std::move(sharedPage), gcList)); + } }; for (auto &page : pageCollection.StickyPages) { addPage(page, true); diff --git a/ydb/core/tablet_flat/flat_part_loader.cpp b/ydb/core/tablet_flat/flat_part_loader.cpp index 6a89b0c348c9..ccfee31a41b0 100644 --- a/ydb/core/tablet_flat/flat_part_loader.cpp +++ b/ydb/core/tablet_flat/flat_part_loader.cpp @@ -321,7 +321,7 @@ TLoader::TFetch TLoader::StagePreloadData() auto partStore = PartView.As(); // Note: preload works only for main group pages - auto total = partStore->PageCollections[0]->Total(); + auto total = partStore->PageCollections[0]->PageCollection->Total(); TVector toLoad(::Reserve(total)); for (TPageId pageId : xrange(total)) { diff --git a/ydb/core/tablet_flat/flat_part_loader.h b/ydb/core/tablet_flat/flat_part_loader.h index c698bc63b731..f8eda25901e3 100644 --- a/ydb/core/tablet_flat/flat_part_loader.h +++ b/ydb/core/tablet_flat/flat_part_loader.h @@ -65,7 +65,7 @@ namespace NTable { auto savedPage = SavedPages.find(pageId); if (savedPage == SavedPages.end()) { - if (auto cachedPage = Cache->GetPage(pageId); cachedPage) { + if (auto cachedPage = Cache->FindPage(pageId); cachedPage) { if (auto sharedPageRef = cachedPage->SharedBody; sharedPageRef && sharedPageRef.Use()) { // Save page in case it's evicted on the next iteration AddSavedPage(pageId, std::move(sharedPageRef)); @@ -110,7 +110,11 @@ namespace NTable { bool sticky = NeedIn(pageType) || pageType == EPage::FlatIndex; AddSavedPage(loaded.PageId, loaded.Page); - Cache->Fill(loaded.PageId, std::move(loaded.Page), sticky); + if (sticky) { + Cache->AddStickyPage(loaded.PageId, std::move(loaded.Page)); + } else { + Cache->AddPage(loaded.PageId, std::move(loaded.Page)); + } } private: diff --git a/ydb/core/tablet_flat/flat_sausagecache.cpp b/ydb/core/tablet_flat/flat_sausagecache.cpp index 879961085de2..a48f7a5f7901 100644 --- a/ydb/core/tablet_flat/flat_sausagecache.cpp +++ b/ydb/core/tablet_flat/flat_sausagecache.cpp @@ -4,12 +4,19 @@ namespace NKikimr { namespace NTabletFlatExecutor { -TPrivatePageCache::TPage::TPage(size_t size, TPageId pageId, TInfo* info) - : LoadState(LoadStateNo) - , Id(pageId) +TSharedPageRef UnUse(TSharedPageRef sharedBody) { + sharedBody.UnUse(); + return sharedBody; +} + +TPrivatePageCache::TPage::TPage(TPageId id, size_t size, TSharedPageRef sharedBody, TInfo* info) + : Id(id) , Size(size) + , SharedBody(UnUse(std::move(sharedBody))) , Info(info) -{} +{ + Y_ENSURE(SharedBody); +} TPrivatePageCache::TInfo::TInfo(TIntrusiveConstPtr pageCollection) : Id(pageCollection->Label()) @@ -28,264 +35,155 @@ TPrivatePageCache::TInfo::TInfo(const TInfo &info) PageMap.resize(info.PageMap.size()); for (const auto& [pageId, page] : info.PageMap) { Y_ASSERT(page); - EnsurePage(pageId)->ProvideSharedBody(page->SharedBody); + AddPage(pageId, page->SharedBody); } } -TIntrusivePtr TPrivatePageCache::GetPageCollection(const TLogoBlobID &id) const { - auto it = PageCollections.find(id); - Y_ENSURE(it != PageCollections.end(), "trying to get unknown page collection"); - return it->second; +TPrivatePageCache::TInfo* TPrivatePageCache::FindPageCollection(const TLogoBlobID &id) const { + auto *pageCollection = PageCollections.FindPtr(id); + return pageCollection ? pageCollection->Get() : nullptr; +} + +TPrivatePageCache::TInfo* TPrivatePageCache::GetPageCollection(const TLogoBlobID &id) const { + auto pageCollection = FindPageCollection(id); + Y_ENSURE(pageCollection, "trying to get unknown page collection"); + return pageCollection; } -void TPrivatePageCache::RegisterPageCollection(TIntrusivePtr info) { +void TPrivatePageCache::AddPageCollection(TIntrusivePtr info) { auto inserted = PageCollections.insert(decltype(PageCollections)::value_type(info->Id, info)); Y_ENSURE(inserted.second, "double registration of page collection is forbidden"); ++Stats.TotalCollections; - for (const auto& [pageId, page] : info->PageMap) { + for (const auto& [pageId, page] : info->GetPageMap()) { Y_ASSERT(page); - Y_ENSURE(page->SharedBody, "new filled page can't be without a shared body"); - + Stats.TotalSharedBody += page->Size; - if (page->PinnedBody) - Stats.TotalPinnedBody += page->Size; - TryUnload(page.Get()); // notify shared cache that we have a page handle SharedCacheTouches[page->Info->Id].insert(page->Id); } } -void TPrivatePageCache::ForgetPageCollection(TIntrusivePtr info) { - for (const auto& [pageId, page] : info->PageMap) { +void TPrivatePageCache::DropPageCollection(TInfo *info) { + for (const auto& [pageId, page] : info->GetPageMap()) { Y_ASSERT(page); - if (page->SharedBody) - Stats.TotalSharedBody -= page->Size; - if (page->PinnedBody) - Stats.TotalPinnedBody -= page->Size; - if (page->PinnedBody && !page->SharedBody) - Stats.TotalExclusive -= page->Size; + Stats.TotalSharedBody -= page->Size; } info->Clear(); - PageCollections.erase(info->Id); SharedCacheTouches.erase(info->Id); + PageCollections.erase(info->Id); --Stats.TotalCollections; } -TPrivatePageCache::TInfo* TPrivatePageCache::Info(TLogoBlobID id) { - auto *x = PageCollections.FindPtr(id); - if (x) - return x->Get(); - else - return nullptr; -} - -void TPrivatePageCache::TryLoad(TPage *page) { - if (page->LoadState == TPage::LoadStateLoaded) { - return; - } - - if (page->LoadState == TPage::LoadStateNo && page->SharedBody) { - if (page->SharedBody.Use()) { - if (Y_LIKELY(!page->PinnedBody)) - Stats.TotalPinnedBody += page->Size; - page->PinnedBody = TPinnedPageRef(page->SharedBody).GetData(); - page->LoadState = TPage::LoadStateLoaded; - return; - } - - page->SharedBody.Drop(); - Stats.TotalSharedBody -= page->Size; - if (Y_UNLIKELY(page->PinnedBody)) { - Stats.TotalExclusive += page->Size; - } - } -} - -void TPrivatePageCache::TPrivatePageCache::TryUnload(TPage *page) { - if (page->LoadState == TPage::LoadStateLoaded) { - page->LoadState = TPage::LoadStateNo; - if (Y_LIKELY(page->PinnedBody)) { - Stats.TotalPinnedBody -= page->Size; - if (!page->SharedBody) { - Stats.TotalExclusive -= page->Size; - } - page->PinnedBody = { }; - } - page->SharedBody.UnUse(); - } -} - -// page may be made free after this call -void TPrivatePageCache::TPrivatePageCache::TryEraseIfUnnecessary(TPage *page) { - if (page->IsUnnecessary()) { - if (Y_UNLIKELY(page->PinnedBody)) { - Stats.TotalPinnedBody -= page->Size; - Stats.TotalExclusive -= page->Size; - page->PinnedBody = { }; - } - const TPageId pageId = page->Id; - auto* info = page->Info; - Y_DEBUG_ABORT_UNLESS(info->PageMap[pageId].Get() == page); - bool erased = info->PageMap.erase(pageId); - Y_ENSURE(erased); +void TPrivatePageCache::ToLoadPage(TPageId pageId, TInfo *info) { + if (ToLoad[info->Id].insert(pageId).second) { + Stats.ToLoadCount++; + Y_ASSERT(!info->IsStickyPage(pageId)); + Stats.ToLoadSize += info->GetPageSize(pageId); } } const TSharedData* TPrivatePageCache::Lookup(TPageId pageId, TInfo *info) { - TPage *page = info->EnsurePage(pageId); - - TryLoad(page); - - if (page->LoadState == TPage::LoadStateLoaded) { - if (page->Empty()) { - Touches.PushBack(page); - Stats.CurrentCacheHits++; - if (!page->IsSticky()) { - Stats.CurrentCacheHitSize += page->Size; - } - } - return &page->PinnedBody; + Y_ENSURE(Pinned, "can be called only in a transaction context"); + auto& pinnedCollection = (*Pinned)[info->Id]; + auto* pinnedPage = pinnedCollection.FindPtr(pageId); + if (pinnedPage) { + // pinned pages do not need to be counted again + return &pinnedPage->PinnedBody; } - if (page->Empty()) { - Y_DEBUG_ABORT_UNLESS(info->GetPageType(page->Id) != EPage::FlatIndex, "flat index pages should have been sticked and preloaded"); - ToLoad.PushBack(page); - Stats.CurrentCacheMisses++; + auto page = info->FindPage(pageId); + if (!page) { + ToLoadPage(pageId, info); + return nullptr; } - return nullptr; -} -void TPrivatePageCache::CountTouches(TPinned &pinned, ui32 &touchedUnpinnedPages, ui64 &touchedUnpinnedMemory, ui64 &touchedPinnedMemory) { - if (pinned.empty()) { - touchedUnpinnedPages += Stats.CurrentCacheHits; - touchedUnpinnedMemory += Stats.CurrentCacheHitSize; - return; + auto sharedBody = page->SharedBody; + if (!sharedBody.Use()) { + DropPage(pageId, info); + ToLoadPage(pageId, info); + return nullptr; } - for (auto &page : Touches) { - bool isPinned = pinned[page.Info->Id].contains(page.Id); + auto emplaced = pinnedCollection.emplace(pageId, TPinnedPage(std::move(sharedBody))); + Y_ENSURE(emplaced.second); - if (!isPinned) { - touchedUnpinnedPages++; - } - - // Note: it seems useless to count sticky pages in tx usage - // also we want to read index from Env - if (!page.IsSticky()) { - if (isPinned) { - touchedPinnedMemory += page.Size; - } else { - touchedUnpinnedMemory += page.Size; - } - } + Stats.NewlyPinnedCount++; + if (!page->IsSticky()) { + Stats.NewlyPinnedSize += page->Size; } + + return &emplaced.first->second.PinnedBody; } -void TPrivatePageCache::PinTouches(TPinned &pinned, ui32 &touchedPages, ui32 &pinnedPages, ui64 &pinnedMemory) { - for (auto &page : Touches) { - if (Y_UNLIKELY(!page.SharedBody)) { - continue; - } +void TPrivatePageCache::BeginTransaction(TPinned* pinned) { + Y_ENSURE(!Pinned); + Y_ENSURE(!ToLoad); - auto &pinnedCollection = pinned[page.Info->Id]; - - // would insert only if first seen - if (auto inserted = pinnedCollection.insert(std::make_pair(page.Id, page.SharedBody)); inserted.second) { - Y_ENSURE(inserted.first->second.IsUsed()); - - pinnedPages++; - // Note: it seems useless to count sticky pages in tx usage - // also we want to read index from Env - if (!page.IsSticky()) { - pinnedMemory += page.Size; - } - } - touchedPages++; - } -} + Y_ENSURE(!Stats.NewlyPinnedCount); + Y_ENSURE(!Stats.NewlyPinnedSize); + Y_ENSURE(!Stats.ToLoadCount); + Y_ENSURE(!Stats.ToLoadSize); -void TPrivatePageCache::CountToLoad(TPinned &pinned, ui32 &toLoadPages, ui64 &toLoadMemory) { - for (auto &page : ToLoad) { - auto &pinnedCollection = pinned[page.Info->Id]; + Y_ENSURE(pinned); + Pinned = pinned; +} - Y_ASSERT(!pinnedCollection.contains(page.Id)); +void TPrivatePageCache::EndTransaction() { + Pinned = nullptr; + ToLoad.clear(); - toLoadPages++; - // Note: it seems useless to count sticky pages in tx usage - // also we want to read index from Env - if (!page.IsSticky()) { - toLoadMemory += page.Size; - } - } + Stats.NewlyPinnedCount = 0; + Stats.NewlyPinnedSize = 0; + Stats.ToLoadCount = 0; + Stats.ToLoadSize = 0; } -THashMap> TPrivatePageCache::GetToLoad() const { - THashMap> result; - for (auto &page : ToLoad) { - result[page.Info].push_back(page.Id); +void TPrivatePageCache::DropPage(TPageId pageId, TInfo *info) { + if (info->DropPage(pageId)) { + Stats.TotalSharedBody -= info->GetPageSize(pageId); } - return result; } -void TPrivatePageCache::ResetTouchesAndToLoad(bool verifyEmpty) { - if (verifyEmpty) { - Y_ENSURE(!Touches); - Y_ENSURE(!Stats.CurrentCacheHits); - Y_ENSURE(!Stats.CurrentCacheHitSize); - Y_ENSURE(!ToLoad); - Y_ENSURE(!Stats.CurrentCacheMisses); - } - - while (Touches) { - TPage *page = Touches.PopBack(); - TryUnload(page); +void TPrivatePageCache::AddPage(TPageId pageId, TSharedPageRef sharedBody, TInfo *info) +{ + if (info->AddPage(pageId, std::move(sharedBody))) { + Stats.TotalSharedBody += info->GetPageSize(pageId); } - Stats.CurrentCacheHits = 0; - Stats.CurrentCacheHitSize = 0; - Stats.CurrentCacheMisses = 0; +} - while (ToLoad) { - TPage *page = ToLoad.PopBack(); - TryEraseIfUnnecessary(page); +THashMap> TPrivatePageCache::GetToLoad() { + THashMap> result; + for (auto& [pageCollectionId, pages_] : ToLoad) { + TVector pages(pages_.begin(), pages_.end()); + std::sort(pages.begin(), pages.end()); + result.emplace(pageCollectionId, std::move(pages)); } + ToLoad.clear(); + return result; } -void TPrivatePageCache::DropSharedBody(TPageId pageId, TInfo *info) { - TPage *page = info->GetPage(pageId); - if (!page) - return; +void TPrivatePageCache::TranslatePinnedToSharedCacheTouches() { + if (!Pinned) { + return; // canceling transaction may have no pinned pages yet + } - if (!page->SharedBody.IsUsed()) { - if (Y_LIKELY(page->SharedBody)) { - Stats.TotalSharedBody -= page->Size; - if (Y_UNLIKELY(page->PinnedBody)) { - Stats.TotalExclusive += page->Size; + for (const auto& [pageCollectionId, pages] : *Pinned) { + if (FindPageCollection(pageCollectionId)) { + auto& touches = SharedCacheTouches[pageCollectionId]; + for (const auto& [pageId, pinnedPageRef] : pages) { + touches.insert(pageId); } - page->SharedBody = { }; } - TryEraseIfUnnecessary(page); } } -void TPrivatePageCache::ProvideBlock(TPageId pageId, TSharedPageRef sharedBody, TInfo *info) -{ - Y_ENSURE(sharedBody.IsUsed()); - TPage *page = info->EnsurePage(pageId); - - if (Y_UNLIKELY(page->SharedBody)) - Stats.TotalSharedBody -= page->Size; - if (Y_UNLIKELY(page->PinnedBody)) - Stats.TotalPinnedBody -= page->Size; - if (Y_UNLIKELY(page->PinnedBody && !page->SharedBody)) - Stats.TotalExclusive -= page->Size; - - page->ProvideSharedBody(std::move(sharedBody)); - Stats.TotalSharedBody += page->Size; +THashMap> TPrivatePageCache::GetSharedCacheTouches() { + return std::move(SharedCacheTouches); } THashMap> TPrivatePageCache::DetachPrivatePageCache() { @@ -299,28 +197,4 @@ THashMap> TPrivatePageCache return ret; } -void TPrivatePageCache::TouchSharedCache(const TPinned &pinned) { - // report all touched pages: - - // 1. these that were touched on this transaction retry - for (auto &page : Touches) { - SharedCacheTouches[page.Info->Id].insert(page.Id); - } - - // 2. and these that were touched on previous transaction retries - for (const auto& [pageCollectionId, pages] : pinned) { - if (auto *info = Info(pageCollectionId)) { - auto& touches = SharedCacheTouches[pageCollectionId]; - for (auto& page : pages) { - touches.insert(page.first); - } - } - } -} - - -THashMap> TPrivatePageCache::GetSharedCacheTouches() { - return std::move(SharedCacheTouches); -} - }} diff --git a/ydb/core/tablet_flat/flat_sausagecache.h b/ydb/core/tablet_flat/flat_sausagecache.h index a1b11734ec0e..6a82ccd72725 100644 --- a/ydb/core/tablet_flat/flat_sausagecache.h +++ b/ydb/core/tablet_flat/flat_sausagecache.h @@ -12,68 +12,50 @@ using namespace NSharedCache; class TPrivatePageCache { public: - using TPinned = THashMap>; + // TODO: provide a way to get TSharedData from used TSharedPageRef directly + struct TPinnedPage { + TSharedPageRef SharedBody; + TSharedData PinnedBody; + + explicit TPinnedPage(TSharedPageRef sharedBody) + : SharedBody(std::move(sharedBody)) + , PinnedBody(TPinnedPageRef(SharedBody).GetData()) + {} + }; + + using TPinned = THashMap>; struct TInfo; struct TStats { - ui64 TotalCollections = 0; // total number of registered collections - ui64 TotalSharedBody = 0; // total number of bytes currently referenced from shared cache - ui64 TotalPinnedBody = 0; // total number of bytes currently pinned in memory - ui64 TotalExclusive = 0; // total number of bytes exclusive to this cache (not from shared cache) - size_t CurrentCacheHits = 0; // = Touches.Size() - ui64 CurrentCacheHitSize = 0; // = Touches.Where(t => !t.Sticky).Sum(t => t.Size) - size_t CurrentCacheMisses = 0; // = ToLoad.Size() - }; + ui64 TotalCollections = 0; + ui64 TotalSharedBody = 0; + + size_t NewlyPinnedCount = 0; + ui64 NewlyPinnedSize = 0; - struct TPage : public TIntrusiveListItem { - enum ELoadState { - LoadStateNo, - LoadStateLoaded, - }; + size_t ToLoadCount = 0; + ui64 ToLoadSize = 0; + }; - ui32 LoadState : 2; - + struct TPage : TNonCopyable { const TPageId Id; const size_t Size; + const TSharedPageRef SharedBody; + const TInfo* const Info; - TInfo* const Info; - TSharedPageRef SharedBody; - TSharedData PinnedBody; - - TPage(size_t size, TPageId pageId, TInfo* info); - - TPage(const TPage&) = delete; - TPage(TPage&&) = delete; - - bool IsUnnecessary() const noexcept { - return ( - LoadState == LoadStateNo && - !SharedBody); - } + TPage(TPageId id, size_t size, TSharedPageRef sharedBody, TInfo* info); bool IsSticky() const noexcept { // Note: because this method doesn't use TPage flags // it may be called from multiple threads later // also it doesn't affect offloading, only touched memory counting - return Info->IsSticky(Id); - } - - void ProvideSharedBody(TSharedPageRef sharedBody) { - // here sharedBody may be unused or invalid - SharedBody = std::move(sharedBody); - SharedBody.UnUse(); - LoadState = LoadStateNo; - PinnedBody = { }; + return Info->IsStickyPage(Id); } }; struct TInfo : public TThrRefBase { - ui32 Total() const noexcept { - return PageMap.size(); - } - - TPage* GetPage(TPageId pageId) const noexcept { + TPage* FindPage(TPageId pageId) const noexcept { return PageMap[pageId].Get(); } @@ -85,35 +67,36 @@ class TPrivatePageCache { return PageCollection->Page(pageId).Size; } - TPage* EnsurePage(TPageId pageId) { - auto* page = GetPage(pageId); - if (!page) { - PageMap.emplace(pageId, THolder(page = new TPage(PageCollection->Page(pageId).Size, pageId, this))); - } - return page; + const TPageMap>& GetPageMap() const noexcept { + return PageMap; } - // Note: this method is only called during a page collection creation - void Fill(TPageId pageId, TSharedPageRef sharedBody, bool sticky) { - if (sticky) { - AddSticky(pageId, sharedBody); - } - EnsurePage(pageId)->ProvideSharedBody(std::move(sharedBody)); + bool IsStickyPage(TPageId pageId) const noexcept { + return StickyPages.contains(pageId); } - void AddSticky(TPageId pageId, TSharedPageRef page) { - Y_ENSURE(page.IsUsed()); - if (StickyPages.emplace(pageId, page).second) { - StickyPagesSize += TPinnedPageRef(page)->size(); - } + ui64 GetStickySize() const noexcept { + return StickyPagesSize; } - bool IsSticky(TPageId pageId) const noexcept { - return StickyPages.contains(pageId); + bool AddPage(TPageId pageId, TSharedPageRef sharedBody) { + return PageMap.emplace(pageId, MakeHolder( + pageId, + GetPageSize(pageId), + std::move(sharedBody), + this)); } - ui64 GetStickySize() const noexcept { - return StickyPagesSize; + void AddStickyPage(TPageId pageId, TSharedPageRef sharedBody) { + Y_ENSURE(sharedBody.IsUsed()); + if (StickyPages.emplace(pageId, sharedBody).second) { + StickyPagesSize += GetPageSize(pageId); + } + AddPage(pageId, std::move(sharedBody)); + } + + bool DropPage(TPageId pageId) { + return PageMap.erase(pageId); } void Clear() { @@ -140,12 +123,14 @@ class TPrivatePageCache { const TLogoBlobID Id; const TIntrusiveConstPtr PageCollection; - TPageMap> PageMap; - explicit TInfo(TIntrusiveConstPtr pack); + explicit TInfo(TIntrusiveConstPtr pageCollection); TInfo(const TInfo &info); private: + // all pages in PageMap have valid unused shared body + TPageMap> PageMap; + // storing sticky pages used refs guarantees that they won't be offload from Shared Cache THashMap StickyPages; ui64 StickyPagesSize = 0; @@ -153,44 +138,40 @@ class TPrivatePageCache { }; public: - TIntrusivePtr GetPageCollection(const TLogoBlobID &id) const; - void RegisterPageCollection(TIntrusivePtr info); - void ForgetPageCollection(TIntrusivePtr info); - - TInfo* Info(TLogoBlobID id); + TInfo* FindPageCollection(const TLogoBlobID &id) const; + TInfo* GetPageCollection(const TLogoBlobID &id) const; + void AddPageCollection(TIntrusivePtr info); + void DropPageCollection(TInfo *info); const TStats& GetStats() const { return Stats; } const TSharedData* Lookup(TPageId pageId, TInfo *info); - void CountTouches(TPinned &pinned, ui32 &touchedUnpinnedPages, ui64 &touchedUnpinnedMemory, ui64 &touchedPinnedMemory); - void PinTouches(TPinned &pinned, ui32 &touchedPages, ui32 &pinnedPages, ui64 &pinnedMemory); - - void CountToLoad(TPinned &pinned, ui32 &toLoadPages, ui64 &toLoadMemory); - THashMap> GetToLoad() const; + // TODO: move this methods somewhere else (probably to TPageCollectionTxEnv) + // and keep page states and counters there + void BeginTransaction(TPinned* pinned); + void EndTransaction(); - void ResetTouchesAndToLoad(bool verifyEmpty); + void DropPage(TPageId pageId, TInfo *info); + void AddPage(TPageId pageId, TSharedPageRef sharedBody, TInfo *info); - void DropSharedBody(TPageId pageId, TInfo *info); - - void ProvideBlock(TPageId pageId, TSharedPageRef sharedBody, TInfo *info); THashMap> DetachPrivatePageCache(); - void TouchSharedCache(const TPinned &pinned); + THashMap> GetToLoad(); + void TranslatePinnedToSharedCacheTouches(); THashMap> GetSharedCacheTouches(); +private: + void ToLoadPage(TPageId pageId, TInfo *info); + private: THashMap> PageCollections; THashMap> SharedCacheTouches; TStats Stats; - TIntrusiveList Touches; - TIntrusiveList ToLoad; - - void TryLoad(TPage *page); - void TryUnload(TPage *page); - void TryEraseIfUnnecessary(TPage *page); + TPinned* Pinned; + THashMap> ToLoad; }; }} diff --git a/ydb/core/tablet_flat/ut/ut_shared_sausagecache.cpp b/ydb/core/tablet_flat/ut/ut_shared_sausagecache.cpp index dd265b28155d..e99cba2e2eac 100644 --- a/ydb/core/tablet_flat/ut/ut_shared_sausagecache.cpp +++ b/ydb/core/tablet_flat/ut/ut_shared_sausagecache.cpp @@ -230,6 +230,30 @@ void SetupSharedCache(TMyEnvBase& env, NKikimrSharedCache::TReplacementPolicy po } } +auto MakeSharedCacheRequestsDeterministic(TTestActorRuntime& runtime) { + // keep page ordered because their order doesn't matter + // but may effect cache policies + return std::make_tuple( + MakeHolder>(runtime, [&](const auto& ev) { + auto &msg = *ev->Get(); + + TVector>> touched; + for (const auto& [pageCollectionId, pages] : msg.Touched) { + touched.emplace_back(pageCollectionId, TVector(pages.begin(), pages.end())); + Sort(touched.back().second); + } + Sort(touched); + + msg.Touched = {}; + for (const auto& [pageCollectionId, pages] : touched) { + msg.Touched.emplace(pageCollectionId, THashSet(pages.begin(), pages.end())); + } + + return false; + }) + ); +} + Y_UNIT_TEST_SUITE(TSharedPageCache) { Y_UNIT_TEST(Limits) { @@ -238,6 +262,7 @@ Y_UNIT_TEST(Limits) { env->SetLogPriority(NKikimrServices::TABLET_EXECUTOR, NActors::NLog::PRI_TRACE); env->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); auto counters = GetSharedPageCounters(env); + auto deterministicRequests = MakeSharedCacheRequestsDeterministic(env.Env); bool bTreeIndex = env->GetAppData().FeatureFlags.GetEnableLocalDBBtreeIndex(); ui32 passiveBytes = bTreeIndex ? 131 : 7772; @@ -306,6 +331,7 @@ Y_UNIT_TEST(Limits_Config) { env->SetLogPriority(NKikimrServices::TABLET_EXECUTOR, NActors::NLog::PRI_TRACE); env->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); auto counters = GetSharedPageCounters(env); + auto deterministicRequests = MakeSharedCacheRequestsDeterministic(env.Env); bool bTreeIndex = env->GetAppData().FeatureFlags.GetEnableLocalDBBtreeIndex(); ui32 passiveBytes = bTreeIndex ? 131 : 7772; @@ -373,6 +399,7 @@ Y_UNIT_TEST(ThreeLeveledLRU) { env->SetLogPriority(NKikimrServices::TABLET_EXECUTOR, NActors::NLog::PRI_TRACE); env->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); auto counters = GetSharedPageCounters(env); + auto deterministicRequests = MakeSharedCacheRequestsDeterministic(env.Env); env.FireDummyTablet(ui32(NFake::TDummy::EFlg::Comp)); env.SendSync(new NFake::TEvExecute{ new TTxInitSchema() }); @@ -478,6 +505,7 @@ Y_UNIT_TEST(S3FIFO) { env->SetLogPriority(NKikimrServices::TABLET_EXECUTOR, NActors::NLog::PRI_TRACE); env->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); auto counters = GetSharedPageCounters(env); + auto deterministicRequests = MakeSharedCacheRequestsDeterministic(env.Env); env.FireDummyTablet(ui32(NFake::TDummy::EFlg::Comp)); env.SendSync(new NFake::TEvExecute{ new TTxInitSchema() }); @@ -583,6 +611,7 @@ Y_UNIT_TEST(ClockPro) { env->SetLogPriority(NKikimrServices::TABLET_EXECUTOR, NActors::NLog::PRI_TRACE); env->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); auto counters = GetSharedPageCounters(env); + auto deterministicRequests = MakeSharedCacheRequestsDeterministic(env.Env); env.FireDummyTablet(ui32(NFake::TDummy::EFlg::Comp)); env.SendSync(new NFake::TEvExecute{ new TTxInitSchema() }); @@ -702,6 +731,7 @@ Y_UNIT_TEST(ReplacementPolicySwitch) { env->SetLogPriority(NKikimrServices::TABLET_EXECUTOR, NActors::NLog::PRI_TRACE); env->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); auto counters = GetSharedPageCounters(env); + auto deterministicRequests = MakeSharedCacheRequestsDeterministic(env.Env); env.FireDummyTablet(ui32(NFake::TDummy::EFlg::Comp)); env.SendSync(new NFake::TEvExecute{ new TTxInitSchema() }); @@ -754,6 +784,7 @@ Y_UNIT_TEST(BigCache_BTreeIndex) { env->SetLogPriority(NKikimrServices::TABLET_EXECUTOR, NActors::NLog::PRI_TRACE); env->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); auto counters = GetSharedPageCounters(env); + auto deterministicRequests = MakeSharedCacheRequestsDeterministic(env.Env); env->GetAppData().FeatureFlags.SetEnableLocalDBBtreeIndex(true); env.FireDummyTablet(ui32(NFake::TDummy::EFlg::Comp)); @@ -821,6 +852,7 @@ Y_UNIT_TEST(BigCache_FlatIndex) { env->SetLogPriority(NKikimrServices::TABLET_EXECUTOR, NActors::NLog::PRI_TRACE); env->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); auto counters = GetSharedPageCounters(env); + auto deterministicRequests = MakeSharedCacheRequestsDeterministic(env.Env); env->GetAppData().FeatureFlags.SetEnableLocalDBBtreeIndex(false); env.FireDummyTablet(ui32(NFake::TDummy::EFlg::Comp)); @@ -888,6 +920,7 @@ Y_UNIT_TEST(MiddleCache_BTreeIndex) { env->SetLogPriority(NKikimrServices::TABLET_EXECUTOR, NActors::NLog::PRI_TRACE); env->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); auto counters = GetSharedPageCounters(env); + auto deterministicRequests = MakeSharedCacheRequestsDeterministic(env.Env); env->GetAppData().FeatureFlags.SetEnableLocalDBBtreeIndex(true); env.FireDummyTablet(ui32(NFake::TDummy::EFlg::Comp)); @@ -955,6 +988,7 @@ Y_UNIT_TEST(MiddleCache_FlatIndex) { env->SetLogPriority(NKikimrServices::TABLET_EXECUTOR, NActors::NLog::PRI_TRACE); env->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); auto counters = GetSharedPageCounters(env); + auto deterministicRequests = MakeSharedCacheRequestsDeterministic(env.Env); env->GetAppData().FeatureFlags.SetEnableLocalDBBtreeIndex(false); env.FireDummyTablet(ui32(NFake::TDummy::EFlg::Comp)); @@ -1022,6 +1056,7 @@ Y_UNIT_TEST(ZeroCache_BTreeIndex) { env->SetLogPriority(NKikimrServices::TABLET_EXECUTOR, NActors::NLog::PRI_TRACE); env->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); auto counters = GetSharedPageCounters(env); + auto deterministicRequests = MakeSharedCacheRequestsDeterministic(env.Env); env->GetAppData().FeatureFlags.SetEnableLocalDBBtreeIndex(true); env.FireDummyTablet(ui32(NFake::TDummy::EFlg::Comp)); @@ -1089,6 +1124,7 @@ Y_UNIT_TEST(ZeroCache_FlatIndex) { env->SetLogPriority(NKikimrServices::TABLET_EXECUTOR, NActors::NLog::PRI_TRACE); env->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); auto counters = GetSharedPageCounters(env); + auto deterministicRequests = MakeSharedCacheRequestsDeterministic(env.Env); env->GetAppData().FeatureFlags.SetEnableLocalDBBtreeIndex(false); env.FireDummyTablet(ui32(NFake::TDummy::EFlg::Comp)); @@ -1558,6 +1594,7 @@ void ManyPartsSetup(TMyEnvBase& env) { Y_UNIT_TEST(One_Transaction_One_Key) { TMyEnvBase env; auto counters = GetSharedPageCounters(env); + auto deterministicRequests = MakeSharedCacheRequestsDeterministic(env.Env); BasicSetup(env); @@ -1572,6 +1609,7 @@ Y_UNIT_TEST(One_Transaction_One_Key) { Y_UNIT_TEST(One_Transaction_Two_Keys) { TMyEnvBase env; auto counters = GetSharedPageCounters(env); + auto deterministicRequests = MakeSharedCacheRequestsDeterministic(env.Env); BasicSetup(env); @@ -1586,6 +1624,7 @@ Y_UNIT_TEST(One_Transaction_Two_Keys) { Y_UNIT_TEST(One_Transaction_Two_Keys_Many_Parts) { TMyEnvBase env; auto counters = GetSharedPageCounters(env); + auto deterministicRequests = MakeSharedCacheRequestsDeterministic(env.Env); ManyPartsSetup(env); @@ -1602,6 +1641,7 @@ Y_UNIT_TEST(One_Transaction_Two_Keys_Many_Parts) { Y_UNIT_TEST(Two_Transactions_One_Key) { TMyEnvBase env; auto counters = GetSharedPageCounters(env); + auto deterministicRequests = MakeSharedCacheRequestsDeterministic(env.Env); BasicSetup(env); @@ -1661,6 +1701,7 @@ Y_UNIT_TEST(Two_Transactions_One_Key) { Y_UNIT_TEST(Two_Transactions_Two_Keys) { TMyEnvBase env; auto counters = GetSharedPageCounters(env); + auto deterministicRequests = MakeSharedCacheRequestsDeterministic(env.Env); BasicSetup(env); @@ -1721,6 +1762,7 @@ Y_UNIT_TEST(Two_Transactions_Two_Keys) { Y_UNIT_TEST(Compaction) { TMyEnvBase env; auto counters = GetSharedPageCounters(env); + auto deterministicRequests = MakeSharedCacheRequestsDeterministic(env.Env); BasicSetup(env); From 656353ad7d790ab345e181510c8fdec9e962a3a5 Mon Sep 17 00:00:00 2001 From: Bulat Date: Fri, 8 Aug 2025 20:43:39 +0300 Subject: [PATCH 52/52] [C++ SDK] Update vector example (#22613) --- .../examples/vector_index_builtin/main.cpp | 6 +- .../vector_index_builtin/vector_index.cpp | 107 +++++++++++++++++- .../vector_index_builtin/vector_index.h | 17 ++- 3 files changed, 123 insertions(+), 7 deletions(-) diff --git a/ydb/public/sdk/cpp/examples/vector_index_builtin/main.cpp b/ydb/public/sdk/cpp/examples/vector_index_builtin/main.cpp index 519d26a15ca0..6e016abb36ce 100644 --- a/ydb/public/sdk/cpp/examples/vector_index_builtin/main.cpp +++ b/ydb/public/sdk/cpp/examples/vector_index_builtin/main.cpp @@ -39,10 +39,10 @@ void VectorExample( {.Id = "8", .Document = "document 8", .Embedding = {0.02, 0.98, 0.1}}, {.Id = "9", .Document = "document 9", .Embedding = {0.0, 1.0, 0.05}}, }; - InsertItems(client, tableName, items); - PrintResults(SearchItems(client, tableName, {1.0, 0.0, 0.0}, "CosineSimilarity", 3)); + InsertItemsAsBytes(client, tableName, items); + PrintResults(SearchItemsAsBytes(client, tableName, {1.0, 0.0, 0.0}, "CosineSimilarity", 3)); AddIndex(driver, client, database, tableName, indexName, "similarity=cosine", 3, 1, 3); - PrintResults(SearchItems(client, tableName, {1.0, 0.0, 0.0}, "CosineSimilarity", 3, indexName)); + PrintResults(SearchItemsAsBytes(client, tableName, {1.0, 0.0, 0.0}, "CosineSimilarity", 3, indexName)); } catch (const std::exception& e) { std::cerr << "Execution failed: " << e.what() << std::endl; } diff --git a/ydb/public/sdk/cpp/examples/vector_index_builtin/vector_index.cpp b/ydb/public/sdk/cpp/examples/vector_index_builtin/vector_index.cpp index 2df8d3e19d85..67cf2557f674 100644 --- a/ydb/public/sdk/cpp/examples/vector_index_builtin/vector_index.cpp +++ b/ydb/public/sdk/cpp/examples/vector_index_builtin/vector_index.cpp @@ -2,6 +2,15 @@ #include +std::string ConvertVectorToBytes(const std::vector& vector) +{ + std::string result; + for (const auto& value : vector) { + const char* bytes = reinterpret_cast(&value); + result += std::string(bytes, sizeof(float)); + } + return result + "\x01"; +} void DropVectorTable(NYdb::NQuery::TQueryClient& client, const std::string& tableName) { @@ -29,7 +38,52 @@ void CreateVectorTable(NYdb::NQuery::TQueryClient& client, const std::string& ta std::cout << "Vector table created: " << tableName << std::endl; } -void InsertItems( +void InsertItemsAsBytes( + NYdb::NQuery::TQueryClient& client, + const std::string& tableName, + const std::vector& items) +{ + std::string query = std::format(R"( + DECLARE $items AS List>; + UPSERT INTO `{0}` + ( + id, + document, + embedding + ) + SELECT + id, + document, + embedding, + FROM AS_TABLE($items); + )", tableName); + + NYdb::TParamsBuilder paramsBuilder; + auto& valueBuilder = paramsBuilder.AddParam("$items"); + valueBuilder.BeginList(); + for (const auto& item : items) { + valueBuilder.AddListItem(); + valueBuilder.BeginStruct(); + valueBuilder.AddMember("id").Utf8(item.Id); + valueBuilder.AddMember("document").Utf8(item.Document); + valueBuilder.AddMember("embedding").String(ConvertVectorToBytes(item.Embedding)); + valueBuilder.EndStruct(); + } + valueBuilder.EndList(); + valueBuilder.Build(); + + NYdb::NStatusHelpers::ThrowOnError(client.RetryQuerySync([params = paramsBuilder.Build(), &query](NYdb::NQuery::TSession session) { + return session.ExecuteQuery(query, NYdb::NQuery::TTxControl::BeginTx(NYdb::NQuery::TTxSettings::SerializableRW()).CommitTx(), params).ExtractValueSync(); + })); + + std::cout << items.size() << " items inserted" << std::endl; +} + +void InsertItemsAsFloatList( NYdb::NQuery::TQueryClient& client, const std::string& tableName, const std::vector& items) @@ -122,7 +176,56 @@ void AddIndex( std::cout << "Table index `" << indexName << "` for table `" << tableName << "` added" << std::endl; } -std::vector SearchItems( +std::vector SearchItemsAsBytes( + NYdb::NQuery::TQueryClient& client, + const std::string& tableName, + const std::vector& embedding, + const std::string& strategy, + std::uint64_t limit, + const std::optional& indexName) +{ + std::string viewIndex = indexName ? "VIEW " + *indexName : ""; + std::string sortOrder = strategy.ends_with("Similarity") ? "DESC" : "ASC"; + + std::string query = std::format(R"( + DECLARE $embedding as String; + SELECT + id, + document, + Knn::{2}(embedding, $embedding) as score + FROM {0} {1} + ORDER BY score + {3} + LIMIT {4}; + )", tableName, viewIndex, strategy, sortOrder, limit); + + auto params = NYdb::TParamsBuilder() + .AddParam("$embedding") + .String(ConvertVectorToBytes(embedding)) + .Build() + .Build(); + + std::vector result; + + NYdb::NStatusHelpers::ThrowOnError(client.RetryQuerySync([params, &query, &result](NYdb::NQuery::TSession session) { + auto execResult = session.ExecuteQuery(query, NYdb::NQuery::TTxControl::BeginTx(NYdb::NQuery::TTxSettings::SerializableRW()).CommitTx(), params).ExtractValueSync(); + if (execResult.IsSuccess()) { + auto parser = execResult.GetResultSetParser(0); + while (parser.TryNextRow()) { + result.push_back({ + .Id = *parser.ColumnParser(0).GetOptionalUtf8(), + .Document = *parser.ColumnParser(1).GetOptionalUtf8(), + .Score = *parser.ColumnParser(2).GetOptionalFloat() + }); + } + } + return execResult; + })); + + return result; +} + +std::vector SearchItemsAsFloatList( NYdb::NQuery::TQueryClient& client, const std::string& tableName, const std::vector& embedding, diff --git a/ydb/public/sdk/cpp/examples/vector_index_builtin/vector_index.h b/ydb/public/sdk/cpp/examples/vector_index_builtin/vector_index.h index 7c4a2302b96c..53479d8e2d2f 100644 --- a/ydb/public/sdk/cpp/examples/vector_index_builtin/vector_index.h +++ b/ydb/public/sdk/cpp/examples/vector_index_builtin/vector_index.h @@ -22,7 +22,12 @@ void DropVectorTable(NYdb::NQuery::TQueryClient& client, const std::string& tabl void CreateVectorTable(NYdb::NQuery::TQueryClient& client, const std::string& tableName); -void InsertItems( +void InsertItemsAsBytes( + NYdb::NQuery::TQueryClient& client, + const std::string& tableName, + const std::vector& items); + +void InsertItemsAsFloatList( NYdb::NQuery::TQueryClient& client, const std::string& tableName, const std::vector& items); @@ -38,7 +43,15 @@ void AddIndex( std::uint64_t levels, std::uint64_t clusters); -std::vector SearchItems( +std::vector SearchItemsAsBytes( + NYdb::NQuery::TQueryClient& client, + const std::string& tableName, + const std::vector& embedding, + const std::string& strategy, + std::uint64_t limit, + const std::optional& indexName = std::nullopt); + +std::vector SearchItemsAsFloatList( NYdb::NQuery::TQueryClient& client, const std::string& tableName, const std::vector& embedding,