refactor process cache management to handle out of order process ebpf events (#2232)

* add basic pod info for process

* refactory process cache management to handle out of order events

* format

* fix

* fix consume retry queue crash\nfix memleak due to inproper reuse in clone event

* fix shrink size

* fix format

* fix containerId buffer

* Add debug log

* fix exit do not clear cache if k8s meta not exist

* fix event lost for using circular buffer

* fix ebpf_process_secure config reload problem

* Use fixed sized common queue to control memory

Improve tcmalloc release mem machanism
Fix crash when FileFlusher has no permission to write

* fix ut

* Update core/common/TimeKeeper.cpp

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: yyuuttaaoo

.clangd

@@ -3,6 +3,8 @@ CompileFlags:
     - -fgnuc-version=9.3.1
     - -Wall
     - -Wextra
+    # - -fno-access-control  # 允许UT访问私有成员
+    # - -DAPSARA_UNIT_TEST_MAIN  # 允许编译UT特殊方法
     - -I/opt/rh/devtoolset-9/root/usr/lib/gcc/x86_64-redhat-linux/9/include
     - -I/opt/logtail/deps/include
   CompilationDatabase: build

core/_thirdparty/coolbpf

@@ -31,6 +31,7 @@
 #include "common/MachineInfoUtil.h"
 #include "common/RuntimeUtil.h"
 #include "common/StringTools.h"
+#include "common/TimeKeeper.h"
 #include "common/TimeUtil.h"
 #include "common/UUIDUtil.h"
 #include "common/version.h"
@@ -81,6 +82,7 @@ Application::Application() : mStartTime(time(nullptr)) {
 }
 
 void Application::Init() {
+    TimeKeeper::GetInstance();
     // change working dir to ./${ILOGTAIL_VERSION}/
     string processExecutionDir = GetProcessExecutionDir();
     AppConfig::GetInstance()->SetProcessExecutionDir(processExecutionDir);
@@ -267,9 +269,6 @@ void Application::Start() { // GCOVR_EXCL_START
     }
 
     time_t curTime = 0, lastConfigCheckTime = 0, lastUpdateMetricTime = 0, lastCheckTagsTime = 0, lastQueueGCTime = 0;
-#ifndef LOGTAIL_NO_TC_MALLOC
-    time_t lastTcmallocReleaseMemTime = 0;
-#endif
     while (true) {
         curTime = time(NULL);
         if (curTime - lastCheckTagsTime >= INT32_FLAG(file_tags_update_interval)) {
@@ -291,9 +290,9 @@ void Application::Start() { // GCOVR_EXCL_START
             lastConfigCheckTime = curTime;
         }
 #ifndef LOGTAIL_NO_TC_MALLOC
-        if (curTime - lastTcmallocReleaseMemTime >= INT32_FLAG(tcmalloc_release_memory_interval)) {
+        if (curTime - gLastTcmallocReleaseMemTime >= INT32_FLAG(tcmalloc_release_memory_interval)) {
             MallocExtension::instance()->ReleaseFreeMemory();
-            lastTcmallocReleaseMemTime = curTime;
+            gLastTcmallocReleaseMemTime = curTime;
         }
 #endif
         if (curTime - lastQueueGCTime >= INT32_FLAG(queue_check_gc_interval_sec)) {
@@ -378,7 +377,7 @@ void Application::Exit() {
     FlusherSLS::RecycleResourceIfNotUsed();
 
     CollectionPipelineManager::GetInstance()->ClearAllPipelines();
-
+    TimeKeeper::GetInstance()->Stop();
 #if defined(_MSC_VER)
     ReleaseWindowsSignalObject();
 #endif

core/application/Application.cpp

@@ -70,7 +70,7 @@
 
 DEFINE_FLAG_BOOL(enable_processor_spl, "", true);
 DEFINE_FLAG_BOOL(enable_ebpf_network_observer, "", false);
-DEFINE_FLAG_BOOL(enable_ebpf_process_secure, "", false);
+DEFINE_FLAG_BOOL(enable_ebpf_process_secure, "", true);
 DEFINE_FLAG_BOOL(enable_ebpf_file_secure, "", false);
 DEFINE_FLAG_BOOL(enable_ebpf_network_secure, "", false);
 

core/collection_pipeline/plugin/PluginRegistry.cpp

@@ -14,7 +14,7 @@
 
 #include "DynamicLibHelper.h"
 
-#include "Logger.h"
+#include "logger/Logger.h"
 
 #if defined(__linux__)
 #include <dlfcn.h>

core/common/CapabilityUtil.cpp

@@ -65,7 +65,7 @@ static void Base64Encoding(istream& is, ostream& os, char makeupChar, const char
     }
 }
 
-string Base64Enconde(const string& message) {
+string Base64Encode(const string& message) {
     istringstream iss(message);
     ostringstream oss;
     Base64Encoding(iss, oss, '=', "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/");

core/common/DynamicLibHelper.cpp

@@ -20,6 +20,6 @@
 
 namespace logtail {
 
-std::string Base64Enconde(const std::string& message);
+std::string Base64Encode(const std::string& message);
 
 } // namespace logtail

core/common/EncodingUtil.cpp

@@ -140,19 +140,10 @@ class Cache {
     bool tryGet(const Key& kIn, Value& vOut) { return tryGetCopy(kIn, vOut); }
 
     bool tryGetCopy(const Key& kIn, Value& vOut) {
-        Guard g(lock_);
-        Value tmp;
-        if (!tryGetRef_nolock(kIn, tmp)) {
-            return false;
-        }
-        vOut = tmp;
-        return true;
-    }
-
-    bool tryGetRef(const Key& kIn, Value& vOut) {
         Guard g(lock_);
         return tryGetRef_nolock(kIn, vOut);
     }
+
     /**
      *	The const reference returned here is only
      *    guaranteed to be valid till the next insert/delete