From 3e71cc172448d0507219a8e013571aec6382164a Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Wed, 23 Jul 2025 16:10:42 -0700 Subject: [PATCH] [CAS] Move CASConfiguration into LLVM Move CASConfiguration into LLVM and add a JSON based configuration file to help constructing CAS from init file `.cas-config`. This is so that tools like lldb/dsymutil can be CAS aware without user explicitly providing the CAS path when invoking the tool. --- clang/include/clang/CAS/CASOptions.h | 65 +++-------- clang/lib/CAS/CASOptions.cpp | 45 ++------ llvm/include/llvm/CAS/CASConfiguration.h | 79 +++++++++++++ llvm/lib/CAS/CASConfiguration.cpp | 122 ++++++++++++++++++++ llvm/lib/CAS/CMakeLists.txt | 1 + llvm/unittests/CAS/CASConfigurationTest.cpp | 63 ++++++++++ llvm/unittests/CAS/CMakeLists.txt | 1 + 7 files changed, 289 insertions(+), 87 deletions(-) create mode 100644 llvm/include/llvm/CAS/CASConfiguration.h create mode 100644 llvm/lib/CAS/CASConfiguration.cpp create mode 100644 llvm/unittests/CAS/CASConfigurationTest.cpp diff --git a/clang/include/clang/CAS/CASOptions.h b/clang/include/clang/CAS/CASOptions.h index 63f5a1f6d36c6..8799d040cee54 100644 --- a/clang/include/clang/CAS/CASOptions.h +++ b/clang/include/clang/CAS/CASOptions.h @@ -15,6 +15,7 @@ #define LLVM_CLANG_CAS_CASOPTIONS_H #include "llvm/ADT/SmallVector.h" +#include "llvm/CAS/CASConfiguration.h" #include "llvm/Support/Error.h" #include #include @@ -30,52 +31,6 @@ namespace clang { class DiagnosticsEngine; -/// Base class for options configuring which CAS to use. Separated for the -/// fields where we don't need special move/copy logic. -/// -/// TODO: Add appropriate options once we support plugins. -class CASConfiguration { -public: - enum CASKind { - UnknownCAS, - InMemoryCAS, - OnDiskCAS, - }; - - /// Kind of CAS to use. - CASKind getKind() const { - return IsFrozen ? UnknownCAS : CASPath.empty() ? InMemoryCAS : OnDiskCAS; - } - - /// Path to a persistent backing store on-disk. This is optional, although \a - /// CASFileSystemRootID is unlikely to work without it. - /// - /// - "" means there is none; falls back to in-memory. - /// - "auto" is an alias for an automatically chosen location in the user's - /// system cache. - std::string CASPath; - - std::string PluginPath; - /// Each entry is a (, ) pair. - std::vector> PluginOptions; - - friend bool operator==(const CASConfiguration &LHS, - const CASConfiguration &RHS) { - return LHS.CASPath == RHS.CASPath && LHS.PluginPath == RHS.PluginPath && - LHS.PluginOptions == RHS.PluginOptions; - } - friend bool operator!=(const CASConfiguration &LHS, - const CASConfiguration &RHS) { - return !(LHS == RHS); - } - -private: - /// Whether the configuration has been "frozen", in order to hide the kind of - /// CAS that's in use. - bool IsFrozen = false; - friend class CASOptions; -}; - /// Options configuring which CAS to use. User-accessible fields should be /// defined in CASConfiguration to enable caching a CAS instance. /// @@ -87,8 +42,18 @@ class CASConfiguration { /// clang::createVFSFromCompilerInvocation() uses the same CAS instance that /// the rest of the compiler job does, without updating all callers. Probably /// it would be better to update all callers and remove it from here. -class CASOptions : public CASConfiguration { +class CASOptions : public llvm::cas::CASConfiguration { public: + enum CASKind { + UnknownCAS, + InMemoryCAS, + OnDiskCAS, + }; + + /// Kind of CAS to use. + CASKind getKind() const { + return IsFrozen ? UnknownCAS : CASPath.empty() ? InMemoryCAS : OnDiskCAS; + } /// Get a CAS & ActionCache defined by the options above. Future calls will /// return the same instances... unless the configuration has changed, in /// which case new ones will be created. @@ -117,8 +82,6 @@ class CASOptions : public CASConfiguration { /// default on-disk CAS, otherwise this is a noop. void ensurePersistentCAS(); - void getResolvedCASPath(llvm::SmallVectorImpl &Result) const; - private: /// Initialize Cached CAS and ActionCache. llvm::Error initCache() const; @@ -133,6 +96,10 @@ class CASOptions : public CASConfiguration { CASConfiguration Config; }; mutable CachedCAS Cache; + + /// Whether the configuration has been "frozen", in order to hide the kind of + /// CAS that's in use. + bool IsFrozen = false; }; } // end namespace clang diff --git a/clang/lib/CAS/CASOptions.cpp b/clang/lib/CAS/CASOptions.cpp index e9d940d5cf2be..381fde3a24820 100644 --- a/clang/lib/CAS/CASOptions.cpp +++ b/clang/lib/CAS/CASOptions.cpp @@ -22,7 +22,7 @@ std::pair, std::shared_ptr> CASOptions::getOrCreateDatabases(DiagnosticsEngine &Diags, bool CreateEmptyDBsOnFailure) const { - if (Cache.Config.IsFrozen) + if (IsFrozen) return {Cache.CAS, Cache.AC}; if (auto E = initCache()) @@ -44,7 +44,7 @@ CASOptions::getOrCreateDatabases() const { } void CASOptions::freezeConfig(DiagnosticsEngine &Diags) { - if (Cache.Config.IsFrozen) + if (IsFrozen) return; // Make sure the cache is initialized. @@ -57,7 +57,7 @@ void CASOptions::freezeConfig(DiagnosticsEngine &Diags) { // scheduled/executed at a level that has access to the configuration. auto &CurrentConfig = static_cast(*this); CurrentConfig = CASConfiguration(); - CurrentConfig.IsFrozen = Cache.Config.IsFrozen = true; + IsFrozen = true; if (Cache.CAS) { // Set the CASPath to the hash schema, since that leaks through CASContext's @@ -90,41 +90,10 @@ llvm::Error CASOptions::initCache() const { Cache.Config = CurrentConfig; StringRef CASPath = Cache.Config.CASPath; - if (!PluginPath.empty()) { - std::pair, std::shared_ptr> DBs; - if (llvm::Error E = - createPluginCASDatabases(PluginPath, CASPath, PluginOptions) - .moveInto(DBs)) { - return E; - } - std::tie(Cache.CAS, Cache.AC) = std::move(DBs); - return llvm::Error::success(); - } - - if (CASPath.empty()) { - Cache.CAS = llvm::cas::createInMemoryCAS(); - Cache.AC = llvm::cas::createInMemoryActionCache(); - return llvm::Error::success(); - } - - SmallString<256> PathBuf; - getResolvedCASPath(PathBuf); - if (CASPath == "auto") { - getDefaultOnDiskCASPath(PathBuf); - CASPath = PathBuf; - } - std::pair, std::unique_ptr> DBs; - if (llvm::Error E = createOnDiskUnifiedCASDatabases(CASPath).moveInto(DBs)) - return E; + auto DBs = Cache.Config.createDatabases(); + if (!DBs) + return DBs.takeError(); - std::tie(Cache.CAS, Cache.AC) = std::move(DBs); + std::tie(Cache.CAS, Cache.AC) = std::move(*DBs); return llvm::Error::success(); } - -void CASOptions::getResolvedCASPath(SmallVectorImpl &Result) const { - if (CASPath == "auto") { - getDefaultOnDiskCASPath(Result); - } else { - Result.assign(CASPath.begin(), CASPath.end()); - } -} diff --git a/llvm/include/llvm/CAS/CASConfiguration.h b/llvm/include/llvm/CAS/CASConfiguration.h new file mode 100644 index 0000000000000..c07b7b546b7cf --- /dev/null +++ b/llvm/include/llvm/CAS/CASConfiguration.h @@ -0,0 +1,79 @@ +//===- CASOptions.h - Options for configuring the CAS -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// Defines the llvm::cas::CASConfiguration interface. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CAS_CASCONFIGURATION_H +#define LLVM_CAS_CASCONFIGURATION_H + +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/VirtualFileSystem.h" +#include +#include + +namespace llvm::cas { + +class ActionCache; +class ObjectStore; + +/// Base class for options configuring which CAS to use. +class CASConfiguration { +public: + /// Path to a persistent backing store on-disk. + /// + /// - "" means there is none; falls back to in-memory. + /// - "auto" is an alias for an automatically chosen location in the user's + /// system cache. + std::string CASPath; + /// Path to the CAS plugin library. + std::string PluginPath; + /// Each entry is a (, ) pair. + std::vector> PluginOptions; + + friend bool operator==(const CASConfiguration &LHS, + const CASConfiguration &RHS) { + return LHS.CASPath == RHS.CASPath && LHS.PluginPath == RHS.PluginPath && + LHS.PluginOptions == RHS.PluginOptions; + } + friend bool operator!=(const CASConfiguration &LHS, + const CASConfiguration &RHS) { + return !(LHS == RHS); + } + + // Get resolved CASPath. + void getResolvedCASPath(llvm::SmallVectorImpl &Result) const; + + // Create CASDatabase from the CASConfiguration. + llvm::Expected, + std::shared_ptr>> + createDatabases() const; + + /// Write CAS configuration file. + void writeConfigurationFile(raw_ostream &OS) const; + + /// Create CASConfiguration from config file content. + static llvm::Expected + createFromConfig(llvm::StringRef Content); + + /// Create CASConfiguration from recurively search config file from a path. + /// + /// Returns the path to configuration file and its corresponding + /// CASConfiguration. + static std::optional> + createFromSearchConfigFile( + StringRef Path, + llvm::IntrusiveRefCntPtr VFS = nullptr); +}; + +} // namespace llvm::cas + +#endif diff --git a/llvm/lib/CAS/CASConfiguration.cpp b/llvm/lib/CAS/CASConfiguration.cpp new file mode 100644 index 0000000000000..c816f20ed5e4b --- /dev/null +++ b/llvm/lib/CAS/CASConfiguration.cpp @@ -0,0 +1,122 @@ +//===- CASConfiguration.cpp -------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/CAS/CASConfiguration.h" +#include "llvm/CAS/ActionCache.h" +#include "llvm/CAS/BuiltinUnifiedCASDatabases.h" +#include "llvm/CAS/ObjectStore.h" +#include "llvm/Support/JSON.h" + +using namespace llvm; +using namespace llvm::cas; + +void CASConfiguration::getResolvedCASPath( + llvm::SmallVectorImpl &Result) const { + if (CASPath == "auto") { + getDefaultOnDiskCASPath(Result); + } else { + Result.assign(CASPath.begin(), CASPath.end()); + } +} + +Expected, std::shared_ptr>> +CASConfiguration::createDatabases() const { + if (!PluginPath.empty()) + return createPluginCASDatabases(PluginPath, CASPath, PluginOptions); + + if (CASPath.empty()) { + return std::pair(createInMemoryCAS(), createInMemoryActionCache()); + } + + SmallString<128> PathBuf; + getResolvedCASPath(PathBuf); + + std::pair, std::unique_ptr> DBs; + return createOnDiskUnifiedCASDatabases(PathBuf); +} + +void CASConfiguration::writeConfigurationFile(raw_ostream &OS) const { + using namespace llvm::json; + Object Root; + Root["CASPath"] = CASPath; + Root["PluginPath"] = PluginPath; + + Array PlugOpts; + for (const auto &Opt : PluginOptions) { + Object Entry; + Entry[Opt.first] = Opt.second; + PlugOpts.emplace_back(std::move(Entry)); + } + Root["PluginOptions"] = std::move(PlugOpts); + + OS << formatv("{0:2}", Value(std::move(Root))); +} + +Expected +CASConfiguration::createFromConfig(StringRef Content) { + auto Parsed = json::parse(Content); + if (!Parsed) + return Parsed.takeError(); + + CASConfiguration Config; + auto *Root = Parsed->getAsObject(); + if (!Root) + return createStringError( + "CASConfiguration file error: top level object missing"); + + if (auto CASPath = Root->getString("CASPath")) + Config.CASPath = *CASPath; + + if (auto PluginPath = Root->getString("PluginPath")) + Config.PluginPath = *PluginPath; + + if (auto *Opts = Root->getArray("PluginOptions")) { + for (auto &Opt : *Opts) { + if (auto *Arg = Opt.getAsObject()) { + for (auto &Entry : *Arg) { + if (auto V = Entry.second.getAsString()) + Config.PluginOptions.emplace_back(Entry.first.str(), *V); + } + } + } + } + + return Config; +} + +std::optional> +CASConfiguration::createFromSearchConfigFile( + StringRef Path, IntrusiveRefCntPtr VFS) { + if (!VFS) + VFS = vfs::getRealFileSystem(); + + while (!Path.empty()) { + SmallString<256> ConfigPath(Path); + sys::path::append(ConfigPath, ".cas-config"); + auto File = VFS->openFileForRead(ConfigPath); + if (!File || !*File) { + Path = sys::path::parent_path(Path); + continue; + } + + auto Buffer = (*File)->getBuffer(ConfigPath); + if (!Buffer || !*Buffer) { + Path = sys::path::parent_path(Path); + continue; + } + + auto Config = createFromConfig((*Buffer)->getBuffer()); + if (!Config) { + consumeError(Config.takeError()); + Path = sys::path::parent_path(Path); + continue; + } + return std::pair{ConfigPath.str().str(), *Config}; + } + return std::nullopt; +} diff --git a/llvm/lib/CAS/CMakeLists.txt b/llvm/lib/CAS/CMakeLists.txt index ca70b0bfee9f0..2f647033a1196 100644 --- a/llvm/lib/CAS/CMakeLists.txt +++ b/llvm/lib/CAS/CMakeLists.txt @@ -7,6 +7,7 @@ add_llvm_component_library(LLVMCAS ActionCaches.cpp BuiltinCAS.cpp BuiltinUnifiedCASDatabases.cpp + CASConfiguration.cpp CASFileSystem.cpp CASNodeSchema.cpp CASOutputBackend.cpp diff --git a/llvm/unittests/CAS/CASConfigurationTest.cpp b/llvm/unittests/CAS/CASConfigurationTest.cpp new file mode 100644 index 0000000000000..0416ab799e215 --- /dev/null +++ b/llvm/unittests/CAS/CASConfigurationTest.cpp @@ -0,0 +1,63 @@ +//===- CASConfiguration.cpp -----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/CAS/CASConfiguration.h" +#include "llvm/Testing/Support/Error.h" +#include "gtest/gtest.h" + +using namespace llvm; +using namespace llvm::cas; + +TEST(CASConfigurationTest, roundTrips) { + auto roundTripConfig = [](CASConfiguration &Config) { + std::string Serialized; + raw_string_ostream OS(Serialized); + Config.writeConfigurationFile(OS); + + std::optional NewConfig; + ASSERT_THAT_ERROR( + CASConfiguration::createFromConfig(Serialized).moveInto(NewConfig), + Succeeded()); + ASSERT_TRUE(Config == *NewConfig); + }; + + CASConfiguration Config; + roundTripConfig(Config); + + Config.CASPath = "/tmp"; + roundTripConfig(Config); + + Config.PluginPath = "/test.plug"; + roundTripConfig(Config); + + Config.PluginOptions.emplace_back("a", "b"); + roundTripConfig(Config); + + Config.PluginOptions.emplace_back("c", "d"); + roundTripConfig(Config); +} + +TEST(CASConfigurationTest, configFileSearch) { + auto VFS = makeIntrusiveRefCnt(); + ASSERT_FALSE(CASConfiguration::createFromSearchConfigFile("/a/b/c/d/e", VFS)); + + // Add an empty file. + VFS->addFile("/a/b/c/.cas-config", 0, + llvm::MemoryBuffer::getMemBufferCopy("")); + ASSERT_FALSE(CASConfiguration::createFromSearchConfigFile("/a/b/c/d/e", VFS)); + + VFS->addFile("/a/b/c/d/.cas-config", 0, + llvm::MemoryBuffer::getMemBufferCopy("{\"CASPath\": \"/tmp\"}")); + CASConfiguration Config; + Config.CASPath = "/tmp"; + auto NewConfig = + CASConfiguration::createFromSearchConfigFile("/a/b/c/d/e", VFS); + ASSERT_TRUE(NewConfig); + ASSERT_TRUE(NewConfig->first == "/a/b/c/d/.cas-config"); + ASSERT_TRUE(Config == NewConfig->second); +} diff --git a/llvm/unittests/CAS/CMakeLists.txt b/llvm/unittests/CAS/CMakeLists.txt index c7f65479e8e1c..c6fb564dd4621 100644 --- a/llvm/unittests/CAS/CMakeLists.txt +++ b/llvm/unittests/CAS/CMakeLists.txt @@ -18,6 +18,7 @@ set(LLVM_LINK_COMPONENTS add_llvm_unittest(CASTests ActionCacheTest.cpp BuiltinUnifiedCASDatabasesTest.cpp + CASConfigurationTest.cpp CASFileSystemTest.cpp CASTestConfig.cpp CASOutputBackendTest.cpp