[CAS] Remove CAS dependency from LLVM_Utils module

Fix the proper laying of the llvm library by splitting CAS reference
away from FileSystem references.

This has some behavior changes:
* The common base class for FileSystem configured with a CAS-backing is
  now `llvm::cas::CASBackedFileSystem` and migrate CAS related
  FileSystem APIs to the new class.
* ThreadSafeFileSystem is folded into CASBackedFileSystem to avoid
  diamond class hierachies.
* llvm::vfs::File needs to be RTTIExtended, so that a user can tell if
  the file is CAS backed when there are overlaying file system.
* When configured with multi-layer CAS backed file system, the file
  content is available in every CAS along the way the file is resolved
  and the returned CASID is from the top layer. This is a more
  deterministic behavior than current one, that only the lowest level
  CAS has the reference, and user actually don't know which CAS it is
  associated with unless know in advance which layer vended the file.

Author: cachemeifyoucan

clang/include/clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h

@@ -14,9 +14,9 @@
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/CAS/ActionCache.h"
+#include "llvm/CAS/CASFileSystem.h"
 #include "llvm/CAS/CASID.h"
 #include "llvm/CAS/CASReference.h"
-#include "llvm/CAS/ThreadSafeFileSystem.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/ErrorOr.h"
 #include "llvm/Support/VirtualFileSystem.h"
@@ -32,8 +32,12 @@ namespace clang {
 namespace tooling {
 namespace dependencies {
 
-class DependencyScanningCASFilesystem : public llvm::cas::ThreadSafeFileSystem {
+class DependencyScanningCASFilesystem
+    : public llvm::RTTIExtends<DependencyScanningCASFilesystem,
+                               llvm::cas::CASBackedFileSystem> {
 public:
+  static const char ID;
+
   DependencyScanningCASFilesystem(
       IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> WorkerFS,
       llvm::cas::ActionCache &Cache);
@@ -59,13 +63,13 @@ class DependencyScanningCASFilesystem : public llvm::cas::ThreadSafeFileSystem {
     return FS->isLocal(Path, Result);
   }
 
-  IntrusiveRefCntPtr<llvm::cas::ThreadSafeFileSystem>
+  IntrusiveRefCntPtr<llvm::cas::CASBackedFileSystem>
   createThreadSafeProxyFS() override;
 
   llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override;
   bool exists(const Twine &Path) override;
-  llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
-  openFileForRead(const Twine &Path) override;
+  llvm::Expected<std::unique_ptr<llvm::cas::CASBackedFile>>
+  openCASBackedFileForRead(const Twine &Path) override;
 
   /// \returns The scanned preprocessor directive tokens of the file that are
   /// used to speed up preprocessing, if available.

clang/lib/Basic/FileManager.cpp

@@ -20,6 +20,7 @@
 #include "clang/Basic/FileSystemStatCache.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/CAS/CASFileSystem.h"
 #include "llvm/CAS/CASReference.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/Support/FileSystem.h"
@@ -536,10 +537,10 @@ FileManager::getBufferForFile(FileEntryRef FE, bool isVolatile,
     auto Result = Entry->File->getBuffer(Filename, FileSize,
                                          RequiresNullTerminator, isVolatile);
     if (CASContents) {
-      auto CASRef = Entry->File->getObjectRefForContent();
-      if (!CASRef)
-        return CASRef.getError();
-      *CASContents = *CASRef;
+      if (auto *CASFile = dyn_cast<llvm::cas::CASBackedFile>(Entry->File.get()))
+        *CASContents = CASFile->getObjectRefForContent();
+      else
+        *CASContents = std::nullopt;
     }
     Entry->closeFile();
     return Result;
@@ -555,25 +556,52 @@ FileManager::getBufferForFileImpl(StringRef Filename, int64_t FileSize,
                                   bool isVolatile, bool RequiresNullTerminator,
                                   bool IsText,
                                   std::optional<cas::ObjectRef> *CASContents) const {
+  auto getBufferImpl = [&](StringRef Name)
+      -> llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> {
+    auto F = FS->openFileForRead(Name);
+    if (!F)
+      return F.getError();
+
+    if (CASContents) {
+      if (auto *CASFile = dyn_cast<llvm::cas::CASBackedFile>(F->get()))
+        *CASContents = CASFile->getObjectRefForContent();
+      else
+        *CASContents = std::nullopt;
+    }
+    return FS->getBufferForFile(Name, FileSize, RequiresNullTerminator,
+                                isVolatile, IsText);
+  };
+
   if (FileSystemOpts.WorkingDir.empty())
-    return FS->getBufferForFile(Filename, FileSize, RequiresNullTerminator,
-                                isVolatile, IsText, CASContents);
+    return getBufferImpl(Filename);
 
   SmallString<128> FilePath(Filename);
   FixupRelativePath(FilePath);
-  return FS->getBufferForFile(FilePath, FileSize, RequiresNullTerminator,
-                              isVolatile, IsText, CASContents);
+  return getBufferImpl(FilePath);
 }
 
 llvm::ErrorOr<std::optional<cas::ObjectRef>>
 FileManager::getObjectRefForFileContent(const Twine &Filename) {
+  auto getObjectRefImpl =
+      [&](const Twine &Name) -> llvm::ErrorOr<std::optional<cas::ObjectRef>> {
+    auto F = FS->openFileForRead(Name);
+    if (!F)
+      return F.getError();
+
+    auto *CASFile = dyn_cast<llvm::cas::CASBackedFile>(F->get());
+    if (!CASFile)
+      return std::nullopt;
+
+    return CASFile->getObjectRefForContent();
+  };
+
   if (FileSystemOpts.WorkingDir.empty())
-    return FS->getObjectRefForFileContent(Filename);
+    return getObjectRefImpl(Filename);
 
   SmallString<128> FilePath;
   Filename.toVector(FilePath);
   FixupRelativePath(FilePath);
-  return FS->getObjectRefForFileContent(FilePath);
+  return getObjectRefImpl(FilePath);
 }
 
 /// getStatValue - Get the 'stat' information for the specified path,

clang/lib/CAS/IncludeTree.cpp

@@ -9,6 +9,7 @@
 #include "clang/CAS/IncludeTree.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallBitVector.h"
+#include "llvm/CAS/CASFileSystem.h"
 #include "llvm/CAS/ObjectStore.h"
 #include "llvm/Support/EndianStream.h"
 #include "llvm/Support/Error.h"
@@ -899,11 +900,11 @@ namespace {
 /// of the preprocessor, for creating \p FileEntries using a file path, while
 /// "replaying" an \p IncludeTreeRoot. It is not intended to be a complete
 /// implementation of a file system.
-class IncludeTreeFileSystem : public llvm::vfs::FileSystem {
+class IncludeTreeFileSystem final : public llvm::cas::CASBackedFileSystem {
   llvm::cas::ObjectStore &CAS;
 
 public:
-  class IncludeTreeFile : public llvm::vfs::File {
+  class IncludeTreeFile final : public llvm::cas::CASBackedFile {
     llvm::vfs::Status Stat;
     StringRef Contents;
     cas::ObjectRef ContentsRef;
@@ -924,10 +925,7 @@ class IncludeTreeFileSystem : public llvm::vfs::FileSystem {
                                               Name.toStringRef(NameBuf));
     }
 
-    llvm::ErrorOr<std::optional<cas::ObjectRef>>
-    getObjectRefForContent() override {
-      return ContentsRef;
-    }
+    cas::ObjectRef getObjectRefForContent() override { return ContentsRef; }
 
     std::error_code close() override { return std::error_code(); }
   };
@@ -972,13 +970,13 @@ class IncludeTreeFileSystem : public llvm::vfs::FileSystem {
     return llvm::errorToErrorCode(fileError(Filename));
   }
 
-  llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
-  openFileForRead(const Twine &Path) override {
+  llvm::Expected<std::unique_ptr<llvm::cas::CASBackedFile>>
+  openCASBackedFileForRead(const Twine &Path) override {
     SmallString<128> Filename;
     getPath(Path, Filename);
     auto MaterializedFile = materialize(Filename);
     if (!MaterializedFile)
-      return llvm::errorToErrorCode(MaterializedFile.takeError());
+      return MaterializedFile.takeError();
     llvm::vfs::Status Stat = makeStatus(
         Filename, MaterializedFile->Contents.size(), MaterializedFile->UniqueID,
         llvm::sys::fs::file_type::regular_file);
@@ -1037,6 +1035,10 @@ class IncludeTreeFileSystem : public llvm::vfs::FileSystem {
   std::error_code setCurrentWorkingDirectory(const Twine &Path) override {
     return llvm::errc::operation_not_permitted;
   }
+  IntrusiveRefCntPtr<llvm::cas::CASBackedFileSystem>
+  createThreadSafeProxyFS() override {
+    llvm::report_fatal_error("not implemented");
+  }
 };
 } // namespace
 

clang/lib/Tooling/DependencyScanning/DependencyScanningCASFilesystem.cpp

@@ -40,6 +40,7 @@ DependencyScanningCASFilesystem::DependencyScanningCASFilesystem(
     : FS(WorkerFS), Entries(EntryAlloc), CAS(WorkerFS->getCAS()), Cache(Cache) {
 }
 
+const char DependencyScanningCASFilesystem::ID = 0;
 DependencyScanningCASFilesystem::~DependencyScanningCASFilesystem() = default;
 
 static Expected<cas::ObjectRef>
@@ -276,19 +277,18 @@ bool DependencyScanningCASFilesystem::exists(const Twine &Path) {
   return getCachingFS().exists(Path);
 }
 
-IntrusiveRefCntPtr<llvm::cas::ThreadSafeFileSystem>
+IntrusiveRefCntPtr<llvm::cas::CASBackedFileSystem>
 DependencyScanningCASFilesystem::createThreadSafeProxyFS() {
   llvm::report_fatal_error("not implemented");
 }
 
 namespace {
 
-class DepScanFile final : public llvm::vfs::File {
+class DepScanFile final : public llvm::cas::CASBackedFile {
 public:
-  DepScanFile(StringRef Buffer, std::optional<cas::ObjectRef> CASContents,
+  DepScanFile(StringRef Buffer, cas::ObjectRef CASContents,
               llvm::vfs::Status Stat)
-      : Buffer(Buffer), CASContents(std::move(CASContents)),
-        Stat(std::move(Stat)) {}
+      : Buffer(Buffer), CASContents(CASContents), Stat(std::move(Stat)) {}
 
   llvm::ErrorOr<llvm::vfs::Status> status() override { return Stat; }
 
@@ -299,36 +299,35 @@ class DepScanFile final : public llvm::vfs::File {
     return llvm::MemoryBuffer::getMemBuffer(Buffer, Name.toStringRef(Storage));
   }
 
-  llvm::ErrorOr<std::optional<cas::ObjectRef>>
-  getObjectRefForContent() override {
-    return CASContents;
-  }
+  cas::ObjectRef getObjectRefForContent() override { return CASContents; }
 
   std::error_code close() override { return {}; }
 
 private:
   StringRef Buffer;
-  std::optional<cas::ObjectRef> CASContents;
+  cas::ObjectRef CASContents;
   llvm::vfs::Status Stat;
 };
 
 } // end anonymous namespace
 
-llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
-DependencyScanningCASFilesystem::openFileForRead(const Twine &Path) {
+Expected<std::unique_ptr<llvm::cas::CASBackedFile>>
+DependencyScanningCASFilesystem::openCASBackedFileForRead(const Twine &Path) {
   LookupPathResult Result = lookupPath(Path);
   if (!Result.Entry) {
     if (std::error_code EC = Result.Status.getError())
-      return EC;
+      return llvm::errorCodeToError(EC);
     assert(Result.Status->getType() ==
            llvm::sys::fs::file_type::directory_file);
-    return std::make_error_code(std::errc::is_a_directory);
+    return llvm::createFileError(
+        Path, std::make_error_code(std::errc::is_a_directory));
   }
   if (Result.Entry->EC)
-    return Result.Entry->EC;
+    return llvm::errorCodeToError(Result.Entry->EC);
 
+  assert(Result.Entry->CASContents);
   return std::make_unique<DepScanFile>(
-      *Result.Entry->Buffer, Result.Entry->CASContents, Result.Entry->Status);
+      *Result.Entry->Buffer, *Result.Entry->CASContents, Result.Entry->Status);
 }
 
 std::optional<ArrayRef<dependency_directives_scan::Directive>>

clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp

@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
+#include "llvm/CAS/CASFileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Threading.h"
 #include <optional>
@@ -33,10 +34,9 @@ DependencyScanningWorkerFilesystem::readFile(StringRef Filename) {
     return MaybeBuffer.getError();
   auto Buffer = std::move(*MaybeBuffer);
 
-  auto MaybeCASContents = File->getObjectRefForContent();
-  if (!MaybeCASContents)
-    return MaybeCASContents.getError();
-  auto CASContents = std::move(*MaybeCASContents);
+  std::optional<cas::ObjectRef> CASContents;
+  if (auto *CASFile = dyn_cast<llvm::cas::CASBackedFile>(File.get()))
+    CASContents = CASFile->getObjectRefForContent();
 
   // If the file size changed between read and stat, pretend it didn't.
   if (Stat.getSize() != Buffer->getBufferSize())
@@ -362,9 +362,8 @@ namespace {
 class DepScanFile final : public llvm::vfs::File {
 public:
   DepScanFile(std::unique_ptr<llvm::MemoryBuffer> Buffer,
-              std::optional<cas::ObjectRef> CASContents, llvm::vfs::Status Stat)
-      : Buffer(std::move(Buffer)), CASContents(std::move(CASContents)),
-        Stat(std::move(Stat)) {}
+              llvm::vfs::Status Stat)
+      : Buffer(std::move(Buffer)), Stat(std::move(Stat)) {}
 
   static llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> create(EntryRef Entry);
 
@@ -376,16 +375,37 @@ class DepScanFile final : public llvm::vfs::File {
     return std::move(Buffer);
   }
 
-  llvm::ErrorOr<std::optional<cas::ObjectRef>>
-  getObjectRefForContent() override {
-    return CASContents;
+  std::error_code close() override { return {}; }
+
+private:
+  std::unique_ptr<llvm::MemoryBuffer> Buffer;
+  llvm::vfs::Status Stat;
+};
+
+class DepScanCASFile final : public llvm::cas::CASBackedFile {
+public:
+  DepScanCASFile(std::unique_ptr<llvm::MemoryBuffer> Buffer,
+                 cas::ObjectRef CASContents, llvm::vfs::Status Stat)
+      : Buffer(std::move(Buffer)), CASContents(CASContents),
+        Stat(std::move(Stat)) {}
+
+  static llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> create(EntryRef Entry);
+
+  llvm::ErrorOr<llvm::vfs::Status> status() override { return Stat; }
+
+  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+  getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator,
+            bool IsVolatile) override {
+    return std::move(Buffer);
   }
 
+  cas::ObjectRef getObjectRefForContent() override { return CASContents; }
+
   std::error_code close() override { return {}; }
 
 private:
   std::unique_ptr<llvm::MemoryBuffer> Buffer;
-  std::optional<cas::ObjectRef> CASContents;
+  cas::ObjectRef CASContents;
   llvm::vfs::Status Stat;
 };
 
@@ -398,14 +418,22 @@ DepScanFile::create(EntryRef Entry) {
   if (Entry.isDirectory())
     return std::make_error_code(std::errc::is_a_directory);
 
-  auto Result = std::make_unique<DepScanFile>(
+  std::unique_ptr<llvm::vfs::File> Result;
+  if (Entry.getObjectRefForContent())
+    Result = std::make_unique<DepScanCASFile>(
       llvm::MemoryBuffer::getMemBuffer(Entry.getContents(),
                                        Entry.getStatus().getName(),
                                        /*RequiresNullTerminator=*/false),
-      Entry.getObjectRefForContent(), Entry.getStatus());
+      *Entry.getObjectRefForContent(), Entry.getStatus());
+  else
+    Result = std::make_unique<DepScanFile>(
+        llvm::MemoryBuffer::getMemBuffer(Entry.getContents(),
+                                         Entry.getStatus().getName(),
+                                         /*RequiresNullTerminator=*/false),
+        Entry.getStatus());
+
 
-  return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>(
-      std::unique_ptr<llvm::vfs::File>(std::move(Result)));
+  return Result;
 }
 
 llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>