Add a VFSFromYAML class and a parser to create it

Provides a way to create a virtual file system using a YAML file that
supports mapping a file to a path on an 'external' file system. The
external file system will typically be the 'real' file system, but for
testing it can be changed.

A future patch will add a clang option to allow the user to specify such
a file and overlay it, but for now this code is only exercised by the
unit tests.

Differential Revision: http://llvm-reviews.chandlerc.com/D2835

llvm-svn: 201905

clang/lib/Basic/VirtualFileSystem.cpp

@@ -10,10 +10,14 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Basic/VirtualFileSystem.h"
+#include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/OwningPtr.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/Atomic.h"
 #include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/YAMLParser.h"
 
 using namespace clang;
 using namespace clang::vfs;
@@ -83,6 +87,7 @@ class RealFile : public File {
   RealFile(int FD) : FD(FD) {
     assert(FD >= 0 && "Invalid or inactive file descriptor");
   }
+
 public:
   ~RealFile();
   ErrorOr<Status> status() LLVM_OVERRIDE;
@@ -91,9 +96,7 @@ public:
                        bool RequiresNullTerminator = true) LLVM_OVERRIDE;
   error_code close() LLVM_OVERRIDE;
 };
-RealFile::~RealFile() {
-  close();
-}
+RealFile::~RealFile() { close(); }
 
 ErrorOr<Status> RealFile::status() {
   assert(FD != -1 && "cannot stat closed file");
@@ -191,3 +194,572 @@ error_code OverlayFileSystem::openFileForRead(const llvm::Twine &Path,
   }
   return error_code(errc::no_such_file_or_directory, system_category());
 }
+
+//===-----------------------------------------------------------------------===/
+// VFSFromYAML implementation
+//===-----------------------------------------------------------------------===/
+
+// Allow DenseMap<StringRef, ...>.  This is useful below because we know all the
+// strings are literals and will outlive the map, and there is no reason to
+// store them.
+namespace llvm {
+  template<>
+  struct DenseMapInfo<StringRef> {
+    // This assumes that "" will never be a valid key.
+    static inline StringRef getEmptyKey() { return StringRef(""); }
+    static inline StringRef getTombstoneKey() { return StringRef(); }
+    static unsigned getHashValue(StringRef Val) { return HashString(Val); }
+    static bool isEqual(StringRef LHS, StringRef RHS) { return LHS == RHS; }
+  };
+}
+
+namespace {
+
+enum EntryKind {
+  EK_Directory,
+  EK_File
+};
+
+/// \brief A single file or directory in the VFS.
+class Entry {
+  EntryKind Kind;
+  std::string Name;
+
+public:
+  virtual ~Entry();
+#if LLVM_HAS_RVALUE_REFERENCES
+  Entry(EntryKind K, std::string Name) : Kind(K), Name(std::move(Name)) {}
+#endif
+  Entry(EntryKind K, StringRef Name) : Kind(K), Name(Name) {}
+  StringRef getName() const { return Name; }
+  EntryKind getKind() const { return Kind; }
+};
+
+class DirectoryEntry : public Entry {
+  std::vector<Entry *> Contents;
+  Status S;
+
+public:
+  virtual ~DirectoryEntry();
+#if LLVM_HAS_RVALUE_REFERENCES
+  DirectoryEntry(std::string Name, std::vector<Entry *> Contents, Status S)
+      : Entry(EK_Directory, std::move(Name)), Contents(std::move(Contents)),
+        S(std::move(S)) {}
+#endif
+  DirectoryEntry(StringRef Name, ArrayRef<Entry *> Contents, const Status &S)
+      : Entry(EK_Directory, Name), Contents(Contents), S(S) {}
+  Status getStatus() { return S; }
+  typedef std::vector<Entry *>::iterator iterator;
+  iterator contents_begin() { return Contents.begin(); }
+  iterator contents_end() { return Contents.end(); }
+  static bool classof(const Entry *E) { return E->getKind() == EK_Directory; }
+};
+
+class FileEntry : public Entry {
+  std::string ExternalContentsPath;
+
+public:
+#if LLVM_HAS_RVALUE_REFERENCES
+  FileEntry(std::string Name, std::string ExternalContentsPath)
+      : Entry(EK_File, std::move(Name)),
+        ExternalContentsPath(std::move(ExternalContentsPath)) {}
+#endif
+  FileEntry(StringRef Name, StringRef ExternalContentsPath)
+      : Entry(EK_File, Name), ExternalContentsPath(ExternalContentsPath) {}
+  StringRef getExternalContentsPath() const { return ExternalContentsPath; }
+  static bool classof(const Entry *E) { return E->getKind() == EK_File; }
+};
+
+/// \brief A virtual file system parsed from a YAML file.
+///
+/// Currently, this class allows creating virtual directories and mapping
+/// virtual file paths to existing external files, available in \c ExternalFS.
+///
+/// The basic structure of the parsed file is:
+/// \verbatim
+/// {
+///   'version': <version number>,
+///   <optional configuration>
+///   'roots': [
+///              <directory entries>
+///            ]
+/// }
+/// \endverbatim
+///
+/// All configuration options are optional.
+///   'case-sensitive': <boolean, default=true>
+///
+/// Virtual directories are represented as
+/// \verbatim
+/// {
+///   'type': 'directory',
+///   'name': <string>,
+///   'contents': [ <file or directory entries> ]
+/// }
+/// \endverbatim
+///
+/// The default attributes for virtual directories are:
+/// \verbatim
+/// MTime = now() when created
+/// Perms = 0777
+/// User = Group = 0
+/// Size = 0
+/// UniqueID = unspecified unique value
+/// \endverbatim
+///
+/// Re-mapped files are represented as
+/// \verbatim
+/// {
+///   'type': 'file',
+///   'name': <string>,
+///   'external-contents': <path to external file>)
+/// }
+/// \endverbatim
+///
+/// and inherit their attributes from the external contents.
+///
+/// In both cases, the 'name' field must be a single path component (containing
+/// no separators).
+class VFSFromYAML : public vfs::FileSystem {
+  std::vector<Entry *> Roots; ///< The root(s) of the virtual file system.
+  /// \brief The file system to use for external references.
+  IntrusiveRefCntPtr<FileSystem> ExternalFS;
+
+  /// @name Configuration
+  /// @{
+
+  /// \brief Whether to perform case-sensitive comparisons.
+  ///
+  /// Currently, case-insensitive matching only works correctly with ASCII.
+  bool CaseSensitive; ///< Whether to perform case-sensitive comparisons.
+  /// @}
+
+  friend class VFSFromYAMLParser;
+
+private:
+  VFSFromYAML(IntrusiveRefCntPtr<FileSystem> ExternalFS)
+      : ExternalFS(ExternalFS), CaseSensitive(true) {}
+
+  /// \brief Looks up \p Path in \c Roots.
+  ErrorOr<Entry *> lookupPath(const Twine &Path);
+
+  /// \brief Looks up the path <tt>[Start, End)</tt> in \p From, possibly
+  /// recursing into the contents of \p From if it is a directory.
+  ErrorOr<Entry *> lookupPath(sys::path::const_iterator Start,
+                              sys::path::const_iterator End, Entry *From);
+
+public:
+  ~VFSFromYAML();
+
+  /// \brief Parses \p Buffer, which is expected to be in YAML format and
+  /// returns a virtual file system representing its contents.
+  ///
+  /// Takes ownership of \p Buffer.
+  static VFSFromYAML *create(MemoryBuffer *Buffer,
+                             SourceMgr::DiagHandlerTy DiagHandler,
+                             IntrusiveRefCntPtr<FileSystem> ExternalFS);
+
+  ErrorOr<Status> status(const Twine &Path) LLVM_OVERRIDE;
+  error_code openFileForRead(const Twine &Path,
+                             OwningPtr<File> &Result) LLVM_OVERRIDE;
+};
+
+/// \brief A helper class to hold the common YAML parsing state.
+class VFSFromYAMLParser {
+  yaml::Stream &Stream;
+
+  void error(yaml::Node *N, const Twine &Msg) {
+    Stream.printError(N, Msg);
+  }
+
+  // false on error
+  bool parseScalarString(yaml::Node *N, StringRef &Result,
+                         SmallVectorImpl<char> &Storage) {
+    yaml::ScalarNode *S = dyn_cast<yaml::ScalarNode>(N);
+    if (!S) {
+      error(N, "expected string");
+      return false;
+    }
+    Result = S->getValue(Storage);
+    return true;
+  }
+
+  // false on error
+  bool parseScalarBool(yaml::Node *N, bool &Result) {
+    SmallString<5> Storage;
+    StringRef Value;
+    if (!parseScalarString(N, Value, Storage))
+      return false;
+
+    if (Value.equals_lower("true") || Value.equals_lower("on") ||
+        Value.equals_lower("yes") || Value == "1") {
+      Result = true;
+      return true;
+    } else if (Value.equals_lower("false") || Value.equals_lower("off") ||
+               Value.equals_lower("no") || Value == "0") {
+      Result = false;
+      return true;
+    }
+
+    error(N, "expected boolean value");
+    return false;
+  }
+
+  struct KeyStatus {
+    KeyStatus(bool Required=false) : Required(Required), Seen(false) {}
+    bool Required;
+    bool Seen;
+  };
+  typedef std::pair<StringRef, KeyStatus> KeyStatusPair;
+
+  // false on error
+  bool checkDuplicateOrUnknownKey(yaml::Node *KeyNode, StringRef Key,
+                                  DenseMap<StringRef, KeyStatus> &Keys) {
+    if (!Keys.count(Key)) {
+      error(KeyNode, "unknown key");
+      return false;
+    }
+    KeyStatus &S = Keys[Key];
+    if (S.Seen) {
+      error(KeyNode, Twine("duplicate key '") + Key + "'");
+      return false;
+    }
+    S.Seen = true;
+    return true;
+  }
+
+  // false on error
+  bool checkMissingKeys(yaml::Node *Obj, DenseMap<StringRef, KeyStatus> &Keys) {
+    for (DenseMap<StringRef, KeyStatus>::iterator I = Keys.begin(),
+         E = Keys.end();
+         I != E; ++I) {
+      if (I->second.Required && !I->second.Seen) {
+        error(Obj, Twine("missing key '") + I->first + "'");
+        return false;
+      }
+    }
+    return true;
+  }
+
+  Entry *parseEntry(yaml::Node *N) {
+    yaml::MappingNode *M = dyn_cast<yaml::MappingNode>(N);
+    if (!M) {
+      error(N, "expected mapping node for file or directory entry");
+      return NULL;
+    }
+
+    KeyStatusPair Fields[] = {
+      KeyStatusPair("name", true),
+      KeyStatusPair("type", true),
+      KeyStatusPair("contents", false),
+      KeyStatusPair("external-contents", false)
+    };
+
+    DenseMap<StringRef, KeyStatus> Keys(
+        &Fields[0], Fields + sizeof(Fields)/sizeof(Fields[0]));
+
+    bool HasContents = false; // external or otherwise
+    std::vector<Entry *> EntryArrayContents;
+    std::string ExternalContentsPath;
+    std::string Name;
+    EntryKind Kind;
+
+    for (yaml::MappingNode::iterator I = M->begin(), E = M->end(); I != E;
+         ++I) {
+      StringRef Key;
+      // Reuse the buffer for key and value, since we don't look at key after
+      // parsing value.
+      SmallString<256> Buffer;
+      if (!parseScalarString(I->getKey(), Key, Buffer))
+        return NULL;
+
+      if (!checkDuplicateOrUnknownKey(I->getKey(), Key, Keys))
+        return NULL;
+
+      StringRef Value;
+      if (Key == "name") {
+        if (!parseScalarString(I->getValue(), Value, Buffer))
+          return NULL;
+        Name = Value;
+        if (sys::path::has_parent_path(Name)) {
+          error(I->getValue(), "unexpected path separator in name");
+          return NULL;
+        }
+      } else if (Key == "type") {
+        if (!parseScalarString(I->getValue(), Value, Buffer))
+          return NULL;
+        if (Value == "file")
+          Kind = EK_File;
+        else if (Value == "directory")
+          Kind = EK_Directory;
+        else {
+          error(I->getValue(), "unknown value for 'type'");
+          return NULL;
+        }
+      } else if (Key == "contents") {
+        if (HasContents) {
+          error(I->getKey(),
+                "entry already has 'contents' or 'external-contents'");
+          return NULL;
+        }
+        HasContents = true;
+        yaml::SequenceNode *Contents =
+            dyn_cast<yaml::SequenceNode>(I->getValue());
+        if (!Contents) {
+          // FIXME: this is only for directories, what about files?
+          error(I->getValue(), "expected array");
+          return NULL;
+        }
+
+        for (yaml::SequenceNode::iterator I = Contents->begin(),
+                                          E = Contents->end();
+             I != E; ++I) {
+          if (Entry *E = parseEntry(&*I))
+            EntryArrayContents.push_back(E);
+          else
+            return NULL;
+        }
+      } else if (Key == "external-contents") {
+        if (HasContents) {
+          error(I->getKey(),
+                "entry already has 'contents' or 'external-contents'");
+          return NULL;
+        }
+        HasContents = true;
+        if (!parseScalarString(I->getValue(), Value, Buffer))
+          return NULL;
+        ExternalContentsPath = Value;
+      } else {
+        llvm_unreachable("key missing from Keys");
+      }
+    }
+
+    if (Stream.failed())
+      return NULL;
+
+    // check for missing keys
+    if (!HasContents) {
+      error(N, "missing key 'contents' or 'external-contents'");
+      return NULL;
+    }
+    if (!checkMissingKeys(N, Keys))
+      return NULL;
+
+    switch (Kind) {
+    case EK_File:
+      return new FileEntry(llvm_move(Name), llvm_move(ExternalContentsPath));
+    case EK_Directory:
+      return new DirectoryEntry(
+          llvm_move(Name), llvm_move(EntryArrayContents),
+          Status("", "", getNextVirtualUniqueID(), sys::TimeValue::now(), 0, 0,
+                 0, file_type::directory_file, sys::fs::all_all));
+    }
+  }
+
+public:
+  VFSFromYAMLParser(yaml::Stream &S) : Stream(S) {}
+
+  // false on error
+  bool parse(yaml::Node *Root, VFSFromYAML *FS) {
+    yaml::MappingNode *Top = dyn_cast<yaml::MappingNode>(Root);
+    if (!Top) {
+      error(Root, "expected mapping node");
+      return false;
+    }
+
+    KeyStatusPair Fields[] = {
+      KeyStatusPair("version", true),
+      KeyStatusPair("case-sensitive", false),
+      KeyStatusPair("roots", true),
+    };
+
+    DenseMap<StringRef, KeyStatus> Keys(
+        &Fields[0], Fields + sizeof(Fields)/sizeof(Fields[0]));
+
+    // Parse configuration and 'roots'
+    for (yaml::MappingNode::iterator I = Top->begin(), E = Top->end(); I != E;
+         ++I) {
+      SmallString<10> KeyBuffer;
+      StringRef Key;
+      if (!parseScalarString(I->getKey(), Key, KeyBuffer))
+        return false;
+
+      if (!checkDuplicateOrUnknownKey(I->getKey(), Key, Keys))
+        return false;
+
+      if (Key == "roots") {
+        yaml::SequenceNode *Roots = dyn_cast<yaml::SequenceNode>(I->getValue());
+        if (!Roots) {
+          error(I->getValue(), "expected array");
+          return false;
+        }
+
+        for (yaml::SequenceNode::iterator I = Roots->begin(), E = Roots->end();
+             I != E; ++I) {
+          if (Entry *E = parseEntry(&*I))
+            FS->Roots.push_back(E);
+          else
+            return false;
+        }
+      } else if (Key == "version") {
+        StringRef VersionString;
+        SmallString<4> Storage;
+        if (!parseScalarString(I->getValue(), VersionString, Storage))
+          return false;
+        int Version;
+        if (VersionString.getAsInteger<int>(10, Version)) {
+          error(I->getValue(), "expected integer");
+          return false;
+        }
+        if (Version < 0) {
+          error(I->getValue(), "invalid version number");
+          return false;
+        }
+        if (Version != 0) {
+          error(I->getValue(), "version mismatch, expected 0");
+          return false;
+        }
+      } else if (Key == "case-sensitive") {
+        if (!parseScalarBool(I->getValue(), FS->CaseSensitive))
+          return false;
+      } else {
+        llvm_unreachable("key missing from Keys");
+      }
+    }
+
+    if (Stream.failed())
+      return false;
+
+    if (!checkMissingKeys(Top, Keys))
+      return false;
+    return true;
+  }
+};
+} // end of anonymous namespace
+
+Entry::~Entry() {}
+DirectoryEntry::~DirectoryEntry() { llvm::DeleteContainerPointers(Contents); }
+
+VFSFromYAML::~VFSFromYAML() { llvm::DeleteContainerPointers(Roots); }
+
+VFSFromYAML *VFSFromYAML::create(MemoryBuffer *Buffer,
+                                 SourceMgr::DiagHandlerTy DiagHandler,
+                                 IntrusiveRefCntPtr<FileSystem> ExternalFS) {
+
+  SourceMgr SM;
+  yaml::Stream Stream(Buffer, SM);
+
+  SM.setDiagHandler(DiagHandler);
+  yaml::document_iterator DI = Stream.begin();
+  yaml::Node *Root = DI->getRoot();
+  if (DI == Stream.end() || !Root) {
+    SM.PrintMessage(SMLoc(), SourceMgr::DK_Error, "expected root node");
+    return NULL;
+  }
+
+  VFSFromYAMLParser P(Stream);
+
+  OwningPtr<VFSFromYAML> FS(new VFSFromYAML(ExternalFS));
+  if (!P.parse(Root, FS.get()))
+    return NULL;
+
+  return FS.take();
+}
+
+ErrorOr<Entry *> VFSFromYAML::lookupPath(const Twine &Path_) {
+  SmallVector<char, 256> Storage;
+  StringRef Path = Path_.toNullTerminatedStringRef(Storage);
+
+  if (Path.empty())
+    return error_code(errc::invalid_argument, system_category());
+
+  sys::path::const_iterator Start = sys::path::begin(Path);
+  sys::path::const_iterator End = sys::path::end(Path);
+  for (std::vector<Entry *>::iterator I = Roots.begin(), E = Roots.end();
+       I != E; ++I) {
+    ErrorOr<Entry *> Result = lookupPath(Start, End, *I);
+    if (Result || Result.getError() != errc::no_such_file_or_directory)
+      return Result;
+  }
+  return error_code(errc::no_such_file_or_directory, system_category());
+}
+
+ErrorOr<Entry *> VFSFromYAML::lookupPath(sys::path::const_iterator Start,
+                                         sys::path::const_iterator End,
+                                         Entry *From) {
+  // FIXME: handle . and ..
+  if (CaseSensitive ? !Start->equals(From->getName())
+                    : !Start->equals_lower(From->getName()))
+    // failure to match
+    return error_code(errc::no_such_file_or_directory, system_category());
+
+  ++Start;
+
+  if (Start == End) {
+    // Match!
+    return From;
+  }
+
+  DirectoryEntry *DE = dyn_cast<DirectoryEntry>(From);
+  if (!DE)
+    return error_code(errc::not_a_directory, system_category());
+
+  for (DirectoryEntry::iterator I = DE->contents_begin(),
+                                E = DE->contents_end();
+       I != E; ++I) {
+    ErrorOr<Entry *> Result = lookupPath(Start, End, *I);
+    if (Result || Result.getError() != errc::no_such_file_or_directory)
+      return Result;
+  }
+  return error_code(errc::no_such_file_or_directory, system_category());
+}
+
+ErrorOr<Status> VFSFromYAML::status(const Twine &Path) {
+  ErrorOr<Entry *> Result = lookupPath(Path);
+  if (!Result)
+    return Result.getError();
+
+  std::string PathStr(Path.str());
+  if (FileEntry *F = dyn_cast<FileEntry>(*Result)) {
+    ErrorOr<Status> S = ExternalFS->status(F->getExternalContentsPath());
+    if (S) {
+      assert(S->getName() == S->getExternalName() &&
+             S->getName() == F->getExternalContentsPath());
+      S->setName(PathStr);
+    }
+    return S;
+  } else { // directory
+    DirectoryEntry *DE = cast<DirectoryEntry>(*Result);
+    Status S = DE->getStatus();
+    S.setName(PathStr);
+    S.setExternalName(PathStr);
+    return S;
+  }
+}
+
+error_code VFSFromYAML::openFileForRead(const Twine &Path,
+                                        OwningPtr<vfs::File> &Result) {
+  ErrorOr<Entry *> E = lookupPath(Path);
+  if (!E)
+    return E.getError();
+
+  FileEntry *F = dyn_cast<FileEntry>(*E);
+  if (!F) // FIXME: errc::not_a_file?
+    return error_code(errc::invalid_argument, system_category());
+
+  return ExternalFS->openFileForRead(Path, Result);
+}
+
+IntrusiveRefCntPtr<FileSystem>
+vfs::getVFSFromYAML(MemoryBuffer *Buffer, SourceMgr::DiagHandlerTy DiagHandler,
+                    IntrusiveRefCntPtr<FileSystem> ExternalFS) {
+  return VFSFromYAML::create(Buffer, DiagHandler, ExternalFS);
+}
+
+UniqueID vfs::getNextVirtualUniqueID() {
+  static volatile sys::cas_flag UID = 0;
+  sys::cas_flag ID = llvm::sys::AtomicIncrement(&UID);
+  // The following assumes that uint64_t max will never collide with a real
+  // dev_t value from the OS.
+  return UniqueID(std::numeric_limits<uint64_t>::max(), ID);
+}