[modules] Add support for #include_next.

#include_next interacts poorly with modules: it depends on where in the list of
include paths the current file was found. Files covered by module maps are not
found in include search paths when building the module (and are not found in
include search paths when @importing the module either), so this isn't really
meaningful. Instead, we fake up the result that #include_next *should* have
given: find the first path that would have resulted in the given file being
picked, and search from there onwards.

llvm-svn: 220177

clang/docs/Modules.rst

@@ -136,6 +136,19 @@ will be automatically mapped to an import of the module ``std.io``. Even with sp
 
   The automatic mapping of ``#include`` to ``import`` also solves an implementation problem: importing a module with a definition of some entity (say, a ``struct Point``) and then parsing a header containing another definition of ``struct Point`` would cause a redefinition error, even if it is the same ``struct Point``. By mapping ``#include`` to ``import``, the compiler can guarantee that it always sees just the already-parsed definition from the module.
 
+While building a module, ``#include_next`` is also supported, with one caveat.
+The usual behavior of ``#include_next`` is to search for the specified filename
+in the list of include paths, starting from the path *after* the one
+in which the current file was found.
+Because files listed in module maps are not found through include paths, a
+different strategy is used for ``#include_next`` directives in such files: the
+list of include paths is searched for the specified header name, to find the
+first include path that would refer to the current file. ``#include_next`` is
+interpreted as if the current file had been found in that path.
+If this search finds a file named by a module map, the ``#include_next``
+directive is translated into an import, just like for a ``#include``
+directive.``
+
 Module maps
 -----------
 The crucial link between modules and headers is described by a *module map*, which describes how a collection of existing headers maps on to the (logical) structure of a module. For example, one could imagine a module ``std`` covering the C standard library. Each of the C standard library headers (``<stdio.h>``, ``<stdlib.h>``, ``<math.h>``, etc.) would contribute to the ``std`` module, by placing their respective APIs into the corresponding submodule (``std.io``, ``std.lib``, ``std.math``, etc.). Having a list of the headers that are part of the ``std`` module allows the compiler to build the ``std`` module as a standalone entity, and having the mapping from header names to (sub)modules allows the automatic translation of ``#include`` directives to module imports.

clang/include/clang/Lex/Preprocessor.h

@@ -567,6 +567,9 @@ public:
   /// expansions going on at the time.
   PreprocessorLexer *getCurrentFileLexer() const;
 
+  /// \brief Return the submodule owning the file being lexed.
+  Module *getCurrentSubmodule() const { return CurSubmodule; }
+
   /// \brief Returns the FileID for the preprocessor predefines.
   FileID getPredefinesFileID() const { return PredefinesFileID; }
 
@@ -1320,6 +1323,7 @@ public:
   /// reference is for system \#include's or not (i.e. using <> instead of "").
   const FileEntry *LookupFile(SourceLocation FilenameLoc, StringRef Filename,
                               bool isAngled, const DirectoryLookup *FromDir,
+                              const FileEntry *FromFile,
                               const DirectoryLookup *&CurDir,
                               SmallVectorImpl<char> *SearchPath,
                               SmallVectorImpl<char> *RelativePath,
@@ -1534,6 +1538,7 @@ private:
   void HandleIncludeDirective(SourceLocation HashLoc,
                               Token &Tok,
                               const DirectoryLookup *LookupFrom = nullptr,
+                              const FileEntry *LookupFromFile = nullptr,
                               bool isImport = false);
   void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
   void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);

clang/lib/Frontend/VerifyDiagnosticConsumer.cpp

@@ -402,8 +402,9 @@ static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM,
 
         // Lookup file via Preprocessor, like a #include.
         const DirectoryLookup *CurDir;
-        const FileEntry *FE = PP->LookupFile(Pos, Filename, false, nullptr,
+        const FileEntry *FE =
-                                             CurDir, nullptr, nullptr, nullptr);
+            PP->LookupFile(Pos, Filename, false, nullptr, nullptr, CurDir,
+                           nullptr, nullptr, nullptr);
         if (!FE) {
           Diags.Report(Pos.getLocWithOffset(PH.C-PH.Begin),
                        diag::err_verify_missing_file) << Filename << KindStr;

clang/lib/Lex/ModuleMap.cpp

@@ -1749,12 +1749,14 @@ void ModuleMapParser::parseHeaderDecl(MMToken::TokenKind LeadingToken,
         Role = ModuleMap::PrivateHeader;
       else
         assert(LeadingToken == MMToken::HeaderKeyword);
-        
+
-      Map.addHeader(ActiveModule, File, Role);
+      // If there is a builtin counterpart to this file, add it now, before
-      
+      // the "real" header, so we build the built-in one first when building
-      // If there is a builtin counterpart to this file, add it now.
+      // the module.
       if (BuiltinFile)
         Map.addHeader(ActiveModule, BuiltinFile, Role);
+
+      Map.addHeader(ActiveModule, File, Role);
     }
   } else if (LeadingToken != MMToken::ExcludeKeyword) {
     // Ignore excluded header files. They're optional anyway.

clang/lib/Lex/PPDirectives.cpp

@@ -534,6 +534,7 @@ const FileEntry *Preprocessor::LookupFile(
     StringRef Filename,
     bool isAngled,
     const DirectoryLookup *FromDir,
+    const FileEntry *FromFile,
     const DirectoryLookup *&CurDir,
     SmallVectorImpl<char> *SearchPath,
     SmallVectorImpl<char> *RelativePath,
@@ -543,7 +544,7 @@ const FileEntry *Preprocessor::LookupFile(
   // stack, record the parent #includes.
   SmallVector<std::pair<const FileEntry *, const DirectoryEntry *>, 16>
       Includers;
-  if (!FromDir) {
+  if (!FromDir && !FromFile) {
     FileID FID = getCurrentFileLexer()->getFileID();
     const FileEntry *FileEnt = SourceMgr.getFileEntryForID(FID);
 
@@ -575,8 +576,30 @@ const FileEntry *Preprocessor::LookupFile(
     }
   }
 
-  // Do a standard file entry lookup.
   CurDir = CurDirLookup;
+
+  if (FromFile) {
+    // We're supposed to start looking from after a particular file. Search
+    // the include path until we find that file or run out of files.
+    const DirectoryLookup *TmpCurDir = CurDir;
+    const DirectoryLookup *TmpFromDir = nullptr;
+    while (const FileEntry *FE = HeaderInfo.LookupFile(
+               Filename, FilenameLoc, isAngled, TmpFromDir, TmpCurDir,
+               Includers, SearchPath, RelativePath, SuggestedModule,
+               SkipCache)) {
+      // Keep looking as if this file did a #include_next.
+      TmpFromDir = TmpCurDir;
+      ++TmpFromDir;
+      if (FE == FromFile) {
+        // Found it.
+        FromDir = TmpFromDir;
+        CurDir = TmpCurDir;
+        break;
+      }
+    }
+  }
+
+  // Do a standard file entry lookup.
   const FileEntry *FE = HeaderInfo.LookupFile(
       Filename, FilenameLoc, isAngled, FromDir, CurDir, Includers, SearchPath,
       RelativePath, SuggestedModule, SkipCache);
@@ -1353,6 +1376,7 @@ static void EnterAnnotationToken(Preprocessor &PP,
 void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc, 
                                           Token &IncludeTok,
                                           const DirectoryLookup *LookupFrom,
+                                          const FileEntry *LookupFromFile,
                                           bool isImport) {
 
   Token FilenameTok;
@@ -1450,8 +1474,8 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc,
   }
   const FileEntry *File = LookupFile(
       FilenameLoc, LangOpts.MSVCCompat ? NormalizedPath.c_str() : Filename,
-      isAngled, LookupFrom, CurDir, Callbacks ? &SearchPath : nullptr,
+      isAngled, LookupFrom, LookupFromFile, CurDir,
-      Callbacks ? &RelativePath : nullptr,
+      Callbacks ? &SearchPath : nullptr, Callbacks ? &RelativePath : nullptr,
       HeaderInfo.getHeaderSearchOpts().ModuleMaps ? &SuggestedModule : nullptr);
 
   if (Callbacks) {
@@ -1465,14 +1489,13 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc,
           HeaderInfo.AddSearchPath(DL, isAngled);
           
           // Try the lookup again, skipping the cache.
-          File = LookupFile(FilenameLoc,
+          File = LookupFile(
-                            LangOpts.MSVCCompat ? NormalizedPath.c_str()
+              FilenameLoc,
-                                                : Filename,
+              LangOpts.MSVCCompat ? NormalizedPath.c_str() : Filename, isAngled,
-                            isAngled, LookupFrom, CurDir, nullptr, nullptr,
+              LookupFrom, LookupFromFile, CurDir, nullptr, nullptr,
-                            HeaderInfo.getHeaderSearchOpts().ModuleMaps
+              HeaderInfo.getHeaderSearchOpts().ModuleMaps ? &SuggestedModule
-                                      ? &SuggestedModule
+                                                          : nullptr,
-                                      : nullptr,
+              /*SkipCache*/ true);
-                            /*SkipCache*/ true);
         }
       }
     }
@@ -1494,8 +1517,10 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc,
       // provide the user with a possible fixit.
       if (isAngled) {
         File = LookupFile(
-            FilenameLoc, LangOpts.MSVCCompat ? NormalizedPath.c_str() : Filename,
+            FilenameLoc,
-            false, LookupFrom, CurDir, Callbacks ? &SearchPath : nullptr,
+            LangOpts.MSVCCompat ? NormalizedPath.c_str() : Filename, false,
+            LookupFrom, LookupFromFile, CurDir,
+            Callbacks ? &SearchPath : nullptr,
             Callbacks ? &RelativePath : nullptr,
             HeaderInfo.getHeaderSearchOpts().ModuleMaps ? &SuggestedModule
                                                         : nullptr);
@@ -1692,9 +1717,16 @@ void Preprocessor::HandleIncludeNextDirective(SourceLocation HashLoc,
   // the current found directory.  If we can't do this, issue a
   // diagnostic.
   const DirectoryLookup *Lookup = CurDirLookup;
+  const FileEntry *LookupFromFile = nullptr;
   if (isInPrimaryFile()) {
     Lookup = nullptr;
     Diag(IncludeNextTok, diag::pp_include_next_in_primary);
+  } else if (CurSubmodule) {
+    // Start looking up in the directory *after* the one in which the current
+    // file would be found, if any.
+    assert(CurPPLexer && "#include_next directive in macro?");
+    LookupFromFile = CurPPLexer->getFileEntry();
+    Lookup = nullptr;
   } else if (!Lookup) {
     Diag(IncludeNextTok, diag::pp_include_next_absolute_path);
   } else {
@@ -1702,7 +1734,8 @@ void Preprocessor::HandleIncludeNextDirective(SourceLocation HashLoc,
     ++Lookup;
   }
 
-  return HandleIncludeDirective(HashLoc, IncludeNextTok, Lookup);
+  return HandleIncludeDirective(HashLoc, IncludeNextTok, Lookup,
+                                LookupFromFile);
 }
 
 /// HandleMicrosoftImportDirective - Implements \#import for Microsoft Mode
@@ -1728,7 +1761,7 @@ void Preprocessor::HandleImportDirective(SourceLocation HashLoc,
       return HandleMicrosoftImportDirective(ImportTok);
     Diag(ImportTok, diag::ext_pp_import_directive);
   }
-  return HandleIncludeDirective(HashLoc, ImportTok, nullptr, true);
+  return HandleIncludeDirective(HashLoc, ImportTok, nullptr, nullptr, true);
 }
 
 /// HandleIncludeMacrosDirective - The -imacros command line option turns into a
@@ -1749,7 +1782,7 @@ void Preprocessor::HandleIncludeMacrosDirective(SourceLocation HashLoc,
 
   // Treat this as a normal #include for checking purposes.  If this is
   // successful, it will push a new lexer onto the include stack.
-  HandleIncludeDirective(HashLoc, IncludeMacrosTok, nullptr, false);
+  HandleIncludeDirective(HashLoc, IncludeMacrosTok);
 
   Token TmpTok;
   do {

clang/lib/Lex/PPMacroExpansion.cpp

@@ -1053,7 +1053,8 @@ static bool HasExtension(const Preprocessor &PP, const IdentifierInfo *II) {
 /// Returns true if successful.
 static bool EvaluateHasIncludeCommon(Token &Tok,
                                      IdentifierInfo *II, Preprocessor &PP,
-                                     const DirectoryLookup *LookupFrom) {
+                                     const DirectoryLookup *LookupFrom,
+                                     const FileEntry *LookupFromFile) {
   // Save the location of the current token.  If a '(' is later found, use
   // that location.  If not, use the end of this location instead.
   SourceLocation LParenLoc = Tok.getLocation();
@@ -1148,8 +1149,8 @@ static bool EvaluateHasIncludeCommon(Token &Tok,
   // Search include directories.
   const DirectoryLookup *CurDir;
   const FileEntry *File =
-      PP.LookupFile(FilenameLoc, Filename, isAngled, LookupFrom, CurDir,
+      PP.LookupFile(FilenameLoc, Filename, isAngled, LookupFrom, LookupFromFile,
-                    nullptr, nullptr, nullptr);
+                    CurDir, nullptr, nullptr, nullptr);
 
   // Get the result value.  A result of true means the file exists.
   return File != nullptr;
@@ -1159,7 +1160,7 @@ static bool EvaluateHasIncludeCommon(Token &Tok,
 /// Returns true if successful.
 static bool EvaluateHasInclude(Token &Tok, IdentifierInfo *II,
                                Preprocessor &PP) {
-  return EvaluateHasIncludeCommon(Tok, II, PP, nullptr);
+  return EvaluateHasIncludeCommon(Tok, II, PP, nullptr, nullptr);
 }
 
 /// EvaluateHasIncludeNext - Process '__has_include_next("path")' expression.
@@ -1169,10 +1170,19 @@ static bool EvaluateHasIncludeNext(Token &Tok,
   // __has_include_next is like __has_include, except that we start
   // searching after the current found directory.  If we can't do this,
   // issue a diagnostic.
+  // FIXME: Factor out duplication wiht
+  // Preprocessor::HandleIncludeNextDirective.
   const DirectoryLookup *Lookup = PP.GetCurDirLookup();
+  const FileEntry *LookupFromFile = nullptr;
   if (PP.isInPrimaryFile()) {
     Lookup = nullptr;
     PP.Diag(Tok, diag::pp_include_next_in_primary);
+  } else if (PP.getCurrentSubmodule()) {
+    // Start looking up in the directory *after* the one in which the current
+    // file would be found, if any.
+    assert(PP.getCurrentLexer() && "#include_next directive in macro?");
+    LookupFromFile = PP.getCurrentLexer()->getFileEntry();
+    Lookup = nullptr;
   } else if (!Lookup) {
     PP.Diag(Tok, diag::pp_include_next_absolute_path);
   } else {
@@ -1180,7 +1190,7 @@ static bool EvaluateHasIncludeNext(Token &Tok,
     ++Lookup;
   }
 
-  return EvaluateHasIncludeCommon(Tok, II, PP, Lookup);
+  return EvaluateHasIncludeCommon(Tok, II, PP, Lookup, LookupFromFile);
 }
 
 /// \brief Process __building_module(identifier) expression.

clang/lib/Lex/Pragma.cpp

@@ -472,9 +472,9 @@ void Preprocessor::HandlePragmaDependency(Token &DependencyTok) {
 
   // Search include directories for this file.
   const DirectoryLookup *CurDir;
-  const FileEntry *File = LookupFile(FilenameTok.getLocation(), Filename,
+  const FileEntry *File =
-                                     isAngled, nullptr, CurDir, nullptr,
+      LookupFile(FilenameTok.getLocation(), Filename, isAngled, nullptr,
-                                     nullptr, nullptr);
+                 nullptr, CurDir, nullptr, nullptr, nullptr);
   if (!File) {
     if (!SuppressIncludeNotFoundError)
       Diag(FilenameTok, diag::err_pp_file_not_found) << Filename;

clang/test/Modules/Inputs/include_next/x/a.h

@@ -0,0 +1,2 @@
+#include_next "a.h"
+enum { ax = 1 };

clang/test/Modules/Inputs/include_next/x/module.modulemap

@@ -0,0 +1,2 @@
+module xa { header "a.h" export * }
+module xb { header "subdir/b.h" export * }

clang/test/Modules/Inputs/include_next/x/subdir/b.h

@@ -0,0 +1,2 @@
+#include_next <b.h>
+enum { bx = 3 };

clang/test/Modules/Inputs/include_next/y/a.h

@@ -0,0 +1 @@
+enum { ay = 2 };

clang/test/Modules/Inputs/include_next/y/b.h

@@ -0,0 +1 @@
+enum { by = 4 };

clang/test/Modules/Inputs/include_next/y/module.modulemap

@@ -0,0 +1,2 @@
+module ya { header "a.h" export * }
+module yb { header "b.h" export * }

clang/test/Modules/cstd.m

@@ -1,5 +1,5 @@
 // RUN: rm -rf %t
-// RUN: %clang -fsyntax-only -isystem %S/Inputs/System/usr/include -fmodules -fmodules-cache-path=%t -D__need_wint_t -Werror=implicit-function-declaration %s
+// RUN: %clang -fsyntax-only -isystem %S/Inputs/System/usr/include -ffreestanding -fmodules -fmodules-cache-path=%t -D__need_wint_t -Werror=implicit-function-declaration %s
 
 @import uses_other_constants;
 const double other_value = DBL_MAX;

clang/test/Modules/include_next.c

@@ -0,0 +1,11 @@
+// RUN: rm -rf %t
+// RUN: %clang_cc1 -I%S/Inputs/include_next/x -I%S/Inputs/include_next/y -verify %s
+// RUN: %clang_cc1 -I%S/Inputs/include_next/x -I%S/Inputs/include_next/y -verify %s -fmodules -fmodules-cache-path=%t
+
+// expected-no-diagnostics
+#include "a.h"
+#include "subdir/b.h"
+_Static_assert(ax == 1, "");
+_Static_assert(ay == 2, "");
+_Static_assert(bx == 3, "");
+_Static_assert(by == 4, "");