Add the clang library to the repo (with some of my changes, too).

24 files changed, 16950 insertions, 0 deletions

diff --git a/clang/lib/Lex/CMakeLists.txt b/clang/lib/Lex/CMakeLists.txt
new file mode 100644
index 0000000..0a2ffdb
--- /dev/null
+++ b/clang/lib/Lex/CMakeLists.txt
@@ -0,0 +1,31 @@
+# TODO: Add -maltivec when ARCH is PowerPC.
+
+set(LLVM_LINK_COMPONENTS support)
+
+set(LLVM_USED_LIBS clangBasic)
+
+add_clang_library(clangLex
+  HeaderMap.cpp
+  HeaderSearch.cpp
+  Lexer.cpp
+  LiteralSupport.cpp
+  MacroArgs.cpp
+  MacroInfo.cpp
+  ModuleMap.cpp
+  PPCaching.cpp
+  PPCallbacks.cpp
+  PPDirectives.cpp
+  PPExpressions.cpp
+  PPLexerChange.cpp
+  PPMacroExpansion.cpp
+  PTHLexer.cpp
+  Pragma.cpp
+  PreprocessingRecord.cpp
+  Preprocessor.cpp
+  PreprocessorLexer.cpp
+  ScratchBuffer.cpp
+  TokenConcatenation.cpp
+  TokenLexer.cpp
+  )
+
+add_dependencies(clangLex ClangDiagnosticLex ClangAttrSpellings)
diff --git a/clang/lib/Lex/HeaderMap.cpp b/clang/lib/Lex/HeaderMap.cpp
new file mode 100644
index 0000000..bbfc1df
--- /dev/null
+++ b/clang/lib/Lex/HeaderMap.cpp
@@ -0,0 +1,228 @@
+//===--- HeaderMap.cpp - A file that acts like dir of symlinks ------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the HeaderMap interface.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Lex/HeaderMap.h"
+#include "clang/Basic/FileManager.h"
+#include "llvm/ADT/OwningPtr.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/DataTypes.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include <cctype>
+#include <cstdio>
+using namespace clang;
+
+//===----------------------------------------------------------------------===//
+// Data Structures and Manifest Constants
+//===----------------------------------------------------------------------===//
+
+enum {
+  HMAP_HeaderMagicNumber = ('h' << 24) | ('m' << 16) | ('a' << 8) | 'p',
+  HMAP_HeaderVersion = 1,
+
+  HMAP_EmptyBucketKey = 0
+};
+
+namespace clang {
+struct HMapBucket {
+  uint32_t Key;          // Offset (into strings) of key.
+
+  uint32_t Prefix;     // Offset (into strings) of value prefix.
+  uint32_t Suffix;     // Offset (into strings) of value suffix.
+};
+
+struct HMapHeader {
+  uint32_t Magic;           // Magic word, also indicates byte order.
+  uint16_t Version;         // Version number -- currently 1.
+  uint16_t Reserved;        // Reserved for future use - zero for now.
+  uint32_t StringsOffset;   // Offset to start of string pool.
+  uint32_t NumEntries;      // Number of entries in the string table.
+  uint32_t NumBuckets;      // Number of buckets (always a power of 2).
+  uint32_t MaxValueLength;  // Length of longest result path (excluding nul).
+  // An array of 'NumBuckets' HMapBucket objects follows this header.
+  // Strings follow the buckets, at StringsOffset.
+};
+} // end namespace clang.
+
+/// HashHMapKey - This is the 'well known' hash function required by the file
+/// format, used to look up keys in the hash table.  The hash table uses simple
+/// linear probing based on this function.
+static inline unsigned HashHMapKey(StringRef Str) {
+  unsigned Result = 0;
+  const char *S = Str.begin(), *End = Str.end();
+
+  for (; S != End; S++)
+    Result += tolower(*S) * 13;
+  return Result;
+}
+
+
+
+//===----------------------------------------------------------------------===//
+// Verification and Construction
+//===----------------------------------------------------------------------===//
+
+/// HeaderMap::Create - This attempts to load the specified file as a header
+/// map.  If it doesn't look like a HeaderMap, it gives up and returns null.
+/// If it looks like a HeaderMap but is obviously corrupted, it puts a reason
+/// into the string error argument and returns null.
+const HeaderMap *HeaderMap::Create(const FileEntry *FE, FileManager &FM) {
+  // If the file is too small to be a header map, ignore it.
+  unsigned FileSize = FE->getSize();
+  if (FileSize <= sizeof(HMapHeader)) return 0;
+
+  OwningPtr<const llvm::MemoryBuffer> FileBuffer(FM.getBufferForFile(FE));
+  if (FileBuffer == 0) return 0;  // Unreadable file?
+  const char *FileStart = FileBuffer->getBufferStart();
+
+  // We know the file is at least as big as the header, check it now.
+  const HMapHeader *Header = reinterpret_cast<const HMapHeader*>(FileStart);
+
+  // Sniff it to see if it's a headermap by checking the magic number and
+  // version.
+  bool NeedsByteSwap;
+  if (Header->Magic == HMAP_HeaderMagicNumber &&
+      Header->Version == HMAP_HeaderVersion)
+    NeedsByteSwap = false;
+  else if (Header->Magic == llvm::ByteSwap_32(HMAP_HeaderMagicNumber) &&
+           Header->Version == llvm::ByteSwap_16(HMAP_HeaderVersion))
+    NeedsByteSwap = true;  // Mixed endianness headermap.
+  else
+    return 0;  // Not a header map.
+
+  if (Header->Reserved != 0) return 0;
+
+  // Okay, everything looks good, create the header map.
+  return new HeaderMap(FileBuffer.take(), NeedsByteSwap);
+}
+
+HeaderMap::~HeaderMap() {
+  delete FileBuffer;
+}
+
+//===----------------------------------------------------------------------===//
+//  Utility Methods
+//===----------------------------------------------------------------------===//
+
+
+/// getFileName - Return the filename of the headermap.
+const char *HeaderMap::getFileName() const {
+  return FileBuffer->getBufferIdentifier();
+}
+
+unsigned HeaderMap::getEndianAdjustedWord(unsigned X) const {
+  if (!NeedsBSwap) return X;
+  return llvm::ByteSwap_32(X);
+}
+
+/// getHeader - Return a reference to the file header, in unbyte-swapped form.
+/// This method cannot fail.
+const HMapHeader &HeaderMap::getHeader() const {
+  // We know the file is at least as big as the header.  Return it.
+  return *reinterpret_cast<const HMapHeader*>(FileBuffer->getBufferStart());
+}
+
+/// getBucket - Return the specified hash table bucket from the header map,
+/// bswap'ing its fields as appropriate.  If the bucket number is not valid,
+/// this return a bucket with an empty key (0).
+HMapBucket HeaderMap::getBucket(unsigned BucketNo) const {
+  HMapBucket Result;
+  Result.Key = HMAP_EmptyBucketKey;
+
+  const HMapBucket *BucketArray =
+    reinterpret_cast<const HMapBucket*>(FileBuffer->getBufferStart() +
+                                        sizeof(HMapHeader));
+
+  const HMapBucket *BucketPtr = BucketArray+BucketNo;
+  if ((char*)(BucketPtr+1) > FileBuffer->getBufferEnd()) {
+    Result.Prefix = 0;
+    Result.Suffix = 0;
+    return Result;  // Invalid buffer, corrupt hmap.
+  }
+
+  // Otherwise, the bucket is valid.  Load the values, bswapping as needed.
+  Result.Key    = getEndianAdjustedWord(BucketPtr->Key);
+  Result.Prefix = getEndianAdjustedWord(BucketPtr->Prefix);
+  Result.Suffix = getEndianAdjustedWord(BucketPtr->Suffix);
+  return Result;
+}
+
+/// getString - Look up the specified string in the string table.  If the string
+/// index is not valid, it returns an empty string.
+const char *HeaderMap::getString(unsigned StrTabIdx) const {
+  // Add the start of the string table to the idx.
+  StrTabIdx += getEndianAdjustedWord(getHeader().StringsOffset);
+
+  // Check for invalid index.
+  if (StrTabIdx >= FileBuffer->getBufferSize())
+    return 0;
+
+  // Otherwise, we have a valid pointer into the file.  Just return it.  We know
+  // that the "string" can not overrun the end of the file, because the buffer
+  // is nul terminated by virtue of being a MemoryBuffer.
+  return FileBuffer->getBufferStart()+StrTabIdx;
+}
+
+//===----------------------------------------------------------------------===//
+// The Main Drivers
+//===----------------------------------------------------------------------===//
+
+/// dump - Print the contents of this headermap to stderr.
+void HeaderMap::dump() const {
+  const HMapHeader &Hdr = getHeader();
+  unsigned NumBuckets = getEndianAdjustedWord(Hdr.NumBuckets);
+
+  fprintf(stderr, "Header Map %s:\n  %d buckets, %d entries\n",
+          getFileName(), NumBuckets,
+          getEndianAdjustedWord(Hdr.NumEntries));
+
+  for (unsigned i = 0; i != NumBuckets; ++i) {
+    HMapBucket B = getBucket(i);
+    if (B.Key == HMAP_EmptyBucketKey) continue;
+
+    const char *Key    = getString(B.Key);
+    const char *Prefix = getString(B.Prefix);
+    const char *Suffix = getString(B.Suffix);
+    fprintf(stderr, "  %d. %s -> '%s' '%s'\n", i, Key, Prefix, Suffix);
+  }
+}
+
+/// LookupFile - Check to see if the specified relative filename is located in
+/// this HeaderMap.  If so, open it and return its FileEntry.
+const FileEntry *HeaderMap::LookupFile(
+    StringRef Filename, FileManager &FM) const {
+  const HMapHeader &Hdr = getHeader();
+  unsigned NumBuckets = getEndianAdjustedWord(Hdr.NumBuckets);
+
+  // If the number of buckets is not a power of two, the headermap is corrupt.
+  // Don't probe infinitely.
+  if (NumBuckets & (NumBuckets-1))
+    return 0;
+
+  // Linearly probe the hash table.
+  for (unsigned Bucket = HashHMapKey(Filename);; ++Bucket) {
+    HMapBucket B = getBucket(Bucket & (NumBuckets-1));
+    if (B.Key == HMAP_EmptyBucketKey) return 0; // Hash miss.
+
+    // See if the key matches.  If not, probe on.
+    if (!Filename.equals_lower(getString(B.Key)))
+      continue;
+
+    // If so, we have a match in the hash table.  Construct the destination
+    // path.
+    SmallString<1024> DestPath;
+    DestPath += getString(B.Prefix);
+    DestPath += getString(B.Suffix);
+    return FM.getFile(DestPath.str());
+  }
+}
diff --git a/clang/lib/Lex/HeaderSearch.cpp b/clang/lib/Lex/HeaderSearch.cpp
new file mode 100644
index 0000000..d688e23
--- /dev/null
+++ b/clang/lib/Lex/HeaderSearch.cpp
@@ -0,0 +1,1035 @@
+//===--- HeaderSearch.cpp - Resolve Header File Locations ---===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file implements the DirectoryLookup and HeaderSearch interfaces.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Lex/HeaderSearch.h"
+#include "clang/Lex/HeaderMap.h"
+#include "clang/Lex/Lexer.h"
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/IdentifierTable.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/Capacity.h"
+#include <cstdio>
+using namespace clang;
+
+const IdentifierInfo *
+HeaderFileInfo::getControllingMacro(ExternalIdentifierLookup *External) {
+  if (ControllingMacro)
+    return ControllingMacro;
+
+  if (!ControllingMacroID || !External)
+    return 0;
+
+  ControllingMacro = External->GetIdentifier(ControllingMacroID);
+  return ControllingMacro;
+}
+
+ExternalHeaderFileInfoSource::~ExternalHeaderFileInfoSource() {}
+
+HeaderSearch::HeaderSearch(FileManager &FM, DiagnosticsEngine &Diags,
+                           const LangOptions &LangOpts, 
+                           const TargetInfo *Target)
+  : FileMgr(FM), Diags(Diags), FrameworkMap(64), 
+    ModMap(FileMgr, *Diags.getClient(), LangOpts, Target)
+{
+  AngledDirIdx = 0;
+  SystemDirIdx = 0;
+  NoCurDirSearch = false;
+
+  ExternalLookup = 0;
+  ExternalSource = 0;
+  NumIncluded = 0;
+  NumMultiIncludeFileOptzn = 0;
+  NumFrameworkLookups = NumSubFrameworkLookups = 0;
+}
+
+HeaderSearch::~HeaderSearch() {
+  // Delete headermaps.
+  for (unsigned i = 0, e = HeaderMaps.size(); i != e; ++i)
+    delete HeaderMaps[i].second;
+}
+
+void HeaderSearch::PrintStats() {
+  fprintf(stderr, "\n*** HeaderSearch Stats:\n");
+  fprintf(stderr, "%d files tracked.\n", (int)FileInfo.size());
+  unsigned NumOnceOnlyFiles = 0, MaxNumIncludes = 0, NumSingleIncludedFiles = 0;
+  for (unsigned i = 0, e = FileInfo.size(); i != e; ++i) {
+    NumOnceOnlyFiles += FileInfo[i].isImport;
+    if (MaxNumIncludes < FileInfo[i].NumIncludes)
+      MaxNumIncludes = FileInfo[i].NumIncludes;
+    NumSingleIncludedFiles += FileInfo[i].NumIncludes == 1;
+  }
+  fprintf(stderr, "  %d #import/#pragma once files.\n", NumOnceOnlyFiles);
+  fprintf(stderr, "  %d included exactly once.\n", NumSingleIncludedFiles);
+  fprintf(stderr, "  %d max times a file is included.\n", MaxNumIncludes);
+
+  fprintf(stderr, "  %d #include/#include_next/#import.\n", NumIncluded);
+  fprintf(stderr, "    %d #includes skipped due to"
+          " the multi-include optimization.\n", NumMultiIncludeFileOptzn);
+
+  fprintf(stderr, "%d framework lookups.\n", NumFrameworkLookups);
+  fprintf(stderr, "%d subframework lookups.\n", NumSubFrameworkLookups);
+}
+
+/// CreateHeaderMap - This method returns a HeaderMap for the specified
+/// FileEntry, uniquing them through the the 'HeaderMaps' datastructure.
+const HeaderMap *HeaderSearch::CreateHeaderMap(const FileEntry *FE) {
+  // We expect the number of headermaps to be small, and almost always empty.
+  // If it ever grows, use of a linear search should be re-evaluated.
+  if (!HeaderMaps.empty()) {
+    for (unsigned i = 0, e = HeaderMaps.size(); i != e; ++i)
+      // Pointer equality comparison of FileEntries works because they are
+      // already uniqued by inode.
+      if (HeaderMaps[i].first == FE)
+        return HeaderMaps[i].second;
+  }
+
+  if (const HeaderMap *HM = HeaderMap::Create(FE, FileMgr)) {
+    HeaderMaps.push_back(std::make_pair(FE, HM));
+    return HM;
+  }
+
+  return 0;
+}
+
+std::string HeaderSearch::getModuleFileName(Module *Module) {
+  // If we don't have a module cache path, we can't do anything.
+  if (ModuleCachePath.empty()) 
+    return std::string();
+
+
+  SmallString<256> Result(ModuleCachePath);
+  llvm::sys::path::append(Result, Module->getTopLevelModule()->Name + ".pcm");
+  return Result.str().str();
+}
+
+std::string HeaderSearch::getModuleFileName(StringRef ModuleName) {
+  // If we don't have a module cache path, we can't do anything.
+  if (ModuleCachePath.empty()) 
+    return std::string();
+  
+  
+  SmallString<256> Result(ModuleCachePath);
+  llvm::sys::path::append(Result, ModuleName + ".pcm");
+  return Result.str().str();
+}
+
+Module *HeaderSearch::lookupModule(StringRef ModuleName, bool AllowSearch) {
+  // Look in the module map to determine if there is a module by this name.
+  Module *Module = ModMap.findModule(ModuleName);
+  if (Module || !AllowSearch)
+    return Module;
+  
+  // Look through the various header search paths to load any avai;able module 
+  // maps, searching for a module map that describes this module.
+  for (unsigned Idx = 0, N = SearchDirs.size(); Idx != N; ++Idx) {
+    if (SearchDirs[Idx].isFramework()) {
+      // Search for or infer a module map for a framework.
+      SmallString<128> FrameworkDirName;
+      FrameworkDirName += SearchDirs[Idx].getFrameworkDir()->getName();
+      llvm::sys::path::append(FrameworkDirName, ModuleName + ".framework");
+      if (const DirectoryEntry *FrameworkDir 
+            = FileMgr.getDirectory(FrameworkDirName)) {
+        bool IsSystem
+          = SearchDirs[Idx].getDirCharacteristic() != SrcMgr::C_User;
+        Module = loadFrameworkModule(ModuleName, FrameworkDir, IsSystem);
+        if (Module)
+          break;
+      }
+    }
+    
+    // FIXME: Figure out how header maps and module maps will work together.
+    
+    // Only deal with normal search directories.
+    if (!SearchDirs[Idx].isNormalDir())
+      continue;
+    
+    // Search for a module map file in this directory.
+    if (loadModuleMapFile(SearchDirs[Idx].getDir()) == LMM_NewlyLoaded) {
+      // We just loaded a module map file; check whether the module is
+      // available now.
+      Module = ModMap.findModule(ModuleName);
+      if (Module)
+        break;
+    }
+              
+    // Search for a module map in a subdirectory with the same name as the
+    // module.
+    SmallString<128> NestedModuleMapDirName;
+    NestedModuleMapDirName = SearchDirs[Idx].getDir()->getName();
+    llvm::sys::path::append(NestedModuleMapDirName, ModuleName);
+    if (loadModuleMapFile(NestedModuleMapDirName) == LMM_NewlyLoaded) {
+      // If we just loaded a module map file, look for the module again.
+      Module = ModMap.findModule(ModuleName);
+      if (Module)
+        break;
+    }
+  }
+  
+  return Module;
+}
+
+//===----------------------------------------------------------------------===//
+// File lookup within a DirectoryLookup scope
+//===----------------------------------------------------------------------===//
+
+/// getName - Return the directory or filename corresponding to this lookup
+/// object.
+const char *DirectoryLookup::getName() const {
+  if (isNormalDir())
+    return getDir()->getName();
+  if (isFramework())
+    return getFrameworkDir()->getName();
+  assert(isHeaderMap() && "Unknown DirectoryLookup");
+  return getHeaderMap()->getFileName();
+}
+
+
+/// LookupFile - Lookup the specified file in this search path, returning it
+/// if it exists or returning null if not.
+const FileEntry *DirectoryLookup::LookupFile(
+    StringRef Filename,
+    HeaderSearch &HS,
+    SmallVectorImpl<char> *SearchPath,
+    SmallVectorImpl<char> *RelativePath,
+    Module **SuggestedModule,
+    bool &InUserSpecifiedSystemFramework) const {
+  InUserSpecifiedSystemFramework = false;
+
+  SmallString<1024> TmpDir;
+  if (isNormalDir()) {
+    // Concatenate the requested file onto the directory.
+    TmpDir = getDir()->getName();
+    llvm::sys::path::append(TmpDir, Filename);
+    if (SearchPath != NULL) {
+      StringRef SearchPathRef(getDir()->getName());
+      SearchPath->clear();
+      SearchPath->append(SearchPathRef.begin(), SearchPathRef.end());
+    }
+    if (RelativePath != NULL) {
+      RelativePath->clear();
+      RelativePath->append(Filename.begin(), Filename.end());
+    }
+    
+    // If we have a module map that might map this header, load it and
+    // check whether we'll have a suggestion for a module.
+    if (SuggestedModule && HS.hasModuleMap(TmpDir, getDir())) {
+      const FileEntry *File = HS.getFileMgr().getFile(TmpDir.str(), 
+                                                      /*openFile=*/false);
+      if (!File)
+        return File;
+      
+      // If there is a module that corresponds to this header, 
+      // suggest it.
+      *SuggestedModule = HS.findModuleForHeader(File);
+      return File;
+    }
+    
+    return HS.getFileMgr().getFile(TmpDir.str(), /*openFile=*/true);
+  }
+
+  if (isFramework())
+    return DoFrameworkLookup(Filename, HS, SearchPath, RelativePath,
+                             SuggestedModule, InUserSpecifiedSystemFramework);
+
+  assert(isHeaderMap() && "Unknown directory lookup");
+  const FileEntry * const Result = getHeaderMap()->LookupFile(
+      Filename, HS.getFileMgr());
+  if (Result) {
+    if (SearchPath != NULL) {
+      StringRef SearchPathRef(getName());
+      SearchPath->clear();
+      SearchPath->append(SearchPathRef.begin(), SearchPathRef.end());
+    }
+    if (RelativePath != NULL) {
+      RelativePath->clear();
+      RelativePath->append(Filename.begin(), Filename.end());
+    }
+  }
+  return Result;
+}
+
+
+/// DoFrameworkLookup - Do a lookup of the specified file in the current
+/// DirectoryLookup, which is a framework directory.
+const FileEntry *DirectoryLookup::DoFrameworkLookup(
+    StringRef Filename,
+    HeaderSearch &HS,
+    SmallVectorImpl<char> *SearchPath,
+    SmallVectorImpl<char> *RelativePath,
+    Module **SuggestedModule,
+    bool &InUserSpecifiedSystemFramework) const
+{
+  FileManager &FileMgr = HS.getFileMgr();
+
+  // Framework names must have a '/' in the filename.
+  size_t SlashPos = Filename.find('/');
+  if (SlashPos == StringRef::npos) return 0;
+
+  // Find out if this is the home for the specified framework, by checking
+  // HeaderSearch.  Possible answers are yes/no and unknown.
+  HeaderSearch::FrameworkCacheEntry &CacheEntry =
+    HS.LookupFrameworkCache(Filename.substr(0, SlashPos));
+
+  // If it is known and in some other directory, fail.
+  if (CacheEntry.Directory && CacheEntry.Directory != getFrameworkDir())
+    return 0;
+
+  // Otherwise, construct the path to this framework dir.
+
+  // FrameworkName = "/System/Library/Frameworks/"
+  SmallString<1024> FrameworkName;
+  FrameworkName += getFrameworkDir()->getName();
+  if (FrameworkName.empty() || FrameworkName.back() != '/')
+    FrameworkName.push_back('/');
+
+  // FrameworkName = "/System/Library/Frameworks/Cocoa"
+  StringRef ModuleName(Filename.begin(), SlashPos);
+  FrameworkName += ModuleName;
+
+  // FrameworkName = "/System/Library/Frameworks/Cocoa.framework/"
+  FrameworkName += ".framework/";
+
+  // If the cache entry was unresolved, populate it now.
+  if (CacheEntry.Directory == 0) {
+    HS.IncrementFrameworkLookupCount();
+
+    // If the framework dir doesn't exist, we fail.
+    const DirectoryEntry *Dir = FileMgr.getDirectory(FrameworkName.str());
+    if (Dir == 0) return 0;
+
+    // Otherwise, if it does, remember that this is the right direntry for this
+    // framework.
+    CacheEntry.Directory = getFrameworkDir();
+
+    // If this is a user search directory, check if the framework has been
+    // user-specified as a system framework.
+    if (getDirCharacteristic() == SrcMgr::C_User) {
+      SmallString<1024> SystemFrameworkMarker(FrameworkName);
+      SystemFrameworkMarker += ".system_framework";
+      if (llvm::sys::fs::exists(SystemFrameworkMarker.str())) {
+        CacheEntry.IsUserSpecifiedSystemFramework = true;
+      }
+    }
+  }
+
+  // Set the 'user-specified system framework' flag.
+  InUserSpecifiedSystemFramework = CacheEntry.IsUserSpecifiedSystemFramework;
+
+  if (RelativePath != NULL) {
+    RelativePath->clear();
+    RelativePath->append(Filename.begin()+SlashPos+1, Filename.end());
+  }
+
+  // If we're allowed to look for modules, try to load or create the module
+  // corresponding to this framework.
+  Module *Module = 0;
+  if (SuggestedModule) {
+    if (const DirectoryEntry *FrameworkDir
+                                        = FileMgr.getDirectory(FrameworkName)) {
+      bool IsSystem = getDirCharacteristic() != SrcMgr::C_User;
+      Module = HS.loadFrameworkModule(ModuleName, FrameworkDir, IsSystem);
+    }
+  }
+  
+  // Check "/System/Library/Frameworks/Cocoa.framework/Headers/file.h"
+  unsigned OrigSize = FrameworkName.size();
+
+  FrameworkName += "Headers/";
+
+  if (SearchPath != NULL) {
+    SearchPath->clear();
+    // Without trailing '/'.
+    SearchPath->append(FrameworkName.begin(), FrameworkName.end()-1);
+  }
+
+  // Determine whether this is the module we're building or not.
+  bool AutomaticImport = Module;  
+  FrameworkName.append(Filename.begin()+SlashPos+1, Filename.end());
+  if (const FileEntry *FE = FileMgr.getFile(FrameworkName.str(),
+                                            /*openFile=*/!AutomaticImport)) {
+    if (AutomaticImport)
+      *SuggestedModule = HS.findModuleForHeader(FE);
+    return FE;
+  }
+
+  // Check "/System/Library/Frameworks/Cocoa.framework/PrivateHeaders/file.h"
+  const char *Private = "Private";
+  FrameworkName.insert(FrameworkName.begin()+OrigSize, Private,
+                       Private+strlen(Private));
+  if (SearchPath != NULL)
+    SearchPath->insert(SearchPath->begin()+OrigSize, Private,
+                       Private+strlen(Private));
+
+  const FileEntry *FE = FileMgr.getFile(FrameworkName.str(), 
+                                        /*openFile=*/!AutomaticImport);
+  if (FE && AutomaticImport)
+    *SuggestedModule = HS.findModuleForHeader(FE);
+  return FE;
+}
+
+void HeaderSearch::setTarget(const TargetInfo &Target) {
+  ModMap.setTarget(Target);
+}
+
+
+//===----------------------------------------------------------------------===//
+// Header File Location.
+//===----------------------------------------------------------------------===//
+
+
+/// LookupFile - Given a "foo" or <foo> reference, look up the indicated file,
+/// return null on failure.  isAngled indicates whether the file reference is
+/// for system #include's or not (i.e. using <> instead of "").  CurFileEnt, if
+/// non-null, indicates where the #including file is, in case a relative search
+/// is needed.
+const FileEntry *HeaderSearch::LookupFile(
+    StringRef Filename,
+    bool isAngled,
+    const DirectoryLookup *FromDir,
+    const DirectoryLookup *&CurDir,
+    const FileEntry *CurFileEnt,
+    SmallVectorImpl<char> *SearchPath,
+    SmallVectorImpl<char> *RelativePath,
+    Module **SuggestedModule,
+    bool SkipCache)
+{
+  if (SuggestedModule)
+    *SuggestedModule = 0;
+    
+  // If 'Filename' is absolute, check to see if it exists and no searching.
+  if (llvm::sys::path::is_absolute(Filename)) {
+    CurDir = 0;
+
+    // If this was an #include_next "/absolute/file", fail.
+    if (FromDir) return 0;
+
+    if (SearchPath != NULL)
+      SearchPath->clear();
+    if (RelativePath != NULL) {
+      RelativePath->clear();
+      RelativePath->append(Filename.begin(), Filename.end());
+    }
+    // Otherwise, just return the file.
+    return FileMgr.getFile(Filename, /*openFile=*/true);
+  }
+
+  // Unless disabled, check to see if the file is in the #includer's
+  // directory.  This has to be based on CurFileEnt, not CurDir, because
+  // CurFileEnt could be a #include of a subdirectory (#include "foo/bar.h") and
+  // a subsequent include of "baz.h" should resolve to "whatever/foo/baz.h".
+  // This search is not done for <> headers.
+  if (CurFileEnt && !isAngled && !NoCurDirSearch) {
+    SmallString<1024> TmpDir;
+    // Concatenate the requested file onto the directory.
+    // FIXME: Portability.  Filename concatenation should be in sys::Path.
+    TmpDir += CurFileEnt->getDir()->getName();
+    TmpDir.push_back('/');
+    TmpDir.append(Filename.begin(), Filename.end());
+    if (const FileEntry *FE = FileMgr.getFile(TmpDir.str(),/*openFile=*/true)) {
+      // Leave CurDir unset.
+      // This file is a system header or C++ unfriendly if the old file is.
+      //
+      // Note that the temporary 'DirInfo' is required here, as either call to
+      // getFileInfo could resize the vector and we don't want to rely on order
+      // of evaluation.
+      unsigned DirInfo = getFileInfo(CurFileEnt).DirInfo;
+      getFileInfo(FE).DirInfo = DirInfo;
+      if (SearchPath != NULL) {
+        StringRef SearchPathRef(CurFileEnt->getDir()->getName());
+        SearchPath->clear();
+        SearchPath->append(SearchPathRef.begin(), SearchPathRef.end());
+      }
+      if (RelativePath != NULL) {
+        RelativePath->clear();
+        RelativePath->append(Filename.begin(), Filename.end());
+      }
+      return FE;
+    }
+  }
+
+  CurDir = 0;
+
+  // If this is a system #include, ignore the user #include locs.
+  unsigned i = isAngled ? AngledDirIdx : 0;
+
+  // If this is a #include_next request, start searching after the directory the
+  // file was found in.
+  if (FromDir)
+    i = FromDir-&SearchDirs[0];
+
+  // Cache all of the lookups performed by this method.  Many headers are
+  // multiply included, and the "pragma once" optimization prevents them from
+  // being relex/pp'd, but they would still have to search through a
+  // (potentially huge) series of SearchDirs to find it.
+  std::pair<unsigned, unsigned> &CacheLookup =
+    LookupFileCache.GetOrCreateValue(Filename).getValue();
+
+  // If the entry has been previously looked up, the first value will be
+  // non-zero.  If the value is equal to i (the start point of our search), then
+  // this is a matching hit.
+  if (!SkipCache && CacheLookup.first == i+1) {
+    // Skip querying potentially lots of directories for this lookup.
+    i = CacheLookup.second;
+  } else {
+    // Otherwise, this is the first query, or the previous query didn't match
+    // our search start.  We will fill in our found location below, so prime the
+    // start point value.
+    CacheLookup.first = i+1;
+  }
+
+  // Check each directory in sequence to see if it contains this file.
+  for (; i != SearchDirs.size(); ++i) {
+    bool InUserSpecifiedSystemFramework = false;
+    const FileEntry *FE =
+      SearchDirs[i].LookupFile(Filename, *this, SearchPath, RelativePath,
+                               SuggestedModule, InUserSpecifiedSystemFramework);
+    if (!FE) continue;
+
+    CurDir = &SearchDirs[i];
+
+    // This file is a system header or C++ unfriendly if the dir is.
+    HeaderFileInfo &HFI = getFileInfo(FE);
+    HFI.DirInfo = CurDir->getDirCharacteristic();
+
+    // If the directory characteristic is User but this framework was
+    // user-specified to be treated as a system framework, promote the
+    // characteristic.
+    if (HFI.DirInfo == SrcMgr::C_User && InUserSpecifiedSystemFramework)
+      HFI.DirInfo = SrcMgr::C_System;
+
+    // If this file is found in a header map and uses the framework style of
+    // includes, then this header is part of a framework we're building.
+    if (CurDir->isIndexHeaderMap()) {
+      size_t SlashPos = Filename.find('/');
+      if (SlashPos != StringRef::npos) {
+        HFI.IndexHeaderMapHeader = 1;
+        HFI.Framework = getUniqueFrameworkName(StringRef(Filename.begin(), 
+                                                         SlashPos));
+      }
+    }
+    
+    // Remember this location for the next lookup we do.
+    CacheLookup.second = i;
+    return FE;
+  }
+
+  // If we are including a file with a quoted include "foo.h" from inside
+  // a header in a framework that is currently being built, and we couldn't
+  // resolve "foo.h" any other way, change the include to <Foo/foo.h>, where
+  // "Foo" is the name of the framework in which the including header was found.
+  if (CurFileEnt && !isAngled && Filename.find('/') == StringRef::npos) {
+    HeaderFileInfo &IncludingHFI = getFileInfo(CurFileEnt);
+    if (IncludingHFI.IndexHeaderMapHeader) {
+      SmallString<128> ScratchFilename;
+      ScratchFilename += IncludingHFI.Framework;
+      ScratchFilename += '/';
+      ScratchFilename += Filename;
+      
+      const FileEntry *Result = LookupFile(ScratchFilename, /*isAngled=*/true,
+                                           FromDir, CurDir, CurFileEnt, 
+                                           SearchPath, RelativePath,
+                                           SuggestedModule);
+      std::pair<unsigned, unsigned> &CacheLookup 
+        = LookupFileCache.GetOrCreateValue(Filename).getValue();
+      CacheLookup.second
+        = LookupFileCache.GetOrCreateValue(ScratchFilename).getValue().second;
+      return Result;
+    }
+  }
+
+  // Otherwise, didn't find it. Remember we didn't find this.
+  CacheLookup.second = SearchDirs.size();
+  return 0;
+}
+
+/// LookupSubframeworkHeader - Look up a subframework for the specified
+/// #include file.  For example, if #include'ing <HIToolbox/HIToolbox.h> from
+/// within ".../Carbon.framework/Headers/Carbon.h", check to see if HIToolbox
+/// is a subframework within Carbon.framework.  If so, return the FileEntry
+/// for the designated file, otherwise return null.
+const FileEntry *HeaderSearch::
+LookupSubframeworkHeader(StringRef Filename,
+                         const FileEntry *ContextFileEnt,
+                         SmallVectorImpl<char> *SearchPath,
+                         SmallVectorImpl<char> *RelativePath) {
+  assert(ContextFileEnt && "No context file?");
+
+  // Framework names must have a '/' in the filename.  Find it.
+  // FIXME: Should we permit '\' on Windows?
+  size_t SlashPos = Filename.find('/');
+  if (SlashPos == StringRef::npos) return 0;
+
+  // Look up the base framework name of the ContextFileEnt.
+  const char *ContextName = ContextFileEnt->getName();
+
+  // If the context info wasn't a framework, couldn't be a subframework.
+  const unsigned DotFrameworkLen = 10;
+  const char *FrameworkPos = strstr(ContextName, ".framework");
+  if (FrameworkPos == 0 || 
+      (FrameworkPos[DotFrameworkLen] != '/' && 
+       FrameworkPos[DotFrameworkLen] != '\\'))
+    return 0;
+
+  SmallString<1024> FrameworkName(ContextName, FrameworkPos+DotFrameworkLen+1);
+
+  // Append Frameworks/HIToolbox.framework/
+  FrameworkName += "Frameworks/";
+  FrameworkName.append(Filename.begin(), Filename.begin()+SlashPos);
+  FrameworkName += ".framework/";
+
+  llvm::StringMapEntry<FrameworkCacheEntry> &CacheLookup =
+    FrameworkMap.GetOrCreateValue(Filename.substr(0, SlashPos));
+
+  // Some other location?
+  if (CacheLookup.getValue().Directory &&
+      CacheLookup.getKeyLength() == FrameworkName.size() &&
+      memcmp(CacheLookup.getKeyData(), &FrameworkName[0],
+             CacheLookup.getKeyLength()) != 0)
+    return 0;
+
+  // Cache subframework.
+  if (CacheLookup.getValue().Directory == 0) {
+    ++NumSubFrameworkLookups;
+
+    // If the framework dir doesn't exist, we fail.
+    const DirectoryEntry *Dir = FileMgr.getDirectory(FrameworkName.str());
+    if (Dir == 0) return 0;
+
+    // Otherwise, if it does, remember that this is the right direntry for this
+    // framework.
+    CacheLookup.getValue().Directory = Dir;
+  }
+
+  const FileEntry *FE = 0;
+
+  if (RelativePath != NULL) {
+    RelativePath->clear();
+    RelativePath->append(Filename.begin()+SlashPos+1, Filename.end());
+  }
+
+  // Check ".../Frameworks/HIToolbox.framework/Headers/HIToolbox.h"
+  SmallString<1024> HeadersFilename(FrameworkName);
+  HeadersFilename += "Headers/";
+  if (SearchPath != NULL) {
+    SearchPath->clear();
+    // Without trailing '/'.
+    SearchPath->append(HeadersFilename.begin(), HeadersFilename.end()-1);
+  }
+
+  HeadersFilename.append(Filename.begin()+SlashPos+1, Filename.end());
+  if (!(FE = FileMgr.getFile(HeadersFilename.str(), /*openFile=*/true))) {
+
+    // Check ".../Frameworks/HIToolbox.framework/PrivateHeaders/HIToolbox.h"
+    HeadersFilename = FrameworkName;
+    HeadersFilename += "PrivateHeaders/";
+    if (SearchPath != NULL) {
+      SearchPath->clear();
+      // Without trailing '/'.
+      SearchPath->append(HeadersFilename.begin(), HeadersFilename.end()-1);
+    }
+
+    HeadersFilename.append(Filename.begin()+SlashPos+1, Filename.end());
+    if (!(FE = FileMgr.getFile(HeadersFilename.str(), /*openFile=*/true)))
+      return 0;
+  }
+
+  // This file is a system header or C++ unfriendly if the old file is.
+  //
+  // Note that the temporary 'DirInfo' is required here, as either call to
+  // getFileInfo could resize the vector and we don't want to rely on order
+  // of evaluation.
+  unsigned DirInfo = getFileInfo(ContextFileEnt).DirInfo;
+  getFileInfo(FE).DirInfo = DirInfo;
+  return FE;
+}
+
+/// \brief Helper static function to normalize a path for injection into
+/// a synthetic header.
+/*static*/ std::string
+HeaderSearch::NormalizeDashIncludePath(StringRef File, FileManager &FileMgr) {
+  // Implicit include paths should be resolved relative to the current
+  // working directory first, and then use the regular header search
+  // mechanism. The proper way to handle this is to have the
+  // predefines buffer located at the current working directory, but
+  // it has no file entry. For now, workaround this by using an
+  // absolute path if we find the file here, and otherwise letting
+  // header search handle it.
+  SmallString<128> Path(File);
+  llvm::sys::fs::make_absolute(Path);
+  bool exists;
+  if (llvm::sys::fs::exists(Path.str(), exists) || !exists)
+    Path = File;
+  else if (exists)
+    FileMgr.getFile(File);
+
+  return Lexer::Stringify(Path.str());
+}
+
+//===----------------------------------------------------------------------===//
+// File Info Management.
+//===----------------------------------------------------------------------===//
+
+/// \brief Merge the header file info provided by \p OtherHFI into the current
+/// header file info (\p HFI)
+static void mergeHeaderFileInfo(HeaderFileInfo &HFI, 
+                                const HeaderFileInfo &OtherHFI) {
+  HFI.isImport |= OtherHFI.isImport;
+  HFI.isPragmaOnce |= OtherHFI.isPragmaOnce;
+  HFI.NumIncludes += OtherHFI.NumIncludes;
+  
+  if (!HFI.ControllingMacro && !HFI.ControllingMacroID) {
+    HFI.ControllingMacro = OtherHFI.ControllingMacro;
+    HFI.ControllingMacroID = OtherHFI.ControllingMacroID;
+  }
+  
+  if (OtherHFI.External) {
+    HFI.DirInfo = OtherHFI.DirInfo;
+    HFI.External = OtherHFI.External;
+    HFI.IndexHeaderMapHeader = OtherHFI.IndexHeaderMapHeader;
+  }
+
+  if (HFI.Framework.empty())
+    HFI.Framework = OtherHFI.Framework;
+  
+  HFI.Resolved = true;
+}
+                                
+/// getFileInfo - Return the HeaderFileInfo structure for the specified
+/// FileEntry.
+HeaderFileInfo &HeaderSearch::getFileInfo(const FileEntry *FE) {
+  if (FE->getUID() >= FileInfo.size())
+    FileInfo.resize(FE->getUID()+1);
+  
+  HeaderFileInfo &HFI = FileInfo[FE->getUID()];
+  if (ExternalSource && !HFI.Resolved)
+    mergeHeaderFileInfo(HFI, ExternalSource->GetHeaderFileInfo(FE));
+  return HFI;
+}
+
+bool HeaderSearch::isFileMultipleIncludeGuarded(const FileEntry *File) {
+  // Check if we've ever seen this file as a header.
+  if (File->getUID() >= FileInfo.size())
+    return false;
+
+  // Resolve header file info from the external source, if needed.
+  HeaderFileInfo &HFI = FileInfo[File->getUID()];
+  if (ExternalSource && !HFI.Resolved)
+    mergeHeaderFileInfo(HFI, ExternalSource->GetHeaderFileInfo(File));
+
+  return HFI.isPragmaOnce || HFI.ControllingMacro || HFI.ControllingMacroID;
+}
+
+void HeaderSearch::setHeaderFileInfoForUID(HeaderFileInfo HFI, unsigned UID) {
+  if (UID >= FileInfo.size())
+    FileInfo.resize(UID+1);
+  HFI.Resolved = true;
+  FileInfo[UID] = HFI;
+}
+
+/// ShouldEnterIncludeFile - Mark the specified file as a target of of a
+/// #include, #include_next, or #import directive.  Return false if #including
+/// the file will have no effect or true if we should include it.
+bool HeaderSearch::ShouldEnterIncludeFile(const FileEntry *File, bool isImport){
+  ++NumIncluded; // Count # of attempted #includes.
+
+  // Get information about this file.
+  HeaderFileInfo &FileInfo = getFileInfo(File);
+
+  // If this is a #import directive, check that we have not already imported
+  // this header.
+  if (isImport) {
+    // If this has already been imported, don't import it again.
+    FileInfo.isImport = true;
+
+    // Has this already been #import'ed or #include'd?
+    if (FileInfo.NumIncludes) return false;
+  } else {
+    // Otherwise, if this is a #include of a file that was previously #import'd
+    // or if this is the second #include of a #pragma once file, ignore it.
+    if (FileInfo.isImport)
+      return false;
+  }
+
+  // Next, check to see if the file is wrapped with #ifndef guards.  If so, and
+  // if the macro that guards it is defined, we know the #include has no effect.
+  if (const IdentifierInfo *ControllingMacro
+      = FileInfo.getControllingMacro(ExternalLookup))
+    if (ControllingMacro->hasMacroDefinition()) {
+      ++NumMultiIncludeFileOptzn;
+      return false;
+    }
+
+  // Increment the number of times this file has been included.
+  ++FileInfo.NumIncludes;
+
+  return true;
+}
+
+size_t HeaderSearch::getTotalMemory() const {
+  return SearchDirs.capacity()
+    + llvm::capacity_in_bytes(FileInfo)
+    + llvm::capacity_in_bytes(HeaderMaps)
+    + LookupFileCache.getAllocator().getTotalMemory()
+    + FrameworkMap.getAllocator().getTotalMemory();
+}
+
+StringRef HeaderSearch::getUniqueFrameworkName(StringRef Framework) {
+  return FrameworkNames.GetOrCreateValue(Framework).getKey();
+}
+
+bool HeaderSearch::hasModuleMap(StringRef FileName, 
+                                const DirectoryEntry *Root) {
+  llvm::SmallVector<const DirectoryEntry *, 2> FixUpDirectories;
+  
+  StringRef DirName = FileName;
+  do {
+    // Get the parent directory name.
+    DirName = llvm::sys::path::parent_path(DirName);
+    if (DirName.empty())
+      return false;
+    
+    // Determine whether this directory exists.
+    const DirectoryEntry *Dir = FileMgr.getDirectory(DirName);
+    if (!Dir)
+      return false;
+    
+    // Try to load the module map file in this directory.
+    switch (loadModuleMapFile(Dir)) {
+    case LMM_NewlyLoaded:
+    case LMM_AlreadyLoaded:
+      // Success. All of the directories we stepped through inherit this module
+      // map file.
+      for (unsigned I = 0, N = FixUpDirectories.size(); I != N; ++I)
+        DirectoryHasModuleMap[FixUpDirectories[I]] = true;
+      
+      return true;
+
+    case LMM_NoDirectory:
+    case LMM_InvalidModuleMap:
+      break;
+    }
+
+    // If we hit the top of our search, we're done.
+    if (Dir == Root)
+      return false;
+        
+    // Keep track of all of the directories we checked, so we can mark them as
+    // having module maps if we eventually do find a module map.
+    FixUpDirectories.push_back(Dir);
+  } while (true);
+}
+
+Module *HeaderSearch::findModuleForHeader(const FileEntry *File) {
+  if (Module *Mod = ModMap.findModuleForHeader(File))
+    return Mod;
+  
+  return 0;
+}
+
+bool HeaderSearch::loadModuleMapFile(const FileEntry *File) {
+  const DirectoryEntry *Dir = File->getDir();
+  
+  llvm::DenseMap<const DirectoryEntry *, bool>::iterator KnownDir
+    = DirectoryHasModuleMap.find(Dir);
+  if (KnownDir != DirectoryHasModuleMap.end())
+    return !KnownDir->second;
+  
+  bool Result = ModMap.parseModuleMapFile(File);
+  if (!Result && llvm::sys::path::filename(File->getName()) == "module.map") {
+    // If the file we loaded was a module.map, look for the corresponding
+    // module_private.map.
+    SmallString<128> PrivateFilename(Dir->getName());
+    llvm::sys::path::append(PrivateFilename, "module_private.map");
+    if (const FileEntry *PrivateFile = FileMgr.getFile(PrivateFilename))
+      Result = ModMap.parseModuleMapFile(PrivateFile);
+  }
+  
+  DirectoryHasModuleMap[Dir] = !Result;  
+  return Result;
+}
+
+Module *HeaderSearch::loadFrameworkModule(StringRef Name, 
+                                          const DirectoryEntry *Dir,
+                                          bool IsSystem) {
+  if (Module *Module = ModMap.findModule(Name))
+    return Module;
+  
+  // Try to load a module map file.
+  switch (loadModuleMapFile(Dir)) {
+  case LMM_InvalidModuleMap:
+    break;
+    
+  case LMM_AlreadyLoaded:
+  case LMM_NoDirectory:
+    return 0;
+    
+  case LMM_NewlyLoaded:
+    return ModMap.findModule(Name);
+  }
+
+  // The top-level framework directory, from which we'll infer a framework
+  // module.
+  const DirectoryEntry *TopFrameworkDir = Dir;
+  
+  // The path from the module we're actually looking for back to the top-level
+  // framework name.
+  llvm::SmallVector<StringRef, 2> SubmodulePath;
+  SubmodulePath.push_back(Name);
+  
+  // Walk the directory structure to find any enclosing frameworks.
+  StringRef DirName = Dir->getName();
+  do {
+    // Get the parent directory name.
+    DirName = llvm::sys::path::parent_path(DirName);
+    if (DirName.empty())
+      break;
+    
+    // Determine whether this directory exists.
+    Dir = FileMgr.getDirectory(DirName);
+    if (!Dir)
+      break;
+    
+    // If this is a framework directory, then we're a subframework of this
+    // framework.
+    if (llvm::sys::path::extension(DirName) == ".framework") {
+      SubmodulePath.push_back(llvm::sys::path::stem(DirName));
+      TopFrameworkDir = Dir;
+    }
+  } while (true);
+  
+  // Try to infer a module map from the top-level framework directory.
+  Module *Result = ModMap.inferFrameworkModule(SubmodulePath.back(), 
+                                               TopFrameworkDir,
+                                               IsSystem,
+                                               /*Parent=*/0);
+  
+  // Follow the submodule path to find the requested (sub)framework module
+  // within the top-level framework module.
+  SubmodulePath.pop_back();
+  while (!SubmodulePath.empty() && Result) {
+    Result = ModMap.lookupModuleQualified(SubmodulePath.back(), Result);
+    SubmodulePath.pop_back();
+  }
+  return Result;
+}
+
+
+HeaderSearch::LoadModuleMapResult 
+HeaderSearch::loadModuleMapFile(StringRef DirName) {
+  if (const DirectoryEntry *Dir = FileMgr.getDirectory(DirName))
+    return loadModuleMapFile(Dir);
+  
+  return LMM_NoDirectory;
+}
+
+HeaderSearch::LoadModuleMapResult 
+HeaderSearch::loadModuleMapFile(const DirectoryEntry *Dir) {
+  llvm::DenseMap<const DirectoryEntry *, bool>::iterator KnownDir
+    = DirectoryHasModuleMap.find(Dir);
+  if (KnownDir != DirectoryHasModuleMap.end())
+    return KnownDir->second? LMM_AlreadyLoaded : LMM_InvalidModuleMap;
+  
+  SmallString<128> ModuleMapFileName;
+  ModuleMapFileName += Dir->getName();
+  unsigned ModuleMapDirNameLen = ModuleMapFileName.size();
+  llvm::sys::path::append(ModuleMapFileName, "module.map");
+  if (const FileEntry *ModuleMapFile = FileMgr.getFile(ModuleMapFileName)) {
+    // We have found a module map file. Try to parse it.
+    if (ModMap.parseModuleMapFile(ModuleMapFile)) {
+      // No suitable module map.
+      DirectoryHasModuleMap[Dir] = false;
+      return LMM_InvalidModuleMap;
+    }
+
+    // This directory has a module map.
+    DirectoryHasModuleMap[Dir] = true;
+    
+    // Check whether there is a private module map that we need to load as well.
+    ModuleMapFileName.erase(ModuleMapFileName.begin() + ModuleMapDirNameLen,
+                            ModuleMapFileName.end());
+    llvm::sys::path::append(ModuleMapFileName, "module_private.map");
+    if (const FileEntry *PrivateModuleMapFile
+                                        = FileMgr.getFile(ModuleMapFileName)) {
+      if (ModMap.parseModuleMapFile(PrivateModuleMapFile)) {
+        // No suitable module map.
+        DirectoryHasModuleMap[Dir] = false;
+        return LMM_InvalidModuleMap;
+      }      
+    }
+    
+    return LMM_NewlyLoaded;
+  }
+  
+  // No suitable module map.
+  DirectoryHasModuleMap[Dir] = false;
+  return LMM_InvalidModuleMap;
+}
+
+void HeaderSearch::collectAllModules(llvm::SmallVectorImpl<Module *> &Modules) {
+  Modules.clear();
+  
+  // Load module maps for each of the header search directories.
+  for (unsigned Idx = 0, N = SearchDirs.size(); Idx != N; ++Idx) {
+    if (SearchDirs[Idx].isFramework()) {
+      llvm::error_code EC;
+      SmallString<128> DirNative;
+      llvm::sys::path::native(SearchDirs[Idx].getFrameworkDir()->getName(),
+                              DirNative);
+      
+      // Search each of the ".framework" directories to load them as modules.
+      bool IsSystem = SearchDirs[Idx].getDirCharacteristic() != SrcMgr::C_User;
+      for (llvm::sys::fs::directory_iterator Dir(DirNative.str(), EC), DirEnd;
+           Dir != DirEnd && !EC; Dir.increment(EC)) {
+        if (llvm::sys::path::extension(Dir->path()) != ".framework")
+          continue;
+        
+        const DirectoryEntry *FrameworkDir = FileMgr.getDirectory(Dir->path());
+        if (!FrameworkDir)
+          continue;
+        
+        // Load this framework module.
+        loadFrameworkModule(llvm::sys::path::stem(Dir->path()), FrameworkDir,
+                            IsSystem);
+      }
+      continue;
+    }
+    
+    // FIXME: Deal with header maps.
+    if (SearchDirs[Idx].isHeaderMap())
+      continue;
+    
+    // Try to load a module map file for the search directory.
+    loadModuleMapFile(SearchDirs[Idx].getDir());
+    
+    // Try to load module map files for immediate subdirectories of this search
+    // directory.
+    llvm::error_code EC;
+    SmallString<128> DirNative;
+    llvm::sys::path::native(SearchDirs[Idx].getDir()->getName(), DirNative);
+    for (llvm::sys::fs::directory_iterator Dir(DirNative.str(), EC), DirEnd;
+         Dir != DirEnd && !EC; Dir.increment(EC)) {
+      loadModuleMapFile(Dir->path());
+    }
+  }
+  
+  // Populate the list of modules.
+  for (ModuleMap::module_iterator M = ModMap.module_begin(), 
+                               MEnd = ModMap.module_end();
+       M != MEnd; ++M) {
+    Modules.push_back(M->getValue());
+  }
+}
+
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
new file mode 100644
index 0000000..535a852
--- /dev/null
+++ b/clang/lib/Lex/Lexer.cpp
@@ -0,0 +1,3234 @@
+//===--- Lexer.cpp - C Language Family Lexer ------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file implements the Lexer and Token interfaces.
+//
+//===----------------------------------------------------------------------===//
+//
+// TODO: GCC Diagnostics emitted by the lexer:
+// PEDWARN: (form feed|vertical tab) in preprocessing directive
+//
+// Universal characters, unicode, char mapping:
+// WARNING: `%.*s' is not in NFKC
+// WARNING: `%.*s' is not in NFC
+//
+// Other:
+// TODO: Options to support:
+//    -fexec-charset,-fwide-exec-charset
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Lex/Lexer.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/LexDiagnostic.h"
+#include "clang/Lex/CodeCompletionHandler.h"
+#include "clang/Basic/SourceManager.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include <cstring>
+using namespace clang;
+
+static void InitCharacterInfo();
+
+//===----------------------------------------------------------------------===//
+// Token Class Implementation
+//===----------------------------------------------------------------------===//
+
+/// isObjCAtKeyword - Return true if we have an ObjC keyword identifier.
+bool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const {
+  if (IdentifierInfo *II = getIdentifierInfo())
+    return II->getObjCKeywordID() == objcKey;
+  return false;
+}
+
+/// getObjCKeywordID - Return the ObjC keyword kind.
+tok::ObjCKeywordKind Token::getObjCKeywordID() const {
+  IdentifierInfo *specId = getIdentifierInfo();
+  return specId ? specId->getObjCKeywordID() : tok::objc_not_keyword;
+}
+
+
+//===----------------------------------------------------------------------===//
+// Lexer Class Implementation
+//===----------------------------------------------------------------------===//
+
+void Lexer::anchor() { }
+
+void Lexer::InitLexer(const char *BufStart, const char *BufPtr,
+                      const char *BufEnd) {
+  InitCharacterInfo();
+
+  BufferStart = BufStart;
+  BufferPtr = BufPtr;
+  BufferEnd = BufEnd;
+
+  assert(BufEnd[0] == 0 &&
+         "We assume that the input buffer has a null character at the end"
+         " to simplify lexing!");
+
+  // Check whether we have a BOM in the beginning of the buffer. If yes - act
+  // accordingly. Right now we support only UTF-8 with and without BOM, so, just
+  // skip the UTF-8 BOM if it's present.
+  if (BufferStart == BufferPtr) {
+    // Determine the size of the BOM.
+    StringRef Buf(BufferStart, BufferEnd - BufferStart);
+    size_t BOMLength = llvm::StringSwitch<size_t>(Buf)
+      .StartsWith("\xEF\xBB\xBF", 3) // UTF-8 BOM
+      .Default(0);
+
+    // Skip the BOM.
+    BufferPtr += BOMLength;
+  }
+
+  Is_PragmaLexer = false;
+  CurrentConflictMarkerState = CMK_None;
+
+  // Start of the file is a start of line.
+  IsAtStartOfLine = true;
+
+  // We are not after parsing a #.
+  ParsingPreprocessorDirective = false;
+
+  // We are not after parsing #include.
+  ParsingFilename = false;
+
+  // We are not in raw mode.  Raw mode disables diagnostics and interpretation
+  // of tokens (e.g. identifiers, thus disabling macro expansion).  It is used
+  // to quickly lex the tokens of the buffer, e.g. when handling a "#if 0" block
+  // or otherwise skipping over tokens.
+  LexingRawMode = false;
+
+  // Default to not keeping comments.
+  ExtendedTokenMode = 0;
+}
+
+/// Lexer constructor - Create a new lexer object for the specified buffer
+/// with the specified preprocessor managing the lexing process.  This lexer
+/// assumes that the associated file buffer and Preprocessor objects will
+/// outlive it, so it doesn't take ownership of either of them.
+Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *InputFile, Preprocessor &PP)
+  : PreprocessorLexer(&PP, FID),
+    FileLoc(PP.getSourceManager().getLocForStartOfFile(FID)),
+    LangOpts(PP.getLangOpts()) {
+
+  InitLexer(InputFile->getBufferStart(), InputFile->getBufferStart(),
+            InputFile->getBufferEnd());
+
+  // Default to keeping comments if the preprocessor wants them.
+  SetCommentRetentionState(PP.getCommentRetentionState());
+}
+
+/// Lexer constructor - Create a new raw lexer object.  This object is only
+/// suitable for calls to 'LexRawToken'.  This lexer assumes that the text
+/// range will outlive it, so it doesn't take ownership of it.
+Lexer::Lexer(SourceLocation fileloc, const LangOptions &langOpts,
+             const char *BufStart, const char *BufPtr, const char *BufEnd)
+  : FileLoc(fileloc), LangOpts(langOpts) {
+
+  InitLexer(BufStart, BufPtr, BufEnd);
+
+  // We *are* in raw mode.
+  LexingRawMode = true;
+}
+
+/// Lexer constructor - Create a new raw lexer object.  This object is only
+/// suitable for calls to 'LexRawToken'.  This lexer assumes that the text
+/// range will outlive it, so it doesn't take ownership of it.
+Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *FromFile,
+             const SourceManager &SM, const LangOptions &langOpts)
+  : FileLoc(SM.getLocForStartOfFile(FID)), LangOpts(langOpts) {
+
+  InitLexer(FromFile->getBufferStart(), FromFile->getBufferStart(),
+            FromFile->getBufferEnd());
+
+  // We *are* in raw mode.
+  LexingRawMode = true;
+}
+
+/// Create_PragmaLexer: Lexer constructor - Create a new lexer object for
+/// _Pragma expansion.  This has a variety of magic semantics that this method
+/// sets up.  It returns a new'd Lexer that must be delete'd when done.
+///
+/// On entrance to this routine, TokStartLoc is a macro location which has a
+/// spelling loc that indicates the bytes to be lexed for the token and an
+/// expansion location that indicates where all lexed tokens should be
+/// "expanded from".
+///
+/// FIXME: It would really be nice to make _Pragma just be a wrapper around a
+/// normal lexer that remaps tokens as they fly by.  This would require making
+/// Preprocessor::Lex virtual.  Given that, we could just dump in a magic lexer
+/// interface that could handle this stuff.  This would pull GetMappedTokenLoc
+/// out of the critical path of the lexer!
+///
+Lexer *Lexer::Create_PragmaLexer(SourceLocation SpellingLoc,
+                                 SourceLocation ExpansionLocStart,
+                                 SourceLocation ExpansionLocEnd,
+                                 unsigned TokLen, Preprocessor &PP) {
+  SourceManager &SM = PP.getSourceManager();
+
+  // Create the lexer as if we were going to lex the file normally.
+  FileID SpellingFID = SM.getFileID(SpellingLoc);
+  const llvm::MemoryBuffer *InputFile = SM.getBuffer(SpellingFID);
+  Lexer *L = new Lexer(SpellingFID, InputFile, PP);
+
+  // Now that the lexer is created, change the start/end locations so that we
+  // just lex the subsection of the file that we want.  This is lexing from a
+  // scratch buffer.
+  const char *StrData = SM.getCharacterData(SpellingLoc);
+
+  L->BufferPtr = StrData;
+  L->BufferEnd = StrData+TokLen;
+  assert(L->BufferEnd[0] == 0 && "Buffer is not nul terminated!");
+
+  // Set the SourceLocation with the remapping information.  This ensures that
+  // GetMappedTokenLoc will remap the tokens as they are lexed.
+  L->FileLoc = SM.createExpansionLoc(SM.getLocForStartOfFile(SpellingFID),
+                                     ExpansionLocStart,
+                                     ExpansionLocEnd, TokLen);
+
+  // Ensure that the lexer thinks it is inside a directive, so that end \n will
+  // return an EOD token.
+  L->ParsingPreprocessorDirective = true;
+
+  // This lexer really is for _Pragma.
+  L->Is_PragmaLexer = true;
+  return L;
+}
+
+
+/// Stringify - Convert the specified string into a C string, with surrounding
+/// ""'s, and with escaped \ and " characters.
+std::string Lexer::Stringify(const std::string &Str, bool Charify) {
+  std::string Result = Str;
+  char Quote = Charify ? '\'' : '"';
+  for (unsigned i = 0, e = Result.size(); i != e; ++i) {
+    if (Result[i] == '\\' || Result[i] == Quote) {
+      Result.insert(Result.begin()+i, '\\');
+      ++i; ++e;
+    }
+  }
+  return Result;
+}
+
+/// Stringify - Convert the specified string into a C string by escaping '\'
+/// and " characters.  This does not add surrounding ""'s to the string.
+void Lexer::Stringify(SmallVectorImpl<char> &Str) {
+  for (unsigned i = 0, e = Str.size(); i != e; ++i) {
+    if (Str[i] == '\\' || Str[i] == '"') {
+      Str.insert(Str.begin()+i, '\\');
+      ++i; ++e;
+    }
+  }
+}
+
+//===----------------------------------------------------------------------===//
+// Token Spelling
+//===----------------------------------------------------------------------===//
+
+/// getSpelling() - Return the 'spelling' of this token.  The spelling of a
+/// token are the characters used to represent the token in the source file
+/// after trigraph expansion and escaped-newline folding.  In particular, this
+/// wants to get the true, uncanonicalized, spelling of things like digraphs
+/// UCNs, etc.
+StringRef Lexer::getSpelling(SourceLocation loc,
+                                   SmallVectorImpl<char> &buffer,
+                                   const SourceManager &SM,
+                                   const LangOptions &options,
+                                   bool *invalid) {
+  // Break down the source location.
+  std::pair<FileID, unsigned> locInfo = SM.getDecomposedLoc(loc);
+
+  // Try to the load the file buffer.
+  bool invalidTemp = false;
+  StringRef file = SM.getBufferData(locInfo.first, &invalidTemp);
+  if (invalidTemp) {
+    if (invalid) *invalid = true;
+    return StringRef();
+  }
+
+  const char *tokenBegin = file.data() + locInfo.second;
+
+  // Lex from the start of the given location.
+  Lexer lexer(SM.getLocForStartOfFile(locInfo.first), options,
+              file.begin(), tokenBegin, file.end());
+  Token token;
+  lexer.LexFromRawLexer(token);
+
+  unsigned length = token.getLength();
+
+  // Common case:  no need for cleaning.
+  if (!token.needsCleaning())
+    return StringRef(tokenBegin, length);
+  
+  // Hard case, we need to relex the characters into the string.
+  buffer.clear();
+  buffer.reserve(length);
+  
+  for (const char *ti = tokenBegin, *te = ti + length; ti != te; ) {
+    unsigned charSize;
+    buffer.push_back(Lexer::getCharAndSizeNoWarn(ti, charSize, options));
+    ti += charSize;
+  }
+
+  return StringRef(buffer.data(), buffer.size());
+}
+
+/// getSpelling() - Return the 'spelling' of this token.  The spelling of a
+/// token are the characters used to represent the token in the source file
+/// after trigraph expansion and escaped-newline folding.  In particular, this
+/// wants to get the true, uncanonicalized, spelling of things like digraphs
+/// UCNs, etc.
+std::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr,
+                               const LangOptions &LangOpts, bool *Invalid) {
+  assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
+  
+  // If this token contains nothing interesting, return it directly.
+  bool CharDataInvalid = false;
+  const char* TokStart = SourceMgr.getCharacterData(Tok.getLocation(), 
+                                                    &CharDataInvalid);
+  if (Invalid)
+    *Invalid = CharDataInvalid;
+  if (CharDataInvalid)
+    return std::string();
+  
+  if (!Tok.needsCleaning())
+    return std::string(TokStart, TokStart+Tok.getLength());
+  
+  std::string Result;
+  Result.reserve(Tok.getLength());
+  
+  // Otherwise, hard case, relex the characters into the string.
+  for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength();
+       Ptr != End; ) {
+    unsigned CharSize;
+    Result.push_back(Lexer::getCharAndSizeNoWarn(Ptr, CharSize, LangOpts));
+    Ptr += CharSize;
+  }
+  assert(Result.size() != unsigned(Tok.getLength()) &&
+         "NeedsCleaning flag set on something that didn't need cleaning!");
+  return Result;
+}
+
+/// getSpelling - This method is used to get the spelling of a token into a
+/// preallocated buffer, instead of as an std::string.  The caller is required
+/// to allocate enough space for the token, which is guaranteed to be at least
+/// Tok.getLength() bytes long.  The actual length of the token is returned.
+///
+/// Note that this method may do two possible things: it may either fill in
+/// the buffer specified with characters, or it may *change the input pointer*
+/// to point to a constant buffer with the data already in it (avoiding a
+/// copy).  The caller is not allowed to modify the returned buffer pointer
+/// if an internal buffer is returned.
+unsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer, 
+                            const SourceManager &SourceMgr,
+                            const LangOptions &LangOpts, bool *Invalid) {
+  assert((int)Tok.getLength() >= 0 && "Token character range is bogus!");
+
+  const char *TokStart = 0;
+  // NOTE: this has to be checked *before* testing for an IdentifierInfo.
+  if (Tok.is(tok::raw_identifier))
+    TokStart = Tok.getRawIdentifierData();
+  else if (const IdentifierInfo *II = Tok.getIdentifierInfo()) {
+    // Just return the string from the identifier table, which is very quick.
+    Buffer = II->getNameStart();
+    return II->getLength();
+  }
+
+  // NOTE: this can be checked even after testing for an IdentifierInfo.
+  if (Tok.isLiteral())
+    TokStart = Tok.getLiteralData();
+
+  if (TokStart == 0) {
+    // Compute the start of the token in the input lexer buffer.
+    bool CharDataInvalid = false;
+    TokStart = SourceMgr.getCharacterData(Tok.getLocation(), &CharDataInvalid);
+    if (Invalid)
+      *Invalid = CharDataInvalid;
+    if (CharDataInvalid) {
+      Buffer = "";
+      return 0;
+    }
+  }
+
+  // If this token contains nothing interesting, return it directly.
+  if (!Tok.needsCleaning()) {
+    Buffer = TokStart;
+    return Tok.getLength();
+  }
+
+  // Otherwise, hard case, relex the characters into the string.
+  char *OutBuf = const_cast<char*>(Buffer);
+  for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength();
+       Ptr != End; ) {
+    unsigned CharSize;
+    *OutBuf++ = Lexer::getCharAndSizeNoWarn(Ptr, CharSize, LangOpts);
+    Ptr += CharSize;
+  }
+  assert(unsigned(OutBuf-Buffer) != Tok.getLength() &&
+         "NeedsCleaning flag set on something that didn't need cleaning!");
+
+  return OutBuf-Buffer;
+}
+
+
+
+static bool isWhitespace(unsigned char c);
+
+/// MeasureTokenLength - Relex the token at the specified location and return
+/// its length in bytes in the input file.  If the token needs cleaning (e.g.
+/// includes a trigraph or an escaped newline) then this count includes bytes
+/// that are part of that.
+unsigned Lexer::MeasureTokenLength(SourceLocation Loc,
+                                   const SourceManager &SM,
+                                   const LangOptions &LangOpts) {
+  // TODO: this could be special cased for common tokens like identifiers, ')',
+  // etc to make this faster, if it mattered.  Just look at StrData[0] to handle
+  // all obviously single-char tokens.  This could use
+  // Lexer::isObviouslySimpleCharacter for example to handle identifiers or
+  // something.
+
+  // If this comes from a macro expansion, we really do want the macro name, not
+  // the token this macro expanded to.
+  Loc = SM.getExpansionLoc(Loc);
+  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
+  bool Invalid = false;
+  StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
+  if (Invalid)
+    return 0;
+
+  const char *StrData = Buffer.data()+LocInfo.second;
+
+  if (isWhitespace(StrData[0]))
+    return 0;
+
+  // Create a lexer starting at the beginning of this token.
+  Lexer TheLexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts,
+                 Buffer.begin(), StrData, Buffer.end());
+  TheLexer.SetCommentRetentionState(true);
+  Token TheTok;
+  TheLexer.LexFromRawLexer(TheTok);
+  return TheTok.getLength();
+}
+
+static SourceLocation getBeginningOfFileToken(SourceLocation Loc,
+                                              const SourceManager &SM,
+                                              const LangOptions &LangOpts) {
+  assert(Loc.isFileID());
+  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
+  if (LocInfo.first.isInvalid())
+    return Loc;
+  
+  bool Invalid = false;
+  StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
+  if (Invalid)
+    return Loc;
+
+  // Back up from the current location until we hit the beginning of a line
+  // (or the buffer). We'll relex from that point.
+  const char *BufStart = Buffer.data();
+  if (LocInfo.second >= Buffer.size())
+    return Loc;
+  
+  const char *StrData = BufStart+LocInfo.second;
+  if (StrData[0] == '\n' || StrData[0] == '\r')
+    return Loc;
+
+  const char *LexStart = StrData;
+  while (LexStart != BufStart) {
+    if (LexStart[0] == '\n' || LexStart[0] == '\r') {
+      ++LexStart;
+      break;
+    }
+
+    --LexStart;
+  }
+  
+  // Create a lexer starting at the beginning of this token.
+  SourceLocation LexerStartLoc = Loc.getLocWithOffset(-LocInfo.second);
+  Lexer TheLexer(LexerStartLoc, LangOpts, BufStart, LexStart, Buffer.end());
+  TheLexer.SetCommentRetentionState(true);
+  
+  // Lex tokens until we find the token that contains the source location.
+  Token TheTok;
+  do {
+    TheLexer.LexFromRawLexer(TheTok);
+    
+    if (TheLexer.getBufferLocation() > StrData) {
+      // Lexing this token has taken the lexer past the source location we're
+      // looking for. If the current token encompasses our source location,
+      // return the beginning of that token.
+      if (TheLexer.getBufferLocation() - TheTok.getLength() <= StrData)
+        return TheTok.getLocation();
+      
+      // We ended up skipping over the source location entirely, which means
+      // that it points into whitespace. We're done here.
+      break;
+    }
+  } while (TheTok.getKind() != tok::eof);
+  
+  // We've passed our source location; just return the original source location.
+  return Loc;
+}
+
+SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc,
+                                          const SourceManager &SM,
+                                          const LangOptions &LangOpts) {
+ if (Loc.isFileID())
+   return getBeginningOfFileToken(Loc, SM, LangOpts);
+ 
+ if (!SM.isMacroArgExpansion(Loc))
+   return Loc;
+
+ SourceLocation FileLoc = SM.getSpellingLoc(Loc);
+ SourceLocation BeginFileLoc = getBeginningOfFileToken(FileLoc, SM, LangOpts);
+ std::pair<FileID, unsigned> FileLocInfo = SM.getDecomposedLoc(FileLoc);
+ std::pair<FileID, unsigned> BeginFileLocInfo
+   = SM.getDecomposedLoc(BeginFileLoc);
+ assert(FileLocInfo.first == BeginFileLocInfo.first &&
+        FileLocInfo.second >= BeginFileLocInfo.second);
+ return Loc.getLocWithOffset(BeginFileLocInfo.second - FileLocInfo.second);
+}
+
+namespace {
+  enum PreambleDirectiveKind {
+    PDK_Skipped,
+    PDK_StartIf,
+    PDK_EndIf,
+    PDK_Unknown
+  };
+}
+
+std::pair<unsigned, bool>
+Lexer::ComputePreamble(const llvm::MemoryBuffer *Buffer,
+                       const LangOptions &LangOpts, unsigned MaxLines) {
+  // Create a lexer starting at the beginning of the file. Note that we use a
+  // "fake" file source location at offset 1 so that the lexer will track our
+  // position within the file.
+  const unsigned StartOffset = 1;
+  SourceLocation StartLoc = SourceLocation::getFromRawEncoding(StartOffset);
+  Lexer TheLexer(StartLoc, LangOpts, Buffer->getBufferStart(),
+                 Buffer->getBufferStart(), Buffer->getBufferEnd());
+  
+  bool InPreprocessorDirective = false;
+  Token TheTok;
+  Token IfStartTok;
+  unsigned IfCount = 0;
+
+  unsigned MaxLineOffset = 0;
+  if (MaxLines) {
+    const char *CurPtr = Buffer->getBufferStart();
+    unsigned CurLine = 0;
+    while (CurPtr != Buffer->getBufferEnd()) {
+      char ch = *CurPtr++;
+      if (ch == '\n') {
+        ++CurLine;
+        if (CurLine == MaxLines)
+          break;
+      }
+    }
+    if (CurPtr != Buffer->getBufferEnd())
+      MaxLineOffset = CurPtr - Buffer->getBufferStart();
+  }
+
+  do {
+    TheLexer.LexFromRawLexer(TheTok);
+
+    if (InPreprocessorDirective) {
+      // If we've hit the end of the file, we're done.
+      if (TheTok.getKind() == tok::eof) {
+        InPreprocessorDirective = false;
+        break;
+      }
+      
+      // If we haven't hit the end of the preprocessor directive, skip this
+      // token.
+      if (!TheTok.isAtStartOfLine())
+        continue;
+        
+      // We've passed the end of the preprocessor directive, and will look
+      // at this token again below.
+      InPreprocessorDirective = false;
+    }
+    
+    // Keep track of the # of lines in the preamble.
+    if (TheTok.isAtStartOfLine()) {
+      unsigned TokOffset = TheTok.getLocation().getRawEncoding() - StartOffset;
+
+      // If we were asked to limit the number of lines in the preamble,
+      // and we're about to exceed that limit, we're done.
+      if (MaxLineOffset && TokOffset >= MaxLineOffset)
+        break;
+    }
+
+    // Comments are okay; skip over them.
+    if (TheTok.getKind() == tok::comment)
+      continue;
+    
+    if (TheTok.isAtStartOfLine() && TheTok.getKind() == tok::hash) {
+      // This is the start of a preprocessor directive. 
+      Token HashTok = TheTok;
+      InPreprocessorDirective = true;
+      
+      // Figure out which directive this is. Since we're lexing raw tokens,
+      // we don't have an identifier table available. Instead, just look at
+      // the raw identifier to recognize and categorize preprocessor directives.
+      TheLexer.LexFromRawLexer(TheTok);
+      if (TheTok.getKind() == tok::raw_identifier && !TheTok.needsCleaning()) {
+        StringRef Keyword(TheTok.getRawIdentifierData(),
+                                TheTok.getLength());
+        PreambleDirectiveKind PDK
+          = llvm::StringSwitch<PreambleDirectiveKind>(Keyword)
+              .Case("include", PDK_Skipped)
+              .Case("__include_macros", PDK_Skipped)
+              .Case("define", PDK_Skipped)
+              .Case("undef", PDK_Skipped)
+              .Case("line", PDK_Skipped)
+              .Case("error", PDK_Skipped)
+              .Case("pragma", PDK_Skipped)
+              .Case("import", PDK_Skipped)
+              .Case("include_next", PDK_Skipped)
+              .Case("warning", PDK_Skipped)
+              .Case("ident", PDK_Skipped)
+              .Case("sccs", PDK_Skipped)
+              .Case("assert", PDK_Skipped)
+              .Case("unassert", PDK_Skipped)
+              .Case("if", PDK_StartIf)
+              .Case("ifdef", PDK_StartIf)
+              .Case("ifndef", PDK_StartIf)
+              .Case("elif", PDK_Skipped)
+              .Case("else", PDK_Skipped)
+              .Case("endif", PDK_EndIf)
+              .Default(PDK_Unknown);
+
+        switch (PDK) {
+        case PDK_Skipped:
+          continue;
+
+        case PDK_StartIf:
+          if (IfCount == 0)
+            IfStartTok = HashTok;
+            
+          ++IfCount;
+          continue;
+            
+        case PDK_EndIf:
+          // Mismatched #endif. The preamble ends here.
+          if (IfCount == 0)
+            break;
+
+          --IfCount;
+          continue;
+            
+        case PDK_Unknown:
+          // We don't know what this directive is; stop at the '#'.
+          break;
+        }
+      }
+      
+      // We only end up here if we didn't recognize the preprocessor
+      // directive or it was one that can't occur in the preamble at this
+      // point. Roll back the current token to the location of the '#'.
+      InPreprocessorDirective = false;
+      TheTok = HashTok;
+    }
+
+    // We hit a token that we don't recognize as being in the
+    // "preprocessing only" part of the file, so we're no longer in
+    // the preamble.
+    break;
+  } while (true);
+  
+  SourceLocation End = IfCount? IfStartTok.getLocation() : TheTok.getLocation();
+  return std::make_pair(End.getRawEncoding() - StartLoc.getRawEncoding(),
+                        IfCount? IfStartTok.isAtStartOfLine()
+                               : TheTok.isAtStartOfLine());
+}
+
+
+/// AdvanceToTokenCharacter - Given a location that specifies the start of a
+/// token, return a new location that specifies a character within the token.
+SourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart,
+                                              unsigned CharNo,
+                                              const SourceManager &SM,
+                                              const LangOptions &LangOpts) {
+  // Figure out how many physical characters away the specified expansion
+  // character is.  This needs to take into consideration newlines and
+  // trigraphs.
+  bool Invalid = false;
+  const char *TokPtr = SM.getCharacterData(TokStart, &Invalid);
+  
+  // If they request the first char of the token, we're trivially done.
+  if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(*TokPtr)))
+    return TokStart;
+  
+  unsigned PhysOffset = 0;
+  
+  // The usual case is that tokens don't contain anything interesting.  Skip
+  // over the uninteresting characters.  If a token only consists of simple
+  // chars, this method is extremely fast.
+  while (Lexer::isObviouslySimpleCharacter(*TokPtr)) {
+    if (CharNo == 0)
+      return TokStart.getLocWithOffset(PhysOffset);
+    ++TokPtr, --CharNo, ++PhysOffset;
+  }
+  
+  // If we have a character that may be a trigraph or escaped newline, use a
+  // lexer to parse it correctly.
+  for (; CharNo; --CharNo) {
+    unsigned Size;
+    Lexer::getCharAndSizeNoWarn(TokPtr, Size, LangOpts);
+    TokPtr += Size;
+    PhysOffset += Size;
+  }
+  
+  // Final detail: if we end up on an escaped newline, we want to return the
+  // location of the actual byte of the token.  For example foo\<newline>bar
+  // advanced by 3 should return the location of b, not of \\.  One compounding
+  // detail of this is that the escape may be made by a trigraph.
+  if (!Lexer::isObviouslySimpleCharacter(*TokPtr))
+    PhysOffset += Lexer::SkipEscapedNewLines(TokPtr)-TokPtr;
+  
+  return TokStart.getLocWithOffset(PhysOffset);
+}
+
+/// \brief Computes the source location just past the end of the
+/// token at this source location.
+///
+/// This routine can be used to produce a source location that
+/// points just past the end of the token referenced by \p Loc, and
+/// is generally used when a diagnostic needs to point just after a
+/// token where it expected something different that it received. If
+/// the returned source location would not be meaningful (e.g., if
+/// it points into a macro), this routine returns an invalid
+/// source location.
+///
+/// \param Offset an offset from the end of the token, where the source
+/// location should refer to. The default offset (0) produces a source
+/// location pointing just past the end of the token; an offset of 1 produces
+/// a source location pointing to the last character in the token, etc.
+SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset,
+                                          const SourceManager &SM,
+                                          const LangOptions &LangOpts) {
+  if (Loc.isInvalid())
+    return SourceLocation();
+
+  if (Loc.isMacroID()) {
+    if (Offset > 0 || !isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc))
+      return SourceLocation(); // Points inside the macro expansion.
+  }
+
+  unsigned Len = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
+  if (Len > Offset)
+    Len = Len - Offset;
+  else
+    return Loc;
+  
+  return Loc.getLocWithOffset(Len);
+}
+
+/// \brief Returns true if the given MacroID location points at the first
+/// token of the macro expansion.
+bool Lexer::isAtStartOfMacroExpansion(SourceLocation loc,
+                                      const SourceManager &SM,
+                                      const LangOptions &LangOpts,
+                                      SourceLocation *MacroBegin) {
+  assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc");
+
+  std::pair<FileID, unsigned> infoLoc = SM.getDecomposedLoc(loc);
+  // FIXME: If the token comes from the macro token paste operator ('##')
+  // this function will always return false;
+  if (infoLoc.second > 0)
+    return false; // Does not point at the start of token.
+
+  SourceLocation expansionLoc =
+    SM.getSLocEntry(infoLoc.first).getExpansion().getExpansionLocStart();
+  if (expansionLoc.isFileID()) {
+    // No other macro expansions, this is the first.
+    if (MacroBegin)
+      *MacroBegin = expansionLoc;
+    return true;
+  }
+
+  return isAtStartOfMacroExpansion(expansionLoc, SM, LangOpts, MacroBegin);
+}
+
+/// \brief Returns true if the given MacroID location points at the last
+/// token of the macro expansion.
+bool Lexer::isAtEndOfMacroExpansion(SourceLocation loc,
+                                    const SourceManager &SM,
+                                    const LangOptions &LangOpts,
+                                    SourceLocation *MacroEnd) {
+  assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc");
+
+  SourceLocation spellLoc = SM.getSpellingLoc(loc);
+  unsigned tokLen = MeasureTokenLength(spellLoc, SM, LangOpts);
+  if (tokLen == 0)
+    return false;
+
+  FileID FID = SM.getFileID(loc);
+  SourceLocation afterLoc = loc.getLocWithOffset(tokLen+1);
+  if (SM.isInFileID(afterLoc, FID))
+    return false; // Still in the same FileID, does not point to the last token.
+
+  // FIXME: If the token comes from the macro token paste operator ('##')
+  // or the stringify operator ('#') this function will always return false;
+
+  SourceLocation expansionLoc =
+    SM.getSLocEntry(FID).getExpansion().getExpansionLocEnd();
+  if (expansionLoc.isFileID()) {
+    // No other macro expansions.
+    if (MacroEnd)
+      *MacroEnd = expansionLoc;
+    return true;
+  }
+
+  return isAtEndOfMacroExpansion(expansionLoc, SM, LangOpts, MacroEnd);
+}
+
+static CharSourceRange makeRangeFromFileLocs(CharSourceRange Range,
+                                             const SourceManager &SM,
+                                             const LangOptions &LangOpts) {
+  SourceLocation Begin = Range.getBegin();
+  SourceLocation End = Range.getEnd();
+  assert(Begin.isFileID() && End.isFileID());
+  if (Range.isTokenRange()) {
+    End = Lexer::getLocForEndOfToken(End, 0, SM,LangOpts);
+    if (End.isInvalid())
+      return CharSourceRange();
+  }
+
+  // Break down the source locations.
+  FileID FID;
+  unsigned BeginOffs;
+  llvm::tie(FID, BeginOffs) = SM.getDecomposedLoc(Begin);
+  if (FID.isInvalid())
+    return CharSourceRange();
+
+  unsigned EndOffs;
+  if (!SM.isInFileID(End, FID, &EndOffs) ||
+      BeginOffs > EndOffs)
+    return CharSourceRange();
+
+  return CharSourceRange::getCharRange(Begin, End);
+}
+
+/// \brief Accepts a range and returns a character range with file locations.
+///
+/// Returns a null range if a part of the range resides inside a macro
+/// expansion or the range does not reside on the same FileID.
+CharSourceRange Lexer::makeFileCharRange(CharSourceRange Range,
+                                         const SourceManager &SM,
+                                         const LangOptions &LangOpts) {
+  SourceLocation Begin = Range.getBegin();
+  SourceLocation End = Range.getEnd();
+  if (Begin.isInvalid() || End.isInvalid())
+    return CharSourceRange();
+
+  if (Begin.isFileID() && End.isFileID())
+    return makeRangeFromFileLocs(Range, SM, LangOpts);
+
+  if (Begin.isMacroID() && End.isFileID()) {
+    if (!isAtStartOfMacroExpansion(Begin, SM, LangOpts, &Begin))
+      return CharSourceRange();
+    Range.setBegin(Begin);
+    return makeRangeFromFileLocs(Range, SM, LangOpts);
+  }
+
+  if (Begin.isFileID() && End.isMacroID()) {
+    if ((Range.isTokenRange() && !isAtEndOfMacroExpansion(End, SM, LangOpts,
+                                                          &End)) ||
+        (Range.isCharRange() && !isAtStartOfMacroExpansion(End, SM, LangOpts,
+                                                           &End)))
+      return CharSourceRange();
+    Range.setEnd(End);
+    return makeRangeFromFileLocs(Range, SM, LangOpts);
+  }
+
+  assert(Begin.isMacroID() && End.isMacroID());
+  SourceLocation MacroBegin, MacroEnd;
+  if (isAtStartOfMacroExpansion(Begin, SM, LangOpts, &MacroBegin) &&
+      ((Range.isTokenRange() && isAtEndOfMacroExpansion(End, SM, LangOpts,
+                                                        &MacroEnd)) ||
+       (Range.isCharRange() && isAtStartOfMacroExpansion(End, SM, LangOpts,
+                                                         &MacroEnd)))) {
+    Range.setBegin(MacroBegin);
+    Range.setEnd(MacroEnd);
+    return makeRangeFromFileLocs(Range, SM, LangOpts);
+  }
+
+  FileID FID;
+  unsigned BeginOffs;
+  llvm::tie(FID, BeginOffs) = SM.getDecomposedLoc(Begin);
+  if (FID.isInvalid())
+    return CharSourceRange();
+
+  unsigned EndOffs;
+  if (!SM.isInFileID(End, FID, &EndOffs) ||
+      BeginOffs > EndOffs)
+    return CharSourceRange();
+
+  const SrcMgr::SLocEntry *E = &SM.getSLocEntry(FID);
+  const SrcMgr::ExpansionInfo &Expansion = E->getExpansion();
+  if (Expansion.isMacroArgExpansion() &&
+      Expansion.getSpellingLoc().isFileID()) {
+    SourceLocation SpellLoc = Expansion.getSpellingLoc();
+    Range.setBegin(SpellLoc.getLocWithOffset(BeginOffs));
+    Range.setEnd(SpellLoc.getLocWithOffset(EndOffs));
+    return makeRangeFromFileLocs(Range, SM, LangOpts);
+  }
+
+  return CharSourceRange();
+}
+
+StringRef Lexer::getSourceText(CharSourceRange Range,
+                               const SourceManager &SM,
+                               const LangOptions &LangOpts,
+                               bool *Invalid) {
+  Range = makeFileCharRange(Range, SM, LangOpts);
+  if (Range.isInvalid()) {
+    if (Invalid) *Invalid = true;
+    return StringRef();
+  }
+
+  // Break down the source location.
+  std::pair<FileID, unsigned> beginInfo = SM.getDecomposedLoc(Range.getBegin());
+  if (beginInfo.first.isInvalid()) {
+    if (Invalid) *Invalid = true;
+    return StringRef();
+  }
+
+  unsigned EndOffs;
+  if (!SM.isInFileID(Range.getEnd(), beginInfo.first, &EndOffs) ||
+      beginInfo.second > EndOffs) {
+    if (Invalid) *Invalid = true;
+    return StringRef();
+  }
+
+  // Try to the load the file buffer.
+  bool invalidTemp = false;
+  StringRef file = SM.getBufferData(beginInfo.first, &invalidTemp);
+  if (invalidTemp) {
+    if (Invalid) *Invalid = true;
+    return StringRef();
+  }
+
+  if (Invalid) *Invalid = false;
+  return file.substr(beginInfo.second, EndOffs - beginInfo.second);
+}
+
+StringRef Lexer::getImmediateMacroName(SourceLocation Loc,
+                                       const SourceManager &SM,
+                                       const LangOptions &LangOpts) {
+  assert(Loc.isMacroID() && "Only reasonble to call this on macros");
+
+  // Find the location of the immediate macro expansion.
+  while (1) {
+    FileID FID = SM.getFileID(Loc);
+    const SrcMgr::SLocEntry *E = &SM.getSLocEntry(FID);
+    const SrcMgr::ExpansionInfo &Expansion = E->getExpansion();
+    Loc = Expansion.getExpansionLocStart();
+    if (!Expansion.isMacroArgExpansion())
+      break;
+
+    // For macro arguments we need to check that the argument did not come
+    // from an inner macro, e.g: "MAC1( MAC2(foo) )"
+    
+    // Loc points to the argument id of the macro definition, move to the
+    // macro expansion.
+    Loc = SM.getImmediateExpansionRange(Loc).first;
+    SourceLocation SpellLoc = Expansion.getSpellingLoc();
+    if (SpellLoc.isFileID())
+      break; // No inner macro.
+
+    // If spelling location resides in the same FileID as macro expansion
+    // location, it means there is no inner macro.
+    FileID MacroFID = SM.getFileID(Loc);
+    if (SM.isInFileID(SpellLoc, MacroFID))
+      break;
+
+    // Argument came from inner macro.
+    Loc = SpellLoc;
+  }
+
+  // Find the spelling location of the start of the non-argument expansion
+  // range. This is where the macro name was spelled in order to begin
+  // expanding this macro.
+  Loc = SM.getSpellingLoc(Loc);
+
+  // Dig out the buffer where the macro name was spelled and the extents of the
+  // name so that we can render it into the expansion note.
+  std::pair<FileID, unsigned> ExpansionInfo = SM.getDecomposedLoc(Loc);
+  unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
+  StringRef ExpansionBuffer = SM.getBufferData(ExpansionInfo.first);
+  return ExpansionBuffer.substr(ExpansionInfo.second, MacroTokenLength);
+}
+
+//===----------------------------------------------------------------------===//
+// Character information.
+//===----------------------------------------------------------------------===//
+
+enum {
+  CHAR_HORZ_WS  = 0x01,  // ' ', '\t', '\f', '\v'.  Note, no '\0'
+  CHAR_VERT_WS  = 0x02,  // '\r', '\n'
+  CHAR_LETTER   = 0x04,  // a-z,A-Z
+  CHAR_NUMBER   = 0x08,  // 0-9
+  CHAR_UNDER    = 0x10,  // _
+  CHAR_PERIOD   = 0x20,  // .
+  CHAR_RAWDEL   = 0x40   // {}[]#<>%:;?*+-/^&|~!=,"'
+};
+
+// Statically initialize CharInfo table based on ASCII character set
+// Reference: FreeBSD 7.2 /usr/share/misc/ascii
+static const unsigned char CharInfo[256] =
+{
+// 0 NUL         1 SOH         2 STX         3 ETX
+// 4 EOT         5 ENQ         6 ACK         7 BEL
+   0           , 0           , 0           , 0           ,
+   0           , 0           , 0           , 0           ,
+// 8 BS          9 HT         10 NL         11 VT
+//12 NP         13 CR         14 SO         15 SI
+   0           , CHAR_HORZ_WS, CHAR_VERT_WS, CHAR_HORZ_WS,
+   CHAR_HORZ_WS, CHAR_VERT_WS, 0           , 0           ,
+//16 DLE        17 DC1        18 DC2        19 DC3
+//20 DC4        21 NAK        22 SYN        23 ETB
+   0           , 0           , 0           , 0           ,
+   0           , 0           , 0           , 0           ,
+//24 CAN        25 EM         26 SUB        27 ESC
+//28 FS         29 GS         30 RS         31 US
+   0           , 0           , 0           , 0           ,
+   0           , 0           , 0           , 0           ,
+//32 SP         33  !         34  "         35  #
+//36  $         37  %         38  &         39  '
+   CHAR_HORZ_WS, CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL ,
+   0           , CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL ,
+//40  (         41  )         42  *         43  +
+//44  ,         45  -         46  .         47  /
+   0           , 0           , CHAR_RAWDEL , CHAR_RAWDEL ,
+   CHAR_RAWDEL , CHAR_RAWDEL , CHAR_PERIOD , CHAR_RAWDEL ,
+//48  0         49  1         50  2         51  3
+//52  4         53  5         54  6         55  7
+   CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER ,
+   CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER ,
+//56  8         57  9         58  :         59  ;
+//60  <         61  =         62  >         63  ?
+   CHAR_NUMBER , CHAR_NUMBER , CHAR_RAWDEL , CHAR_RAWDEL ,
+   CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL ,
+//64  @         65  A         66  B         67  C
+//68  D         69  E         70  F         71  G
+   0           , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
+   CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
+//72  H         73  I         74  J         75  K
+//76  L         77  M         78  N         79  O
+   CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
+   CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
+//80  P         81  Q         82  R         83  S
+//84  T         85  U         86  V         87  W
+   CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
+   CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
+//88  X         89  Y         90  Z         91  [
+//92  \         93  ]         94  ^         95  _
+   CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_RAWDEL ,
+   0           , CHAR_RAWDEL , CHAR_RAWDEL , CHAR_UNDER  ,
+//96  `         97  a         98  b         99  c
+//100  d       101  e        102  f        103  g
+   0           , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
+   CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
+//104  h       105  i        106  j        107  k
+//108  l       109  m        110  n        111  o
+   CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
+   CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
+//112  p       113  q        114  r        115  s
+//116  t       117  u        118  v        119  w
+   CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
+   CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER ,
+//120  x       121  y        122  z        123  {
+//124  |       125  }        126  ~        127 DEL
+   CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_RAWDEL ,
+   CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL , 0
+};
+
+static void InitCharacterInfo() {
+  static bool isInited = false;
+  if (isInited) return;
+  // check the statically-initialized CharInfo table
+  assert(CHAR_HORZ_WS == CharInfo[(int)' ']);
+  assert(CHAR_HORZ_WS == CharInfo[(int)'\t']);
+  assert(CHAR_HORZ_WS == CharInfo[(int)'\f']);
+  assert(CHAR_HORZ_WS == CharInfo[(int)'\v']);
+  assert(CHAR_VERT_WS == CharInfo[(int)'\n']);
+  assert(CHAR_VERT_WS == CharInfo[(int)'\r']);
+  assert(CHAR_UNDER   == CharInfo[(int)'_']);
+  assert(CHAR_PERIOD  == CharInfo[(int)'.']);
+  for (unsigned i = 'a'; i <= 'z'; ++i) {
+    assert(CHAR_LETTER == CharInfo[i]);
+    assert(CHAR_LETTER == CharInfo[i+'A'-'a']);
+  }
+  for (unsigned i = '0'; i <= '9'; ++i)
+    assert(CHAR_NUMBER == CharInfo[i]);
+    
+  isInited = true;
+}
+
+
+/// isIdentifierHead - Return true if this is the first character of an
+/// identifier, which is [a-zA-Z_].
+static inline bool isIdentifierHead(unsigned char c) {
+  return (CharInfo[c] & (CHAR_LETTER|CHAR_UNDER)) ? true : false;
+}
+
+/// isIdentifierBody - Return true if this is the body character of an
+/// identifier, which is [a-zA-Z0-9_].
+static inline bool isIdentifierBody(unsigned char c) {
+  return (CharInfo[c] & (CHAR_LETTER|CHAR_NUMBER|CHAR_UNDER)) ? true : false;
+}
+
+/// isHorizontalWhitespace - Return true if this character is horizontal
+/// whitespace: ' ', '\t', '\f', '\v'.  Note that this returns false for '\0'.
+static inline bool isHorizontalWhitespace(unsigned char c) {
+  return (CharInfo[c] & CHAR_HORZ_WS) ? true : false;
+}
+
+/// isVerticalWhitespace - Return true if this character is vertical
+/// whitespace: '\n', '\r'.  Note that this returns false for '\0'.
+static inline bool isVerticalWhitespace(unsigned char c) {
+  return (CharInfo[c] & CHAR_VERT_WS) ? true : false;
+}
+
+/// isWhitespace - Return true if this character is horizontal or vertical
+/// whitespace: ' ', '\t', '\f', '\v', '\n', '\r'.  Note that this returns false
+/// for '\0'.
+static inline bool isWhitespace(unsigned char c) {
+  return (CharInfo[c] & (CHAR_HORZ_WS|CHAR_VERT_WS)) ? true : false;
+}
+
+/// isNumberBody - Return true if this is the body character of an
+/// preprocessing number, which is [a-zA-Z0-9_.].
+static inline bool isNumberBody(unsigned char c) {
+  return (CharInfo[c] & (CHAR_LETTER|CHAR_NUMBER|CHAR_UNDER|CHAR_PERIOD)) ?
+    true : false;
+}
+
+/// isRawStringDelimBody - Return true if this is the body character of a
+/// raw string delimiter.
+static inline bool isRawStringDelimBody(unsigned char c) {
+  return (CharInfo[c] &
+          (CHAR_LETTER|CHAR_NUMBER|CHAR_UNDER|CHAR_PERIOD|CHAR_RAWDEL)) ?
+    true : false;
+}
+
+
+//===----------------------------------------------------------------------===//
+// Diagnostics forwarding code.
+//===----------------------------------------------------------------------===//
+
+/// GetMappedTokenLoc - If lexing out of a 'mapped buffer', where we pretend the
+/// lexer buffer was all expanded at a single point, perform the mapping.
+/// This is currently only used for _Pragma implementation, so it is the slow
+/// path of the hot getSourceLocation method.  Do not allow it to be inlined.
+static LLVM_ATTRIBUTE_NOINLINE SourceLocation GetMappedTokenLoc(
+    Preprocessor &PP, SourceLocation FileLoc, unsigned CharNo, unsigned TokLen);
+static SourceLocation GetMappedTokenLoc(Preprocessor &PP,
+                                        SourceLocation FileLoc,
+                                        unsigned CharNo, unsigned TokLen) {
+  assert(FileLoc.isMacroID() && "Must be a macro expansion");
+
+  // Otherwise, we're lexing "mapped tokens".  This is used for things like
+  // _Pragma handling.  Combine the expansion location of FileLoc with the
+  // spelling location.
+  SourceManager &SM = PP.getSourceManager();
+
+  // Create a new SLoc which is expanded from Expansion(FileLoc) but whose
+  // characters come from spelling(FileLoc)+Offset.
+  SourceLocation SpellingLoc = SM.getSpellingLoc(FileLoc);
+  SpellingLoc = SpellingLoc.getLocWithOffset(CharNo);
+
+  // Figure out the expansion loc range, which is the range covered by the
+  // original _Pragma(...) sequence.
+  std::pair<SourceLocation,SourceLocation> II =
+    SM.getImmediateExpansionRange(FileLoc);
+
+  return SM.createExpansionLoc(SpellingLoc, II.first, II.second, TokLen);
+}
+
+/// getSourceLocation - Return a source location identifier for the specified
+/// offset in the current file.
+SourceLocation Lexer::getSourceLocation(const char *Loc,
+                                        unsigned TokLen) const {
+  assert(Loc >= BufferStart && Loc <= BufferEnd &&
+         "Location out of range for this buffer!");
+
+  // In the normal case, we're just lexing from a simple file buffer, return
+  // the file id from FileLoc with the offset specified.
+  unsigned CharNo = Loc-BufferStart;
+  if (FileLoc.isFileID())
+    return FileLoc.getLocWithOffset(CharNo);
+
+  // Otherwise, this is the _Pragma lexer case, which pretends that all of the
+  // tokens are lexed from where the _Pragma was defined.
+  assert(PP && "This doesn't work on raw lexers");
+  return GetMappedTokenLoc(*PP, FileLoc, CharNo, TokLen);
+}
+
+/// Diag - Forwarding function for diagnostics.  This translate a source
+/// position in the current buffer into a SourceLocation object for rendering.
+DiagnosticBuilder Lexer::Diag(const char *Loc, unsigned DiagID) const {
+  return PP->Diag(getSourceLocation(Loc), DiagID);
+}
+
+//===----------------------------------------------------------------------===//
+// Trigraph and Escaped Newline Handling Code.
+//===----------------------------------------------------------------------===//
+
+/// GetTrigraphCharForLetter - Given a character that occurs after a ?? pair,
+/// return the decoded trigraph letter it corresponds to, or '\0' if nothing.
+static char GetTrigraphCharForLetter(char Letter) {
+  switch (Letter) {
+  default:   return 0;
+  case '=':  return '#';
+  case ')':  return ']';
+  case '(':  return '[';
+  case '!':  return '|';
+  case '\'': return '^';
+  case '>':  return '}';
+  case '/':  return '\\';
+  case '<':  return '{';
+  case '-':  return '~';
+  }
+}
+
+/// DecodeTrigraphChar - If the specified character is a legal trigraph when
+/// prefixed with ??, emit a trigraph warning.  If trigraphs are enabled,
+/// return the result character.  Finally, emit a warning about trigraph use
+/// whether trigraphs are enabled or not.
+static char DecodeTrigraphChar(const char *CP, Lexer *L) {
+  char Res = GetTrigraphCharForLetter(*CP);
+  if (!Res || !L) return Res;
+
+  if (!L->getLangOpts().Trigraphs) {
+    if (!L->isLexingRawMode())
+      L->Diag(CP-2, diag::trigraph_ignored);
+    return 0;
+  }
+
+  if (!L->isLexingRawMode())
+    L->Diag(CP-2, diag::trigraph_converted) << StringRef(&Res, 1);
+  return Res;
+}
+
+/// getEscapedNewLineSize - Return the size of the specified escaped newline,
+/// or 0 if it is not an escaped newline. P[-1] is known to be a "\" or a
+/// trigraph equivalent on entry to this function.
+unsigned Lexer::getEscapedNewLineSize(const char *Ptr) {
+  unsigned Size = 0;
+  while (isWhitespace(Ptr[Size])) {
+    ++Size;
+
+    if (Ptr[Size-1] != '\n' && Ptr[Size-1] != '\r')
+      continue;
+
+    // If this is a \r\n or \n\r, skip the other half.
+    if ((Ptr[Size] == '\r' || Ptr[Size] == '\n') &&
+        Ptr[Size-1] != Ptr[Size])
+      ++Size;
+
+    return Size;
+  }
+
+  // Not an escaped newline, must be a \t or something else.
+  return 0;
+}
+
+/// SkipEscapedNewLines - If P points to an escaped newline (or a series of
+/// them), skip over them and return the first non-escaped-newline found,
+/// otherwise return P.
+const char *Lexer::SkipEscapedNewLines(const char *P) {
+  while (1) {
+    const char *AfterEscape;
+    if (*P == '\\') {
+      AfterEscape = P+1;
+    } else if (*P == '?') {
+      // If not a trigraph for escape, bail out.
+      if (P[1] != '?' || P[2] != '/')
+        return P;
+      AfterEscape = P+3;
+    } else {
+      return P;
+    }
+
+    unsigned NewLineSize = Lexer::getEscapedNewLineSize(AfterEscape);
+    if (NewLineSize == 0) return P;
+    P = AfterEscape+NewLineSize;
+  }
+}
+
+/// \brief Checks that the given token is the first token that occurs after the
+/// given location (this excludes comments and whitespace). Returns the location
+/// immediately after the specified token. If the token is not found or the
+/// location is inside a macro, the returned source location will be invalid.
+SourceLocation Lexer::findLocationAfterToken(SourceLocation Loc,
+                                        tok::TokenKind TKind,
+                                        const SourceManager &SM,
+                                        const LangOptions &LangOpts,
+                                        bool SkipTrailingWhitespaceAndNewLine) {
+  if (Loc.isMacroID()) {
+    if (!Lexer::isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc))
+      return SourceLocation();
+  }
+  Loc = Lexer::getLocForEndOfToken(Loc, 0, SM, LangOpts);
+
+  // Break down the source location.
+  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
+
+  // Try to load the file buffer.
+  bool InvalidTemp = false;
+  llvm::StringRef File = SM.getBufferData(LocInfo.first, &InvalidTemp);
+  if (InvalidTemp)
+    return SourceLocation();
+
+  const char *TokenBegin = File.data() + LocInfo.second;
+
+  // Lex from the start of the given location.
+  Lexer lexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, File.begin(),
+                                      TokenBegin, File.end());
+  // Find the token.
+  Token Tok;
+  lexer.LexFromRawLexer(Tok);
+  if (Tok.isNot(TKind))
+    return SourceLocation();
+  SourceLocation TokenLoc = Tok.getLocation();
+
+  // Calculate how much whitespace needs to be skipped if any.
+  unsigned NumWhitespaceChars = 0;
+  if (SkipTrailingWhitespaceAndNewLine) {
+    const char *TokenEnd = SM.getCharacterData(TokenLoc) +
+                           Tok.getLength();
+    unsigned char C = *TokenEnd;
+    while (isHorizontalWhitespace(C)) {
+      C = *(++TokenEnd);
+      NumWhitespaceChars++;
+    }
+    if (isVerticalWhitespace(C))
+      NumWhitespaceChars++;
+  }
+
+  return TokenLoc.getLocWithOffset(Tok.getLength() + NumWhitespaceChars);
+}
+
+/// getCharAndSizeSlow - Peek a single 'character' from the specified buffer,
+/// get its size, and return it.  This is tricky in several cases:
+///   1. If currently at the start of a trigraph, we warn about the trigraph,
+///      then either return the trigraph (skipping 3 chars) or the '?',
+///      depending on whether trigraphs are enabled or not.
+///   2. If this is an escaped newline (potentially with whitespace between
+///      the backslash and newline), implicitly skip the newline and return
+///      the char after it.
+///   3. If this is a UCN, return it.  FIXME: C++ UCN's?
+///
+/// This handles the slow/uncommon case of the getCharAndSize method.  Here we
+/// know that we can accumulate into Size, and that we have already incremented
+/// Ptr by Size bytes.
+///
+/// NOTE: When this method is updated, getCharAndSizeSlowNoWarn (below) should
+/// be updated to match.
+///
+char Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size,
+                               Token *Tok) {
+  // If we have a slash, look for an escaped newline.
+  if (Ptr[0] == '\\') {
+    ++Size;
+    ++Ptr;
+Slash:
+    // Common case, backslash-char where the char is not whitespace.
+    if (!isWhitespace(Ptr[0])) return '\\';
+
+    // See if we have optional whitespace characters between the slash and
+    // newline.
+    if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) {
+      // Remember that this token needs to be cleaned.
+      if (Tok) Tok->setFlag(Token::NeedsCleaning);
+
+      // Warn if there was whitespace between the backslash and newline.
+      if (Ptr[0] != '\n' && Ptr[0] != '\r' && Tok && !isLexingRawMode())
+        Diag(Ptr, diag::backslash_newline_space);
+
+      // Found backslash<whitespace><newline>.  Parse the char after it.
+      Size += EscapedNewLineSize;
+      Ptr  += EscapedNewLineSize;
+
+      // If the char that we finally got was a \n, then we must have had
+      // something like \<newline><newline>.  We don't want to consume the
+      // second newline.
+      if (*Ptr == '\n' || *Ptr == '\r' || *Ptr == '\0')
+        return ' ';
+
+      // Use slow version to accumulate a correct size field.
+      return getCharAndSizeSlow(Ptr, Size, Tok);
+    }
+
+    // Otherwise, this is not an escaped newline, just return the slash.
+    return '\\';
+  }
+
+  // If this is a trigraph, process it.
+  if (Ptr[0] == '?' && Ptr[1] == '?') {
+    // If this is actually a legal trigraph (not something like "??x"), emit
+    // a trigraph warning.  If so, and if trigraphs are enabled, return it.
+    if (char C = DecodeTrigraphChar(Ptr+2, Tok ? this : 0)) {
+      // Remember that this token needs to be cleaned.
+      if (Tok) Tok->setFlag(Token::NeedsCleaning);
+
+      Ptr += 3;
+      Size += 3;
+      if (C == '\\') goto Slash;
+      return C;
+    }
+  }
+
+  // If this is neither, return a single character.
+  ++Size;
+  return *Ptr;
+}
+
+
+/// getCharAndSizeSlowNoWarn - Handle the slow/uncommon case of the
+/// getCharAndSizeNoWarn method.  Here we know that we can accumulate into Size,
+/// and that we have already incremented Ptr by Size bytes.
+///
+/// NOTE: When this method is updated, getCharAndSizeSlow (above) should
+/// be updated to match.
+char Lexer::getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size,
+                                     const LangOptions &LangOpts) {
+  // If we have a slash, look for an escaped newline.
+  if (Ptr[0] == '\\') {
+    ++Size;
+    ++Ptr;
+Slash:
+    // Common case, backslash-char where the char is not whitespace.
+    if (!isWhitespace(Ptr[0])) return '\\';
+
+    // See if we have optional whitespace characters followed by a newline.
+    if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) {
+      // Found backslash<whitespace><newline>.  Parse the char after it.
+      Size += EscapedNewLineSize;
+      Ptr  += EscapedNewLineSize;
+
+      // If the char that we finally got was a \n, then we must have had
+      // something like \<newline><newline>.  We don't want to consume the
+      // second newline.
+      if (*Ptr == '\n' || *Ptr == '\r' || *Ptr == '\0')
+        return ' ';
+
+      // Use slow version to accumulate a correct size field.
+      return getCharAndSizeSlowNoWarn(Ptr, Size, LangOpts);
+    }
+
+    // Otherwise, this is not an escaped newline, just return the slash.
+    return '\\';
+  }
+
+  // If this is a trigraph, process it.
+  if (LangOpts.Trigraphs && Ptr[0] == '?' && Ptr[1] == '?') {
+    // If this is actually a legal trigraph (not something like "??x"), return
+    // it.
+    if (char C = GetTrigraphCharForLetter(Ptr[2])) {
+      Ptr += 3;
+      Size += 3;
+      if (C == '\\') goto Slash;
+      return C;
+    }
+  }
+
+  // If this is neither, return a single character.
+  ++Size;
+  return *Ptr;
+}
+
+//===----------------------------------------------------------------------===//
+// Helper methods for lexing.
+//===----------------------------------------------------------------------===//
+
+/// \brief Routine that indiscriminately skips bytes in the source file.
+void Lexer::SkipBytes(unsigned Bytes, bool StartOfLine) {
+  BufferPtr += Bytes;
+  if (BufferPtr > BufferEnd)
+    BufferPtr = BufferEnd;
+  IsAtStartOfLine = StartOfLine;
+}
+
+void Lexer::LexIdentifier(Token &Result, const char *CurPtr) {
+  // Match [_A-Za-z0-9]*, we have already matched [_A-Za-z$]
+  unsigned Size;
+  unsigned char C = *CurPtr++;
+  while (isIdentifierBody(C))
+    C = *CurPtr++;
+
+  --CurPtr;   // Back up over the skipped character.
+
+  // Fast path, no $,\,? in identifier found.  '\' might be an escaped newline
+  // or UCN, and ? might be a trigraph for '\', an escaped newline or UCN.
+  // FIXME: UCNs.
+  //
+  // TODO: Could merge these checks into a CharInfo flag to make the comparison
+  // cheaper
+  if (C != '\\' && C != '?' && (C != '$' || !LangOpts.DollarIdents)) {
+FinishIdentifier:
+    const char *IdStart = BufferPtr;
+    FormTokenWithChars(Result, CurPtr, tok::raw_identifier);
+    Result.setRawIdentifierData(IdStart);
+
+    // If we are in raw mode, return this identifier raw.  There is no need to
+    // look up identifier information or attempt to macro expand it.
+    if (LexingRawMode)
+      return;
+
+    // Fill in Result.IdentifierInfo and update the token kind,
+    // looking up the identifier in the identifier table.
+    IdentifierInfo *II = PP->LookUpIdentifierInfo(Result);
+
+    // Finally, now that we know we have an identifier, pass this off to the
+    // preprocessor, which may macro expand it or something.
+    if (II->isHandleIdentifierCase())
+      PP->HandleIdentifier(Result);
+    
+    return;
+  }
+
+  // Otherwise, $,\,? in identifier found.  Enter slower path.
+
+  C = getCharAndSize(CurPtr, Size);
+  while (1) {
+    if (C == '$') {
+      // If we hit a $ and they are not supported in identifiers, we are done.
+      if (!LangOpts.DollarIdents) goto FinishIdentifier;
+
+      // Otherwise, emit a diagnostic and continue.
+      if (!isLexingRawMode())
+        Diag(CurPtr, diag::ext_dollar_in_identifier);
+      CurPtr = ConsumeChar(CurPtr, Size, Result);
+      C = getCharAndSize(CurPtr, Size);
+      continue;
+    } else if (!isIdentifierBody(C)) { // FIXME: UCNs.
+      // Found end of identifier.
+      goto FinishIdentifier;
+    }
+
+    // Otherwise, this character is good, consume it.
+    CurPtr = ConsumeChar(CurPtr, Size, Result);
+
+    C = getCharAndSize(CurPtr, Size);
+    while (isIdentifierBody(C)) { // FIXME: UCNs.
+      CurPtr = ConsumeChar(CurPtr, Size, Result);
+      C = getCharAndSize(CurPtr, Size);
+    }
+  }
+}
+
+/// isHexaLiteral - Return true if Start points to a hex constant.
+/// in microsoft mode (where this is supposed to be several different tokens).
+static bool isHexaLiteral(const char *Start, const LangOptions &LangOpts) {
+  unsigned Size;
+  char C1 = Lexer::getCharAndSizeNoWarn(Start, Size, LangOpts);
+  if (C1 != '0')
+    return false;
+  char C2 = Lexer::getCharAndSizeNoWarn(Start + Size, Size, LangOpts);
+  return (C2 == 'x' || C2 == 'X');
+}
+
+/// LexNumericConstant - Lex the remainder of a integer or floating point
+/// constant. From[-1] is the first character lexed.  Return the end of the
+/// constant.
+void Lexer::LexNumericConstant(Token &Result, const char *CurPtr) {
+  unsigned Size;
+  char C = getCharAndSize(CurPtr, Size);
+  char PrevCh = 0;
+  while (isNumberBody(C)) { // FIXME: UCNs.
+    CurPtr = ConsumeChar(CurPtr, Size, Result);
+    PrevCh = C;
+    C = getCharAndSize(CurPtr, Size);
+  }
+
+  // If we fell out, check for a sign, due to 1e+12.  If we have one, continue.
+  if ((C == '-' || C == '+') && (PrevCh == 'E' || PrevCh == 'e')) {
+    // If we are in Microsoft mode, don't continue if the constant is hex.
+    // For example, MSVC will accept the following as 3 tokens: 0x1234567e+1
+    if (!LangOpts.MicrosoftExt || !isHexaLiteral(BufferPtr, LangOpts))
+      return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result));
+  }
+
+  // If we have a hex FP constant, continue.
+  if ((C == '-' || C == '+') && (PrevCh == 'P' || PrevCh == 'p'))
+    return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result));
+
+  // Update the location of token as well as BufferPtr.
+  const char *TokStart = BufferPtr;
+  FormTokenWithChars(Result, CurPtr, tok::numeric_constant);
+  Result.setLiteralData(TokStart);
+}
+
+/// LexUDSuffix - Lex the ud-suffix production for user-defined literal suffixes
+/// in C++11, or warn on a ud-suffix in C++98.
+const char *Lexer::LexUDSuffix(Token &Result, const char *CurPtr) {
+  assert(getLangOpts().CPlusPlus);
+
+  // Maximally munch an identifier. FIXME: UCNs.
+  unsigned Size;
+  char C = getCharAndSize(CurPtr, Size);
+  if (isIdentifierHead(C)) {
+    if (!getLangOpts().CPlusPlus0x) {
+      if (!isLexingRawMode())
+        Diag(CurPtr,
+             C == '_' ? diag::warn_cxx11_compat_user_defined_literal
+                      : diag::warn_cxx11_compat_reserved_user_defined_literal)
+          << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " ");
+      return CurPtr;
+    }
+
+    // C++11 [lex.ext]p10, [usrlit.suffix]p1: A program containing a ud-suffix
+    // that does not start with an underscore is ill-formed. As a conforming
+    // extension, we treat all such suffixes as if they had whitespace before
+    // them.
+    if (C != '_') {
+      if (!isLexingRawMode())
+        Diag(CurPtr, getLangOpts().MicrosoftMode ? 
+            diag::ext_ms_reserved_user_defined_literal :
+            diag::ext_reserved_user_defined_literal)
+          << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " ");
+      return CurPtr;
+    }
+
+    Result.setFlag(Token::HasUDSuffix);
+    do {
+      CurPtr = ConsumeChar(CurPtr, Size, Result);
+      C = getCharAndSize(CurPtr, Size);
+    } while (isIdentifierBody(C));
+  }
+  return CurPtr;
+}
+
+/// LexStringLiteral - Lex the remainder of a string literal, after having lexed
+/// either " or L" or u8" or u" or U".
+void Lexer::LexStringLiteral(Token &Result, const char *CurPtr,
+                             tok::TokenKind Kind) {
+  const char *NulCharacter = 0; // Does this string contain the \0 character?
+
+  if (!isLexingRawMode() &&
+      (Kind == tok::utf8_string_literal ||
+       Kind == tok::utf16_string_literal ||
+       Kind == tok::utf32_string_literal))
+    Diag(BufferPtr, diag::warn_cxx98_compat_unicode_literal);
+
+  char C = getAndAdvanceChar(CurPtr, Result);
+  while (C != '"') {
+    // Skip escaped characters.  Escaped newlines will already be processed by
+    // getAndAdvanceChar.
+    if (C == '\\')
+      C = getAndAdvanceChar(CurPtr, Result);
+    
+    if (C == '\n' || C == '\r' ||             // Newline.
+        (C == 0 && CurPtr-1 == BufferEnd)) {  // End of file.
+      if (!isLexingRawMode() && !LangOpts.AsmPreprocessor)
+        Diag(BufferPtr, diag::warn_unterminated_string);
+      FormTokenWithChars(Result, CurPtr-1, tok::unknown);
+      return;
+    }
+    
+    if (C == 0) {
+      if (isCodeCompletionPoint(CurPtr-1)) {
+        PP->CodeCompleteNaturalLanguage();
+        FormTokenWithChars(Result, CurPtr-1, tok::unknown);
+        return cutOffLexing();
+      }
+
+      NulCharacter = CurPtr-1;
+    }
+    C = getAndAdvanceChar(CurPtr, Result);
+  }
+
+  // If we are in C++11, lex the optional ud-suffix.
+  if (getLangOpts().CPlusPlus)
+    CurPtr = LexUDSuffix(Result, CurPtr);
+
+  // If a nul character existed in the string, warn about it.
+  if (NulCharacter && !isLexingRawMode())
+    Diag(NulCharacter, diag::null_in_string);
+
+  // Update the location of the token as well as the BufferPtr instance var.
+  const char *TokStart = BufferPtr;
+  FormTokenWithChars(Result, CurPtr, Kind);
+  Result.setLiteralData(TokStart);
+}
+
+/// LexRawStringLiteral - Lex the remainder of a raw string literal, after
+/// having lexed R", LR", u8R", uR", or UR".
+void Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr,
+                                tok::TokenKind Kind) {
+  // This function doesn't use getAndAdvanceChar because C++0x [lex.pptoken]p3:
+  //  Between the initial and final double quote characters of the raw string,
+  //  any transformations performed in phases 1 and 2 (trigraphs,
+  //  universal-character-names, and line splicing) are reverted.
+
+  if (!isLexingRawMode())
+    Diag(BufferPtr, diag::warn_cxx98_compat_raw_string_literal);
+
+  unsigned PrefixLen = 0;
+
+  while (PrefixLen != 16 && isRawStringDelimBody(CurPtr[PrefixLen]))
+    ++PrefixLen;
+
+  // If the last character was not a '(', then we didn't lex a valid delimiter.
+  if (CurPtr[PrefixLen] != '(') {
+    if (!isLexingRawMode()) {
+      const char *PrefixEnd = &CurPtr[PrefixLen];
+      if (PrefixLen == 16) {
+        Diag(PrefixEnd, diag::err_raw_delim_too_long);
+      } else {
+        Diag(PrefixEnd, diag::err_invalid_char_raw_delim)
+          << StringRef(PrefixEnd, 1);
+      }
+    }
+
+    // Search for the next '"' in hopes of salvaging the lexer. Unfortunately,
+    // it's possible the '"' was intended to be part of the raw string, but
+    // there's not much we can do about that.
+    while (1) {
+      char C = *CurPtr++;
+
+      if (C == '"')
+        break;
+      if (C == 0 && CurPtr-1 == BufferEnd) {
+        --CurPtr;
+        break;
+      }
+    }
+
+    FormTokenWithChars(Result, CurPtr, tok::unknown);
+    return;
+  }
+
+  // Save prefix and move CurPtr past it
+  const char *Prefix = CurPtr;
+  CurPtr += PrefixLen + 1; // skip over prefix and '('
+
+  while (1) {
+    char C = *CurPtr++;
+
+    if (C == ')') {
+      // Check for prefix match and closing quote.
+      if (strncmp(CurPtr, Prefix, PrefixLen) == 0 && CurPtr[PrefixLen] == '"') {
+        CurPtr += PrefixLen + 1; // skip over prefix and '"'
+        break;
+      }
+    } else if (C == 0 && CurPtr-1 == BufferEnd) { // End of file.
+      if (!isLexingRawMode())
+        Diag(BufferPtr, diag::err_unterminated_raw_string)
+          << StringRef(Prefix, PrefixLen);
+      FormTokenWithChars(Result, CurPtr-1, tok::unknown);
+      return;
+    }
+  }
+
+  // If we are in C++11, lex the optional ud-suffix.
+  if (getLangOpts().CPlusPlus)
+    CurPtr = LexUDSuffix(Result, CurPtr);
+
+  // Update the location of token as well as BufferPtr.
+  const char *TokStart = BufferPtr;
+  FormTokenWithChars(Result, CurPtr, Kind);
+  Result.setLiteralData(TokStart);
+}
+
+/// LexAngledStringLiteral - Lex the remainder of an angled string literal,
+/// after having lexed the '<' character.  This is used for #include filenames.
+void Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) {
+  const char *NulCharacter = 0; // Does this string contain the \0 character?
+  const char *AfterLessPos = CurPtr;
+  char C = getAndAdvanceChar(CurPtr, Result);
+  while (C != '>') {
+    // Skip escaped characters.
+    if (C == '\\') {
+      // Skip the escaped character.
+      C = getAndAdvanceChar(CurPtr, Result);
+    } else if (C == '\n' || C == '\r' ||             // Newline.
+               (C == 0 && (CurPtr-1 == BufferEnd ||  // End of file.
+                           isCodeCompletionPoint(CurPtr-1)))) {
+      // If the filename is unterminated, then it must just be a lone <
+      // character.  Return this as such.
+      FormTokenWithChars(Result, AfterLessPos, tok::less);
+      return;
+    } else if (C == 0) {
+      NulCharacter = CurPtr-1;
+    }
+    C = getAndAdvanceChar(CurPtr, Result);
+  }
+
+  // If a nul character existed in the string, warn about it.
+  if (NulCharacter && !isLexingRawMode())
+    Diag(NulCharacter, diag::null_in_string);
+
+  // Update the location of token as well as BufferPtr.
+  const char *TokStart = BufferPtr;
+  FormTokenWithChars(Result, CurPtr, tok::angle_string_literal);
+  Result.setLiteralData(TokStart);
+}
+
+
+/// LexCharConstant - Lex the remainder of a character constant, after having
+/// lexed either ' or L' or u' or U'.
+void Lexer::LexCharConstant(Token &Result, const char *CurPtr,
+                            tok::TokenKind Kind) {
+  const char *NulCharacter = 0; // Does this character contain the \0 character?
+
+  if (!isLexingRawMode() &&
+      (Kind == tok::utf16_char_constant || Kind == tok::utf32_char_constant))
+    Diag(BufferPtr, diag::warn_cxx98_compat_unicode_literal);
+
+  char C = getAndAdvanceChar(CurPtr, Result);
+  if (C == '\'') {
+    if (!isLexingRawMode() && !LangOpts.AsmPreprocessor)
+      Diag(BufferPtr, diag::err_empty_character);
+    FormTokenWithChars(Result, CurPtr, tok::unknown);
+    return;
+  }
+
+  while (C != '\'') {
+    // Skip escaped characters.
+    if (C == '\\') {
+      // Skip the escaped character.
+      // FIXME: UCN's
+      C = getAndAdvanceChar(CurPtr, Result);
+    } else if (C == '\n' || C == '\r' ||             // Newline.
+               (C == 0 && CurPtr-1 == BufferEnd)) {  // End of file.
+      if (!isLexingRawMode() && !LangOpts.AsmPreprocessor)
+        Diag(BufferPtr, diag::warn_unterminated_char);
+      FormTokenWithChars(Result, CurPtr-1, tok::unknown);
+      return;
+    } else if (C == 0) {
+      if (isCodeCompletionPoint(CurPtr-1)) {
+        PP->CodeCompleteNaturalLanguage();
+        FormTokenWithChars(Result, CurPtr-1, tok::unknown);
+        return cutOffLexing();
+      }
+
+      NulCharacter = CurPtr-1;
+    }
+    C = getAndAdvanceChar(CurPtr, Result);
+  }
+
+  // If we are in C++11, lex the optional ud-suffix.
+  if (getLangOpts().CPlusPlus)
+    CurPtr = LexUDSuffix(Result, CurPtr);
+
+  // If a nul character existed in the character, warn about it.
+  if (NulCharacter && !isLexingRawMode())
+    Diag(NulCharacter, diag::null_in_char);
+
+  // Update the location of token as well as BufferPtr.
+  const char *TokStart = BufferPtr;
+  FormTokenWithChars(Result, CurPtr, Kind);
+  Result.setLiteralData(TokStart);
+}
+
+/// SkipWhitespace - Efficiently skip over a series of whitespace characters.
+/// Update BufferPtr to point to the next non-whitespace character and return.
+///
+/// This method forms a token and returns true if KeepWhitespaceMode is enabled.
+///
+bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr) {
+  // Whitespace - Skip it, then return the token after the whitespace.
+  unsigned char Char = *CurPtr;  // Skip consequtive spaces efficiently.
+  while (1) {
+    // Skip horizontal whitespace very aggressively.
+    while (isHorizontalWhitespace(Char))
+      Char = *++CurPtr;
+
+    // Otherwise if we have something other than whitespace, we're done.
+    if (Char != '\n' && Char != '\r')
+      break;
+
+    if (ParsingPreprocessorDirective) {
+      // End of preprocessor directive line, let LexTokenInternal handle this.
+      BufferPtr = CurPtr;
+      return false;
+    }
+
+    // ok, but handle newline.
+    // The returned token is at the start of the line.
+    Result.setFlag(Token::StartOfLine);
+    // No leading whitespace seen so far.
+    Result.clearFlag(Token::LeadingSpace);
+    Char = *++CurPtr;
+  }
+
+  // If this isn't immediately after a newline, there is leading space.
+  char PrevChar = CurPtr[-1];
+  if (PrevChar != '\n' && PrevChar != '\r')
+    Result.setFlag(Token::LeadingSpace);
+
+  // If the client wants us to return whitespace, return it now.
+  if (isKeepWhitespaceMode()) {
+    FormTokenWithChars(Result, CurPtr, tok::unknown);
+    return true;
+  }
+
+  BufferPtr = CurPtr;
+  return false;
+}
+
+// SkipBCPLComment - We have just read the // characters from input.  Skip until
+// we find the newline character thats terminate the comment.  Then update
+/// BufferPtr and return.
+///
+/// If we're in KeepCommentMode or any CommentHandler has inserted
+/// some tokens, this will store the first token and return true.
+bool Lexer::SkipBCPLComment(Token &Result, const char *CurPtr) {
+  // If BCPL comments aren't explicitly enabled for this language, emit an
+  // extension warning.
+  if (!LangOpts.BCPLComment && !isLexingRawMode()) {
+    Diag(BufferPtr, diag::ext_bcpl_comment);
+
+    // Mark them enabled so we only emit one warning for this translation
+    // unit.
+    LangOpts.BCPLComment = true;
+  }
+
+  // Scan over the body of the comment.  The common case, when scanning, is that
+  // the comment contains normal ascii characters with nothing interesting in
+  // them.  As such, optimize for this case with the inner loop.
+  char C;
+  do {
+    C = *CurPtr;
+    // Skip over characters in the fast loop.
+    while (C != 0 &&                // Potentially EOF.
+           C != '\n' && C != '\r')  // Newline or DOS-style newline.
+      C = *++CurPtr;
+
+    const char *NextLine = CurPtr;
+    if (C != 0) {
+      // We found a newline, see if it's escaped.
+      const char *EscapePtr = CurPtr-1;
+      while (isHorizontalWhitespace(*EscapePtr)) // Skip whitespace.
+        --EscapePtr;
+
+      if (*EscapePtr == '\\') // Escaped newline.
+        CurPtr = EscapePtr;
+      else if (EscapePtr[0] == '/' && EscapePtr[-1] == '?' &&
+               EscapePtr[-2] == '?') // Trigraph-escaped newline.
+        CurPtr = EscapePtr-2;
+      else
+        break; // This is a newline, we're done.
+
+      C = *CurPtr;
+    }
+
+    // Otherwise, this is a hard case.  Fall back on getAndAdvanceChar to
+    // properly decode the character.  Read it in raw mode to avoid emitting
+    // diagnostics about things like trigraphs.  If we see an escaped newline,
+    // we'll handle it below.
+    const char *OldPtr = CurPtr;
+    bool OldRawMode = isLexingRawMode();
+    LexingRawMode = true;
+    C = getAndAdvanceChar(CurPtr, Result);
+    LexingRawMode = OldRawMode;
+
+    // If we only read only one character, then no special handling is needed.
+    // We're done and can skip forward to the newline.
+    if (C != 0 && CurPtr == OldPtr+1) {
+      CurPtr = NextLine;
+      break;
+    }
+
+    // If we read multiple characters, and one of those characters was a \r or
+    // \n, then we had an escaped newline within the comment.  Emit diagnostic
+    // unless the next line is also a // comment.
+    if (CurPtr != OldPtr+1 && C != '/' && CurPtr[0] != '/') {
+      for (; OldPtr != CurPtr; ++OldPtr)
+        if (OldPtr[0] == '\n' || OldPtr[0] == '\r') {
+          // Okay, we found a // comment that ends in a newline, if the next
+          // line is also a // comment, but has spaces, don't emit a diagnostic.
+          if (isWhitespace(C)) {
+            const char *ForwardPtr = CurPtr;
+            while (isWhitespace(*ForwardPtr))  // Skip whitespace.
+              ++ForwardPtr;
+            if (ForwardPtr[0] == '/' && ForwardPtr[1] == '/')
+              break;
+          }
+
+          if (!isLexingRawMode())
+            Diag(OldPtr-1, diag::ext_multi_line_bcpl_comment);
+          break;
+        }
+    }
+
+    if (CurPtr == BufferEnd+1) { 
+      --CurPtr; 
+      break; 
+    }
+
+    if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) {
+      PP->CodeCompleteNaturalLanguage();
+      cutOffLexing();
+      return false;
+    }
+
+  } while (C != '\n' && C != '\r');
+
+  // Found but did not consume the newline.  Notify comment handlers about the
+  // comment unless we're in a #if 0 block.
+  if (PP && !isLexingRawMode() &&
+      PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr),
+                                            getSourceLocation(CurPtr)))) {
+    BufferPtr = CurPtr;
+    return true; // A token has to be returned.
+  }
+
+  // If we are returning comments as tokens, return this comment as a token.
+  if (inKeepCommentMode())
+    return SaveBCPLComment(Result, CurPtr);
+
+  // If we are inside a preprocessor directive and we see the end of line,
+  // return immediately, so that the lexer can return this as an EOD token.
+  if (ParsingPreprocessorDirective || CurPtr == BufferEnd) {
+    BufferPtr = CurPtr;
+    return false;
+  }
+
+  // Otherwise, eat the \n character.  We don't care if this is a \n\r or
+  // \r\n sequence.  This is an efficiency hack (because we know the \n can't
+  // contribute to another token), it isn't needed for correctness.  Note that
+  // this is ok even in KeepWhitespaceMode, because we would have returned the
+  /// comment above in that mode.
+  ++CurPtr;
+
+  // The next returned token is at the start of the line.
+  Result.setFlag(Token::StartOfLine);
+  // No leading whitespace seen so far.
+  Result.clearFlag(Token::LeadingSpace);
+  BufferPtr = CurPtr;
+  return false;
+}
+
+/// SaveBCPLComment - If in save-comment mode, package up this BCPL comment in
+/// an appropriate way and return it.
+bool Lexer::SaveBCPLComment(Token &Result, const char *CurPtr) {
+  // If we're not in a preprocessor directive, just return the // comment
+  // directly.
+  FormTokenWithChars(Result, CurPtr, tok::comment);
+
+  if (!ParsingPreprocessorDirective)
+    return true;
+
+  // If this BCPL-style comment is in a macro definition, transmogrify it into
+  // a C-style block comment.
+  bool Invalid = false;
+  std::string Spelling = PP->getSpelling(Result, &Invalid);
+  if (Invalid)
+    return true;
+  
+  assert(Spelling[0] == '/' && Spelling[1] == '/' && "Not bcpl comment?");
+  Spelling[1] = '*';   // Change prefix to "/*".
+  Spelling += "*/";    // add suffix.
+
+  Result.setKind(tok::comment);
+  PP->CreateString(&Spelling[0], Spelling.size(), Result,
+                   Result.getLocation(), Result.getLocation());
+  return true;
+}
+
+/// isBlockCommentEndOfEscapedNewLine - Return true if the specified newline
+/// character (either \n or \r) is part of an escaped newline sequence.  Issue a
+/// diagnostic if so.  We know that the newline is inside of a block comment.
+static bool isEndOfBlockCommentWithEscapedNewLine(const char *CurPtr,
+                                                  Lexer *L) {
+  assert(CurPtr[0] == '\n' || CurPtr[0] == '\r');
+
+  // Back up off the newline.
+  --CurPtr;
+
+  // If this is a two-character newline sequence, skip the other character.
+  if (CurPtr[0] == '\n' || CurPtr[0] == '\r') {
+    // \n\n or \r\r -> not escaped newline.
+    if (CurPtr[0] == CurPtr[1])
+      return false;
+    // \n\r or \r\n -> skip the newline.
+    --CurPtr;
+  }
+
+  // If we have horizontal whitespace, skip over it.  We allow whitespace
+  // between the slash and newline.
+  bool HasSpace = false;
+  while (isHorizontalWhitespace(*CurPtr) || *CurPtr == 0) {
+    --CurPtr;
+    HasSpace = true;
+  }
+
+  // If we have a slash, we know this is an escaped newline.
+  if (*CurPtr == '\\') {
+    if (CurPtr[-1] != '*') return false;
+  } else {
+    // It isn't a slash, is it the ?? / trigraph?
+    if (CurPtr[0] != '/' || CurPtr[-1] != '?' || CurPtr[-2] != '?' ||
+        CurPtr[-3] != '*')
+      return false;
+
+    // This is the trigraph ending the comment.  Emit a stern warning!
+    CurPtr -= 2;
+
+    // If no trigraphs are enabled, warn that we ignored this trigraph and
+    // ignore this * character.
+    if (!L->getLangOpts().Trigraphs) {
+      if (!L->isLexingRawMode())
+        L->Diag(CurPtr, diag::trigraph_ignored_block_comment);
+      return false;
+    }
+    if (!L->isLexingRawMode())
+      L->Diag(CurPtr, diag::trigraph_ends_block_comment);
+  }
+
+  // Warn about having an escaped newline between the */ characters.
+  if (!L->isLexingRawMode())
+    L->Diag(CurPtr, diag::escaped_newline_block_comment_end);
+
+  // If there was space between the backslash and newline, warn about it.
+  if (HasSpace && !L->isLexingRawMode())
+    L->Diag(CurPtr, diag::backslash_newline_space);
+
+  return true;
+}
+
+#ifdef __SSE2__
+#include <emmintrin.h>
+#elif __ALTIVEC__
+#include <altivec.h>
+#undef bool
+#endif
+
+/// SkipBlockComment - We have just read the /* characters from input.  Read
+/// until we find the */ characters that terminate the comment.  Note that we
+/// don't bother decoding trigraphs or escaped newlines in block comments,
+/// because they cannot cause the comment to end.  The only thing that can
+/// happen is the comment could end with an escaped newline between the */ end
+/// of comment.
+///
+/// If we're in KeepCommentMode or any CommentHandler has inserted
+/// some tokens, this will store the first token and return true.
+bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) {
+  // Scan one character past where we should, looking for a '/' character.  Once
+  // we find it, check to see if it was preceded by a *.  This common
+  // optimization helps people who like to put a lot of * characters in their
+  // comments.
+
+  // The first character we get with newlines and trigraphs skipped to handle
+  // the degenerate /*/ case below correctly if the * has an escaped newline
+  // after it.
+  unsigned CharSize;
+  unsigned char C = getCharAndSize(CurPtr, CharSize);
+  CurPtr += CharSize;
+  if (C == 0 && CurPtr == BufferEnd+1) {
+    if (!isLexingRawMode())
+      Diag(BufferPtr, diag::err_unterminated_block_comment);
+    --CurPtr;
+
+    // KeepWhitespaceMode should return this broken comment as a token.  Since
+    // it isn't a well formed comment, just return it as an 'unknown' token.
+    if (isKeepWhitespaceMode()) {
+      FormTokenWithChars(Result, CurPtr, tok::unknown);
+      return true;
+    }
+
+    BufferPtr = CurPtr;
+    return false;
+  }
+
+  // Check to see if the first character after the '/*' is another /.  If so,
+  // then this slash does not end the block comment, it is part of it.
+  if (C == '/')
+    C = *CurPtr++;
+
+  while (1) {
+    // Skip over all non-interesting characters until we find end of buffer or a
+    // (probably ending) '/' character.
+    if (CurPtr + 24 < BufferEnd &&
+        // If there is a code-completion point avoid the fast scan because it
+        // doesn't check for '\0'.
+        !(PP && PP->getCodeCompletionFileLoc() == FileLoc)) {
+      // While not aligned to a 16-byte boundary.
+      while (C != '/' && ((intptr_t)CurPtr & 0x0F) != 0)
+        C = *CurPtr++;
+
+      if (C == '/') goto FoundSlash;
+
+#ifdef __SSE2__
+      __m128i Slashes = _mm_set1_epi8('/');
+      while (CurPtr+16 <= BufferEnd) {
+        int cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(*(__m128i*)CurPtr, Slashes));
+        if (cmp != 0) {
+          // Adjust the pointer to point directly after the first slash. It's
+          // not necessary to set C here, it will be overwritten at the end of
+          // the outer loop.
+          CurPtr += llvm::CountTrailingZeros_32(cmp) + 1;
+          goto FoundSlash;
+        }
+        CurPtr += 16;
+      }
+#elif __ALTIVEC__
+      __vector unsigned char Slashes = {
+        '/', '/', '/', '/',  '/', '/', '/', '/',
+        '/', '/', '/', '/',  '/', '/', '/', '/'
+      };
+      while (CurPtr+16 <= BufferEnd &&
+             !vec_any_eq(*(vector unsigned char*)CurPtr, Slashes))
+        CurPtr += 16;
+#else
+      // Scan for '/' quickly.  Many block comments are very large.
+      while (CurPtr[0] != '/' &&
+             CurPtr[1] != '/' &&
+             CurPtr[2] != '/' &&
+             CurPtr[3] != '/' &&
+             CurPtr+4 < BufferEnd) {
+        CurPtr += 4;
+      }
+#endif
+
+      // It has to be one of the bytes scanned, increment to it and read one.
+      C = *CurPtr++;
+    }
+
+    // Loop to scan the remainder.
+    while (C != '/' && C != '\0')
+      C = *CurPtr++;
+
+    if (C == '/') {
+  FoundSlash:
+      if (CurPtr[-2] == '*')  // We found the final */.  We're done!
+        break;
+
+      if ((CurPtr[-2] == '\n' || CurPtr[-2] == '\r')) {
+        if (isEndOfBlockCommentWithEscapedNewLine(CurPtr-2, this)) {
+          // We found the final */, though it had an escaped newline between the
+          // * and /.  We're done!
+          break;
+        }
+      }
+      if (CurPtr[0] == '*' && CurPtr[1] != '/') {
+        // If this is a /* inside of the comment, emit a warning.  Don't do this
+        // if this is a /*/, which will end the comment.  This misses cases with
+        // embedded escaped newlines, but oh well.
+        if (!isLexingRawMode())
+          Diag(CurPtr-1, diag::warn_nested_block_comment);
+      }
+    } else if (C == 0 && CurPtr == BufferEnd+1) {
+      if (!isLexingRawMode())
+        Diag(BufferPtr, diag::err_unterminated_block_comment);
+      // Note: the user probably forgot a */.  We could continue immediately
+      // after the /*, but this would involve lexing a lot of what really is the
+      // comment, which surely would confuse the parser.
+      --CurPtr;
+
+      // KeepWhitespaceMode should return this broken comment as a token.  Since
+      // it isn't a well formed comment, just return it as an 'unknown' token.
+      if (isKeepWhitespaceMode()) {
+        FormTokenWithChars(Result, CurPtr, tok::unknown);
+        return true;
+      }
+
+      BufferPtr = CurPtr;
+      return false;
+    } else if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) {
+      PP->CodeCompleteNaturalLanguage();
+      cutOffLexing();
+      return false;
+    }
+
+    C = *CurPtr++;
+  }
+
+  // Notify comment handlers about the comment unless we're in a #if 0 block.
+  if (PP && !isLexingRawMode() &&
+      PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr),
+                                            getSourceLocation(CurPtr)))) {
+    BufferPtr = CurPtr;
+    return true; // A token has to be returned.
+  }
+
+  // If we are returning comments as tokens, return this comment as a token.
+  if (inKeepCommentMode()) {
+    FormTokenWithChars(Result, CurPtr, tok::comment);
+    return true;
+  }
+
+  // It is common for the tokens immediately after a /**/ comment to be
+  // whitespace.  Instead of going through the big switch, handle it
+  // efficiently now.  This is safe even in KeepWhitespaceMode because we would
+  // have already returned above with the comment as a token.
+  if (isHorizontalWhitespace(*CurPtr)) {
+    Result.setFlag(Token::LeadingSpace);
+    SkipWhitespace(Result, CurPtr+1);
+    return false;
+  }
+
+  // Otherwise, just return so that the next character will be lexed as a token.
+  BufferPtr = CurPtr;
+  Result.setFlag(Token::LeadingSpace);
+  return false;
+}
+
+//===----------------------------------------------------------------------===//
+// Primary Lexing Entry Points
+//===----------------------------------------------------------------------===//
+
+/// ReadToEndOfLine - Read the rest of the current preprocessor line as an
+/// uninterpreted string.  This switches the lexer out of directive mode.
+std::string Lexer::ReadToEndOfLine() {
+  assert(ParsingPreprocessorDirective && ParsingFilename == false &&
+         "Must be in a preprocessing directive!");
+  std::string Result;
+  Token Tmp;
+
+  // CurPtr - Cache BufferPtr in an automatic variable.
+  const char *CurPtr = BufferPtr;
+  while (1) {
+    char Char = getAndAdvanceChar(CurPtr, Tmp);
+    switch (Char) {
+    default:
+      Result += Char;
+      break;
+    case 0:  // Null.
+      // Found end of file?
+      if (CurPtr-1 != BufferEnd) {
+        if (isCodeCompletionPoint(CurPtr-1)) {
+          PP->CodeCompleteNaturalLanguage();
+          cutOffLexing();
+          return Result;
+        }
+
+        // Nope, normal character, continue.
+        Result += Char;
+        break;
+      }
+      // FALL THROUGH.
+    case '\r':
+    case '\n':
+      // Okay, we found the end of the line. First, back up past the \0, \r, \n.
+      assert(CurPtr[-1] == Char && "Trigraphs for newline?");
+      BufferPtr = CurPtr-1;
+
+      // Next, lex the character, which should handle the EOD transition.
+      Lex(Tmp);
+      if (Tmp.is(tok::code_completion)) {
+        if (PP)
+          PP->CodeCompleteNaturalLanguage();
+        Lex(Tmp);
+      }
+      assert(Tmp.is(tok::eod) && "Unexpected token!");
+
+      // Finally, we're done, return the string we found.
+      return Result;
+    }
+  }
+}
+
+/// LexEndOfFile - CurPtr points to the end of this file.  Handle this
+/// condition, reporting diagnostics and handling other edge cases as required.
+/// This returns true if Result contains a token, false if PP.Lex should be
+/// called again.
+bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) {
+  // If we hit the end of the file while parsing a preprocessor directive,
+  // end the preprocessor directive first.  The next token returned will
+  // then be the end of file.
+  if (ParsingPreprocessorDirective) {
+    // Done parsing the "line".
+    ParsingPreprocessorDirective = false;
+    // Update the location of token as well as BufferPtr.
+    FormTokenWithChars(Result, CurPtr, tok::eod);
+
+    // Restore comment saving mode, in case it was disabled for directive.
+    SetCommentRetentionState(PP->getCommentRetentionState());
+    return true;  // Have a token.
+  }
+ 
+  // If we are in raw mode, return this event as an EOF token.  Let the caller
+  // that put us in raw mode handle the event.
+  if (isLexingRawMode()) {
+    Result.startToken();
+    BufferPtr = BufferEnd;
+    FormTokenWithChars(Result, BufferEnd, tok::eof);
+    return true;
+  }
+  
+  // Issue diagnostics for unterminated #if and missing newline.
+
+  // If we are in a #if directive, emit an error.
+  while (!ConditionalStack.empty()) {
+    if (PP->getCodeCompletionFileLoc() != FileLoc)
+      PP->Diag(ConditionalStack.back().IfLoc,
+               diag::err_pp_unterminated_conditional);
+    ConditionalStack.pop_back();
+  }
+
+  // C99 5.1.1.2p2: If the file is non-empty and didn't end in a newline, issue
+  // a pedwarn.
+  if (CurPtr != BufferStart && (CurPtr[-1] != '\n' && CurPtr[-1] != '\r'))
+    Diag(BufferEnd, LangOpts.CPlusPlus0x ? // C++11 [lex.phases] 2.2 p2
+         diag::warn_cxx98_compat_no_newline_eof : diag::ext_no_newline_eof)
+    << FixItHint::CreateInsertion(getSourceLocation(BufferEnd), "\n");
+
+  BufferPtr = CurPtr;
+
+  // Finally, let the preprocessor handle this.
+  return PP->HandleEndOfFile(Result);
+}
+
+/// isNextPPTokenLParen - Return 1 if the next unexpanded token lexed from
+/// the specified lexer will return a tok::l_paren token, 0 if it is something
+/// else and 2 if there are no more tokens in the buffer controlled by the
+/// lexer.
+unsigned Lexer::isNextPPTokenLParen() {
+  assert(!LexingRawMode && "How can we expand a macro from a skipping buffer?");
+
+  // Switch to 'skipping' mode.  This will ensure that we can lex a token
+  // without emitting diagnostics, disables macro expansion, and will cause EOF
+  // to return an EOF token instead of popping the include stack.
+  LexingRawMode = true;
+
+  // Save state that can be changed while lexing so that we can restore it.
+  const char *TmpBufferPtr = BufferPtr;
+  bool inPPDirectiveMode = ParsingPreprocessorDirective;
+
+  Token Tok;
+  Tok.startToken();
+  LexTokenInternal(Tok);
+
+  // Restore state that may have changed.
+  BufferPtr = TmpBufferPtr;
+  ParsingPreprocessorDirective = inPPDirectiveMode;
+
+  // Restore the lexer back to non-skipping mode.
+  LexingRawMode = false;
+
+  if (Tok.is(tok::eof))
+    return 2;
+  return Tok.is(tok::l_paren);
+}
+
+/// FindConflictEnd - Find the end of a version control conflict marker.
+static const char *FindConflictEnd(const char *CurPtr, const char *BufferEnd,
+                                   ConflictMarkerKind CMK) {
+  const char *Terminator = CMK == CMK_Perforce ? "<<<<\n" : ">>>>>>>";
+  size_t TermLen = CMK == CMK_Perforce ? 5 : 7;
+  StringRef RestOfBuffer(CurPtr+TermLen, BufferEnd-CurPtr-TermLen);
+  size_t Pos = RestOfBuffer.find(Terminator);
+  while (Pos != StringRef::npos) {
+    // Must occur at start of line.
+    if (RestOfBuffer[Pos-1] != '\r' &&
+        RestOfBuffer[Pos-1] != '\n') {
+      RestOfBuffer = RestOfBuffer.substr(Pos+TermLen);
+      Pos = RestOfBuffer.find(Terminator);
+      continue;
+    }
+    return RestOfBuffer.data()+Pos;
+  }
+  return 0;
+}
+
+/// IsStartOfConflictMarker - If the specified pointer is the start of a version
+/// control conflict marker like '<<<<<<<', recognize it as such, emit an error
+/// and recover nicely.  This returns true if it is a conflict marker and false
+/// if not.
+bool Lexer::IsStartOfConflictMarker(const char *CurPtr) {
+  // Only a conflict marker if it starts at the beginning of a line.
+  if (CurPtr != BufferStart &&
+      CurPtr[-1] != '\n' && CurPtr[-1] != '\r')
+    return false;
+  
+  // Check to see if we have <<<<<<< or >>>>.
+  if ((BufferEnd-CurPtr < 8 || StringRef(CurPtr, 7) != "<<<<<<<") &&
+      (BufferEnd-CurPtr < 6 || StringRef(CurPtr, 5) != ">>>> "))
+    return false;
+
+  // If we have a situation where we don't care about conflict markers, ignore
+  // it.
+  if (CurrentConflictMarkerState || isLexingRawMode())
+    return false;
+  
+  ConflictMarkerKind Kind = *CurPtr == '<' ? CMK_Normal : CMK_Perforce;
+
+  // Check to see if there is an ending marker somewhere in the buffer at the
+  // start of a line to terminate this conflict marker.
+  if (FindConflictEnd(CurPtr, BufferEnd, Kind)) {
+    // We found a match.  We are really in a conflict marker.
+    // Diagnose this, and ignore to the end of line.
+    Diag(CurPtr, diag::err_conflict_marker);
+    CurrentConflictMarkerState = Kind;
+    
+    // Skip ahead to the end of line.  We know this exists because the
+    // end-of-conflict marker starts with \r or \n.
+    while (*CurPtr != '\r' && *CurPtr != '\n') {
+      assert(CurPtr != BufferEnd && "Didn't find end of line");
+      ++CurPtr;
+    }
+    BufferPtr = CurPtr;
+    return true;
+  }
+  
+  // No end of conflict marker found.
+  return false;
+}
+
+
+/// HandleEndOfConflictMarker - If this is a '====' or '||||' or '>>>>', or if
+/// it is '<<<<' and the conflict marker started with a '>>>>' marker, then it
+/// is the end of a conflict marker.  Handle it by ignoring up until the end of
+/// the line.  This returns true if it is a conflict marker and false if not.
+bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) {
+  // Only a conflict marker if it starts at the beginning of a line.
+  if (CurPtr != BufferStart &&
+      CurPtr[-1] != '\n' && CurPtr[-1] != '\r')
+    return false;
+  
+  // If we have a situation where we don't care about conflict markers, ignore
+  // it.
+  if (!CurrentConflictMarkerState || isLexingRawMode())
+    return false;
+  
+  // Check to see if we have the marker (4 characters in a row).
+  for (unsigned i = 1; i != 4; ++i)
+    if (CurPtr[i] != CurPtr[0])
+      return false;
+  
+  // If we do have it, search for the end of the conflict marker.  This could
+  // fail if it got skipped with a '#if 0' or something.  Note that CurPtr might
+  // be the end of conflict marker.
+  if (const char *End = FindConflictEnd(CurPtr, BufferEnd,
+                                        CurrentConflictMarkerState)) {
+    CurPtr = End;
+    
+    // Skip ahead to the end of line.
+    while (CurPtr != BufferEnd && *CurPtr != '\r' && *CurPtr != '\n')
+      ++CurPtr;
+    
+    BufferPtr = CurPtr;
+    
+    // No longer in the conflict marker.
+    CurrentConflictMarkerState = CMK_None;
+    return true;
+  }
+  
+  return false;
+}
+
+bool Lexer::isCodeCompletionPoint(const char *CurPtr) const {
+  if (PP && PP->isCodeCompletionEnabled()) {
+    SourceLocation Loc = FileLoc.getLocWithOffset(CurPtr-BufferStart);
+    return Loc == PP->getCodeCompletionLoc();
+  }
+
+  return false;
+}
+
+
+/// LexTokenInternal - This implements a simple C family lexer.  It is an
+/// extremely performance critical piece of code.  This assumes that the buffer
+/// has a null character at the end of the file.  This returns a preprocessing
+/// token, not a normal token, as such, it is an internal interface.  It assumes
+/// that the Flags of result have been cleared before calling this.
+void Lexer::LexTokenInternal(Token &Result) {
+LexNextToken:
+  // New token, can't need cleaning yet.
+  Result.clearFlag(Token::NeedsCleaning);
+  Result.setIdentifierInfo(0);
+
+  // CurPtr - Cache BufferPtr in an automatic variable.
+  const char *CurPtr = BufferPtr;
+
+  // Small amounts of horizontal whitespace is very common between tokens.
+  if ((*CurPtr == ' ') || (*CurPtr == '\t')) {
+    ++CurPtr;
+    while ((*CurPtr == ' ') || (*CurPtr == '\t'))
+      ++CurPtr;
+
+    // If we are keeping whitespace and other tokens, just return what we just
+    // skipped.  The next lexer invocation will return the token after the
+    // whitespace.
+    if (isKeepWhitespaceMode()) {
+      FormTokenWithChars(Result, CurPtr, tok::unknown);
+      return;
+    }
+
+    BufferPtr = CurPtr;
+    Result.setFlag(Token::LeadingSpace);
+  }
+
+  unsigned SizeTmp, SizeTmp2;   // Temporaries for use in cases below.
+
+  // Read a character, advancing over it.
+  char Char = getAndAdvanceChar(CurPtr, Result);
+  tok::TokenKind Kind;
+
+  switch (Char) {
+  case 0:  // Null.
+    // Found end of file?
+    if (CurPtr-1 == BufferEnd) {
+      // Read the PP instance variable into an automatic variable, because
+      // LexEndOfFile will often delete 'this'.
+      Preprocessor *PPCache = PP;
+      if (LexEndOfFile(Result, CurPtr-1))  // Retreat back into the file.
+        return;   // Got a token to return.
+      assert(PPCache && "Raw buffer::LexEndOfFile should return a token");
+      return PPCache->Lex(Result);
+    }
+
+    // Check if we are performing code completion.
+    if (isCodeCompletionPoint(CurPtr-1)) {
+      // Return the code-completion token.
+      Result.startToken();
+      FormTokenWithChars(Result, CurPtr, tok::code_completion);
+      return;
+    }
+
+    if (!isLexingRawMode())
+      Diag(CurPtr-1, diag::null_in_file);
+    Result.setFlag(Token::LeadingSpace);
+    if (SkipWhitespace(Result, CurPtr))
+      return; // KeepWhitespaceMode
+
+    goto LexNextToken;   // GCC isn't tail call eliminating.
+      
+  case 26:  // DOS & CP/M EOF: "^Z".
+    // If we're in Microsoft extensions mode, treat this as end of file.
+    if (LangOpts.MicrosoftExt) {
+      // Read the PP instance variable into an automatic variable, because
+      // LexEndOfFile will often delete 'this'.
+      Preprocessor *PPCache = PP;
+      if (LexEndOfFile(Result, CurPtr-1))  // Retreat back into the file.
+        return;   // Got a token to return.
+      assert(PPCache && "Raw buffer::LexEndOfFile should return a token");
+      return PPCache->Lex(Result);
+    }
+    // If Microsoft extensions are disabled, this is just random garbage.
+    Kind = tok::unknown;
+    break;
+      
+  case '\n':
+  case '\r':
+    // If we are inside a preprocessor directive and we see the end of line,
+    // we know we are done with the directive, so return an EOD token.
+    if (ParsingPreprocessorDirective) {
+      // Done parsing the "line".
+      ParsingPreprocessorDirective = false;
+
+      // Restore comment saving mode, in case it was disabled for directive.
+      SetCommentRetentionState(PP->getCommentRetentionState());
+
+      // Since we consumed a newline, we are back at the start of a line.
+      IsAtStartOfLine = true;
+
+      Kind = tok::eod;
+      break;
+    }
+    // The returned token is at the start of the line.
+    Result.setFlag(Token::StartOfLine);
+    // No leading whitespace seen so far.
+    Result.clearFlag(Token::LeadingSpace);
+
+    if (SkipWhitespace(Result, CurPtr))
+      return; // KeepWhitespaceMode
+    goto LexNextToken;   // GCC isn't tail call eliminating.
+  case ' ':
+  case '\t':
+  case '\f':
+  case '\v':
+  SkipHorizontalWhitespace:
+    Result.setFlag(Token::LeadingSpace);
+    if (SkipWhitespace(Result, CurPtr))
+      return; // KeepWhitespaceMode
+
+  SkipIgnoredUnits:
+    CurPtr = BufferPtr;
+
+    // If the next token is obviously a // or /* */ comment, skip it efficiently
+    // too (without going through the big switch stmt).
+    if (CurPtr[0] == '/' && CurPtr[1] == '/' && !inKeepCommentMode() &&
+        LangOpts.BCPLComment && !LangOpts.TraditionalCPP) {
+      if (SkipBCPLComment(Result, CurPtr+2))
+        return; // There is a token to return.
+      goto SkipIgnoredUnits;
+    } else if (CurPtr[0] == '/' && CurPtr[1] == '*' && !inKeepCommentMode()) {
+      if (SkipBlockComment(Result, CurPtr+2))
+        return; // There is a token to return.
+      goto SkipIgnoredUnits;
+    } else if (isHorizontalWhitespace(*CurPtr)) {
+      goto SkipHorizontalWhitespace;
+    }
+    goto LexNextToken;   // GCC isn't tail call eliminating.
+      
+  // C99 6.4.4.1: Integer Constants.
+  // C99 6.4.4.2: Floating Constants.
+  case '0': case '1': case '2': case '3': case '4':
+  case '5': case '6': case '7': case '8': case '9':
+    // Notify MIOpt that we read a non-whitespace/non-comment token.
+    MIOpt.ReadToken();
+    return LexNumericConstant(Result, CurPtr);
+
+  case 'u':   // Identifier (uber) or C++0x UTF-8 or UTF-16 string literal
+    // Notify MIOpt that we read a non-whitespace/non-comment token.
+    MIOpt.ReadToken();
+
+    if (LangOpts.CPlusPlus0x) {
+      Char = getCharAndSize(CurPtr, SizeTmp);
+
+      // UTF-16 string literal
+      if (Char == '"')
+        return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result),
+                                tok::utf16_string_literal);
+
+      // UTF-16 character constant
+      if (Char == '\'')
+        return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result),
+                               tok::utf16_char_constant);
+
+      // UTF-16 raw string literal
+      if (Char == 'R' && getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"')
+        return LexRawStringLiteral(Result,
+                               ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
+                                           SizeTmp2, Result),
+                               tok::utf16_string_literal);
+
+      if (Char == '8') {
+        char Char2 = getCharAndSize(CurPtr + SizeTmp, SizeTmp2);
+
+        // UTF-8 string literal
+        if (Char2 == '"')
+          return LexStringLiteral(Result,
+                               ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
+                                           SizeTmp2, Result),
+                               tok::utf8_string_literal);
+
+        if (Char2 == 'R') {
+          unsigned SizeTmp3;
+          char Char3 = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3);
+          // UTF-8 raw string literal
+          if (Char3 == '"') {
+            return LexRawStringLiteral(Result,
+                   ConsumeChar(ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
+                                           SizeTmp2, Result),
+                               SizeTmp3, Result),
+                   tok::utf8_string_literal);
+          }
+        }
+      }
+    }
+
+    // treat u like the start of an identifier.
+    return LexIdentifier(Result, CurPtr);
+
+  case 'U':   // Identifier (Uber) or C++0x UTF-32 string literal
+    // Notify MIOpt that we read a non-whitespace/non-comment token.
+    MIOpt.ReadToken();
+
+    if (LangOpts.CPlusPlus0x) {
+      Char = getCharAndSize(CurPtr, SizeTmp);
+
+      // UTF-32 string literal
+      if (Char == '"')
+        return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result),
+                                tok::utf32_string_literal);
+
+      // UTF-32 character constant
+      if (Char == '\'')
+        return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result),
+                               tok::utf32_char_constant);
+
+      // UTF-32 raw string literal
+      if (Char == 'R' && getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"')
+        return LexRawStringLiteral(Result,
+                               ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
+                                           SizeTmp2, Result),
+                               tok::utf32_string_literal);
+    }
+
+    // treat U like the start of an identifier.
+    return LexIdentifier(Result, CurPtr);
+
+  case 'R': // Identifier or C++0x raw string literal
+    // Notify MIOpt that we read a non-whitespace/non-comment token.
+    MIOpt.ReadToken();
+
+    if (LangOpts.CPlusPlus0x) {
+      Char = getCharAndSize(CurPtr, SizeTmp);
+
+      if (Char == '"')
+        return LexRawStringLiteral(Result,
+                                   ConsumeChar(CurPtr, SizeTmp, Result),
+                                   tok::string_literal);
+    }
+
+    // treat R like the start of an identifier.
+    return LexIdentifier(Result, CurPtr);
+
+  case 'L':   // Identifier (Loony) or wide literal (L'x' or L"xyz").
+    // Notify MIOpt that we read a non-whitespace/non-comment token.
+    MIOpt.ReadToken();
+    Char = getCharAndSize(CurPtr, SizeTmp);
+
+    // Wide string literal.
+    if (Char == '"')
+      return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result),
+                              tok::wide_string_literal);
+
+    // Wide raw string literal.
+    if (LangOpts.CPlusPlus0x && Char == 'R' &&
+        getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"')
+      return LexRawStringLiteral(Result,
+                               ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
+                                           SizeTmp2, Result),
+                               tok::wide_string_literal);
+
+    // Wide character constant.
+    if (Char == '\'')
+      return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result),
+                             tok::wide_char_constant);
+    // FALL THROUGH, treating L like the start of an identifier.
+
+  // C99 6.4.2: Identifiers.
+  case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G':
+  case 'H': case 'I': case 'J': case 'K':    /*'L'*/case 'M': case 'N':
+  case 'O': case 'P': case 'Q':    /*'R'*/case 'S': case 'T':    /*'U'*/
+  case 'V': case 'W': case 'X': case 'Y': case 'Z':
+  case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g':
+  case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n':
+  case 'o': case 'p': case 'q': case 'r': case 's': case 't':    /*'u'*/
+  case 'v': case 'w': case 'x': case 'y': case 'z':
+  case '_':
+    // Notify MIOpt that we read a non-whitespace/non-comment token.
+    MIOpt.ReadToken();
+    return LexIdentifier(Result, CurPtr);
+
+  case '$':   // $ in identifiers.
+    if (LangOpts.DollarIdents) {
+      if (!isLexingRawMode())
+        Diag(CurPtr-1, diag::ext_dollar_in_identifier);
+      // Notify MIOpt that we read a non-whitespace/non-comment token.
+      MIOpt.ReadToken();
+      return LexIdentifier(Result, CurPtr);
+    }
+
+    Kind = tok::unknown;
+    break;
+
+  // C99 6.4.4: Character Constants.
+  case '\'':
+    // Notify MIOpt that we read a non-whitespace/non-comment token.
+    MIOpt.ReadToken();
+    return LexCharConstant(Result, CurPtr, tok::char_constant);
+
+  // C99 6.4.5: String Literals.
+  case '"':
+    // Notify MIOpt that we read a non-whitespace/non-comment token.
+    MIOpt.ReadToken();
+    return LexStringLiteral(Result, CurPtr, tok::string_literal);
+
+  // C99 6.4.6: Punctuators.
+  case '?':
+    Kind = tok::question;
+    break;
+  case '[':
+    Kind = tok::l_square;
+    break;
+  case ']':
+    Kind = tok::r_square;
+    break;
+  case '(':
+    Kind = tok::l_paren;
+    break;
+  case ')':
+    Kind = tok::r_paren;
+    break;
+  case '{':
+    Kind = tok::l_brace;
+    break;
+  case '}':
+    Kind = tok::r_brace;
+    break;
+  case '.':
+    Char = getCharAndSize(CurPtr, SizeTmp);
+    if (Char >= '0' && Char <= '9') {
+      // Notify MIOpt that we read a non-whitespace/non-comment token.
+      MIOpt.ReadToken();
+
+      return LexNumericConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result));
+    } else if (LangOpts.CPlusPlus && Char == '*') {
+      Kind = tok::periodstar;
+      CurPtr += SizeTmp;
+    } else if (Char == '.' &&
+               getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '.') {
+      Kind = tok::ellipsis;
+      CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
+                           SizeTmp2, Result);
+    } else {
+      Kind = tok::period;
+    }
+    break;
+  case '&':
+    Char = getCharAndSize(CurPtr, SizeTmp);
+    if (Char == '&') {
+      Kind = tok::ampamp;
+      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+    } else if (Char == '=') {
+      Kind = tok::ampequal;
+      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+    } else {
+      Kind = tok::amp;
+    }
+    break;
+  case '*':
+    if (getCharAndSize(CurPtr, SizeTmp) == '=') {
+      Kind = tok::starequal;
+      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+    } else {
+      Kind = tok::star;
+    }
+    break;
+  case '+':
+    Char = getCharAndSize(CurPtr, SizeTmp);
+    if (Char == '+') {
+      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+      Kind = tok::plusplus;
+    } else if (Char == '=') {
+      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+      Kind = tok::plusequal;
+    } else {
+      Kind = tok::plus;
+    }
+    break;
+  case '-':
+    Char = getCharAndSize(CurPtr, SizeTmp);
+    if (Char == '-') {      // --
+      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+      Kind = tok::minusminus;
+    } else if (Char == '>' && LangOpts.CPlusPlus &&
+               getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '*') {  // C++ ->*
+      CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
+                           SizeTmp2, Result);
+      Kind = tok::arrowstar;
+    } else if (Char == '>') {   // ->
+      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+      Kind = tok::arrow;
+    } else if (Char == '=') {   // -=
+      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+      Kind = tok::minusequal;
+    } else {
+      Kind = tok::minus;
+    }
+    break;
+  case '~':
+    Kind = tok::tilde;
+    break;
+  case '!':
+    if (getCharAndSize(CurPtr, SizeTmp) == '=') {
+      Kind = tok::exclaimequal;
+      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+    } else {
+      Kind = tok::exclaim;
+    }
+    break;
+  case '/':
+    // 6.4.9: Comments
+    Char = getCharAndSize(CurPtr, SizeTmp);
+    if (Char == '/') {         // BCPL comment.
+      // Even if BCPL comments are disabled (e.g. in C89 mode), we generally
+      // want to lex this as a comment.  There is one problem with this though,
+      // that in one particular corner case, this can change the behavior of the
+      // resultant program.  For example, In  "foo //**/ bar", C89 would lex
+      // this as "foo / bar" and langauges with BCPL comments would lex it as
+      // "foo".  Check to see if the character after the second slash is a '*'.
+      // If so, we will lex that as a "/" instead of the start of a comment.
+      // However, we never do this in -traditional-cpp mode.
+      if ((LangOpts.BCPLComment ||
+           getCharAndSize(CurPtr+SizeTmp, SizeTmp2) != '*') &&
+          !LangOpts.TraditionalCPP) {
+        if (SkipBCPLComment(Result, ConsumeChar(CurPtr, SizeTmp, Result)))
+          return; // There is a token to return.
+
+        // It is common for the tokens immediately after a // comment to be
+        // whitespace (indentation for the next line).  Instead of going through
+        // the big switch, handle it efficiently now.
+        goto SkipIgnoredUnits;
+      }
+    }
+
+    if (Char == '*') {  // /**/ comment.
+      if (SkipBlockComment(Result, ConsumeChar(CurPtr, SizeTmp, Result)))
+        return; // There is a token to return.
+      goto LexNextToken;   // GCC isn't tail call eliminating.
+    }
+
+    if (Char == '=') {
+      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+      Kind = tok::slashequal;
+    } else {
+      Kind = tok::slash;
+    }
+    break;
+  case '%':
+    Char = getCharAndSize(CurPtr, SizeTmp);
+    if (Char == '=') {
+      Kind = tok::percentequal;
+      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+    } else if (LangOpts.Digraphs && Char == '>') {
+      Kind = tok::r_brace;                             // '%>' -> '}'
+      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+    } else if (LangOpts.Digraphs && Char == ':') {
+      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+      Char = getCharAndSize(CurPtr, SizeTmp);
+      if (Char == '%' && getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == ':') {
+        Kind = tok::hashhash;                          // '%:%:' -> '##'
+        CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
+                             SizeTmp2, Result);
+      } else if (Char == '@' && LangOpts.MicrosoftExt) {// %:@ -> #@ -> Charize
+        CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+        if (!isLexingRawMode())
+          Diag(BufferPtr, diag::ext_charize_microsoft);
+        Kind = tok::hashat;
+      } else {                                         // '%:' -> '#'
+        // We parsed a # character.  If this occurs at the start of the line,
+        // it's actually the start of a preprocessing directive.  Callback to
+        // the preprocessor to handle it.
+        // FIXME: -fpreprocessed mode??
+        if (Result.isAtStartOfLine() && !LexingRawMode && !Is_PragmaLexer) {
+          FormTokenWithChars(Result, CurPtr, tok::hash);
+          PP->HandleDirective(Result);
+
+          // As an optimization, if the preprocessor didn't switch lexers, tail
+          // recurse.
+          if (PP->isCurrentLexer(this)) {
+            // Start a new token. If this is a #include or something, the PP may
+            // want us starting at the beginning of the line again.  If so, set
+            // the StartOfLine flag and clear LeadingSpace.
+            if (IsAtStartOfLine) {
+              Result.setFlag(Token::StartOfLine);
+              Result.clearFlag(Token::LeadingSpace);
+              IsAtStartOfLine = false;
+            }
+            goto LexNextToken;   // GCC isn't tail call eliminating.
+          }
+
+          return PP->Lex(Result);
+        }
+
+        Kind = tok::hash;
+      }
+    } else {
+      Kind = tok::percent;
+    }
+    break;
+  case '<':
+    Char = getCharAndSize(CurPtr, SizeTmp);
+    if (ParsingFilename) {
+      return LexAngledStringLiteral(Result, CurPtr);
+    } else if (Char == '<') {
+      char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2);
+      if (After == '=') {
+        Kind = tok::lesslessequal;
+        CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
+                             SizeTmp2, Result);
+      } else if (After == '<' && IsStartOfConflictMarker(CurPtr-1)) {
+        // If this is actually a '<<<<<<<' version control conflict marker,
+        // recognize it as such and recover nicely.
+        goto LexNextToken;
+      } else if (After == '<' && HandleEndOfConflictMarker(CurPtr-1)) {
+        // If this is '<<<<' and we're in a Perforce-style conflict marker,
+        // ignore it.
+        goto LexNextToken;
+      } else if (LangOpts.CUDA && After == '<') {
+        Kind = tok::lesslessless;
+        CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
+                             SizeTmp2, Result);
+      } else {
+        CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+        Kind = tok::lessless;
+      }
+    } else if (Char == '=') {
+      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+      Kind = tok::lessequal;
+    } else if (LangOpts.Digraphs && Char == ':') {     // '<:' -> '['
+      if (LangOpts.CPlusPlus0x &&
+          getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == ':') {
+        // C++0x [lex.pptoken]p3:
+        //  Otherwise, if the next three characters are <:: and the subsequent
+        //  character is neither : nor >, the < is treated as a preprocessor
+        //  token by itself and not as the first character of the alternative
+        //  token <:.
+        unsigned SizeTmp3;
+        char After = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3);
+        if (After != ':' && After != '>') {
+          Kind = tok::less;
+          if (!isLexingRawMode())
+            Diag(BufferPtr, diag::warn_cxx98_compat_less_colon_colon);
+          break;
+        }
+      }
+
+      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+      Kind = tok::l_square;
+    } else if (LangOpts.Digraphs && Char == '%') {     // '<%' -> '{'
+      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+      Kind = tok::l_brace;
+    } else {
+      Kind = tok::less;
+    }
+    break;
+  case '>':
+    Char = getCharAndSize(CurPtr, SizeTmp);
+    if (Char == '=') {
+      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+      Kind = tok::greaterequal;
+    } else if (Char == '>') {
+      char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2);
+      if (After == '=') {
+        CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
+                             SizeTmp2, Result);
+        Kind = tok::greatergreaterequal;
+      } else if (After == '>' && IsStartOfConflictMarker(CurPtr-1)) {
+        // If this is actually a '>>>>' conflict marker, recognize it as such
+        // and recover nicely.
+        goto LexNextToken;
+      } else if (After == '>' && HandleEndOfConflictMarker(CurPtr-1)) {
+        // If this is '>>>>>>>' and we're in a conflict marker, ignore it.
+        goto LexNextToken;
+      } else if (LangOpts.CUDA && After == '>') {
+        Kind = tok::greatergreatergreater;
+        CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
+                             SizeTmp2, Result);
+      } else {
+        CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+        Kind = tok::greatergreater;
+      }
+      
+    } else {
+      Kind = tok::greater;
+    }
+    break;
+  case '^':
+    Char = getCharAndSize(CurPtr, SizeTmp);
+    if (Char == '=') {
+      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+      Kind = tok::caretequal;
+    } else {
+      Kind = tok::caret;
+    }
+    break;
+  case '|':
+    Char = getCharAndSize(CurPtr, SizeTmp);
+    if (Char == '=') {
+      Kind = tok::pipeequal;
+      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+    } else if (Char == '|') {
+      // If this is '|||||||' and we're in a conflict marker, ignore it.
+      if (CurPtr[1] == '|' && HandleEndOfConflictMarker(CurPtr-1))
+        goto LexNextToken;
+      Kind = tok::pipepipe;
+      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+    } else {
+      Kind = tok::pipe;
+    }
+    break;
+  case ':':
+    Char = getCharAndSize(CurPtr, SizeTmp);
+    if (LangOpts.Digraphs && Char == '>') {
+      Kind = tok::r_square; // ':>' -> ']'
+      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+    } else if (LangOpts.CPlusPlus && Char == ':') {
+      Kind = tok::coloncolon;
+      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+    } else {
+      Kind = tok::colon;
+    }
+    break;
+  case ';':
+    Kind = tok::semi;
+    break;
+  case '=':
+    Char = getCharAndSize(CurPtr, SizeTmp);
+    if (Char == '=') {
+      // If this is '====' and we're in a conflict marker, ignore it.
+      if (CurPtr[1] == '=' && HandleEndOfConflictMarker(CurPtr-1))
+        goto LexNextToken;
+      
+      Kind = tok::equalequal;
+      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+    } else {
+      Kind = tok::equal;
+    }
+    break;
+  case ',':
+    Kind = tok::comma;
+    break;
+  case '#':
+    Char = getCharAndSize(CurPtr, SizeTmp);
+    if (Char == '#') {
+      Kind = tok::hashhash;
+      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+    } else if (Char == '@' && LangOpts.MicrosoftExt) {  // #@ -> Charize
+      Kind = tok::hashat;
+      if (!isLexingRawMode())
+        Diag(BufferPtr, diag::ext_charize_microsoft);
+      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+    } else {
+      // We parsed a # character.  If this occurs at the start of the line,
+      // it's actually the start of a preprocessing directive.  Callback to
+      // the preprocessor to handle it.
+      // FIXME: -fpreprocessed mode??
+      if (Result.isAtStartOfLine() && !LexingRawMode && !Is_PragmaLexer) {
+        FormTokenWithChars(Result, CurPtr, tok::hash);
+        PP->HandleDirective(Result);
+
+        // As an optimization, if the preprocessor didn't switch lexers, tail
+        // recurse.
+        if (PP->isCurrentLexer(this)) {
+          // Start a new token.  If this is a #include or something, the PP may
+          // want us starting at the beginning of the line again.  If so, set
+          // the StartOfLine flag and clear LeadingSpace.
+          if (IsAtStartOfLine) {
+            Result.setFlag(Token::StartOfLine);
+            Result.clearFlag(Token::LeadingSpace);
+            IsAtStartOfLine = false;
+          }
+          goto LexNextToken;   // GCC isn't tail call eliminating.
+        }
+        return PP->Lex(Result);
+      }
+
+      Kind = tok::hash;
+    }
+    break;
+
+  case '@':
+    // Objective C support.
+    if (CurPtr[-1] == '@' && LangOpts.ObjC1)
+      Kind = tok::at;
+    else
+      Kind = tok::unknown;
+    break;
+
+  case '\\':
+    // FIXME: UCN's.
+    // FALL THROUGH.
+  default:
+    Kind = tok::unknown;
+    break;
+  }
+
+  // Notify MIOpt that we read a non-whitespace/non-comment token.
+  MIOpt.ReadToken();
+
+  // Update the location of token as well as BufferPtr.
+  FormTokenWithChars(Result, CurPtr, Kind);
+}
diff --git a/clang/lib/Lex/LiteralSupport.cpp b/clang/lib/Lex/LiteralSupport.cpp
new file mode 100644
index 0000000..c1d228b
--- /dev/null
+++ b/clang/lib/Lex/LiteralSupport.cpp
@@ -0,0 +1,1400 @@
+//===--- LiteralSupport.cpp - Code to parse and process literals ----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the NumericLiteralParser, CharLiteralParser, and
+// StringLiteralParser interfaces.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Lex/LiteralSupport.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/LexDiagnostic.h"
+#include "clang/Basic/TargetInfo.h"
+#include "clang/Basic/ConvertUTF.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/ErrorHandling.h"
+using namespace clang;
+
+/// HexDigitValue - Return the value of the specified hex digit, or -1 if it's
+/// not valid.
+static int HexDigitValue(char C) {
+  if (C >= '0' && C <= '9') return C-'0';
+  if (C >= 'a' && C <= 'f') return C-'a'+10;
+  if (C >= 'A' && C <= 'F') return C-'A'+10;
+  return -1;
+}
+
+static unsigned getCharWidth(tok::TokenKind kind, const TargetInfo &Target) {
+  switch (kind) {
+  default: llvm_unreachable("Unknown token type!");
+  case tok::char_constant:
+  case tok::string_literal:
+  case tok::utf8_string_literal:
+    return Target.getCharWidth();
+  case tok::wide_char_constant:
+  case tok::wide_string_literal:
+    return Target.getWCharWidth();
+  case tok::utf16_char_constant:
+  case tok::utf16_string_literal:
+    return Target.getChar16Width();
+  case tok::utf32_char_constant:
+  case tok::utf32_string_literal:
+    return Target.getChar32Width();
+  }
+}
+
+/// ProcessCharEscape - Parse a standard C escape sequence, which can occur in
+/// either a character or a string literal.
+static unsigned ProcessCharEscape(const char *&ThisTokBuf,
+                                  const char *ThisTokEnd, bool &HadError,
+                                  FullSourceLoc Loc, unsigned CharWidth,
+                                  DiagnosticsEngine *Diags) {
+  // Skip the '\' char.
+  ++ThisTokBuf;
+
+  // We know that this character can't be off the end of the buffer, because
+  // that would have been \", which would not have been the end of string.
+  unsigned ResultChar = *ThisTokBuf++;
+  switch (ResultChar) {
+  // These map to themselves.
+  case '\\': case '\'': case '"': case '?': break;
+
+    // These have fixed mappings.
+  case 'a':
+    // TODO: K&R: the meaning of '\\a' is different in traditional C
+    ResultChar = 7;
+    break;
+  case 'b':
+    ResultChar = 8;
+    break;
+  case 'e':
+    if (Diags)
+      Diags->Report(Loc, diag::ext_nonstandard_escape) << "e";
+    ResultChar = 27;
+    break;
+  case 'E':
+    if (Diags)
+      Diags->Report(Loc, diag::ext_nonstandard_escape) << "E";
+    ResultChar = 27;
+    break;
+  case 'f':
+    ResultChar = 12;
+    break;
+  case 'n':
+    ResultChar = 10;
+    break;
+  case 'r':
+    ResultChar = 13;
+    break;
+  case 't':
+    ResultChar = 9;
+    break;
+  case 'v':
+    ResultChar = 11;
+    break;
+  case 'x': { // Hex escape.
+    ResultChar = 0;
+    if (ThisTokBuf == ThisTokEnd || !isxdigit(*ThisTokBuf)) {
+      if (Diags)
+        Diags->Report(Loc, diag::err_hex_escape_no_digits);
+      HadError = 1;
+      break;
+    }
+
+    // Hex escapes are a maximal series of hex digits.
+    bool Overflow = false;
+    for (; ThisTokBuf != ThisTokEnd; ++ThisTokBuf) {
+      int CharVal = HexDigitValue(ThisTokBuf[0]);
+      if (CharVal == -1) break;
+      // About to shift out a digit?
+      Overflow |= (ResultChar & 0xF0000000) ? true : false;
+      ResultChar <<= 4;
+      ResultChar |= CharVal;
+    }
+
+    // See if any bits will be truncated when evaluated as a character.
+    if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) {
+      Overflow = true;
+      ResultChar &= ~0U >> (32-CharWidth);
+    }
+
+    // Check for overflow.
+    if (Overflow && Diags)   // Too many digits to fit in
+      Diags->Report(Loc, diag::warn_hex_escape_too_large);
+    break;
+  }
+  case '0': case '1': case '2': case '3':
+  case '4': case '5': case '6': case '7': {
+    // Octal escapes.
+    --ThisTokBuf;
+    ResultChar = 0;
+
+    // Octal escapes are a series of octal digits with maximum length 3.
+    // "\0123" is a two digit sequence equal to "\012" "3".
+    unsigned NumDigits = 0;
+    do {
+      ResultChar <<= 3;
+      ResultChar |= *ThisTokBuf++ - '0';
+      ++NumDigits;
+    } while (ThisTokBuf != ThisTokEnd && NumDigits < 3 &&
+             ThisTokBuf[0] >= '0' && ThisTokBuf[0] <= '7');
+
+    // Check for overflow.  Reject '\777', but not L'\777'.
+    if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) {
+      if (Diags)
+        Diags->Report(Loc, diag::warn_octal_escape_too_large);
+      ResultChar &= ~0U >> (32-CharWidth);
+    }
+    break;
+  }
+
+    // Otherwise, these are not valid escapes.
+  case '(': case '{': case '[': case '%':
+    // GCC accepts these as extensions.  We warn about them as such though.
+    if (Diags)
+      Diags->Report(Loc, diag::ext_nonstandard_escape)
+        << std::string()+(char)ResultChar;
+    break;
+  default:
+    if (Diags == 0)
+      break;
+      
+    if (isgraph(ResultChar))
+      Diags->Report(Loc, diag::ext_unknown_escape)
+        << std::string()+(char)ResultChar;
+    else
+      Diags->Report(Loc, diag::ext_unknown_escape)
+        << "x"+llvm::utohexstr(ResultChar);
+    break;
+  }
+
+  return ResultChar;
+}
+
+/// ProcessUCNEscape - Read the Universal Character Name, check constraints and
+/// return the UTF32.
+static bool ProcessUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
+                             const char *ThisTokEnd,
+                             uint32_t &UcnVal, unsigned short &UcnLen,
+                             FullSourceLoc Loc, DiagnosticsEngine *Diags, 
+                             const LangOptions &Features,
+                             bool in_char_string_literal = false) {
+  if (!Features.CPlusPlus && !Features.C99 && Diags)
+    Diags->Report(Loc, diag::warn_ucn_not_valid_in_c89);
+
+  const char *UcnBegin = ThisTokBuf;
+
+  // Skip the '\u' char's.
+  ThisTokBuf += 2;
+
+  if (ThisTokBuf == ThisTokEnd || !isxdigit(*ThisTokBuf)) {
+    if (Diags)
+      Diags->Report(Loc, diag::err_ucn_escape_no_digits);
+    return false;
+  }
+  UcnLen = (ThisTokBuf[-1] == 'u' ? 4 : 8);
+  unsigned short UcnLenSave = UcnLen;
+  for (; ThisTokBuf != ThisTokEnd && UcnLenSave; ++ThisTokBuf, UcnLenSave--) {
+    int CharVal = HexDigitValue(ThisTokBuf[0]);
+    if (CharVal == -1) break;
+    UcnVal <<= 4;
+    UcnVal |= CharVal;
+  }
+  // If we didn't consume the proper number of digits, there is a problem.
+  if (UcnLenSave) {
+    if (Diags) {
+      SourceLocation L =
+        Lexer::AdvanceToTokenCharacter(Loc, UcnBegin - ThisTokBegin,
+                                       Loc.getManager(), Features);
+      Diags->Report(L, diag::err_ucn_escape_incomplete);
+    }
+    return false;
+  }
+
+  // Check UCN constraints (C99 6.4.3p2) [C++11 lex.charset p2]
+  if ((0xD800 <= UcnVal && UcnVal <= 0xDFFF) || // surrogate codepoints
+      UcnVal > 0x10FFFF) {                      // maximum legal UTF32 value
+    if (Diags)
+      Diags->Report(Loc, diag::err_ucn_escape_invalid);
+    return false;
+  }
+
+  // C++11 allows UCNs that refer to control characters and basic source
+  // characters inside character and string literals
+  if (UcnVal < 0xa0 &&
+      (UcnVal != 0x24 && UcnVal != 0x40 && UcnVal != 0x60)) {  // $, @, `
+    bool IsError = (!Features.CPlusPlus0x || !in_char_string_literal);
+    if (Diags) {
+      SourceLocation UcnBeginLoc =
+        Lexer::AdvanceToTokenCharacter(Loc, UcnBegin - ThisTokBegin,
+                                       Loc.getManager(), Features);
+      char BasicSCSChar = UcnVal;
+      if (UcnVal >= 0x20 && UcnVal < 0x7f)
+        Diags->Report(UcnBeginLoc, IsError ? diag::err_ucn_escape_basic_scs :
+                      diag::warn_cxx98_compat_literal_ucn_escape_basic_scs)
+          << StringRef(&BasicSCSChar, 1);
+      else
+        Diags->Report(UcnBeginLoc, IsError ? diag::err_ucn_control_character :
+                      diag::warn_cxx98_compat_literal_ucn_control_character);
+    }
+    if (IsError)
+      return false;
+  }
+
+  return true;
+}
+
+/// EncodeUCNEscape - Read the Universal Character Name, check constraints and
+/// convert the UTF32 to UTF8 or UTF16. This is a subroutine of
+/// StringLiteralParser. When we decide to implement UCN's for identifiers,
+/// we will likely rework our support for UCN's.
+static void EncodeUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
+                            const char *ThisTokEnd,
+                            char *&ResultBuf, bool &HadError,
+                            FullSourceLoc Loc, unsigned CharByteWidth,
+                            DiagnosticsEngine *Diags,
+                            const LangOptions &Features) {
+  typedef uint32_t UTF32;
+  UTF32 UcnVal = 0;
+  unsigned short UcnLen = 0;
+  if (!ProcessUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, UcnVal, UcnLen,
+                        Loc, Diags, Features, true)) {
+    HadError = 1;
+    return;
+  }
+
+  assert((CharByteWidth == 1 || CharByteWidth == 2 || CharByteWidth) &&
+         "only character widths of 1, 2, or 4 bytes supported");
+
+  (void)UcnLen;
+  assert((UcnLen== 4 || UcnLen== 8) && "only ucn length of 4 or 8 supported");
+
+  if (CharByteWidth == 4) {
+    // FIXME: Make the type of the result buffer correct instead of
+    // using reinterpret_cast.
+    UTF32 *ResultPtr = reinterpret_cast<UTF32*>(ResultBuf);
+    *ResultPtr = UcnVal;
+    ResultBuf += 4;
+    return;
+  }
+
+  if (CharByteWidth == 2) {
+    // FIXME: Make the type of the result buffer correct instead of
+    // using reinterpret_cast.
+    UTF16 *ResultPtr = reinterpret_cast<UTF16*>(ResultBuf);
+
+    if (UcnVal < (UTF32)0xFFFF) {
+      *ResultPtr = UcnVal;
+      ResultBuf += 2;
+      return;
+    }
+
+    // Convert to UTF16.
+    UcnVal -= 0x10000;
+    *ResultPtr     = 0xD800 + (UcnVal >> 10);
+    *(ResultPtr+1) = 0xDC00 + (UcnVal & 0x3FF);
+    ResultBuf += 4;
+    return;
+  }
+
+  assert(CharByteWidth == 1 && "UTF-8 encoding is only for 1 byte characters");
+
+  // Now that we've parsed/checked the UCN, we convert from UTF32->UTF8.
+  // The conversion below was inspired by:
+  //   http://www.unicode.org/Public/PROGRAMS/CVTUTF/ConvertUTF.c
+  // First, we determine how many bytes the result will require.
+  typedef uint8_t UTF8;
+
+  unsigned short bytesToWrite = 0;
+  if (UcnVal < (UTF32)0x80)
+    bytesToWrite = 1;
+  else if (UcnVal < (UTF32)0x800)
+    bytesToWrite = 2;
+  else if (UcnVal < (UTF32)0x10000)
+    bytesToWrite = 3;
+  else
+    bytesToWrite = 4;
+
+  const unsigned byteMask = 0xBF;
+  const unsigned byteMark = 0x80;
+
+  // Once the bits are split out into bytes of UTF8, this is a mask OR-ed
+  // into the first byte, depending on how many bytes follow.
+  static const UTF8 firstByteMark[5] = {
+    0x00, 0x00, 0xC0, 0xE0, 0xF0
+  };
+  // Finally, we write the bytes into ResultBuf.
+  ResultBuf += bytesToWrite;
+  switch (bytesToWrite) { // note: everything falls through.
+    case 4: *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
+    case 3: *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
+    case 2: *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
+    case 1: *--ResultBuf = (UTF8) (UcnVal | firstByteMark[bytesToWrite]);
+  }
+  // Update the buffer.
+  ResultBuf += bytesToWrite;
+}
+
+
+///       integer-constant: [C99 6.4.4.1]
+///         decimal-constant integer-suffix
+///         octal-constant integer-suffix
+///         hexadecimal-constant integer-suffix
+///       user-defined-integer-literal: [C++11 lex.ext]
+///         decimal-literal ud-suffix
+///         octal-literal ud-suffix
+///         hexadecimal-literal ud-suffix
+///       decimal-constant:
+///         nonzero-digit
+///         decimal-constant digit
+///       octal-constant:
+///         0
+///         octal-constant octal-digit
+///       hexadecimal-constant:
+///         hexadecimal-prefix hexadecimal-digit
+///         hexadecimal-constant hexadecimal-digit
+///       hexadecimal-prefix: one of
+///         0x 0X
+///       integer-suffix:
+///         unsigned-suffix [long-suffix]
+///         unsigned-suffix [long-long-suffix]
+///         long-suffix [unsigned-suffix]
+///         long-long-suffix [unsigned-sufix]
+///       nonzero-digit:
+///         1 2 3 4 5 6 7 8 9
+///       octal-digit:
+///         0 1 2 3 4 5 6 7
+///       hexadecimal-digit:
+///         0 1 2 3 4 5 6 7 8 9
+///         a b c d e f
+///         A B C D E F
+///       unsigned-suffix: one of
+///         u U
+///       long-suffix: one of
+///         l L
+///       long-long-suffix: one of
+///         ll LL
+///
+///       floating-constant: [C99 6.4.4.2]
+///         TODO: add rules...
+///
+NumericLiteralParser::
+NumericLiteralParser(const char *begin, const char *end,
+                     SourceLocation TokLoc, Preprocessor &pp)
+  : PP(pp), ThisTokBegin(begin), ThisTokEnd(end) {
+
+  // This routine assumes that the range begin/end matches the regex for integer
+  // and FP constants (specifically, the 'pp-number' regex), and assumes that
+  // the byte at "*end" is both valid and not part of the regex.  Because of
+  // this, it doesn't have to check for 'overscan' in various places.
+  assert(!isalnum(*end) && *end != '.' && *end != '_' &&
+         "Lexer didn't maximally munch?");
+
+  s = DigitsBegin = begin;
+  saw_exponent = false;
+  saw_period = false;
+  saw_ud_suffix = false;
+  isLong = false;
+  isUnsigned = false;
+  isLongLong = false;
+  isFloat = false;
+  isImaginary = false;
+  isMicrosoftInteger = false;
+  hadError = false;
+
+  if (*s == '0') { // parse radix
+    ParseNumberStartingWithZero(TokLoc);
+    if (hadError)
+      return;
+  } else { // the first digit is non-zero
+    radix = 10;
+    s = SkipDigits(s);
+    if (s == ThisTokEnd) {
+      // Done.
+    } else if (isxdigit(*s) && !(*s == 'e' || *s == 'E')) {
+      PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-begin),
+              diag::err_invalid_decimal_digit) << StringRef(s, 1);
+      hadError = true;
+      return;
+    } else if (*s == '.') {
+      s++;
+      saw_period = true;
+      s = SkipDigits(s);
+    }
+    if ((*s == 'e' || *s == 'E')) { // exponent
+      const char *Exponent = s;
+      s++;
+      saw_exponent = true;
+      if (*s == '+' || *s == '-')  s++; // sign
+      const char *first_non_digit = SkipDigits(s);
+      if (first_non_digit != s) {
+        s = first_non_digit;
+      } else {
+        PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent-begin),
+                diag::err_exponent_has_no_digits);
+        hadError = true;
+        return;
+      }
+    }
+  }
+
+  SuffixBegin = s;
+
+  // Parse the suffix.  At this point we can classify whether we have an FP or
+  // integer constant.
+  bool isFPConstant = isFloatingLiteral();
+
+  // Loop over all of the characters of the suffix.  If we see something bad,
+  // we break out of the loop.
+  for (; s != ThisTokEnd; ++s) {
+    switch (*s) {
+    case 'f':      // FP Suffix for "float"
+    case 'F':
+      if (!isFPConstant) break;  // Error for integer constant.
+      if (isFloat || isLong) break; // FF, LF invalid.
+      isFloat = true;
+      continue;  // Success.
+    case 'u':
+    case 'U':
+      if (isFPConstant) break;  // Error for floating constant.
+      if (isUnsigned) break;    // Cannot be repeated.
+      isUnsigned = true;
+      continue;  // Success.
+    case 'l':
+    case 'L':
+      if (isLong || isLongLong) break;  // Cannot be repeated.
+      if (isFloat) break;               // LF invalid.
+
+      // Check for long long.  The L's need to be adjacent and the same case.
+      if (s+1 != ThisTokEnd && s[1] == s[0]) {
+        if (isFPConstant) break;        // long long invalid for floats.
+        isLongLong = true;
+        ++s;  // Eat both of them.
+      } else {
+        isLong = true;
+      }
+      continue;  // Success.
+    case 'i':
+    case 'I':
+      if (PP.getLangOpts().MicrosoftExt) {
+        if (isFPConstant || isLong || isLongLong) break;
+
+        // Allow i8, i16, i32, i64, and i128.
+        if (s + 1 != ThisTokEnd) {
+          switch (s[1]) {
+            case '8':
+              s += 2; // i8 suffix
+              isMicrosoftInteger = true;
+              break;
+            case '1':
+              if (s + 2 == ThisTokEnd) break;
+              if (s[2] == '6') {
+                s += 3; // i16 suffix
+                isMicrosoftInteger = true;
+              }
+              else if (s[2] == '2') {
+                if (s + 3 == ThisTokEnd) break;
+                if (s[3] == '8') {
+                  s += 4; // i128 suffix
+                  isMicrosoftInteger = true;
+                }
+              }
+              break;
+            case '3':
+              if (s + 2 == ThisTokEnd) break;
+              if (s[2] == '2') {
+                s += 3; // i32 suffix
+                isLong = true;
+                isMicrosoftInteger = true;
+              }
+              break;
+            case '6':
+              if (s + 2 == ThisTokEnd) break;
+              if (s[2] == '4') {
+                s += 3; // i64 suffix
+                isLongLong = true;
+                isMicrosoftInteger = true;
+              }
+              break;
+            default:
+              break;
+          }
+          break;
+        }
+      }
+      // fall through.
+    case 'j':
+    case 'J':
+      if (isImaginary) break;   // Cannot be repeated.
+      PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-begin),
+              diag::ext_imaginary_constant);
+      isImaginary = true;
+      continue;  // Success.
+    }
+    // If we reached here, there was an error or a ud-suffix.
+    break;
+  }
+
+  if (s != ThisTokEnd) {
+    if (PP.getLangOpts().CPlusPlus0x && s == SuffixBegin && *s == '_') {
+      // We have a ud-suffix! By C++11 [lex.ext]p10, ud-suffixes not starting
+      // with an '_' are ill-formed.
+      saw_ud_suffix = true;
+      return;
+    }
+
+    // Report an error if there are any.
+    PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, SuffixBegin-begin),
+            isFPConstant ? diag::err_invalid_suffix_float_constant :
+                           diag::err_invalid_suffix_integer_constant)
+      << StringRef(SuffixBegin, ThisTokEnd-SuffixBegin);
+    hadError = true;
+    return;
+  }
+}
+
+/// ParseNumberStartingWithZero - This method is called when the first character
+/// of the number is found to be a zero.  This means it is either an octal
+/// number (like '04') or a hex number ('0x123a') a binary number ('0b1010') or
+/// a floating point number (01239.123e4).  Eat the prefix, determining the
+/// radix etc.
+void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) {
+  assert(s[0] == '0' && "Invalid method call");
+  s++;
+
+  // Handle a hex number like 0x1234.
+  if ((*s == 'x' || *s == 'X') && (isxdigit(s[1]) || s[1] == '.')) {
+    s++;
+    radix = 16;
+    DigitsBegin = s;
+    s = SkipHexDigits(s);
+    bool noSignificand = (s == DigitsBegin);
+    if (s == ThisTokEnd) {
+      // Done.
+    } else if (*s == '.') {
+      s++;
+      saw_period = true;
+      const char *floatDigitsBegin = s;
+      s = SkipHexDigits(s);
+      noSignificand &= (floatDigitsBegin == s);
+    }
+
+    if (noSignificand) {
+      PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin), \
+        diag::err_hexconstant_requires_digits);
+      hadError = true;
+      return;
+    }
+
+    // A binary exponent can appear with or with a '.'. If dotted, the
+    // binary exponent is required.
+    if (*s == 'p' || *s == 'P') {
+      const char *Exponent = s;
+      s++;
+      saw_exponent = true;
+      if (*s == '+' || *s == '-')  s++; // sign
+      const char *first_non_digit = SkipDigits(s);
+      if (first_non_digit == s) {
+        PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent-ThisTokBegin),
+                diag::err_exponent_has_no_digits);
+        hadError = true;
+        return;
+      }
+      s = first_non_digit;
+
+      if (!PP.getLangOpts().HexFloats)
+        PP.Diag(TokLoc, diag::ext_hexconstant_invalid);
+    } else if (saw_period) {
+      PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin),
+              diag::err_hexconstant_requires_exponent);
+      hadError = true;
+    }
+    return;
+  }
+
+  // Handle simple binary numbers 0b01010
+  if (*s == 'b' || *s == 'B') {
+    // 0b101010 is a GCC extension.
+    PP.Diag(TokLoc, diag::ext_binary_literal);
+    ++s;
+    radix = 2;
+    DigitsBegin = s;
+    s = SkipBinaryDigits(s);
+    if (s == ThisTokEnd) {
+      // Done.
+    } else if (isxdigit(*s)) {
+      PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin),
+              diag::err_invalid_binary_digit) << StringRef(s, 1);
+      hadError = true;
+    }
+    // Other suffixes will be diagnosed by the caller.
+    return;
+  }
+
+  // For now, the radix is set to 8. If we discover that we have a
+  // floating point constant, the radix will change to 10. Octal floating
+  // point constants are not permitted (only decimal and hexadecimal).
+  radix = 8;
+  DigitsBegin = s;
+  s = SkipOctalDigits(s);
+  if (s == ThisTokEnd)
+    return; // Done, simple octal number like 01234
+
+  // If we have some other non-octal digit that *is* a decimal digit, see if
+  // this is part of a floating point number like 094.123 or 09e1.
+  if (isdigit(*s)) {
+    const char *EndDecimal = SkipDigits(s);
+    if (EndDecimal[0] == '.' || EndDecimal[0] == 'e' || EndDecimal[0] == 'E') {
+      s = EndDecimal;
+      radix = 10;
+    }
+  }
+
+  // If we have a hex digit other than 'e' (which denotes a FP exponent) then
+  // the code is using an incorrect base.
+  if (isxdigit(*s) && *s != 'e' && *s != 'E') {
+    PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin),
+            diag::err_invalid_octal_digit) << StringRef(s, 1);
+    hadError = true;
+    return;
+  }
+
+  if (*s == '.') {
+    s++;
+    radix = 10;
+    saw_period = true;
+    s = SkipDigits(s); // Skip suffix.
+  }
+  if (*s == 'e' || *s == 'E') { // exponent
+    const char *Exponent = s;
+    s++;
+    radix = 10;
+    saw_exponent = true;
+    if (*s == '+' || *s == '-')  s++; // sign
+    const char *first_non_digit = SkipDigits(s);
+    if (first_non_digit != s) {
+      s = first_non_digit;
+    } else {
+      PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent-ThisTokBegin),
+              diag::err_exponent_has_no_digits);
+      hadError = true;
+      return;
+    }
+  }
+}
+
+
+/// GetIntegerValue - Convert this numeric literal value to an APInt that
+/// matches Val's input width.  If there is an overflow, set Val to the low bits
+/// of the result and return true.  Otherwise, return false.
+bool NumericLiteralParser::GetIntegerValue(llvm::APInt &Val) {
+  // Fast path: Compute a conservative bound on the maximum number of
+  // bits per digit in this radix. If we can't possibly overflow a
+  // uint64 based on that bound then do the simple conversion to
+  // integer. This avoids the expensive overflow checking below, and
+  // handles the common cases that matter (small decimal integers and
+  // hex/octal values which don't overflow).
+  unsigned MaxBitsPerDigit = 1;
+  while ((1U << MaxBitsPerDigit) < radix)
+    MaxBitsPerDigit += 1;
+  if ((SuffixBegin - DigitsBegin) * MaxBitsPerDigit <= 64) {
+    uint64_t N = 0;
+    for (s = DigitsBegin; s != SuffixBegin; ++s)
+      N = N*radix + HexDigitValue(*s);
+
+    // This will truncate the value to Val's input width. Simply check
+    // for overflow by comparing.
+    Val = N;
+    return Val.getZExtValue() != N;
+  }
+
+  Val = 0;
+  s = DigitsBegin;
+
+  llvm::APInt RadixVal(Val.getBitWidth(), radix);
+  llvm::APInt CharVal(Val.getBitWidth(), 0);
+  llvm::APInt OldVal = Val;
+
+  bool OverflowOccurred = false;
+  while (s < SuffixBegin) {
+    unsigned C = HexDigitValue(*s++);
+
+    // If this letter is out of bound for this radix, reject it.
+    assert(C < radix && "NumericLiteralParser ctor should have rejected this");
+
+    CharVal = C;
+
+    // Add the digit to the value in the appropriate radix.  If adding in digits
+    // made the value smaller, then this overflowed.
+    OldVal = Val;
+
+    // Multiply by radix, did overflow occur on the multiply?
+    Val *= RadixVal;
+    OverflowOccurred |= Val.udiv(RadixVal) != OldVal;
+
+    // Add value, did overflow occur on the value?
+    //   (a + b) ult b  <=> overflow
+    Val += CharVal;
+    OverflowOccurred |= Val.ult(CharVal);
+  }
+  return OverflowOccurred;
+}
+
+llvm::APFloat::opStatus
+NumericLiteralParser::GetFloatValue(llvm::APFloat &Result) {
+  using llvm::APFloat;
+
+  unsigned n = std::min(SuffixBegin - ThisTokBegin, ThisTokEnd - ThisTokBegin);
+  return Result.convertFromString(StringRef(ThisTokBegin, n),
+                                  APFloat::rmNearestTiesToEven);
+}
+
+
+///       user-defined-character-literal: [C++11 lex.ext]
+///         character-literal ud-suffix
+///       ud-suffix:
+///         identifier
+///       character-literal: [C++11 lex.ccon]
+///         ' c-char-sequence '
+///         u' c-char-sequence '
+///         U' c-char-sequence '
+///         L' c-char-sequence '
+///       c-char-sequence:
+///         c-char
+///         c-char-sequence c-char
+///       c-char:
+///         any member of the source character set except the single-quote ',
+///           backslash \, or new-line character
+///         escape-sequence
+///         universal-character-name
+///       escape-sequence:
+///         simple-escape-sequence
+///         octal-escape-sequence
+///         hexadecimal-escape-sequence
+///       simple-escape-sequence:
+///         one of \' \" \? \\ \a \b \f \n \r \t \v
+///       octal-escape-sequence:
+///         \ octal-digit
+///         \ octal-digit octal-digit
+///         \ octal-digit octal-digit octal-digit
+///       hexadecimal-escape-sequence:
+///         \x hexadecimal-digit
+///         hexadecimal-escape-sequence hexadecimal-digit
+///       universal-character-name: [C++11 lex.charset]
+///         \u hex-quad
+///         \U hex-quad hex-quad
+///       hex-quad:
+///         hex-digit hex-digit hex-digit hex-digit
+///
+CharLiteralParser::CharLiteralParser(const char *begin, const char *end,
+                                     SourceLocation Loc, Preprocessor &PP,
+                                     tok::TokenKind kind) {
+  // At this point we know that the character matches the regex "(L|u|U)?'.*'".
+  HadError = false;
+
+  Kind = kind;
+
+  const char *TokBegin = begin;
+
+  // Skip over wide character determinant.
+  if (Kind != tok::char_constant) {
+    ++begin;
+  }
+
+  // Skip over the entry quote.
+  assert(begin[0] == '\'' && "Invalid token lexed");
+  ++begin;
+
+  // Remove an optional ud-suffix.
+  if (end[-1] != '\'') {
+    const char *UDSuffixEnd = end;
+    do {
+      --end;
+    } while (end[-1] != '\'');
+    UDSuffixBuf.assign(end, UDSuffixEnd);
+    UDSuffixOffset = end - TokBegin;
+  }
+
+  // Trim the ending quote.
+  assert(end != begin && "Invalid token lexed");
+  --end;
+
+  // FIXME: The "Value" is an uint64_t so we can handle char literals of
+  // up to 64-bits.
+  // FIXME: This extensively assumes that 'char' is 8-bits.
+  assert(PP.getTargetInfo().getCharWidth() == 8 &&
+         "Assumes char is 8 bits");
+  assert(PP.getTargetInfo().getIntWidth() <= 64 &&
+         (PP.getTargetInfo().getIntWidth() & 7) == 0 &&
+         "Assumes sizeof(int) on target is <= 64 and a multiple of char");
+  assert(PP.getTargetInfo().getWCharWidth() <= 64 &&
+         "Assumes sizeof(wchar) on target is <= 64");
+
+  SmallVector<uint32_t,4> codepoint_buffer;
+  codepoint_buffer.resize(end-begin);
+  uint32_t *buffer_begin = &codepoint_buffer.front();
+  uint32_t *buffer_end = buffer_begin + codepoint_buffer.size();
+
+  // Unicode escapes representing characters that cannot be correctly
+  // represented in a single code unit are disallowed in character literals
+  // by this implementation.
+  uint32_t largest_character_for_kind;
+  if (tok::wide_char_constant == Kind) {
+    largest_character_for_kind = 0xFFFFFFFFu >> (32-PP.getTargetInfo().getWCharWidth());
+  } else if (tok::utf16_char_constant == Kind) {
+    largest_character_for_kind = 0xFFFF;
+  } else if (tok::utf32_char_constant == Kind) {
+    largest_character_for_kind = 0x10FFFF;
+  } else {
+    largest_character_for_kind = 0x7Fu;
+  }
+
+  while (begin!=end) {
+    // Is this a span of non-escape characters?
+    if (begin[0] != '\\') {
+      char const *start = begin;
+      do {
+        ++begin;
+      } while (begin != end && *begin != '\\');
+
+      char const *tmp_in_start = start;
+      uint32_t *tmp_out_start = buffer_begin;
+      ConversionResult res =
+      ConvertUTF8toUTF32(reinterpret_cast<UTF8 const **>(&start),
+                         reinterpret_cast<UTF8 const *>(begin),
+                         &buffer_begin,buffer_end,strictConversion);
+      if (res!=conversionOK) {
+        // If we see bad encoding for unprefixed character literals, warn and 
+        // simply copy the byte values, for compatibility with gcc and 
+        // older versions of clang.
+        bool NoErrorOnBadEncoding = isAscii();
+        unsigned Msg = diag::err_bad_character_encoding;
+        if (NoErrorOnBadEncoding)
+          Msg = diag::warn_bad_character_encoding;
+        PP.Diag(Loc, Msg);
+        if (NoErrorOnBadEncoding) {
+          start = tmp_in_start;
+          buffer_begin = tmp_out_start;
+          for ( ; start != begin; ++start, ++buffer_begin)
+            *buffer_begin = static_cast<uint8_t>(*start);
+        } else {
+          HadError = true;
+        }
+      } else {
+        for (; tmp_out_start <buffer_begin; ++tmp_out_start) {
+          if (*tmp_out_start > largest_character_for_kind) {
+            HadError = true;
+            PP.Diag(Loc, diag::err_character_too_large);
+          }
+        }
+      }
+
+      continue;
+    }
+    // Is this a Universal Character Name excape?
+    if (begin[1] == 'u' || begin[1] == 'U') {
+      unsigned short UcnLen = 0;
+      if (!ProcessUCNEscape(TokBegin, begin, end, *buffer_begin, UcnLen,
+                            FullSourceLoc(Loc, PP.getSourceManager()),
+                            &PP.getDiagnostics(), PP.getLangOpts(),
+                            true))
+      {
+        HadError = true;
+      } else if (*buffer_begin > largest_character_for_kind) {
+        HadError = true;
+        PP.Diag(Loc,diag::err_character_too_large);
+      }
+
+      ++buffer_begin;
+      continue;
+    }
+    unsigned CharWidth = getCharWidth(Kind, PP.getTargetInfo());
+    uint64_t result =
+    ProcessCharEscape(begin, end, HadError,
+                      FullSourceLoc(Loc,PP.getSourceManager()),
+                      CharWidth, &PP.getDiagnostics());
+    *buffer_begin++ = result;
+  }
+
+  unsigned NumCharsSoFar = buffer_begin-&codepoint_buffer.front();
+
+  if (NumCharsSoFar > 1) {
+    if (isWide())
+      PP.Diag(Loc, diag::warn_extraneous_char_constant);
+    else if (isAscii() && NumCharsSoFar == 4)
+      PP.Diag(Loc, diag::ext_four_char_character_literal);
+    else if (isAscii())
+      PP.Diag(Loc, diag::ext_multichar_character_literal);
+    else
+      PP.Diag(Loc, diag::err_multichar_utf_character_literal);
+    IsMultiChar = true;
+  } else
+    IsMultiChar = false;
+
+  llvm::APInt LitVal(PP.getTargetInfo().getIntWidth(), 0);
+
+  // Narrow character literals act as though their value is concatenated
+  // in this implementation, but warn on overflow.
+  bool multi_char_too_long = false;
+  if (isAscii() && isMultiChar()) {
+    LitVal = 0;
+    for (size_t i=0;i<NumCharsSoFar;++i) {
+      // check for enough leading zeros to shift into
+      multi_char_too_long |= (LitVal.countLeadingZeros() < 8);
+      LitVal <<= 8;
+      LitVal = LitVal + (codepoint_buffer[i] & 0xFF);
+    }
+  } else if (NumCharsSoFar > 0) {
+    // otherwise just take the last character
+    LitVal = buffer_begin[-1];
+  }
+
+  if (!HadError && multi_char_too_long) {
+    PP.Diag(Loc,diag::warn_char_constant_too_large);
+  }
+
+  // Transfer the value from APInt to uint64_t
+  Value = LitVal.getZExtValue();
+
+  // If this is a single narrow character, sign extend it (e.g. '\xFF' is "-1")
+  // if 'char' is signed for this target (C99 6.4.4.4p10).  Note that multiple
+  // character constants are not sign extended in the this implementation:
+  // '\xFF\xFF' = 65536 and '\x0\xFF' = 255, which matches GCC.
+  if (isAscii() && NumCharsSoFar == 1 && (Value & 128) &&
+      PP.getLangOpts().CharIsSigned)
+    Value = (signed char)Value;
+}
+
+
+///       string-literal: [C++0x lex.string]
+///         encoding-prefix " [s-char-sequence] "
+///         encoding-prefix R raw-string
+///       encoding-prefix:
+///         u8
+///         u
+///         U
+///         L
+///       s-char-sequence:
+///         s-char
+///         s-char-sequence s-char
+///       s-char:
+///         any member of the source character set except the double-quote ",
+///           backslash \, or new-line character
+///         escape-sequence
+///         universal-character-name
+///       raw-string:
+///         " d-char-sequence ( r-char-sequence ) d-char-sequence "
+///       r-char-sequence:
+///         r-char
+///         r-char-sequence r-char
+///       r-char:
+///         any member of the source character set, except a right parenthesis )
+///           followed by the initial d-char-sequence (which may be empty)
+///           followed by a double quote ".
+///       d-char-sequence:
+///         d-char
+///         d-char-sequence d-char
+///       d-char:
+///         any member of the basic source character set except:
+///           space, the left parenthesis (, the right parenthesis ),
+///           the backslash \, and the control characters representing horizontal
+///           tab, vertical tab, form feed, and newline.
+///       escape-sequence: [C++0x lex.ccon]
+///         simple-escape-sequence
+///         octal-escape-sequence
+///         hexadecimal-escape-sequence
+///       simple-escape-sequence:
+///         one of \' \" \? \\ \a \b \f \n \r \t \v
+///       octal-escape-sequence:
+///         \ octal-digit
+///         \ octal-digit octal-digit
+///         \ octal-digit octal-digit octal-digit
+///       hexadecimal-escape-sequence:
+///         \x hexadecimal-digit
+///         hexadecimal-escape-sequence hexadecimal-digit
+///       universal-character-name:
+///         \u hex-quad
+///         \U hex-quad hex-quad
+///       hex-quad:
+///         hex-digit hex-digit hex-digit hex-digit
+///
+StringLiteralParser::
+StringLiteralParser(const Token *StringToks, unsigned NumStringToks,
+                    Preprocessor &PP, bool Complain)
+  : SM(PP.getSourceManager()), Features(PP.getLangOpts()),
+    Target(PP.getTargetInfo()), Diags(Complain ? &PP.getDiagnostics() : 0),
+    MaxTokenLength(0), SizeBound(0), CharByteWidth(0), Kind(tok::unknown),
+    ResultPtr(ResultBuf.data()), hadError(false), Pascal(false) {
+  init(StringToks, NumStringToks);
+}
+
+void StringLiteralParser::init(const Token *StringToks, unsigned NumStringToks){
+  // The literal token may have come from an invalid source location (e.g. due
+  // to a PCH error), in which case the token length will be 0.
+  if (NumStringToks == 0 || StringToks[0].getLength() < 2) {
+    hadError = true;
+    return;
+  }
+
+  // Scan all of the string portions, remember the max individual token length,
+  // computing a bound on the concatenated string length, and see whether any
+  // piece is a wide-string.  If any of the string portions is a wide-string
+  // literal, the result is a wide-string literal [C99 6.4.5p4].
+  assert(NumStringToks && "expected at least one token");
+  MaxTokenLength = StringToks[0].getLength();
+  assert(StringToks[0].getLength() >= 2 && "literal token is invalid!");
+  SizeBound = StringToks[0].getLength()-2;  // -2 for "".
+  Kind = StringToks[0].getKind();
+
+  hadError = false;
+
+  // Implement Translation Phase #6: concatenation of string literals
+  /// (C99 5.1.1.2p1).  The common case is only one string fragment.
+  for (unsigned i = 1; i != NumStringToks; ++i) {
+    if (StringToks[i].getLength() < 2) {
+      hadError = true;
+      return;
+    }
+
+    // The string could be shorter than this if it needs cleaning, but this is a
+    // reasonable bound, which is all we need.
+    assert(StringToks[i].getLength() >= 2 && "literal token is invalid!");
+    SizeBound += StringToks[i].getLength()-2;  // -2 for "".
+
+    // Remember maximum string piece length.
+    if (StringToks[i].getLength() > MaxTokenLength)
+      MaxTokenLength = StringToks[i].getLength();
+
+    // Remember if we see any wide or utf-8/16/32 strings.
+    // Also check for illegal concatenations.
+    if (StringToks[i].isNot(Kind) && StringToks[i].isNot(tok::string_literal)) {
+      if (isAscii()) {
+        Kind = StringToks[i].getKind();
+      } else {
+        if (Diags)
+          Diags->Report(FullSourceLoc(StringToks[i].getLocation(), SM),
+                        diag::err_unsupported_string_concat);
+        hadError = true;
+      }
+    }
+  }
+
+  // Include space for the null terminator.
+  ++SizeBound;
+
+  // TODO: K&R warning: "traditional C rejects string constant concatenation"
+
+  // Get the width in bytes of char/wchar_t/char16_t/char32_t
+  CharByteWidth = getCharWidth(Kind, Target);
+  assert((CharByteWidth & 7) == 0 && "Assumes character size is byte multiple");
+  CharByteWidth /= 8;
+
+  // The output buffer size needs to be large enough to hold wide characters.
+  // This is a worst-case assumption which basically corresponds to L"" "long".
+  SizeBound *= CharByteWidth;
+
+  // Size the temporary buffer to hold the result string data.
+  ResultBuf.resize(SizeBound);
+
+  // Likewise, but for each string piece.
+  SmallString<512> TokenBuf;
+  TokenBuf.resize(MaxTokenLength);
+
+  // Loop over all the strings, getting their spelling, and expanding them to
+  // wide strings as appropriate.
+  ResultPtr = &ResultBuf[0];   // Next byte to fill in.
+
+  Pascal = false;
+
+  SourceLocation UDSuffixTokLoc;
+
+  for (unsigned i = 0, e = NumStringToks; i != e; ++i) {
+    const char *ThisTokBuf = &TokenBuf[0];
+    // Get the spelling of the token, which eliminates trigraphs, etc.  We know
+    // that ThisTokBuf points to a buffer that is big enough for the whole token
+    // and 'spelled' tokens can only shrink.
+    bool StringInvalid = false;
+    unsigned ThisTokLen = 
+      Lexer::getSpelling(StringToks[i], ThisTokBuf, SM, Features,
+                         &StringInvalid);
+    if (StringInvalid) {
+      hadError = true;
+      continue;
+    }
+
+    const char *ThisTokBegin = ThisTokBuf;
+    const char *ThisTokEnd = ThisTokBuf+ThisTokLen;
+
+    // Remove an optional ud-suffix.
+    if (ThisTokEnd[-1] != '"') {
+      const char *UDSuffixEnd = ThisTokEnd;
+      do {
+        --ThisTokEnd;
+      } while (ThisTokEnd[-1] != '"');
+
+      StringRef UDSuffix(ThisTokEnd, UDSuffixEnd - ThisTokEnd);
+
+      if (UDSuffixBuf.empty()) {
+        UDSuffixBuf.assign(UDSuffix);
+        UDSuffixToken = i;
+        UDSuffixOffset = ThisTokEnd - ThisTokBuf;
+        UDSuffixTokLoc = StringToks[i].getLocation();
+      } else if (!UDSuffixBuf.equals(UDSuffix)) {
+        // C++11 [lex.ext]p8: At the end of phase 6, if a string literal is the
+        // result of a concatenation involving at least one user-defined-string-
+        // literal, all the participating user-defined-string-literals shall
+        // have the same ud-suffix.
+        if (Diags) {
+          SourceLocation TokLoc = StringToks[i].getLocation();
+          Diags->Report(TokLoc, diag::err_string_concat_mixed_suffix)
+            << UDSuffixBuf << UDSuffix
+            << SourceRange(UDSuffixTokLoc, UDSuffixTokLoc)
+            << SourceRange(TokLoc, TokLoc);
+        }
+        hadError = true;
+      }
+    }
+
+    // Strip the end quote.
+    --ThisTokEnd;
+
+    // TODO: Input character set mapping support.
+
+    // Skip marker for wide or unicode strings.
+    if (ThisTokBuf[0] == 'L' || ThisTokBuf[0] == 'u' || ThisTokBuf[0] == 'U') {
+      ++ThisTokBuf;
+      // Skip 8 of u8 marker for utf8 strings.
+      if (ThisTokBuf[0] == '8')
+        ++ThisTokBuf;
+    }
+
+    // Check for raw string
+    if (ThisTokBuf[0] == 'R') {
+      ThisTokBuf += 2; // skip R"
+
+      const char *Prefix = ThisTokBuf;
+      while (ThisTokBuf[0] != '(')
+        ++ThisTokBuf;
+      ++ThisTokBuf; // skip '('
+
+      // Remove same number of characters from the end
+      ThisTokEnd -= ThisTokBuf - Prefix;
+      assert(ThisTokEnd >= ThisTokBuf && "malformed raw string literal");
+
+      // Copy the string over
+      if (CopyStringFragment(StringRef(ThisTokBuf, ThisTokEnd - ThisTokBuf)))
+        if (DiagnoseBadString(StringToks[i]))
+          hadError = true;
+    } else {
+      assert(ThisTokBuf[0] == '"' && "Expected quote, lexer broken?");
+      ++ThisTokBuf; // skip "
+
+      // Check if this is a pascal string
+      if (Features.PascalStrings && ThisTokBuf + 1 != ThisTokEnd &&
+          ThisTokBuf[0] == '\\' && ThisTokBuf[1] == 'p') {
+
+        // If the \p sequence is found in the first token, we have a pascal string
+        // Otherwise, if we already have a pascal string, ignore the first \p
+        if (i == 0) {
+          ++ThisTokBuf;
+          Pascal = true;
+        } else if (Pascal)
+          ThisTokBuf += 2;
+      }
+
+      while (ThisTokBuf != ThisTokEnd) {
+        // Is this a span of non-escape characters?
+        if (ThisTokBuf[0] != '\\') {
+          const char *InStart = ThisTokBuf;
+          do {
+            ++ThisTokBuf;
+          } while (ThisTokBuf != ThisTokEnd && ThisTokBuf[0] != '\\');
+
+          // Copy the character span over.
+          if (CopyStringFragment(StringRef(InStart, ThisTokBuf - InStart)))
+            if (DiagnoseBadString(StringToks[i]))
+              hadError = true;
+          continue;
+        }
+        // Is this a Universal Character Name escape?
+        if (ThisTokBuf[1] == 'u' || ThisTokBuf[1] == 'U') {
+          EncodeUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd,
+                          ResultPtr, hadError,
+                          FullSourceLoc(StringToks[i].getLocation(), SM),
+                          CharByteWidth, Diags, Features);
+          continue;
+        }
+        // Otherwise, this is a non-UCN escape character.  Process it.
+        unsigned ResultChar =
+          ProcessCharEscape(ThisTokBuf, ThisTokEnd, hadError,
+                            FullSourceLoc(StringToks[i].getLocation(), SM),
+                            CharByteWidth*8, Diags);
+
+        if (CharByteWidth == 4) {
+          // FIXME: Make the type of the result buffer correct instead of
+          // using reinterpret_cast.
+          UTF32 *ResultWidePtr = reinterpret_cast<UTF32*>(ResultPtr);
+          *ResultWidePtr = ResultChar;
+          ResultPtr += 4;
+        } else if (CharByteWidth == 2) {
+          // FIXME: Make the type of the result buffer correct instead of
+          // using reinterpret_cast.
+          UTF16 *ResultWidePtr = reinterpret_cast<UTF16*>(ResultPtr);
+          *ResultWidePtr = ResultChar & 0xFFFF;
+          ResultPtr += 2;
+        } else {
+          assert(CharByteWidth == 1 && "Unexpected char width");
+          *ResultPtr++ = ResultChar & 0xFF;
+        }
+      }
+    }
+  }
+
+  if (Pascal) {
+    if (CharByteWidth == 4) {
+      // FIXME: Make the type of the result buffer correct instead of
+      // using reinterpret_cast.
+      UTF32 *ResultWidePtr = reinterpret_cast<UTF32*>(ResultBuf.data());
+      ResultWidePtr[0] = GetNumStringChars() - 1;
+    } else if (CharByteWidth == 2) {
+      // FIXME: Make the type of the result buffer correct instead of
+      // using reinterpret_cast.
+      UTF16 *ResultWidePtr = reinterpret_cast<UTF16*>(ResultBuf.data());
+      ResultWidePtr[0] = GetNumStringChars() - 1;
+    } else {
+      assert(CharByteWidth == 1 && "Unexpected char width");
+      ResultBuf[0] = GetNumStringChars() - 1;
+    }
+
+    // Verify that pascal strings aren't too large.
+    if (GetStringLength() > 256) {
+      if (Diags) 
+        Diags->Report(FullSourceLoc(StringToks[0].getLocation(), SM),
+                      diag::err_pascal_string_too_long)
+          << SourceRange(StringToks[0].getLocation(),
+                         StringToks[NumStringToks-1].getLocation());
+      hadError = true;
+      return;
+    }
+  } else if (Diags) {
+    // Complain if this string literal has too many characters.
+    unsigned MaxChars = Features.CPlusPlus? 65536 : Features.C99 ? 4095 : 509;
+    
+    if (GetNumStringChars() > MaxChars)
+      Diags->Report(FullSourceLoc(StringToks[0].getLocation(), SM),
+                    diag::ext_string_too_long)
+        << GetNumStringChars() << MaxChars
+        << (Features.CPlusPlus ? 2 : Features.C99 ? 1 : 0)
+        << SourceRange(StringToks[0].getLocation(),
+                       StringToks[NumStringToks-1].getLocation());
+  }
+}
+
+
+/// copyStringFragment - This function copies from Start to End into ResultPtr.
+/// Performs widening for multi-byte characters.
+bool StringLiteralParser::CopyStringFragment(StringRef Fragment) {
+  assert(CharByteWidth==1 || CharByteWidth==2 || CharByteWidth==4);
+  ConversionResult result = conversionOK;
+  // Copy the character span over.
+  if (CharByteWidth == 1) {
+    if (!isLegalUTF8String(reinterpret_cast<const UTF8*>(Fragment.begin()),
+                           reinterpret_cast<const UTF8*>(Fragment.end())))
+      result = sourceIllegal;
+    memcpy(ResultPtr, Fragment.data(), Fragment.size());
+    ResultPtr += Fragment.size();
+  } else if (CharByteWidth == 2) {
+    UTF8 const *sourceStart = (UTF8 const *)Fragment.data();
+    // FIXME: Make the type of the result buffer correct instead of
+    // using reinterpret_cast.
+    UTF16 *targetStart = reinterpret_cast<UTF16*>(ResultPtr);
+    ConversionFlags flags = strictConversion;
+    result = ConvertUTF8toUTF16(
+	    &sourceStart,sourceStart + Fragment.size(),
+        &targetStart,targetStart + 2*Fragment.size(),flags);
+    if (result==conversionOK)
+      ResultPtr = reinterpret_cast<char*>(targetStart);
+  } else if (CharByteWidth == 4) {
+    UTF8 const *sourceStart = (UTF8 const *)Fragment.data();
+    // FIXME: Make the type of the result buffer correct instead of
+    // using reinterpret_cast.
+    UTF32 *targetStart = reinterpret_cast<UTF32*>(ResultPtr);
+    ConversionFlags flags = strictConversion;
+    result = ConvertUTF8toUTF32(
+        &sourceStart,sourceStart + Fragment.size(),
+        &targetStart,targetStart + 4*Fragment.size(),flags);
+    if (result==conversionOK)
+      ResultPtr = reinterpret_cast<char*>(targetStart);
+  }
+  assert((result != targetExhausted)
+         && "ConvertUTF8toUTFXX exhausted target buffer");
+  return result != conversionOK;
+}
+
+bool StringLiteralParser::DiagnoseBadString(const Token &Tok) {
+  // If we see bad encoding for unprefixed string literals, warn and
+  // simply copy the byte values, for compatibility with gcc and older
+  // versions of clang.
+  bool NoErrorOnBadEncoding = isAscii();
+  unsigned Msg = NoErrorOnBadEncoding ? diag::warn_bad_string_encoding :
+                                        diag::err_bad_string_encoding;
+  if (Diags)
+    Diags->Report(FullSourceLoc(Tok.getLocation(), SM), Msg);
+  return !NoErrorOnBadEncoding;
+}
+
+/// getOffsetOfStringByte - This function returns the offset of the
+/// specified byte of the string data represented by Token.  This handles
+/// advancing over escape sequences in the string.
+unsigned StringLiteralParser::getOffsetOfStringByte(const Token &Tok,
+                                                    unsigned ByteNo) const {
+  // Get the spelling of the token.
+  SmallString<32> SpellingBuffer;
+  SpellingBuffer.resize(Tok.getLength());
+
+  bool StringInvalid = false;
+  const char *SpellingPtr = &SpellingBuffer[0];
+  unsigned TokLen = Lexer::getSpelling(Tok, SpellingPtr, SM, Features,
+                                       &StringInvalid);
+  if (StringInvalid)
+    return 0;
+
+  assert(SpellingPtr[0] != 'L' && SpellingPtr[0] != 'u' &&
+         SpellingPtr[0] != 'U' && "Doesn't handle wide or utf strings yet");
+
+
+  const char *SpellingStart = SpellingPtr;
+  const char *SpellingEnd = SpellingPtr+TokLen;
+
+  // Skip over the leading quote.
+  assert(SpellingPtr[0] == '"' && "Should be a string literal!");
+  ++SpellingPtr;
+
+  // Skip over bytes until we find the offset we're looking for.
+  while (ByteNo) {
+    assert(SpellingPtr < SpellingEnd && "Didn't find byte offset!");
+
+    // Step over non-escapes simply.
+    if (*SpellingPtr != '\\') {
+      ++SpellingPtr;
+      --ByteNo;
+      continue;
+    }
+
+    // Otherwise, this is an escape character.  Advance over it.
+    bool HadError = false;
+    ProcessCharEscape(SpellingPtr, SpellingEnd, HadError,
+                      FullSourceLoc(Tok.getLocation(), SM),
+                      CharByteWidth*8, Diags);
+    assert(!HadError && "This method isn't valid on erroneous strings");
+    --ByteNo;
+  }
+
+  return SpellingPtr-SpellingStart;
+}
diff --git a/clang/lib/Lex/MacroArgs.cpp b/clang/lib/Lex/MacroArgs.cpp
new file mode 100644
index 0000000..e2b251a
--- /dev/null
+++ b/clang/lib/Lex/MacroArgs.cpp
@@ -0,0 +1,317 @@
+//===--- TokenLexer.cpp - Lex from a token stream -------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the TokenLexer interface.
+//
+//===----------------------------------------------------------------------===//
+
+#include "MacroArgs.h"
+#include "clang/Lex/MacroInfo.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/LexDiagnostic.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/SaveAndRestore.h"
+#include <algorithm>
+
+using namespace clang;
+
+/// MacroArgs ctor function - This destroys the vector passed in.
+MacroArgs *MacroArgs::create(const MacroInfo *MI,
+                             llvm::ArrayRef<Token> UnexpArgTokens,
+                             bool VarargsElided, Preprocessor &PP) {
+  assert(MI->isFunctionLike() &&
+         "Can't have args for an object-like macro!");
+  MacroArgs **ResultEnt = 0;
+  unsigned ClosestMatch = ~0U;
+  
+  // See if we have an entry with a big enough argument list to reuse on the
+  // free list.  If so, reuse it.
+  for (MacroArgs **Entry = &PP.MacroArgCache; *Entry;
+       Entry = &(*Entry)->ArgCache)
+    if ((*Entry)->NumUnexpArgTokens >= UnexpArgTokens.size() &&
+        (*Entry)->NumUnexpArgTokens < ClosestMatch) {
+      ResultEnt = Entry;
+      
+      // If we have an exact match, use it.
+      if ((*Entry)->NumUnexpArgTokens == UnexpArgTokens.size())
+        break;
+      // Otherwise, use the best fit.
+      ClosestMatch = (*Entry)->NumUnexpArgTokens;
+    }
+  
+  MacroArgs *Result;
+  if (ResultEnt == 0) {
+    // Allocate memory for a MacroArgs object with the lexer tokens at the end.
+    Result = (MacroArgs*)malloc(sizeof(MacroArgs) + 
+                                UnexpArgTokens.size() * sizeof(Token));
+    // Construct the MacroArgs object.
+    new (Result) MacroArgs(UnexpArgTokens.size(), VarargsElided);
+  } else {
+    Result = *ResultEnt;
+    // Unlink this node from the preprocessors singly linked list.
+    *ResultEnt = Result->ArgCache;
+    Result->NumUnexpArgTokens = UnexpArgTokens.size();
+    Result->VarargsElided = VarargsElided;
+  }
+
+  // Copy the actual unexpanded tokens to immediately after the result ptr.
+  if (!UnexpArgTokens.empty())
+    std::copy(UnexpArgTokens.begin(), UnexpArgTokens.end(), 
+              const_cast<Token*>(Result->getUnexpArgument(0)));
+
+  return Result;
+}
+
+/// destroy - Destroy and deallocate the memory for this object.
+///
+void MacroArgs::destroy(Preprocessor &PP) {
+  StringifiedArgs.clear();
+
+  // Don't clear PreExpArgTokens, just clear the entries.  Clearing the entries
+  // would deallocate the element vectors.
+  for (unsigned i = 0, e = PreExpArgTokens.size(); i != e; ++i)
+    PreExpArgTokens[i].clear();
+  
+  // Add this to the preprocessor's free list.
+  ArgCache = PP.MacroArgCache;
+  PP.MacroArgCache = this;
+}
+
+/// deallocate - This should only be called by the Preprocessor when managing
+/// its freelist.
+MacroArgs *MacroArgs::deallocate() {
+  MacroArgs *Next = ArgCache;
+  
+  // Run the dtor to deallocate the vectors.
+  this->~MacroArgs();
+  // Release the memory for the object.
+  free(this);
+  
+  return Next;
+}
+
+
+/// getArgLength - Given a pointer to an expanded or unexpanded argument,
+/// return the number of tokens, not counting the EOF, that make up the
+/// argument.
+unsigned MacroArgs::getArgLength(const Token *ArgPtr) {
+  unsigned NumArgTokens = 0;
+  for (; ArgPtr->isNot(tok::eof); ++ArgPtr)
+    ++NumArgTokens;
+  return NumArgTokens;
+}
+
+
+/// getUnexpArgument - Return the unexpanded tokens for the specified formal.
+///
+const Token *MacroArgs::getUnexpArgument(unsigned Arg) const {
+  // The unexpanded argument tokens start immediately after the MacroArgs object
+  // in memory.
+  const Token *Start = (const Token *)(this+1);
+  const Token *Result = Start;
+  // Scan to find Arg.
+  for (; Arg; ++Result) {
+    assert(Result < Start+NumUnexpArgTokens && "Invalid arg #");
+    if (Result->is(tok::eof))
+      --Arg;
+  }
+  assert(Result < Start+NumUnexpArgTokens && "Invalid arg #");
+  return Result;
+}
+
+
+/// ArgNeedsPreexpansion - If we can prove that the argument won't be affected
+/// by pre-expansion, return false.  Otherwise, conservatively return true.
+bool MacroArgs::ArgNeedsPreexpansion(const Token *ArgTok,
+                                     Preprocessor &PP) const {
+  // If there are no identifiers in the argument list, or if the identifiers are
+  // known to not be macros, pre-expansion won't modify it.
+  for (; ArgTok->isNot(tok::eof); ++ArgTok)
+    if (IdentifierInfo *II = ArgTok->getIdentifierInfo()) {
+      if (II->hasMacroDefinition() && PP.getMacroInfo(II)->isEnabled())
+        // Return true even though the macro could be a function-like macro
+        // without a following '(' token.
+        return true;
+    }
+  return false;
+}
+
+/// getPreExpArgument - Return the pre-expanded form of the specified
+/// argument.
+const std::vector<Token> &
+MacroArgs::getPreExpArgument(unsigned Arg, const MacroInfo *MI, 
+                             Preprocessor &PP) {
+  assert(Arg < MI->getNumArgs() && "Invalid argument number!");
+
+  // If we have already computed this, return it.
+  if (PreExpArgTokens.size() < MI->getNumArgs())
+    PreExpArgTokens.resize(MI->getNumArgs());
+  
+  std::vector<Token> &Result = PreExpArgTokens[Arg];
+  if (!Result.empty()) return Result;
+
+  SaveAndRestore<bool> PreExpandingMacroArgs(PP.InMacroArgPreExpansion, true);
+
+  const Token *AT = getUnexpArgument(Arg);
+  unsigned NumToks = getArgLength(AT)+1;  // Include the EOF.
+
+  // Otherwise, we have to pre-expand this argument, populating Result.  To do
+  // this, we set up a fake TokenLexer to lex from the unexpanded argument
+  // list.  With this installed, we lex expanded tokens until we hit the EOF
+  // token at the end of the unexp list.
+  PP.EnterTokenStream(AT, NumToks, false /*disable expand*/,
+                      false /*owns tokens*/);
+
+  // Lex all of the macro-expanded tokens into Result.
+  do {
+    Result.push_back(Token());
+    Token &Tok = Result.back();
+    PP.Lex(Tok);
+  } while (Result.back().isNot(tok::eof));
+
+  // Pop the token stream off the top of the stack.  We know that the internal
+  // pointer inside of it is to the "end" of the token stream, but the stack
+  // will not otherwise be popped until the next token is lexed.  The problem is
+  // that the token may be lexed sometime after the vector of tokens itself is
+  // destroyed, which would be badness.
+  if (PP.InCachingLexMode())
+    PP.ExitCachingLexMode();
+  PP.RemoveTopOfLexerStack();
+  return Result;
+}
+
+
+/// StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of
+/// tokens into the literal string token that should be produced by the C #
+/// preprocessor operator.  If Charify is true, then it should be turned into
+/// a character literal for the Microsoft charize (#@) extension.
+///
+Token MacroArgs::StringifyArgument(const Token *ArgToks,
+                                   Preprocessor &PP, bool Charify,
+                                   SourceLocation ExpansionLocStart,
+                                   SourceLocation ExpansionLocEnd) {
+  Token Tok;
+  Tok.startToken();
+  Tok.setKind(Charify ? tok::char_constant : tok::string_literal);
+
+  const Token *ArgTokStart = ArgToks;
+
+  // Stringify all the tokens.
+  SmallString<128> Result;
+  Result += "\"";
+
+  bool isFirst = true;
+  for (; ArgToks->isNot(tok::eof); ++ArgToks) {
+    const Token &Tok = *ArgToks;
+    if (!isFirst && (Tok.hasLeadingSpace() || Tok.isAtStartOfLine()))
+      Result += ' ';
+    isFirst = false;
+
+    // If this is a string or character constant, escape the token as specified
+    // by 6.10.3.2p2.
+    if (Tok.is(tok::string_literal) ||       // "foo"
+        Tok.is(tok::wide_string_literal) ||  // L"foo"
+        Tok.is(tok::utf8_string_literal) ||  // u8"foo"
+        Tok.is(tok::utf16_string_literal) || // u"foo"
+        Tok.is(tok::utf32_string_literal) || // U"foo"
+        Tok.is(tok::char_constant) ||        // 'x'
+        Tok.is(tok::wide_char_constant) ||   // L'x'.
+        Tok.is(tok::utf16_char_constant) ||  // u'x'.
+        Tok.is(tok::utf32_char_constant)) {  // U'x'.
+      bool Invalid = false;
+      std::string TokStr = PP.getSpelling(Tok, &Invalid);
+      if (!Invalid) {
+        std::string Str = Lexer::Stringify(TokStr);
+        Result.append(Str.begin(), Str.end());
+      }
+    } else if (Tok.is(tok::code_completion)) {
+      PP.CodeCompleteNaturalLanguage();
+    } else {
+      // Otherwise, just append the token.  Do some gymnastics to get the token
+      // in place and avoid copies where possible.
+      unsigned CurStrLen = Result.size();
+      Result.resize(CurStrLen+Tok.getLength());
+      const char *BufPtr = &Result[CurStrLen];
+      bool Invalid = false;
+      unsigned ActualTokLen = PP.getSpelling(Tok, BufPtr, &Invalid);
+
+      if (!Invalid) {
+        // If getSpelling returned a pointer to an already uniqued version of
+        // the string instead of filling in BufPtr, memcpy it onto our string.
+        if (BufPtr != &Result[CurStrLen])
+          memcpy(&Result[CurStrLen], BufPtr, ActualTokLen);
+
+        // If the token was dirty, the spelling may be shorter than the token.
+        if (ActualTokLen != Tok.getLength())
+          Result.resize(CurStrLen+ActualTokLen);
+      }
+    }
+  }
+
+  // If the last character of the string is a \, and if it isn't escaped, this
+  // is an invalid string literal, diagnose it as specified in C99.
+  if (Result.back() == '\\') {
+    // Count the number of consequtive \ characters.  If even, then they are
+    // just escaped backslashes, otherwise it's an error.
+    unsigned FirstNonSlash = Result.size()-2;
+    // Guaranteed to find the starting " if nothing else.
+    while (Result[FirstNonSlash] == '\\')
+      --FirstNonSlash;
+    if ((Result.size()-1-FirstNonSlash) & 1) {
+      // Diagnose errors for things like: #define F(X) #X   /   F(\)
+      PP.Diag(ArgToks[-1], diag::pp_invalid_string_literal);
+      Result.pop_back();  // remove one of the \'s.
+    }
+  }
+  Result += '"';
+
+  // If this is the charify operation and the result is not a legal character
+  // constant, diagnose it.
+  if (Charify) {
+    // First step, turn double quotes into single quotes:
+    Result[0] = '\'';
+    Result[Result.size()-1] = '\'';
+
+    // Check for bogus character.
+    bool isBad = false;
+    if (Result.size() == 3)
+      isBad = Result[1] == '\'';   // ''' is not legal. '\' already fixed above.
+    else
+      isBad = (Result.size() != 4 || Result[1] != '\\');  // Not '\x'
+
+    if (isBad) {
+      PP.Diag(ArgTokStart[0], diag::err_invalid_character_to_charify);
+      Result = "' '";  // Use something arbitrary, but legal.
+    }
+  }
+
+  PP.CreateString(&Result[0], Result.size(), Tok,
+                  ExpansionLocStart, ExpansionLocEnd);
+  return Tok;
+}
+
+/// getStringifiedArgument - Compute, cache, and return the specified argument
+/// that has been 'stringified' as required by the # operator.
+const Token &MacroArgs::getStringifiedArgument(unsigned ArgNo,
+                                               Preprocessor &PP,
+                                               SourceLocation ExpansionLocStart,
+                                               SourceLocation ExpansionLocEnd) {
+  assert(ArgNo < NumUnexpArgTokens && "Invalid argument number!");
+  if (StringifiedArgs.empty()) {
+    StringifiedArgs.resize(getNumArguments());
+    memset((void*)&StringifiedArgs[0], 0,
+           sizeof(StringifiedArgs[0])*getNumArguments());
+  }
+  if (StringifiedArgs[ArgNo].isNot(tok::string_literal))
+    StringifiedArgs[ArgNo] = StringifyArgument(getUnexpArgument(ArgNo), PP,
+                                               /*Charify=*/false,
+                                               ExpansionLocStart,
+                                               ExpansionLocEnd);
+  return StringifiedArgs[ArgNo];
+}
diff --git a/clang/lib/Lex/MacroArgs.h b/clang/lib/Lex/MacroArgs.h
new file mode 100644
index 0000000..cf86d71
--- /dev/null
+++ b/clang/lib/Lex/MacroArgs.h
@@ -0,0 +1,125 @@
+//===--- MacroArgs.h - Formal argument info for Macros ----------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the MacroArgs interface.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_MACROARGS_H
+#define LLVM_CLANG_MACROARGS_H
+
+#include "llvm/ADT/ArrayRef.h"
+
+#include <vector>
+
+namespace clang {
+  class MacroInfo;
+  class Preprocessor;
+  class Token;
+  class SourceLocation;
+
+/// MacroArgs - An instance of this class captures information about
+/// the formal arguments specified to a function-like macro invocation.
+class MacroArgs {
+  /// NumUnexpArgTokens - The number of raw, unexpanded tokens for the
+  /// arguments.  All of the actual argument tokens are allocated immediately
+  /// after the MacroArgs object in memory.  This is all of the arguments
+  /// concatenated together, with 'EOF' markers at the end of each argument.
+  unsigned NumUnexpArgTokens;
+
+  /// VarargsElided - True if this is a C99 style varargs macro invocation and
+  /// there was no argument specified for the "..." argument.  If the argument
+  /// was specified (even empty) or this isn't a C99 style varargs function, or
+  /// if in strict mode and the C99 varargs macro had only a ... argument, this
+  /// is false.
+  bool VarargsElided;
+  
+  /// PreExpArgTokens - Pre-expanded tokens for arguments that need them.  Empty
+  /// if not yet computed.  This includes the EOF marker at the end of the
+  /// stream.
+  std::vector<std::vector<Token> > PreExpArgTokens;
+
+  /// StringifiedArgs - This contains arguments in 'stringified' form.  If the
+  /// stringified form of an argument has not yet been computed, this is empty.
+  std::vector<Token> StringifiedArgs;
+
+  /// ArgCache - This is a linked list of MacroArgs objects that the
+  /// Preprocessor owns which we use to avoid thrashing malloc/free.
+  MacroArgs *ArgCache;
+  
+  MacroArgs(unsigned NumToks, bool varargsElided)
+    : NumUnexpArgTokens(NumToks), VarargsElided(varargsElided), ArgCache(0) {}
+  ~MacroArgs() {}
+public:
+  /// MacroArgs ctor function - Create a new MacroArgs object with the specified
+  /// macro and argument info.
+  static MacroArgs *create(const MacroInfo *MI,
+                           llvm::ArrayRef<Token> UnexpArgTokens,
+                           bool VarargsElided, Preprocessor &PP);
+
+  /// destroy - Destroy and deallocate the memory for this object.
+  ///
+  void destroy(Preprocessor &PP);
+
+  /// ArgNeedsPreexpansion - If we can prove that the argument won't be affected
+  /// by pre-expansion, return false.  Otherwise, conservatively return true.
+  bool ArgNeedsPreexpansion(const Token *ArgTok, Preprocessor &PP) const;
+
+  /// getUnexpArgument - Return a pointer to the first token of the unexpanded
+  /// token list for the specified formal.
+  ///
+  const Token *getUnexpArgument(unsigned Arg) const;
+
+  /// getArgLength - Given a pointer to an expanded or unexpanded argument,
+  /// return the number of tokens, not counting the EOF, that make up the
+  /// argument.
+  static unsigned getArgLength(const Token *ArgPtr);
+
+  /// getPreExpArgument - Return the pre-expanded form of the specified
+  /// argument.
+  const std::vector<Token> &
+    getPreExpArgument(unsigned Arg, const MacroInfo *MI, Preprocessor &PP);
+
+  /// getStringifiedArgument - Compute, cache, and return the specified argument
+  /// that has been 'stringified' as required by the # operator.
+  const Token &getStringifiedArgument(unsigned ArgNo, Preprocessor &PP,
+                                      SourceLocation ExpansionLocStart,
+                                      SourceLocation ExpansionLocEnd);
+
+  /// getNumArguments - Return the number of arguments passed into this macro
+  /// invocation.
+  unsigned getNumArguments() const { return NumUnexpArgTokens; }
+
+
+  /// isVarargsElidedUse - Return true if this is a C99 style varargs macro
+  /// invocation and there was no argument specified for the "..." argument.  If
+  /// the argument was specified (even empty) or this isn't a C99 style varargs
+  /// function, or if in strict mode and the C99 varargs macro had only a ...
+  /// argument, this returns false.
+  bool isVarargsElidedUse() const { return VarargsElided; }
+
+  /// StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of
+  /// tokens into the literal string token that should be produced by the C #
+  /// preprocessor operator.  If Charify is true, then it should be turned into
+  /// a character literal for the Microsoft charize (#@) extension.
+  ///
+  static Token StringifyArgument(const Token *ArgToks,
+                                 Preprocessor &PP, bool Charify,
+                                 SourceLocation ExpansionLocStart,
+                                 SourceLocation ExpansionLocEnd);
+  
+  
+  /// deallocate - This should only be called by the Preprocessor when managing
+  /// its freelist.
+  MacroArgs *deallocate();
+};
+
+}  // end namespace clang
+
+#endif
diff --git a/clang/lib/Lex/MacroInfo.cpp b/clang/lib/Lex/MacroInfo.cpp
new file mode 100644
index 0000000..3d0c9a1
--- /dev/null
+++ b/clang/lib/Lex/MacroInfo.cpp
@@ -0,0 +1,133 @@
+//===--- MacroInfo.cpp - Information about #defined identifiers -----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the MacroInfo interface.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Lex/MacroInfo.h"
+#include "clang/Lex/Preprocessor.h"
+using namespace clang;
+
+MacroInfo::MacroInfo(SourceLocation DefLoc) : Location(DefLoc) {
+  IsFunctionLike = false;
+  IsC99Varargs = false;
+  IsGNUVarargs = false;
+  IsBuiltinMacro = false;
+  IsFromAST = false;
+  ChangedAfterLoad = false;
+  IsDisabled = false;
+  IsUsed = false;
+  IsAllowRedefinitionsWithoutWarning = false;
+  IsWarnIfUnused = false;
+  IsDefinitionLengthCached = false;
+  IsPublic = true;
+  
+  ArgumentList = 0;
+  NumArguments = 0;
+}
+
+MacroInfo::MacroInfo(const MacroInfo &MI, llvm::BumpPtrAllocator &PPAllocator) {
+  Location = MI.Location;
+  EndLocation = MI.EndLocation;
+  ReplacementTokens = MI.ReplacementTokens;
+  IsFunctionLike = MI.IsFunctionLike;
+  IsC99Varargs = MI.IsC99Varargs;
+  IsGNUVarargs = MI.IsGNUVarargs;
+  IsBuiltinMacro = MI.IsBuiltinMacro;
+  IsFromAST = MI.IsFromAST;
+  ChangedAfterLoad = MI.ChangedAfterLoad;
+  IsDisabled = MI.IsDisabled;
+  IsUsed = MI.IsUsed;
+  IsAllowRedefinitionsWithoutWarning = MI.IsAllowRedefinitionsWithoutWarning;
+  IsWarnIfUnused = MI.IsWarnIfUnused;
+  IsDefinitionLengthCached = MI.IsDefinitionLengthCached;
+  DefinitionLength = MI.DefinitionLength;
+  IsPublic = MI.IsPublic;
+  
+  ArgumentList = 0;
+  NumArguments = 0;
+  setArgumentList(MI.ArgumentList, MI.NumArguments, PPAllocator);
+}
+
+unsigned MacroInfo::getDefinitionLengthSlow(SourceManager &SM) const {
+  assert(!IsDefinitionLengthCached);
+  IsDefinitionLengthCached = true;
+
+  if (ReplacementTokens.empty())
+    return (DefinitionLength = 0);
+
+  const Token &firstToken = ReplacementTokens.front();
+  const Token &lastToken = ReplacementTokens.back();
+  SourceLocation macroStart = firstToken.getLocation();
+  SourceLocation macroEnd = lastToken.getLocation();
+  assert(macroStart.isValid() && macroEnd.isValid());
+  assert((macroStart.isFileID() || firstToken.is(tok::comment)) &&
+         "Macro defined in macro?");
+  assert((macroEnd.isFileID() || lastToken.is(tok::comment)) &&
+         "Macro defined in macro?");
+  std::pair<FileID, unsigned>
+      startInfo = SM.getDecomposedExpansionLoc(macroStart);
+  std::pair<FileID, unsigned>
+      endInfo = SM.getDecomposedExpansionLoc(macroEnd);
+  assert(startInfo.first == endInfo.first &&
+         "Macro definition spanning multiple FileIDs ?");
+  assert(startInfo.second <= endInfo.second);
+  DefinitionLength = endInfo.second - startInfo.second;
+  DefinitionLength += lastToken.getLength();
+
+  return DefinitionLength;
+}
+
+/// isIdenticalTo - Return true if the specified macro definition is equal to
+/// this macro in spelling, arguments, and whitespace.  This is used to emit
+/// duplicate definition warnings.  This implements the rules in C99 6.10.3.
+///
+bool MacroInfo::isIdenticalTo(const MacroInfo &Other, Preprocessor &PP) const {
+  // Check # tokens in replacement, number of args, and various flags all match.
+  if (ReplacementTokens.size() != Other.ReplacementTokens.size() ||
+      getNumArgs() != Other.getNumArgs() ||
+      isFunctionLike() != Other.isFunctionLike() ||
+      isC99Varargs() != Other.isC99Varargs() ||
+      isGNUVarargs() != Other.isGNUVarargs())
+    return false;
+
+  // Check arguments.
+  for (arg_iterator I = arg_begin(), OI = Other.arg_begin(), E = arg_end();
+       I != E; ++I, ++OI)
+    if (*I != *OI) return false;
+
+  // Check all the tokens.
+  for (unsigned i = 0, e = ReplacementTokens.size(); i != e; ++i) {
+    const Token &A = ReplacementTokens[i];
+    const Token &B = Other.ReplacementTokens[i];
+    if (A.getKind() != B.getKind())
+      return false;
+
+    // If this isn't the first first token, check that the whitespace and
+    // start-of-line characteristics match.
+    if (i != 0 &&
+        (A.isAtStartOfLine() != B.isAtStartOfLine() ||
+         A.hasLeadingSpace() != B.hasLeadingSpace()))
+      return false;
+
+    // If this is an identifier, it is easy.
+    if (A.getIdentifierInfo() || B.getIdentifierInfo()) {
+      if (A.getIdentifierInfo() != B.getIdentifierInfo())
+        return false;
+      continue;
+    }
+
+    // Otherwise, check the spelling.
+    if (PP.getSpelling(A) != PP.getSpelling(B))
+      return false;
+  }
+
+  return true;
+}
diff --git a/clang/lib/Lex/Makefile b/clang/lib/Lex/Makefile
new file mode 100644
index 0000000..d80fb55
--- /dev/null
+++ b/clang/lib/Lex/Makefile
@@ -0,0 +1,24 @@
+##===- clang/lib/Lex/Makefile ------------------------------*- Makefile -*-===##
+# 
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+# 
+##===----------------------------------------------------------------------===##
+#
+#  This implements the Lexer library for the C-Language front-end.
+#
+##===----------------------------------------------------------------------===##
+
+CLANG_LEVEL := ../..
+include $(CLANG_LEVEL)/../../Makefile.config
+
+LIBRARYNAME := clangLex
+
+ifeq ($(ARCH),PowerPC)
+CXX.Flags += -maltivec
+endif
+
+include $(CLANG_LEVEL)/Makefile
+
diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp
new file mode 100644
index 0000000..5304311
--- /dev/null
+++ b/clang/lib/Lex/ModuleMap.cpp
@@ -0,0 +1,1437 @@
+//===--- ModuleMap.cpp - Describe the layout of modules ---------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the ModuleMap implementation, which describes the layout
+// of a module as it relates to headers.
+//
+//===----------------------------------------------------------------------===//
+#include "clang/Lex/ModuleMap.h"
+#include "clang/Lex/Lexer.h"
+#include "clang/Lex/LiteralSupport.h"
+#include "clang/Lex/LexDiagnostic.h"
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/TargetInfo.h"
+#include "clang/Basic/TargetOptions.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Host.h"
+#include "llvm/Support/PathV2.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSwitch.h"
+using namespace clang;
+
+Module::ExportDecl 
+ModuleMap::resolveExport(Module *Mod, 
+                         const Module::UnresolvedExportDecl &Unresolved,
+                         bool Complain) {
+  // We may have just a wildcard.
+  if (Unresolved.Id.empty()) {
+    assert(Unresolved.Wildcard && "Invalid unresolved export");
+    return Module::ExportDecl(0, true);
+  }
+  
+  // Find the starting module.
+  Module *Context = lookupModuleUnqualified(Unresolved.Id[0].first, Mod);
+  if (!Context) {
+    if (Complain)
+      Diags->Report(Unresolved.Id[0].second, 
+                    diag::err_mmap_missing_module_unqualified)
+        << Unresolved.Id[0].first << Mod->getFullModuleName();
+    
+    return Module::ExportDecl();
+  }
+
+  // Dig into the module path.
+  for (unsigned I = 1, N = Unresolved.Id.size(); I != N; ++I) {
+    Module *Sub = lookupModuleQualified(Unresolved.Id[I].first,
+                                        Context);
+    if (!Sub) {
+      if (Complain)
+        Diags->Report(Unresolved.Id[I].second, 
+                      diag::err_mmap_missing_module_qualified)
+          << Unresolved.Id[I].first << Context->getFullModuleName()
+          << SourceRange(Unresolved.Id[0].second, Unresolved.Id[I-1].second);
+      
+      return Module::ExportDecl();      
+    }
+    
+    Context = Sub;
+  }
+  
+  return Module::ExportDecl(Context, Unresolved.Wildcard);
+}
+
+ModuleMap::ModuleMap(FileManager &FileMgr, const DiagnosticConsumer &DC,
+                     const LangOptions &LangOpts, const TargetInfo *Target)
+  : LangOpts(LangOpts), Target(Target), BuiltinIncludeDir(0)
+{
+  IntrusiveRefCntPtr<DiagnosticIDs> DiagIDs(new DiagnosticIDs);
+  Diags = IntrusiveRefCntPtr<DiagnosticsEngine>(
+            new DiagnosticsEngine(DiagIDs));
+  Diags->setClient(DC.clone(*Diags), /*ShouldOwnClient=*/true);
+  SourceMgr = new SourceManager(*Diags, FileMgr);
+}
+
+ModuleMap::~ModuleMap() {
+  for (llvm::StringMap<Module *>::iterator I = Modules.begin(), 
+                                        IEnd = Modules.end();
+       I != IEnd; ++I) {
+    delete I->getValue();
+  }
+  
+  delete SourceMgr;
+}
+
+void ModuleMap::setTarget(const TargetInfo &Target) {
+  assert((!this->Target || this->Target == &Target) && 
+         "Improper target override");
+  this->Target = &Target;
+}
+
+Module *ModuleMap::findModuleForHeader(const FileEntry *File) {
+  llvm::DenseMap<const FileEntry *, Module *>::iterator Known
+    = Headers.find(File);
+  if (Known != Headers.end()) {
+    // If a header corresponds to an unavailable module, don't report
+    // that it maps to anything.
+    if (!Known->second->isAvailable())
+      return 0;
+
+    return Known->second;
+  }
+  
+  const DirectoryEntry *Dir = File->getDir();
+  llvm::SmallVector<const DirectoryEntry *, 2> SkippedDirs;
+  StringRef DirName = Dir->getName();
+
+  // Keep walking up the directory hierarchy, looking for a directory with
+  // an umbrella header.
+  do {    
+    llvm::DenseMap<const DirectoryEntry *, Module *>::iterator KnownDir
+      = UmbrellaDirs.find(Dir);
+    if (KnownDir != UmbrellaDirs.end()) {
+      Module *Result = KnownDir->second;
+      
+      // Search up the module stack until we find a module with an umbrella
+      // directory.
+      Module *UmbrellaModule = Result;
+      while (!UmbrellaModule->getUmbrellaDir() && UmbrellaModule->Parent)
+        UmbrellaModule = UmbrellaModule->Parent;
+
+      if (UmbrellaModule->InferSubmodules) {
+        // Infer submodules for each of the directories we found between
+        // the directory of the umbrella header and the directory where 
+        // the actual header is located.
+        bool Explicit = UmbrellaModule->InferExplicitSubmodules;
+        
+        for (unsigned I = SkippedDirs.size(); I != 0; --I) {
+          // Find or create the module that corresponds to this directory name.
+          StringRef Name = llvm::sys::path::stem(SkippedDirs[I-1]->getName());
+          Result = findOrCreateModule(Name, Result, /*IsFramework=*/false,
+                                      Explicit).first;
+          
+          // Associate the module and the directory.
+          UmbrellaDirs[SkippedDirs[I-1]] = Result;
+
+          // If inferred submodules export everything they import, add a 
+          // wildcard to the set of exports.
+          if (UmbrellaModule->InferExportWildcard && Result->Exports.empty())
+            Result->Exports.push_back(Module::ExportDecl(0, true));
+        }
+        
+        // Infer a submodule with the same name as this header file.
+        StringRef Name = llvm::sys::path::stem(File->getName());
+        Result = findOrCreateModule(Name, Result, /*IsFramework=*/false,
+                                    Explicit).first;
+        
+        // If inferred submodules export everything they import, add a 
+        // wildcard to the set of exports.
+        if (UmbrellaModule->InferExportWildcard && Result->Exports.empty())
+          Result->Exports.push_back(Module::ExportDecl(0, true));
+      } else {
+        // Record each of the directories we stepped through as being part of
+        // the module we found, since the umbrella header covers them all.
+        for (unsigned I = 0, N = SkippedDirs.size(); I != N; ++I)
+          UmbrellaDirs[SkippedDirs[I]] = Result;
+      }
+      
+      Headers[File] = Result;
+
+      // If a header corresponds to an unavailable module, don't report
+      // that it maps to anything.
+      if (!Result->isAvailable())
+        return 0;
+
+      return Result;
+    }
+    
+    SkippedDirs.push_back(Dir);
+    
+    // Retrieve our parent path.
+    DirName = llvm::sys::path::parent_path(DirName);
+    if (DirName.empty())
+      break;
+    
+    // Resolve the parent path to a directory entry.
+    Dir = SourceMgr->getFileManager().getDirectory(DirName);
+  } while (Dir);
+  
+  return 0;
+}
+
+bool ModuleMap::isHeaderInUnavailableModule(const FileEntry *Header) {
+  llvm::DenseMap<const FileEntry *, Module *>::iterator Known
+    = Headers.find(Header);
+  if (Known != Headers.end())
+    return !Known->second->isAvailable();
+  
+  const DirectoryEntry *Dir = Header->getDir();
+  llvm::SmallVector<const DirectoryEntry *, 2> SkippedDirs;
+  StringRef DirName = Dir->getName();
+
+  // Keep walking up the directory hierarchy, looking for a directory with
+  // an umbrella header.
+  do {    
+    llvm::DenseMap<const DirectoryEntry *, Module *>::iterator KnownDir
+      = UmbrellaDirs.find(Dir);
+    if (KnownDir != UmbrellaDirs.end()) {
+      Module *Found = KnownDir->second;
+      if (!Found->isAvailable())
+        return true;
+
+      // Search up the module stack until we find a module with an umbrella
+      // directory.
+      Module *UmbrellaModule = Found;
+      while (!UmbrellaModule->getUmbrellaDir() && UmbrellaModule->Parent)
+        UmbrellaModule = UmbrellaModule->Parent;
+
+      if (UmbrellaModule->InferSubmodules) {
+        for (unsigned I = SkippedDirs.size(); I != 0; --I) {
+          // Find or create the module that corresponds to this directory name.
+          StringRef Name = llvm::sys::path::stem(SkippedDirs[I-1]->getName());
+          Found = lookupModuleQualified(Name, Found);
+          if (!Found)
+            return false;
+          if (!Found->isAvailable())
+            return true;
+        }
+        
+        // Infer a submodule with the same name as this header file.
+        StringRef Name = llvm::sys::path::stem(Header->getName());
+        Found = lookupModuleQualified(Name, Found);
+        if (!Found)
+          return false;
+      }
+
+      return !Found->isAvailable();
+    }
+    
+    SkippedDirs.push_back(Dir);
+    
+    // Retrieve our parent path.
+    DirName = llvm::sys::path::parent_path(DirName);
+    if (DirName.empty())
+      break;
+    
+    // Resolve the parent path to a directory entry.
+    Dir = SourceMgr->getFileManager().getDirectory(DirName);
+  } while (Dir);
+  
+  return false;
+}
+
+Module *ModuleMap::findModule(StringRef Name) {
+  llvm::StringMap<Module *>::iterator Known = Modules.find(Name);
+  if (Known != Modules.end())
+    return Known->getValue();
+  
+  return 0;
+}
+
+Module *ModuleMap::lookupModuleUnqualified(StringRef Name, Module *Context) {
+  for(; Context; Context = Context->Parent) {
+    if (Module *Sub = lookupModuleQualified(Name, Context))
+      return Sub;
+  }
+  
+  return findModule(Name);
+}
+
+Module *ModuleMap::lookupModuleQualified(StringRef Name, Module *Context) {
+  if (!Context)
+    return findModule(Name);
+  
+  return Context->findSubmodule(Name);
+}
+
+std::pair<Module *, bool> 
+ModuleMap::findOrCreateModule(StringRef Name, Module *Parent, bool IsFramework,
+                              bool IsExplicit) {
+  // Try to find an existing module with this name.
+  if (Module *Sub = lookupModuleQualified(Name, Parent))
+    return std::make_pair(Sub, false);
+  
+  // Create a new module with this name.
+  Module *Result = new Module(Name, SourceLocation(), Parent, IsFramework, 
+                              IsExplicit);
+  if (!Parent)
+    Modules[Name] = Result;
+  return std::make_pair(Result, true);
+}
+
+Module *
+ModuleMap::inferFrameworkModule(StringRef ModuleName, 
+                                const DirectoryEntry *FrameworkDir,
+                                bool IsSystem,
+                                Module *Parent) {
+  // Check whether we've already found this module.
+  if (Module *Mod = lookupModuleQualified(ModuleName, Parent))
+    return Mod;
+  
+  FileManager &FileMgr = SourceMgr->getFileManager();
+  
+  // Look for an umbrella header.
+  SmallString<128> UmbrellaName = StringRef(FrameworkDir->getName());
+  llvm::sys::path::append(UmbrellaName, "Headers");
+  llvm::sys::path::append(UmbrellaName, ModuleName + ".h");
+  const FileEntry *UmbrellaHeader = FileMgr.getFile(UmbrellaName);
+  
+  // FIXME: If there's no umbrella header, we could probably scan the
+  // framework to load *everything*. But, it's not clear that this is a good
+  // idea.
+  if (!UmbrellaHeader)
+    return 0;
+  
+  Module *Result = new Module(ModuleName, SourceLocation(), Parent,
+                              /*IsFramework=*/true, /*IsExplicit=*/false);
+  if (IsSystem)
+    Result->IsSystem = IsSystem;
+  
+  if (!Parent)
+    Modules[ModuleName] = Result;
+  
+  // umbrella header "umbrella-header-name"
+  Result->Umbrella = UmbrellaHeader;
+  Headers[UmbrellaHeader] = Result;
+  UmbrellaDirs[UmbrellaHeader->getDir()] = Result;
+  
+  // export *
+  Result->Exports.push_back(Module::ExportDecl(0, true));
+  
+  // module * { export * }
+  Result->InferSubmodules = true;
+  Result->InferExportWildcard = true;
+  
+  // Look for subframeworks.
+  llvm::error_code EC;
+  SmallString<128> SubframeworksDirName
+    = StringRef(FrameworkDir->getName());
+  llvm::sys::path::append(SubframeworksDirName, "Frameworks");
+  SmallString<128> SubframeworksDirNameNative;
+  llvm::sys::path::native(SubframeworksDirName.str(),
+                          SubframeworksDirNameNative);
+  for (llvm::sys::fs::directory_iterator 
+         Dir(SubframeworksDirNameNative.str(), EC), DirEnd;
+       Dir != DirEnd && !EC; Dir.increment(EC)) {
+    if (!StringRef(Dir->path()).endswith(".framework"))
+      continue;
+    
+    if (const DirectoryEntry *SubframeworkDir
+          = FileMgr.getDirectory(Dir->path())) {
+      // FIXME: Do we want to warn about subframeworks without umbrella headers?
+      inferFrameworkModule(llvm::sys::path::stem(Dir->path()), SubframeworkDir,
+                           IsSystem, Result);
+    }
+  }
+
+  return Result;
+}
+
+void ModuleMap::setUmbrellaHeader(Module *Mod, const FileEntry *UmbrellaHeader){
+  Headers[UmbrellaHeader] = Mod;
+  Mod->Umbrella = UmbrellaHeader;
+  UmbrellaDirs[UmbrellaHeader->getDir()] = Mod;
+}
+
+void ModuleMap::setUmbrellaDir(Module *Mod, const DirectoryEntry *UmbrellaDir) {
+  Mod->Umbrella = UmbrellaDir;
+  UmbrellaDirs[UmbrellaDir] = Mod;
+}
+
+void ModuleMap::addHeader(Module *Mod, const FileEntry *Header) {
+  Mod->Headers.push_back(Header);
+  Headers[Header] = Mod;
+}
+
+const FileEntry *
+ModuleMap::getContainingModuleMapFile(Module *Module) {
+  if (Module->DefinitionLoc.isInvalid() || !SourceMgr)
+    return 0;
+
+  return SourceMgr->getFileEntryForID(
+           SourceMgr->getFileID(Module->DefinitionLoc));
+}
+
+void ModuleMap::dump() {
+  llvm::errs() << "Modules:";
+  for (llvm::StringMap<Module *>::iterator M = Modules.begin(), 
+                                        MEnd = Modules.end(); 
+       M != MEnd; ++M)
+    M->getValue()->print(llvm::errs(), 2);
+  
+  llvm::errs() << "Headers:";
+  for (llvm::DenseMap<const FileEntry *, Module *>::iterator 
+            H = Headers.begin(),
+         HEnd = Headers.end();
+       H != HEnd; ++H) {
+    llvm::errs() << "  \"" << H->first->getName() << "\" -> " 
+                 << H->second->getFullModuleName() << "\n";
+  }
+}
+
+bool ModuleMap::resolveExports(Module *Mod, bool Complain) {
+  bool HadError = false;
+  for (unsigned I = 0, N = Mod->UnresolvedExports.size(); I != N; ++I) {
+    Module::ExportDecl Export = resolveExport(Mod, Mod->UnresolvedExports[I], 
+                                              Complain);
+    if (Export.getPointer() || Export.getInt())
+      Mod->Exports.push_back(Export);
+    else
+      HadError = true;
+  }
+  Mod->UnresolvedExports.clear();
+  return HadError;
+}
+
+Module *ModuleMap::inferModuleFromLocation(FullSourceLoc Loc) {
+  if (Loc.isInvalid())
+    return 0;
+  
+  // Use the expansion location to determine which module we're in.
+  FullSourceLoc ExpansionLoc = Loc.getExpansionLoc();
+  if (!ExpansionLoc.isFileID())
+    return 0;  
+  
+  
+  const SourceManager &SrcMgr = Loc.getManager();
+  FileID ExpansionFileID = ExpansionLoc.getFileID();
+  
+  while (const FileEntry *ExpansionFile
+           = SrcMgr.getFileEntryForID(ExpansionFileID)) {
+    // Find the module that owns this header (if any).
+    if (Module *Mod = findModuleForHeader(ExpansionFile))
+      return Mod;
+    
+    // No module owns this header, so look up the inclusion chain to see if
+    // any included header has an associated module.
+    SourceLocation IncludeLoc = SrcMgr.getIncludeLoc(ExpansionFileID);
+    if (IncludeLoc.isInvalid())
+      return 0;
+    
+    ExpansionFileID = SrcMgr.getFileID(IncludeLoc);
+  }
+  
+  return 0;
+}
+
+//----------------------------------------------------------------------------//
+// Module map file parser
+//----------------------------------------------------------------------------//
+
+namespace clang {
+  /// \brief A token in a module map file.
+  struct MMToken {
+    enum TokenKind {
+      Comma,
+      EndOfFile,
+      HeaderKeyword,
+      Identifier,
+      ExplicitKeyword,
+      ExportKeyword,
+      FrameworkKeyword,
+      ModuleKeyword,
+      Period,
+      UmbrellaKeyword,
+      RequiresKeyword,
+      Star,
+      StringLiteral,
+      LBrace,
+      RBrace,
+      LSquare,
+      RSquare
+    } Kind;
+    
+    unsigned Location;
+    unsigned StringLength;
+    const char *StringData;
+    
+    void clear() {
+      Kind = EndOfFile;
+      Location = 0;
+      StringLength = 0;
+      StringData = 0;
+    }
+    
+    bool is(TokenKind K) const { return Kind == K; }
+    
+    SourceLocation getLocation() const {
+      return SourceLocation::getFromRawEncoding(Location);
+    }
+    
+    StringRef getString() const {
+      return StringRef(StringData, StringLength);
+    }
+  };
+  
+  class ModuleMapParser {
+    Lexer &L;
+    SourceManager &SourceMgr;
+    DiagnosticsEngine &Diags;
+    ModuleMap &Map;
+    
+    /// \brief The directory that this module map resides in.
+    const DirectoryEntry *Directory;
+
+    /// \brief The directory containing Clang-supplied headers.
+    const DirectoryEntry *BuiltinIncludeDir;
+
+    /// \brief Whether an error occurred.
+    bool HadError;
+    
+    /// \brief Default target information, used only for string literal
+    /// parsing.
+    OwningPtr<TargetInfo> Target;
+    
+    /// \brief Stores string data for the various string literals referenced
+    /// during parsing.
+    llvm::BumpPtrAllocator StringData;
+    
+    /// \brief The current token.
+    MMToken Tok;
+    
+    /// \brief The active module.
+    Module *ActiveModule;
+    
+    /// \brief Consume the current token and return its location.
+    SourceLocation consumeToken();
+    
+    /// \brief Skip tokens until we reach the a token with the given kind
+    /// (or the end of the file).
+    void skipUntil(MMToken::TokenKind K);
+
+    typedef llvm::SmallVector<std::pair<std::string, SourceLocation>, 2>
+      ModuleId;
+    bool parseModuleId(ModuleId &Id);
+    void parseModuleDecl();
+    void parseRequiresDecl();
+    void parseHeaderDecl(SourceLocation UmbrellaLoc);
+    void parseUmbrellaDirDecl(SourceLocation UmbrellaLoc);
+    void parseExportDecl();
+    void parseInferredSubmoduleDecl(bool Explicit);
+    
+    const DirectoryEntry *getOverriddenHeaderSearchDir();
+    
+  public:
+    explicit ModuleMapParser(Lexer &L, SourceManager &SourceMgr, 
+                             DiagnosticsEngine &Diags,
+                             ModuleMap &Map,
+                             const DirectoryEntry *Directory,
+                             const DirectoryEntry *BuiltinIncludeDir)
+      : L(L), SourceMgr(SourceMgr), Diags(Diags), Map(Map), 
+        Directory(Directory), BuiltinIncludeDir(BuiltinIncludeDir), 
+        HadError(false), ActiveModule(0)
+    {
+      TargetOptions TargetOpts;
+      TargetOpts.Triple = llvm::sys::getDefaultTargetTriple();
+      Target.reset(TargetInfo::CreateTargetInfo(Diags, TargetOpts));
+      
+      Tok.clear();
+      consumeToken();
+    }
+    
+    bool parseModuleMapFile();
+  };
+}
+
+SourceLocation ModuleMapParser::consumeToken() {
+retry:
+  SourceLocation Result = Tok.getLocation();
+  Tok.clear();
+  
+  Token LToken;
+  L.LexFromRawLexer(LToken);
+  Tok.Location = LToken.getLocation().getRawEncoding();
+  switch (LToken.getKind()) {
+  case tok::raw_identifier:
+    Tok.StringData = LToken.getRawIdentifierData();
+    Tok.StringLength = LToken.getLength();
+    Tok.Kind = llvm::StringSwitch<MMToken::TokenKind>(Tok.getString())
+                 .Case("header", MMToken::HeaderKeyword)
+                 .Case("explicit", MMToken::ExplicitKeyword)
+                 .Case("export", MMToken::ExportKeyword)
+                 .Case("framework", MMToken::FrameworkKeyword)
+                 .Case("module", MMToken::ModuleKeyword)
+                 .Case("requires", MMToken::RequiresKeyword)
+                 .Case("umbrella", MMToken::UmbrellaKeyword)
+                 .Default(MMToken::Identifier);
+    break;
+
+  case tok::comma:
+    Tok.Kind = MMToken::Comma;
+    break;
+
+  case tok::eof:
+    Tok.Kind = MMToken::EndOfFile;
+    break;
+      
+  case tok::l_brace:
+    Tok.Kind = MMToken::LBrace;
+    break;
+
+  case tok::l_square:
+    Tok.Kind = MMToken::LSquare;
+    break;
+      
+  case tok::period:
+    Tok.Kind = MMToken::Period;
+    break;
+      
+  case tok::r_brace:
+    Tok.Kind = MMToken::RBrace;
+    break;
+      
+  case tok::r_square:
+    Tok.Kind = MMToken::RSquare;
+    break;
+      
+  case tok::star:
+    Tok.Kind = MMToken::Star;
+    break;
+      
+  case tok::string_literal: {
+    if (LToken.hasUDSuffix()) {
+      Diags.Report(LToken.getLocation(), diag::err_invalid_string_udl);
+      HadError = true;
+      goto retry;
+    }
+
+    // Parse the string literal.
+    LangOptions LangOpts;
+    StringLiteralParser StringLiteral(&LToken, 1, SourceMgr, LangOpts, *Target);
+    if (StringLiteral.hadError)
+      goto retry;
+    
+    // Copy the string literal into our string data allocator.
+    unsigned Length = StringLiteral.GetStringLength();
+    char *Saved = StringData.Allocate<char>(Length + 1);
+    memcpy(Saved, StringLiteral.GetString().data(), Length);
+    Saved[Length] = 0;
+    
+    // Form the token.
+    Tok.Kind = MMToken::StringLiteral;
+    Tok.StringData = Saved;
+    Tok.StringLength = Length;
+    break;
+  }
+      
+  case tok::comment:
+    goto retry;
+      
+  default:
+    Diags.Report(LToken.getLocation(), diag::err_mmap_unknown_token);
+    HadError = true;
+    goto retry;
+  }
+  
+  return Result;
+}
+
+void ModuleMapParser::skipUntil(MMToken::TokenKind K) {
+  unsigned braceDepth = 0;
+  unsigned squareDepth = 0;
+  do {
+    switch (Tok.Kind) {
+    case MMToken::EndOfFile:
+      return;
+
+    case MMToken::LBrace:
+      if (Tok.is(K) && braceDepth == 0 && squareDepth == 0)
+        return;
+        
+      ++braceDepth;
+      break;
+
+    case MMToken::LSquare:
+      if (Tok.is(K) && braceDepth == 0 && squareDepth == 0)
+        return;
+      
+      ++squareDepth;
+      break;
+
+    case MMToken::RBrace:
+      if (braceDepth > 0)
+        --braceDepth;
+      else if (Tok.is(K))
+        return;
+      break;
+
+    case MMToken::RSquare:
+      if (squareDepth > 0)
+        --squareDepth;
+      else if (Tok.is(K))
+        return;
+      break;
+
+    default:
+      if (braceDepth == 0 && squareDepth == 0 && Tok.is(K))
+        return;
+      break;
+    }
+    
+   consumeToken();
+  } while (true);
+}
+
+/// \brief Parse a module-id.
+///
+///   module-id:
+///     identifier
+///     identifier '.' module-id
+///
+/// \returns true if an error occurred, false otherwise.
+bool ModuleMapParser::parseModuleId(ModuleId &Id) {
+  Id.clear();
+  do {
+    if (Tok.is(MMToken::Identifier)) {
+      Id.push_back(std::make_pair(Tok.getString(), Tok.getLocation()));
+      consumeToken();
+    } else {
+      Diags.Report(Tok.getLocation(), diag::err_mmap_expected_module_name);
+      return true;
+    }
+    
+    if (!Tok.is(MMToken::Period))
+      break;
+    
+    consumeToken();
+  } while (true);
+  
+  return false;
+}
+
+namespace {
+  /// \brief Enumerates the known attributes.
+  enum AttributeKind {
+    /// \brief An unknown attribute.
+    AT_unknown,
+    /// \brief The 'system' attribute.
+    AT_system
+  };
+}
+
+/// \brief Parse a module declaration.
+///
+///   module-declaration:
+///     'explicit'[opt] 'framework'[opt] 'module' module-id attributes[opt] 
+///       { module-member* }
+///
+///   attributes:
+///     attribute attributes
+///     attribute
+///
+///   attribute:
+///     [ identifier ]
+///
+///   module-member:
+///     requires-declaration
+///     header-declaration
+///     submodule-declaration
+///     export-declaration
+///
+///   submodule-declaration:
+///     module-declaration
+///     inferred-submodule-declaration
+void ModuleMapParser::parseModuleDecl() {
+  assert(Tok.is(MMToken::ExplicitKeyword) || Tok.is(MMToken::ModuleKeyword) ||
+         Tok.is(MMToken::FrameworkKeyword));
+  // Parse 'explicit' or 'framework' keyword, if present.
+  SourceLocation ExplicitLoc;
+  bool Explicit = false;
+  bool Framework = false;
+
+  // Parse 'explicit' keyword, if present.
+  if (Tok.is(MMToken::ExplicitKeyword)) {
+    ExplicitLoc = consumeToken();
+    Explicit = true;
+  }
+
+  // Parse 'framework' keyword, if present.
+  if (Tok.is(MMToken::FrameworkKeyword)) {
+    consumeToken();
+    Framework = true;
+  } 
+  
+  // Parse 'module' keyword.
+  if (!Tok.is(MMToken::ModuleKeyword)) {
+    Diags.Report(Tok.getLocation(), diag::err_mmap_expected_module);
+    consumeToken();
+    HadError = true;
+    return;
+  }
+  consumeToken(); // 'module' keyword
+
+  // If we have a wildcard for the module name, this is an inferred submodule.
+  // Parse it. 
+  if (Tok.is(MMToken::Star))
+    return parseInferredSubmoduleDecl(Explicit);
+  
+  // Parse the module name.
+  ModuleId Id;
+  if (parseModuleId(Id)) {
+    HadError = true;
+    return;
+  }
+  
+  if (ActiveModule) {
+    if (Id.size() > 1) {
+      Diags.Report(Id.front().second, diag::err_mmap_nested_submodule_id)
+        << SourceRange(Id.front().second, Id.back().second);
+      
+      HadError = true;
+      return;
+    }
+  } else if (Id.size() == 1 && Explicit) {
+    // Top-level modules can't be explicit.
+    Diags.Report(ExplicitLoc, diag::err_mmap_explicit_top_level);
+    Explicit = false;
+    ExplicitLoc = SourceLocation();
+    HadError = true;
+  }
+  
+  Module *PreviousActiveModule = ActiveModule;  
+  if (Id.size() > 1) {
+    // This module map defines a submodule. Go find the module of which it
+    // is a submodule.
+    ActiveModule = 0;
+    for (unsigned I = 0, N = Id.size() - 1; I != N; ++I) {
+      if (Module *Next = Map.lookupModuleQualified(Id[I].first, ActiveModule)) {
+        ActiveModule = Next;
+        continue;
+      }
+      
+      if (ActiveModule) {
+        Diags.Report(Id[I].second, diag::err_mmap_missing_module_qualified)
+          << Id[I].first << ActiveModule->getTopLevelModule();
+      } else {
+        Diags.Report(Id[I].second, diag::err_mmap_expected_module_name);
+      }
+      HadError = true;
+      return;
+    }
+  } 
+  
+  StringRef ModuleName = Id.back().first;
+  SourceLocation ModuleNameLoc = Id.back().second;
+  
+  // Parse the optional attribute list.
+  bool IsSystem = false;
+  while (Tok.is(MMToken::LSquare)) {
+    // Consume the '['.
+    SourceLocation LSquareLoc = consumeToken();
+    
+    // Check whether we have an attribute name here.
+    if (!Tok.is(MMToken::Identifier)) {
+      Diags.Report(Tok.getLocation(), diag::err_mmap_expected_attribute);
+      skipUntil(MMToken::RSquare);
+      if (Tok.is(MMToken::RSquare))
+        consumeToken();
+      continue;
+    }
+    
+    // Decode the attribute name.
+    AttributeKind Attribute 
+      = llvm::StringSwitch<AttributeKind>(Tok.getString())
+        .Case("system", AT_system)
+        .Default(AT_unknown);
+    switch (Attribute) {
+    case AT_unknown:
+      Diags.Report(Tok.getLocation(), diag::warn_mmap_unknown_attribute)
+        << Tok.getString();
+      break;
+        
+    case AT_system:
+      IsSystem = true;
+      break;
+    }
+    consumeToken();
+    
+    // Consume the ']'.
+    if (!Tok.is(MMToken::RSquare)) {
+      Diags.Report(Tok.getLocation(), diag::err_mmap_expected_rsquare);
+      Diags.Report(LSquareLoc, diag::note_mmap_lsquare_match);
+      skipUntil(MMToken::RSquare);
+    }
+
+    if (Tok.is(MMToken::RSquare))
+      consumeToken();
+  }
+  
+  // Parse the opening brace.
+  if (!Tok.is(MMToken::LBrace)) {
+    Diags.Report(Tok.getLocation(), diag::err_mmap_expected_lbrace)
+      << ModuleName;
+    HadError = true;
+    return;
+  }  
+  SourceLocation LBraceLoc = consumeToken();
+  
+  // Determine whether this (sub)module has already been defined.
+  if (Module *Existing = Map.lookupModuleQualified(ModuleName, ActiveModule)) {
+    if (Existing->DefinitionLoc.isInvalid() && !ActiveModule) {
+      // Skip the module definition.
+      skipUntil(MMToken::RBrace);
+      if (Tok.is(MMToken::RBrace))
+        consumeToken();
+      else {
+        Diags.Report(Tok.getLocation(), diag::err_mmap_expected_rbrace);
+        Diags.Report(LBraceLoc, diag::note_mmap_lbrace_match);
+        HadError = true;        
+      }
+      return;
+    }
+    
+    Diags.Report(ModuleNameLoc, diag::err_mmap_module_redefinition)
+      << ModuleName;
+    Diags.Report(Existing->DefinitionLoc, diag::note_mmap_prev_definition);
+    
+    // Skip the module definition.
+    skipUntil(MMToken::RBrace);
+    if (Tok.is(MMToken::RBrace))
+      consumeToken();
+    
+    HadError = true;
+    return;
+  }
+
+  // Start defining this module.
+  ActiveModule = Map.findOrCreateModule(ModuleName, ActiveModule, Framework,
+                                        Explicit).first;
+  ActiveModule->DefinitionLoc = ModuleNameLoc;
+  if (IsSystem)
+    ActiveModule->IsSystem = true;
+  
+  bool Done = false;
+  do {
+    switch (Tok.Kind) {
+    case MMToken::EndOfFile:
+    case MMToken::RBrace:
+      Done = true;
+      break;
+        
+    case MMToken::ExplicitKeyword:
+    case MMToken::FrameworkKeyword:
+    case MMToken::ModuleKeyword:
+      parseModuleDecl();
+      break;
+        
+    case MMToken::ExportKeyword:
+      parseExportDecl();
+      break;
+        
+    case MMToken::RequiresKeyword:
+      parseRequiresDecl();
+      break;
+
+    case MMToken::UmbrellaKeyword: {
+      SourceLocation UmbrellaLoc = consumeToken();
+      if (Tok.is(MMToken::HeaderKeyword))
+        parseHeaderDecl(UmbrellaLoc);
+      else
+        parseUmbrellaDirDecl(UmbrellaLoc);
+      break;
+    }
+        
+    case MMToken::HeaderKeyword:
+      parseHeaderDecl(SourceLocation());
+      break;
+        
+    default:
+      Diags.Report(Tok.getLocation(), diag::err_mmap_expected_member);
+      consumeToken();
+      break;        
+    }
+  } while (!Done);
+
+  if (Tok.is(MMToken::RBrace))
+    consumeToken();
+  else {
+    Diags.Report(Tok.getLocation(), diag::err_mmap_expected_rbrace);
+    Diags.Report(LBraceLoc, diag::note_mmap_lbrace_match);
+    HadError = true;
+  }
+
+  // We're done parsing this module. Pop back to the previous module.
+  ActiveModule = PreviousActiveModule;
+}
+
+/// \brief Parse a requires declaration.
+///
+///   requires-declaration:
+///     'requires' feature-list
+///
+///   feature-list:
+///     identifier ',' feature-list
+///     identifier
+void ModuleMapParser::parseRequiresDecl() {
+  assert(Tok.is(MMToken::RequiresKeyword));
+
+  // Parse 'requires' keyword.
+  consumeToken();
+
+  // Parse the feature-list.
+  do {
+    if (!Tok.is(MMToken::Identifier)) {
+      Diags.Report(Tok.getLocation(), diag::err_mmap_expected_feature);
+      HadError = true;
+      return;
+    }
+
+    // Consume the feature name.
+    std::string Feature = Tok.getString();
+    consumeToken();
+
+    // Add this feature.
+    ActiveModule->addRequirement(Feature, Map.LangOpts, *Map.Target);
+
+    if (!Tok.is(MMToken::Comma))
+      break;
+
+    // Consume the comma.
+    consumeToken();
+  } while (true);
+}
+
+/// \brief Append to \p Paths the set of paths needed to get to the 
+/// subframework in which the given module lives.
+static void appendSubframeworkPaths(Module *Mod,
+                                    llvm::SmallVectorImpl<char> &Path) {
+  // Collect the framework names from the given module to the top-level module.
+  llvm::SmallVector<StringRef, 2> Paths;
+  for (; Mod; Mod = Mod->Parent) {
+    if (Mod->IsFramework)
+      Paths.push_back(Mod->Name);
+  }
+  
+  if (Paths.empty())
+    return;
+  
+  // Add Frameworks/Name.framework for each subframework.
+  for (unsigned I = Paths.size() - 1; I != 0; --I) {
+    llvm::sys::path::append(Path, "Frameworks");
+    llvm::sys::path::append(Path, Paths[I-1] + ".framework");
+  }
+}
+
+/// \brief Determine whether the given file name is the name of a builtin
+/// header, supplied by Clang to replace, override, or augment existing system
+/// headers.
+static bool isBuiltinHeader(StringRef FileName) {
+  return llvm::StringSwitch<bool>(FileName)
+      .Case("float.h", true)
+      .Case("iso646.h", true)
+      .Case("limits.h", true)
+      .Case("stdalign.h", true)
+      .Case("stdarg.h", true)
+      .Case("stdbool.h", true)
+      .Case("stddef.h", true)
+      .Case("stdint.h", true)
+      .Case("tgmath.h", true)
+      .Case("unwind.h", true)
+      .Default(false);
+}
+
+/// \brief Parse a header declaration.
+///
+///   header-declaration:
+///     'umbrella'[opt] 'header' string-literal
+void ModuleMapParser::parseHeaderDecl(SourceLocation UmbrellaLoc) {
+  assert(Tok.is(MMToken::HeaderKeyword));
+  consumeToken();
+
+  bool Umbrella = UmbrellaLoc.isValid();
+  
+  // Parse the header name.
+  if (!Tok.is(MMToken::StringLiteral)) {
+    Diags.Report(Tok.getLocation(), diag::err_mmap_expected_header) 
+      << "header";
+    HadError = true;
+    return;
+  }
+  std::string FileName = Tok.getString();
+  SourceLocation FileNameLoc = consumeToken();
+  
+  // Check whether we already have an umbrella.
+  if (Umbrella && ActiveModule->Umbrella) {
+    Diags.Report(FileNameLoc, diag::err_mmap_umbrella_clash)
+      << ActiveModule->getFullModuleName();
+    HadError = true;
+    return;
+  }
+
+  // Look for this file.
+  const FileEntry *File = 0;
+  const FileEntry *BuiltinFile = 0;
+  SmallString<128> PathName;
+  if (llvm::sys::path::is_absolute(FileName)) {
+    PathName = FileName;
+    File = SourceMgr.getFileManager().getFile(PathName);
+  } else if (const DirectoryEntry *Dir = getOverriddenHeaderSearchDir()) {
+    PathName = Dir->getName();
+    llvm::sys::path::append(PathName, FileName);
+    File = SourceMgr.getFileManager().getFile(PathName);
+  } else {
+    // Search for the header file within the search directory.
+    PathName = Directory->getName();
+    unsigned PathLength = PathName.size();
+    
+    if (ActiveModule->isPartOfFramework()) {
+      appendSubframeworkPaths(ActiveModule, PathName);
+      
+      // Check whether this file is in the public headers.
+      llvm::sys::path::append(PathName, "Headers");
+      llvm::sys::path::append(PathName, FileName);
+      File = SourceMgr.getFileManager().getFile(PathName);
+      
+      if (!File) {
+        // Check whether this file is in the private headers.
+        PathName.resize(PathLength);
+        llvm::sys::path::append(PathName, "PrivateHeaders");
+        llvm::sys::path::append(PathName, FileName);
+        File = SourceMgr.getFileManager().getFile(PathName);
+      }
+    } else {
+      // Lookup for normal headers.
+      llvm::sys::path::append(PathName, FileName);
+      File = SourceMgr.getFileManager().getFile(PathName);
+      
+      // If this is a system module with a top-level header, this header
+      // may have a counterpart (or replacement) in the set of headers
+      // supplied by Clang. Find that builtin header.
+      if (ActiveModule->IsSystem && !Umbrella && BuiltinIncludeDir &&
+          BuiltinIncludeDir != Directory && isBuiltinHeader(FileName)) {
+        SmallString<128> BuiltinPathName(BuiltinIncludeDir->getName());
+        llvm::sys::path::append(BuiltinPathName, FileName);
+        BuiltinFile = SourceMgr.getFileManager().getFile(BuiltinPathName);
+        
+        // If Clang supplies this header but the underlying system does not,
+        // just silently swap in our builtin version. Otherwise, we'll end
+        // up adding both (later).
+        if (!File && BuiltinFile) {
+          File = BuiltinFile;
+          BuiltinFile = 0;
+        }
+      }
+    }
+  }
+  
+  // FIXME: We shouldn't be eagerly stat'ing every file named in a module map.
+  // Come up with a lazy way to do this.
+  if (File) {
+    if (const Module *OwningModule = Map.Headers[File]) {
+      Diags.Report(FileNameLoc, diag::err_mmap_header_conflict)
+        << FileName << OwningModule->getFullModuleName();
+      HadError = true;
+    } else if (Umbrella) {
+      const DirectoryEntry *UmbrellaDir = File->getDir();
+      if ((OwningModule = Map.UmbrellaDirs[UmbrellaDir])) {
+        Diags.Report(UmbrellaLoc, diag::err_mmap_umbrella_clash)
+          << OwningModule->getFullModuleName();
+        HadError = true;
+      } else {
+        // Record this umbrella header.
+        Map.setUmbrellaHeader(ActiveModule, File);
+      }
+    } else {
+      // Record this header.
+      Map.addHeader(ActiveModule, File);
+      
+      // If there is a builtin counterpart to this file, add it now.
+      if (BuiltinFile)
+        Map.addHeader(ActiveModule, BuiltinFile);
+    }
+  } else {
+    Diags.Report(FileNameLoc, diag::err_mmap_header_not_found)
+      << Umbrella << FileName;
+    HadError = true;
+  }
+}
+
+/// \brief Parse an umbrella directory declaration.
+///
+///   umbrella-dir-declaration:
+///     umbrella string-literal
+void ModuleMapParser::parseUmbrellaDirDecl(SourceLocation UmbrellaLoc) {
+  // Parse the directory name.
+  if (!Tok.is(MMToken::StringLiteral)) {
+    Diags.Report(Tok.getLocation(), diag::err_mmap_expected_header) 
+      << "umbrella";
+    HadError = true;
+    return;
+  }
+
+  std::string DirName = Tok.getString();
+  SourceLocation DirNameLoc = consumeToken();
+  
+  // Check whether we already have an umbrella.
+  if (ActiveModule->Umbrella) {
+    Diags.Report(DirNameLoc, diag::err_mmap_umbrella_clash)
+      << ActiveModule->getFullModuleName();
+    HadError = true;
+    return;
+  }
+
+  // Look for this file.
+  const DirectoryEntry *Dir = 0;
+  if (llvm::sys::path::is_absolute(DirName))
+    Dir = SourceMgr.getFileManager().getDirectory(DirName);
+  else {
+    SmallString<128> PathName;
+    PathName = Directory->getName();
+    llvm::sys::path::append(PathName, DirName);
+    Dir = SourceMgr.getFileManager().getDirectory(PathName);
+  }
+  
+  if (!Dir) {
+    Diags.Report(DirNameLoc, diag::err_mmap_umbrella_dir_not_found)
+      << DirName;
+    HadError = true;
+    return;
+  }
+  
+  if (Module *OwningModule = Map.UmbrellaDirs[Dir]) {
+    Diags.Report(UmbrellaLoc, diag::err_mmap_umbrella_clash)
+      << OwningModule->getFullModuleName();
+    HadError = true;
+    return;
+  } 
+  
+  // Record this umbrella directory.
+  Map.setUmbrellaDir(ActiveModule, Dir);
+}
+
+/// \brief Parse a module export declaration.
+///
+///   export-declaration:
+///     'export' wildcard-module-id
+///
+///   wildcard-module-id:
+///     identifier
+///     '*'
+///     identifier '.' wildcard-module-id
+void ModuleMapParser::parseExportDecl() {
+  assert(Tok.is(MMToken::ExportKeyword));
+  SourceLocation ExportLoc = consumeToken();
+  
+  // Parse the module-id with an optional wildcard at the end.
+  ModuleId ParsedModuleId;
+  bool Wildcard = false;
+  do {
+    if (Tok.is(MMToken::Identifier)) {
+      ParsedModuleId.push_back(std::make_pair(Tok.getString(), 
+                                              Tok.getLocation()));
+      consumeToken();
+      
+      if (Tok.is(MMToken::Period)) {
+        consumeToken();
+        continue;
+      } 
+      
+      break;
+    }
+    
+    if(Tok.is(MMToken::Star)) {
+      Wildcard = true;
+      consumeToken();
+      break;
+    }
+    
+    Diags.Report(Tok.getLocation(), diag::err_mmap_export_module_id);
+    HadError = true;
+    return;
+  } while (true);
+  
+  Module::UnresolvedExportDecl Unresolved = { 
+    ExportLoc, ParsedModuleId, Wildcard 
+  };
+  ActiveModule->UnresolvedExports.push_back(Unresolved);
+}
+
+void ModuleMapParser::parseInferredSubmoduleDecl(bool Explicit) {
+  assert(Tok.is(MMToken::Star));
+  SourceLocation StarLoc = consumeToken();
+  bool Failed = false;
+  
+  // Inferred modules must be submodules.
+  if (!ActiveModule) {
+    Diags.Report(StarLoc, diag::err_mmap_top_level_inferred_submodule);
+    Failed = true;
+  }
+  
+  // Inferred modules must have umbrella directories.
+  if (!Failed && !ActiveModule->getUmbrellaDir()) {
+    Diags.Report(StarLoc, diag::err_mmap_inferred_no_umbrella);
+    Failed = true;
+  }
+  
+  // Check for redefinition of an inferred module.
+  if (!Failed && ActiveModule->InferSubmodules) {
+    Diags.Report(StarLoc, diag::err_mmap_inferred_redef);
+    if (ActiveModule->InferredSubmoduleLoc.isValid())
+      Diags.Report(ActiveModule->InferredSubmoduleLoc,
+                   diag::note_mmap_prev_definition);
+    Failed = true;
+  }
+  
+  // If there were any problems with this inferred submodule, skip its body.
+  if (Failed) {
+    if (Tok.is(MMToken::LBrace)) {
+      consumeToken();
+      skipUntil(MMToken::RBrace);
+      if (Tok.is(MMToken::RBrace))
+        consumeToken();
+    }
+    HadError = true;
+    return;
+  }
+  
+  // Note that we have an inferred submodule.
+  ActiveModule->InferSubmodules = true;
+  ActiveModule->InferredSubmoduleLoc = StarLoc;
+  ActiveModule->InferExplicitSubmodules = Explicit;
+  
+  // Parse the opening brace.
+  if (!Tok.is(MMToken::LBrace)) {
+    Diags.Report(Tok.getLocation(), diag::err_mmap_expected_lbrace_wildcard);
+    HadError = true;
+    return;
+  }  
+  SourceLocation LBraceLoc = consumeToken();
+
+  // Parse the body of the inferred submodule.
+  bool Done = false;
+  do {
+    switch (Tok.Kind) {
+    case MMToken::EndOfFile:
+    case MMToken::RBrace:
+      Done = true;
+      break;
+      
+    case MMToken::ExportKeyword: {
+      consumeToken();
+      if (Tok.is(MMToken::Star)) 
+        ActiveModule->InferExportWildcard = true;
+      else
+        Diags.Report(Tok.getLocation(), 
+                     diag::err_mmap_expected_export_wildcard);
+      consumeToken();
+      break;
+    }
+      
+    case MMToken::ExplicitKeyword:
+    case MMToken::ModuleKeyword:
+    case MMToken::HeaderKeyword:
+    case MMToken::UmbrellaKeyword:
+    default:
+      Diags.Report(Tok.getLocation(), diag::err_mmap_expected_wildcard_member);
+      consumeToken();
+      break;        
+    }
+  } while (!Done);
+  
+  if (Tok.is(MMToken::RBrace))
+    consumeToken();
+  else {
+    Diags.Report(Tok.getLocation(), diag::err_mmap_expected_rbrace);
+    Diags.Report(LBraceLoc, diag::note_mmap_lbrace_match);
+    HadError = true;
+  }
+}
+
+/// \brief If there is a specific header search directory due the presence
+/// of an umbrella directory, retrieve that directory. Otherwise, returns null.
+const DirectoryEntry *ModuleMapParser::getOverriddenHeaderSearchDir() {
+  for (Module *Mod = ActiveModule; Mod; Mod = Mod->Parent) {
+    // If we have an umbrella directory, use that.
+    if (Mod->hasUmbrellaDir())
+      return Mod->getUmbrellaDir();
+    
+    // If we have a framework directory, stop looking.
+    if (Mod->IsFramework)
+      return 0;
+  }
+  
+  return 0;
+}
+
+/// \brief Parse a module map file.
+///
+///   module-map-file:
+///     module-declaration*
+bool ModuleMapParser::parseModuleMapFile() {
+  do {
+    switch (Tok.Kind) {
+    case MMToken::EndOfFile:
+      return HadError;
+      
+    case MMToken::ExplicitKeyword:
+    case MMToken::ModuleKeyword:
+    case MMToken::FrameworkKeyword:
+      parseModuleDecl();
+      break;
+      
+    case MMToken::Comma:
+    case MMToken::ExportKeyword:
+    case MMToken::HeaderKeyword:
+    case MMToken::Identifier:
+    case MMToken::LBrace:
+    case MMToken::LSquare:
+    case MMToken::Period:
+    case MMToken::RBrace:
+    case MMToken::RSquare:
+    case MMToken::RequiresKeyword:
+    case MMToken::Star:
+    case MMToken::StringLiteral:
+    case MMToken::UmbrellaKeyword:
+      Diags.Report(Tok.getLocation(), diag::err_mmap_expected_module);
+      HadError = true;
+      consumeToken();
+      break;
+    }
+  } while (true);
+}
+
+bool ModuleMap::parseModuleMapFile(const FileEntry *File) {
+  assert(Target != 0 && "Missing target information");
+  FileID ID = SourceMgr->createFileID(File, SourceLocation(), SrcMgr::C_User);
+  const llvm::MemoryBuffer *Buffer = SourceMgr->getBuffer(ID);
+  if (!Buffer)
+    return true;
+  
+  // Parse this module map file.
+  Lexer L(ID, SourceMgr->getBuffer(ID), *SourceMgr, MMapLangOpts);
+  Diags->getClient()->BeginSourceFile(MMapLangOpts);
+  ModuleMapParser Parser(L, *SourceMgr, *Diags, *this, File->getDir(),
+                         BuiltinIncludeDir);
+  bool Result = Parser.parseModuleMapFile();
+  Diags->getClient()->EndSourceFile();
+  
+  return Result;
+}
diff --git a/clang/lib/Lex/PPCaching.cpp b/clang/lib/Lex/PPCaching.cpp
new file mode 100644
index 0000000..6f4c189
--- /dev/null
+++ b/clang/lib/Lex/PPCaching.cpp
@@ -0,0 +1,118 @@
+//===--- PPCaching.cpp - Handle caching lexed tokens ----------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements pieces of the Preprocessor interface that manage the
+// caching of lexed tokens.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Lex/Preprocessor.h"
+using namespace clang;
+
+/// EnableBacktrackAtThisPos - From the point that this method is called, and
+/// until CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
+/// keeps track of the lexed tokens so that a subsequent Backtrack() call will
+/// make the Preprocessor re-lex the same tokens.
+///
+/// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
+/// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
+/// be combined with the EnableBacktrackAtThisPos calls in reverse order.
+void Preprocessor::EnableBacktrackAtThisPos() {
+  BacktrackPositions.push_back(CachedLexPos);
+  EnterCachingLexMode();
+}
+
+/// CommitBacktrackedTokens - Disable the last EnableBacktrackAtThisPos call.
+void Preprocessor::CommitBacktrackedTokens() {
+  assert(!BacktrackPositions.empty()
+         && "EnableBacktrackAtThisPos was not called!");
+  BacktrackPositions.pop_back();
+}
+
+/// Backtrack - Make Preprocessor re-lex the tokens that were lexed since
+/// EnableBacktrackAtThisPos() was previously called.
+void Preprocessor::Backtrack() {
+  assert(!BacktrackPositions.empty()
+         && "EnableBacktrackAtThisPos was not called!");
+  CachedLexPos = BacktrackPositions.back();
+  BacktrackPositions.pop_back();
+  recomputeCurLexerKind();
+}
+
+void Preprocessor::CachingLex(Token &Result) {
+  if (!InCachingLexMode())
+    return;
+
+  if (CachedLexPos < CachedTokens.size()) {
+    Result = CachedTokens[CachedLexPos++];
+    return;
+  }
+
+  ExitCachingLexMode();
+  Lex(Result);
+
+  if (isBacktrackEnabled()) {
+    // Cache the lexed token.
+    EnterCachingLexMode();
+    CachedTokens.push_back(Result);
+    ++CachedLexPos;
+    return;
+  }
+
+  if (CachedLexPos < CachedTokens.size()) {
+    EnterCachingLexMode();
+  } else {
+    // All cached tokens were consumed.
+    CachedTokens.clear();
+    CachedLexPos = 0;
+  }
+}
+
+void Preprocessor::EnterCachingLexMode() {
+  if (InCachingLexMode())
+    return;
+
+  PushIncludeMacroStack();
+  CurLexerKind = CLK_CachingLexer;
+}
+
+
+const Token &Preprocessor::PeekAhead(unsigned N) {
+  assert(CachedLexPos + N > CachedTokens.size() && "Confused caching.");
+  ExitCachingLexMode();
+  for (unsigned C = CachedLexPos + N - CachedTokens.size(); C > 0; --C) {
+    CachedTokens.push_back(Token());
+    Lex(CachedTokens.back());
+  }
+  EnterCachingLexMode();
+  return CachedTokens.back();
+}
+
+void Preprocessor::AnnotatePreviousCachedTokens(const Token &Tok) {
+  assert(Tok.isAnnotation() && "Expected annotation token");
+  assert(CachedLexPos != 0 && "Expected to have some cached tokens");
+  assert(CachedTokens[CachedLexPos-1].getLastLoc() == Tok.getAnnotationEndLoc()
+         && "The annotation should be until the most recent cached token");
+
+  // Start from the end of the cached tokens list and look for the token
+  // that is the beginning of the annotation token.
+  for (CachedTokensTy::size_type i = CachedLexPos; i != 0; --i) {
+    CachedTokensTy::iterator AnnotBegin = CachedTokens.begin() + i-1;
+    if (AnnotBegin->getLocation() == Tok.getLocation()) {
+      assert((BacktrackPositions.empty() || BacktrackPositions.back() < i) &&
+             "The backtrack pos points inside the annotated tokens!");
+      // Replace the cached tokens with the single annotation token.
+      if (i < CachedLexPos)
+        CachedTokens.erase(AnnotBegin + 1, CachedTokens.begin() + CachedLexPos);
+      *AnnotBegin = Tok;
+      CachedLexPos = i;
+      return;
+    }
+  }
+}
diff --git a/clang/lib/Lex/PPCallbacks.cpp b/clang/lib/Lex/PPCallbacks.cpp
new file mode 100644
index 0000000..952b926
--- /dev/null
+++ b/clang/lib/Lex/PPCallbacks.cpp
@@ -0,0 +1,14 @@
+//===--- PPCallbacks.cpp - Callbacks for Preprocessor actions ---*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Lex/PPCallbacks.h"
+
+using namespace clang;
+
+void PPChainedCallbacks::anchor() { }
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
new file mode 100644
index 0000000..625a204
--- /dev/null
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -0,0 +1,2075 @@
+//===--- PPDirectives.cpp - Directive Handling for Preprocessor -----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file implements # directive processing for the Preprocessor.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/LiteralSupport.h"
+#include "clang/Lex/HeaderSearch.h"
+#include "clang/Lex/MacroInfo.h"
+#include "clang/Lex/LexDiagnostic.h"
+#include "clang/Lex/CodeCompletionHandler.h"
+#include "clang/Lex/ModuleLoader.h"
+#include "clang/Lex/Pragma.h"
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/SourceManager.h"
+#include "llvm/ADT/APInt.h"
+#include "llvm/Support/ErrorHandling.h"
+using namespace clang;
+
+//===----------------------------------------------------------------------===//
+// Utility Methods for Preprocessor Directive Handling.
+//===----------------------------------------------------------------------===//
+
+MacroInfo *Preprocessor::AllocateMacroInfo() {
+  MacroInfoChain *MIChain;
+
+  if (MICache) {
+    MIChain = MICache;
+    MICache = MICache->Next;
+  }
+  else {
+    MIChain = BP.Allocate<MacroInfoChain>();
+  }
+
+  MIChain->Next = MIChainHead;
+  MIChain->Prev = 0;
+  if (MIChainHead)
+    MIChainHead->Prev = MIChain;
+  MIChainHead = MIChain;
+
+  return &(MIChain->MI);
+}
+
+MacroInfo *Preprocessor::AllocateMacroInfo(SourceLocation L) {
+  MacroInfo *MI = AllocateMacroInfo();
+  new (MI) MacroInfo(L);
+  return MI;
+}
+
+MacroInfo *Preprocessor::CloneMacroInfo(const MacroInfo &MacroToClone) {
+  MacroInfo *MI = AllocateMacroInfo();
+  new (MI) MacroInfo(MacroToClone, BP);
+  return MI;
+}
+
+/// ReleaseMacroInfo - Release the specified MacroInfo.  This memory will
+///  be reused for allocating new MacroInfo objects.
+void Preprocessor::ReleaseMacroInfo(MacroInfo *MI) {
+  MacroInfoChain *MIChain = (MacroInfoChain*) MI;
+  if (MacroInfoChain *Prev = MIChain->Prev) {
+    MacroInfoChain *Next = MIChain->Next;
+    Prev->Next = Next;
+    if (Next)
+      Next->Prev = Prev;
+  }
+  else {
+    assert(MIChainHead == MIChain);
+    MIChainHead = MIChain->Next;
+    MIChainHead->Prev = 0;
+  }
+  MIChain->Next = MICache;
+  MICache = MIChain;
+
+  MI->Destroy();
+}
+
+/// DiscardUntilEndOfDirective - Read and discard all tokens remaining on the
+/// current line until the tok::eod token is found.
+void Preprocessor::DiscardUntilEndOfDirective() {
+  Token Tmp;
+  do {
+    LexUnexpandedToken(Tmp);
+    assert(Tmp.isNot(tok::eof) && "EOF seen while discarding directive tokens");
+  } while (Tmp.isNot(tok::eod));
+}
+
+/// ReadMacroName - Lex and validate a macro name, which occurs after a
+/// #define or #undef.  This sets the token kind to eod and discards the rest
+/// of the macro line if the macro name is invalid.  isDefineUndef is 1 if
+/// this is due to a a #define, 2 if #undef directive, 0 if it is something
+/// else (e.g. #ifdef).
+void Preprocessor::ReadMacroName(Token &MacroNameTok, char isDefineUndef) {
+  // Read the token, don't allow macro expansion on it.
+  LexUnexpandedToken(MacroNameTok);
+
+  if (MacroNameTok.is(tok::code_completion)) {
+    if (CodeComplete)
+      CodeComplete->CodeCompleteMacroName(isDefineUndef == 1);
+    setCodeCompletionReached();
+    LexUnexpandedToken(MacroNameTok);
+  }
+  
+  // Missing macro name?
+  if (MacroNameTok.is(tok::eod)) {
+    Diag(MacroNameTok, diag::err_pp_missing_macro_name);
+    return;
+  }
+
+  IdentifierInfo *II = MacroNameTok.getIdentifierInfo();
+  if (II == 0) {
+    bool Invalid = false;
+    std::string Spelling = getSpelling(MacroNameTok, &Invalid);
+    if (Invalid)
+      return;
+
+    const IdentifierInfo &Info = Identifiers.get(Spelling);
+
+    // Allow #defining |and| and friends in microsoft mode.
+    if (Info.isCPlusPlusOperatorKeyword() && getLangOpts().MicrosoftMode) {
+      MacroNameTok.setIdentifierInfo(getIdentifierInfo(Spelling));
+      return;
+    }
+
+    if (Info.isCPlusPlusOperatorKeyword())
+      // C++ 2.5p2: Alternative tokens behave the same as its primary token
+      // except for their spellings.
+      Diag(MacroNameTok, diag::err_pp_operator_used_as_macro_name) << Spelling;
+    else
+      Diag(MacroNameTok, diag::err_pp_macro_not_identifier);
+    // Fall through on error.
+  } else if (isDefineUndef && II->getPPKeywordID() == tok::pp_defined) {
+    // Error if defining "defined": C99 6.10.8.4.
+    Diag(MacroNameTok, diag::err_defined_macro_name);
+  } else if (isDefineUndef && II->hasMacroDefinition() &&
+             getMacroInfo(II)->isBuiltinMacro()) {
+    // Error if defining "__LINE__" and other builtins: C99 6.10.8.4.
+    if (isDefineUndef == 1)
+      Diag(MacroNameTok, diag::pp_redef_builtin_macro);
+    else
+      Diag(MacroNameTok, diag::pp_undef_builtin_macro);
+  } else {
+    // Okay, we got a good identifier node.  Return it.
+    return;
+  }
+
+  // Invalid macro name, read and discard the rest of the line.  Then set the
+  // token kind to tok::eod.
+  MacroNameTok.setKind(tok::eod);
+  return DiscardUntilEndOfDirective();
+}
+
+/// CheckEndOfDirective - Ensure that the next token is a tok::eod token.  If
+/// not, emit a diagnostic and consume up until the eod.  If EnableMacros is
+/// true, then we consider macros that expand to zero tokens as being ok.
+void Preprocessor::CheckEndOfDirective(const char *DirType, bool EnableMacros) {
+  Token Tmp;
+  // Lex unexpanded tokens for most directives: macros might expand to zero
+  // tokens, causing us to miss diagnosing invalid lines.  Some directives (like
+  // #line) allow empty macros.
+  if (EnableMacros)
+    Lex(Tmp);
+  else
+    LexUnexpandedToken(Tmp);
+
+  // There should be no tokens after the directive, but we allow them as an
+  // extension.
+  while (Tmp.is(tok::comment))  // Skip comments in -C mode.
+    LexUnexpandedToken(Tmp);
+
+  if (Tmp.isNot(tok::eod)) {
+    // Add a fixit in GNU/C99/C++ mode.  Don't offer a fixit for strict-C89,
+    // or if this is a macro-style preprocessing directive, because it is more
+    // trouble than it is worth to insert /**/ and check that there is no /**/
+    // in the range also.
+    FixItHint Hint;
+    if ((LangOpts.GNUMode || LangOpts.C99 || LangOpts.CPlusPlus) &&
+        !CurTokenLexer)
+      Hint = FixItHint::CreateInsertion(Tmp.getLocation(),"//");
+    Diag(Tmp, diag::ext_pp_extra_tokens_at_eol) << DirType << Hint;
+    DiscardUntilEndOfDirective();
+  }
+}
+
+
+
+/// SkipExcludedConditionalBlock - We just read a #if or related directive and
+/// decided that the subsequent tokens are in the #if'd out portion of the
+/// file.  Lex the rest of the file, until we see an #endif.  If
+/// FoundNonSkipPortion is true, then we have already emitted code for part of
+/// this #if directive, so #else/#elif blocks should never be entered. If ElseOk
+/// is true, then #else directives are ok, if not, then we have already seen one
+/// so a #else directive is a duplicate.  When this returns, the caller can lex
+/// the first valid token.
+void Preprocessor::SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
+                                                bool FoundNonSkipPortion,
+                                                bool FoundElse,
+                                                SourceLocation ElseLoc) {
+  ++NumSkipped;
+  assert(CurTokenLexer == 0 && CurPPLexer && "Lexing a macro, not a file?");
+
+  CurPPLexer->pushConditionalLevel(IfTokenLoc, /*isSkipping*/false,
+                                 FoundNonSkipPortion, FoundElse);
+
+  if (CurPTHLexer) {
+    PTHSkipExcludedConditionalBlock();
+    return;
+  }
+
+  // Enter raw mode to disable identifier lookup (and thus macro expansion),
+  // disabling warnings, etc.
+  CurPPLexer->LexingRawMode = true;
+  Token Tok;
+  while (1) {
+    CurLexer->Lex(Tok);
+
+    if (Tok.is(tok::code_completion)) {
+      if (CodeComplete)
+        CodeComplete->CodeCompleteInConditionalExclusion();
+      setCodeCompletionReached();
+      continue;
+    }
+    
+    // If this is the end of the buffer, we have an error.
+    if (Tok.is(tok::eof)) {
+      // Emit errors for each unterminated conditional on the stack, including
+      // the current one.
+      while (!CurPPLexer->ConditionalStack.empty()) {
+        if (CurLexer->getFileLoc() != CodeCompletionFileLoc)
+          Diag(CurPPLexer->ConditionalStack.back().IfLoc,
+               diag::err_pp_unterminated_conditional);
+        CurPPLexer->ConditionalStack.pop_back();
+      }
+
+      // Just return and let the caller lex after this #include.
+      break;
+    }
+
+    // If this token is not a preprocessor directive, just skip it.
+    if (Tok.isNot(tok::hash) || !Tok.isAtStartOfLine())
+      continue;
+
+    // We just parsed a # character at the start of a line, so we're in
+    // directive mode.  Tell the lexer this so any newlines we see will be
+    // converted into an EOD token (this terminates the macro).
+    CurPPLexer->ParsingPreprocessorDirective = true;
+    if (CurLexer) CurLexer->SetCommentRetentionState(false);
+
+
+    // Read the next token, the directive flavor.
+    LexUnexpandedToken(Tok);
+
+    // If this isn't an identifier directive (e.g. is "# 1\n" or "#\n", or
+    // something bogus), skip it.
+    if (Tok.isNot(tok::raw_identifier)) {
+      CurPPLexer->ParsingPreprocessorDirective = false;
+      // Restore comment saving mode.
+      if (CurLexer) CurLexer->SetCommentRetentionState(KeepComments);
+      continue;
+    }
+
+    // If the first letter isn't i or e, it isn't intesting to us.  We know that
+    // this is safe in the face of spelling differences, because there is no way
+    // to spell an i/e in a strange way that is another letter.  Skipping this
+    // allows us to avoid looking up the identifier info for #define/#undef and
+    // other common directives.
+    const char *RawCharData = Tok.getRawIdentifierData();
+
+    char FirstChar = RawCharData[0];
+    if (FirstChar >= 'a' && FirstChar <= 'z' &&
+        FirstChar != 'i' && FirstChar != 'e') {
+      CurPPLexer->ParsingPreprocessorDirective = false;
+      // Restore comment saving mode.
+      if (CurLexer) CurLexer->SetCommentRetentionState(KeepComments);
+      continue;
+    }
+
+    // Get the identifier name without trigraphs or embedded newlines.  Note
+    // that we can't use Tok.getIdentifierInfo() because its lookup is disabled
+    // when skipping.
+    char DirectiveBuf[20];
+    StringRef Directive;
+    if (!Tok.needsCleaning() && Tok.getLength() < 20) {
+      Directive = StringRef(RawCharData, Tok.getLength());
+    } else {
+      std::string DirectiveStr = getSpelling(Tok);
+      unsigned IdLen = DirectiveStr.size();
+      if (IdLen >= 20) {
+        CurPPLexer->ParsingPreprocessorDirective = false;
+        // Restore comment saving mode.
+        if (CurLexer) CurLexer->SetCommentRetentionState(KeepComments);
+        continue;
+      }
+      memcpy(DirectiveBuf, &DirectiveStr[0], IdLen);
+      Directive = StringRef(DirectiveBuf, IdLen);
+    }
+
+    if (Directive.startswith("if")) {
+      StringRef Sub = Directive.substr(2);
+      if (Sub.empty() ||   // "if"
+          Sub == "def" ||   // "ifdef"
+          Sub == "ndef") {  // "ifndef"
+        // We know the entire #if/#ifdef/#ifndef block will be skipped, don't
+        // bother parsing the condition.
+        DiscardUntilEndOfDirective();
+        CurPPLexer->pushConditionalLevel(Tok.getLocation(), /*wasskipping*/true,
+                                       /*foundnonskip*/false,
+                                       /*foundelse*/false);
+      }
+    } else if (Directive[0] == 'e') {
+      StringRef Sub = Directive.substr(1);
+      if (Sub == "ndif") {  // "endif"
+        CheckEndOfDirective("endif");
+        PPConditionalInfo CondInfo;
+        CondInfo.WasSkipping = true; // Silence bogus warning.
+        bool InCond = CurPPLexer->popConditionalLevel(CondInfo);
+        (void)InCond;  // Silence warning in no-asserts mode.
+        assert(!InCond && "Can't be skipping if not in a conditional!");
+
+        // If we popped the outermost skipping block, we're done skipping!
+        if (!CondInfo.WasSkipping) {
+          if (Callbacks)
+            Callbacks->Endif(Tok.getLocation(), CondInfo.IfLoc);
+          break;
+        }
+      } else if (Sub == "lse") { // "else".
+        // #else directive in a skipping conditional.  If not in some other
+        // skipping conditional, and if #else hasn't already been seen, enter it
+        // as a non-skipping conditional.
+        PPConditionalInfo &CondInfo = CurPPLexer->peekConditionalLevel();
+
+        // If this is a #else with a #else before it, report the error.
+        if (CondInfo.FoundElse) Diag(Tok, diag::pp_err_else_after_else);
+
+        // Note that we've seen a #else in this conditional.
+        CondInfo.FoundElse = true;
+
+        // If the conditional is at the top level, and the #if block wasn't
+        // entered, enter the #else block now.
+        if (!CondInfo.WasSkipping && !CondInfo.FoundNonSkip) {
+          CondInfo.FoundNonSkip = true;
+          CheckEndOfDirective("else");
+          if (Callbacks)
+            Callbacks->Else(Tok.getLocation(), CondInfo.IfLoc);
+          break;
+        } else {
+          DiscardUntilEndOfDirective();  // C99 6.10p4.
+        }
+      } else if (Sub == "lif") {  // "elif".
+        PPConditionalInfo &CondInfo = CurPPLexer->peekConditionalLevel();
+
+        bool ShouldEnter;
+        const SourceLocation ConditionalBegin = CurPPLexer->getSourceLocation();
+        // If this is in a skipping block or if we're already handled this #if
+        // block, don't bother parsing the condition.
+        if (CondInfo.WasSkipping || CondInfo.FoundNonSkip) {
+          DiscardUntilEndOfDirective();
+          ShouldEnter = false;
+        } else {
+          // Restore the value of LexingRawMode so that identifiers are
+          // looked up, etc, inside the #elif expression.
+          assert(CurPPLexer->LexingRawMode && "We have to be skipping here!");
+          CurPPLexer->LexingRawMode = false;
+          IdentifierInfo *IfNDefMacro = 0;
+          ShouldEnter = EvaluateDirectiveExpression(IfNDefMacro);
+          CurPPLexer->LexingRawMode = true;
+        }
+        const SourceLocation ConditionalEnd = CurPPLexer->getSourceLocation();
+
+        // If this is a #elif with a #else before it, report the error.
+        if (CondInfo.FoundElse) Diag(Tok, diag::pp_err_elif_after_else);
+
+        // If this condition is true, enter it!
+        if (ShouldEnter) {
+          CondInfo.FoundNonSkip = true;
+          if (Callbacks)
+            Callbacks->Elif(Tok.getLocation(),
+                            SourceRange(ConditionalBegin, ConditionalEnd),
+                            CondInfo.IfLoc);
+          break;
+        }
+      }
+    }
+
+    CurPPLexer->ParsingPreprocessorDirective = false;
+    // Restore comment saving mode.
+    if (CurLexer) CurLexer->SetCommentRetentionState(KeepComments);
+  }
+
+  // Finally, if we are out of the conditional (saw an #endif or ran off the end
+  // of the file, just stop skipping and return to lexing whatever came after
+  // the #if block.
+  CurPPLexer->LexingRawMode = false;
+
+  if (Callbacks) {
+    SourceLocation BeginLoc = ElseLoc.isValid() ? ElseLoc : IfTokenLoc;
+    Callbacks->SourceRangeSkipped(SourceRange(BeginLoc, Tok.getLocation()));
+  }
+}
+
+void Preprocessor::PTHSkipExcludedConditionalBlock() {
+
+  while (1) {
+    assert(CurPTHLexer);
+    assert(CurPTHLexer->LexingRawMode == false);
+
+    // Skip to the next '#else', '#elif', or #endif.
+    if (CurPTHLexer->SkipBlock()) {
+      // We have reached an #endif.  Both the '#' and 'endif' tokens
+      // have been consumed by the PTHLexer.  Just pop off the condition level.
+      PPConditionalInfo CondInfo;
+      bool InCond = CurPTHLexer->popConditionalLevel(CondInfo);
+      (void)InCond;  // Silence warning in no-asserts mode.
+      assert(!InCond && "Can't be skipping if not in a conditional!");
+      break;
+    }
+
+    // We have reached a '#else' or '#elif'.  Lex the next token to get
+    // the directive flavor.
+    Token Tok;
+    LexUnexpandedToken(Tok);
+
+    // We can actually look up the IdentifierInfo here since we aren't in
+    // raw mode.
+    tok::PPKeywordKind K = Tok.getIdentifierInfo()->getPPKeywordID();
+
+    if (K == tok::pp_else) {
+      // #else: Enter the else condition.  We aren't in a nested condition
+      //  since we skip those. We're always in the one matching the last
+      //  blocked we skipped.
+      PPConditionalInfo &CondInfo = CurPTHLexer->peekConditionalLevel();
+      // Note that we've seen a #else in this conditional.
+      CondInfo.FoundElse = true;
+
+      // If the #if block wasn't entered then enter the #else block now.
+      if (!CondInfo.FoundNonSkip) {
+        CondInfo.FoundNonSkip = true;
+
+        // Scan until the eod token.
+        CurPTHLexer->ParsingPreprocessorDirective = true;
+        DiscardUntilEndOfDirective();
+        CurPTHLexer->ParsingPreprocessorDirective = false;
+
+        break;
+      }
+
+      // Otherwise skip this block.
+      continue;
+    }
+
+    assert(K == tok::pp_elif);
+    PPConditionalInfo &CondInfo = CurPTHLexer->peekConditionalLevel();
+
+    // If this is a #elif with a #else before it, report the error.
+    if (CondInfo.FoundElse)
+      Diag(Tok, diag::pp_err_elif_after_else);
+
+    // If this is in a skipping block or if we're already handled this #if
+    // block, don't bother parsing the condition.  We just skip this block.
+    if (CondInfo.FoundNonSkip)
+      continue;
+
+    // Evaluate the condition of the #elif.
+    IdentifierInfo *IfNDefMacro = 0;
+    CurPTHLexer->ParsingPreprocessorDirective = true;
+    bool ShouldEnter = EvaluateDirectiveExpression(IfNDefMacro);
+    CurPTHLexer->ParsingPreprocessorDirective = false;
+
+    // If this condition is true, enter it!
+    if (ShouldEnter) {
+      CondInfo.FoundNonSkip = true;
+      break;
+    }
+
+    // Otherwise, skip this block and go to the next one.
+    continue;
+  }
+}
+
+/// LookupFile - Given a "foo" or <foo> reference, look up the indicated file,
+/// return null on failure.  isAngled indicates whether the file reference is
+/// for system #include's or not (i.e. using <> instead of "").
+const FileEntry *Preprocessor::LookupFile(
+    StringRef Filename,
+    bool isAngled,
+    const DirectoryLookup *FromDir,
+    const DirectoryLookup *&CurDir,
+    SmallVectorImpl<char> *SearchPath,
+    SmallVectorImpl<char> *RelativePath,
+    Module **SuggestedModule,
+    bool SkipCache) {
+  // If the header lookup mechanism may be relative to the current file, pass in
+  // info about where the current file is.
+  const FileEntry *CurFileEnt = 0;
+  if (!FromDir) {
+    FileID FID = getCurrentFileLexer()->getFileID();
+    CurFileEnt = SourceMgr.getFileEntryForID(FID);
+
+    // If there is no file entry associated with this file, it must be the
+    // predefines buffer.  Any other file is not lexed with a normal lexer, so
+    // it won't be scanned for preprocessor directives.   If we have the
+    // predefines buffer, resolve #include references (which come from the
+    // -include command line argument) as if they came from the main file, this
+    // affects file lookup etc.
+    if (CurFileEnt == 0) {
+      FID = SourceMgr.getMainFileID();
+      CurFileEnt = SourceMgr.getFileEntryForID(FID);
+    }
+  }
+
+  // Do a standard file entry lookup.
+  CurDir = CurDirLookup;
+  const FileEntry *FE = HeaderInfo.LookupFile(
+      Filename, isAngled, FromDir, CurDir, CurFileEnt,
+      SearchPath, RelativePath, SuggestedModule, SkipCache);
+  if (FE) return FE;
+
+  // Otherwise, see if this is a subframework header.  If so, this is relative
+  // to one of the headers on the #include stack.  Walk the list of the current
+  // headers on the #include stack and pass them to HeaderInfo.
+  // FIXME: SuggestedModule!
+  if (IsFileLexer()) {
+    if ((CurFileEnt = SourceMgr.getFileEntryForID(CurPPLexer->getFileID())))
+      if ((FE = HeaderInfo.LookupSubframeworkHeader(Filename, CurFileEnt,
+                                                    SearchPath, RelativePath)))
+        return FE;
+  }
+
+  for (unsigned i = 0, e = IncludeMacroStack.size(); i != e; ++i) {
+    IncludeStackInfo &ISEntry = IncludeMacroStack[e-i-1];
+    if (IsFileLexer(ISEntry)) {
+      if ((CurFileEnt =
+           SourceMgr.getFileEntryForID(ISEntry.ThePPLexer->getFileID())))
+        if ((FE = HeaderInfo.LookupSubframeworkHeader(
+                Filename, CurFileEnt, SearchPath, RelativePath)))
+          return FE;
+    }
+  }
+
+  // Otherwise, we really couldn't find the file.
+  return 0;
+}
+
+
+//===----------------------------------------------------------------------===//
+// Preprocessor Directive Handling.
+//===----------------------------------------------------------------------===//
+
+/// HandleDirective - This callback is invoked when the lexer sees a # token
+/// at the start of a line.  This consumes the directive, modifies the
+/// lexer/preprocessor state, and advances the lexer(s) so that the next token
+/// read is the correct one.
+void Preprocessor::HandleDirective(Token &Result) {
+  // FIXME: Traditional: # with whitespace before it not recognized by K&R?
+
+  // We just parsed a # character at the start of a line, so we're in directive
+  // mode.  Tell the lexer this so any newlines we see will be converted into an
+  // EOD token (which terminates the directive).
+  CurPPLexer->ParsingPreprocessorDirective = true;
+
+  ++NumDirectives;
+
+  // We are about to read a token.  For the multiple-include optimization FA to
+  // work, we have to remember if we had read any tokens *before* this
+  // pp-directive.
+  bool ReadAnyTokensBeforeDirective =CurPPLexer->MIOpt.getHasReadAnyTokensVal();
+
+  // Save the '#' token in case we need to return it later.
+  Token SavedHash = Result;
+
+  // Read the next token, the directive flavor.  This isn't expanded due to
+  // C99 6.10.3p8.
+  LexUnexpandedToken(Result);
+
+  // C99 6.10.3p11: Is this preprocessor directive in macro invocation?  e.g.:
+  //   #define A(x) #x
+  //   A(abc
+  //     #warning blah
+  //   def)
+  // If so, the user is relying on undefined behavior, emit a diagnostic. Do
+  // not support this for #include-like directives, since that can result in
+  // terrible diagnostics, and does not work in GCC.
+  if (InMacroArgs) {
+    if (IdentifierInfo *II = Result.getIdentifierInfo()) {
+      switch (II->getPPKeywordID()) {
+      case tok::pp_include:
+      case tok::pp_import:
+      case tok::pp_include_next:
+      case tok::pp___include_macros:
+        Diag(Result, diag::err_embedded_include) << II->getName();
+        DiscardUntilEndOfDirective();
+        return;
+      default:
+        break;
+      }
+    }
+    Diag(Result, diag::ext_embedded_directive);
+  }
+
+TryAgain:
+  switch (Result.getKind()) {
+  case tok::eod:
+    return;   // null directive.
+  case tok::comment:
+    // Handle stuff like "# /*foo*/ define X" in -E -C mode.
+    LexUnexpandedToken(Result);
+    goto TryAgain;
+  case tok::code_completion:
+    if (CodeComplete)
+      CodeComplete->CodeCompleteDirective(
+                                    CurPPLexer->getConditionalStackDepth() > 0);
+    setCodeCompletionReached();
+    return;
+  case tok::numeric_constant:  // # 7  GNU line marker directive.
+    if (getLangOpts().AsmPreprocessor)
+      break;  // # 4 is not a preprocessor directive in .S files.
+    return HandleDigitDirective(Result);
+  default:
+    IdentifierInfo *II = Result.getIdentifierInfo();
+    if (II == 0) break;  // Not an identifier.
+
+    // Ask what the preprocessor keyword ID is.
+    switch (II->getPPKeywordID()) {
+    default: break;
+    // C99 6.10.1 - Conditional Inclusion.
+    case tok::pp_if:
+      return HandleIfDirective(Result, ReadAnyTokensBeforeDirective);
+    case tok::pp_ifdef:
+      return HandleIfdefDirective(Result, false, true/*not valid for miopt*/);
+    case tok::pp_ifndef:
+      return HandleIfdefDirective(Result, true, ReadAnyTokensBeforeDirective);
+    case tok::pp_elif:
+      return HandleElifDirective(Result);
+    case tok::pp_else:
+      return HandleElseDirective(Result);
+    case tok::pp_endif:
+      return HandleEndifDirective(Result);
+
+    // C99 6.10.2 - Source File Inclusion.
+    case tok::pp_include:
+      // Handle #include.
+      return HandleIncludeDirective(SavedHash.getLocation(), Result);
+    case tok::pp___include_macros:
+      // Handle -imacros.
+      return HandleIncludeMacrosDirective(SavedHash.getLocation(), Result); 
+
+    // C99 6.10.3 - Macro Replacement.
+    case tok::pp_define:
+      return HandleDefineDirective(Result);
+    case tok::pp_undef:
+      return HandleUndefDirective(Result);
+
+    // C99 6.10.4 - Line Control.
+    case tok::pp_line:
+      return HandleLineDirective(Result);
+
+    // C99 6.10.5 - Error Directive.
+    case tok::pp_error:
+      return HandleUserDiagnosticDirective(Result, false);
+
+    // C99 6.10.6 - Pragma Directive.
+    case tok::pp_pragma:
+      return HandlePragmaDirective(PIK_HashPragma);
+
+    // GNU Extensions.
+    case tok::pp_import:
+      return HandleImportDirective(SavedHash.getLocation(), Result);
+    case tok::pp_include_next:
+      return HandleIncludeNextDirective(SavedHash.getLocation(), Result);
+
+    case tok::pp_warning:
+      Diag(Result, diag::ext_pp_warning_directive);
+      return HandleUserDiagnosticDirective(Result, true);
+    case tok::pp_ident:
+      return HandleIdentSCCSDirective(Result);
+    case tok::pp_sccs:
+      return HandleIdentSCCSDirective(Result);
+    case tok::pp_assert:
+      //isExtension = true;  // FIXME: implement #assert
+      break;
+    case tok::pp_unassert:
+      //isExtension = true;  // FIXME: implement #unassert
+      break;
+        
+    case tok::pp___public_macro:
+      if (getLangOpts().Modules)
+        return HandleMacroPublicDirective(Result);
+      break;
+        
+    case tok::pp___private_macro:
+      if (getLangOpts().Modules)
+        return HandleMacroPrivateDirective(Result);
+      break;
+    }
+    break;
+  }
+
+  // If this is a .S file, treat unknown # directives as non-preprocessor
+  // directives.  This is important because # may be a comment or introduce
+  // various pseudo-ops.  Just return the # token and push back the following
+  // token to be lexed next time.
+  if (getLangOpts().AsmPreprocessor) {
+    Token *Toks = new Token[2];
+    // Return the # and the token after it.
+    Toks[0] = SavedHash;
+    Toks[1] = Result;
+    
+    // If the second token is a hashhash token, then we need to translate it to
+    // unknown so the token lexer doesn't try to perform token pasting.
+    if (Result.is(tok::hashhash))
+      Toks[1].setKind(tok::unknown);
+    
+    // Enter this token stream so that we re-lex the tokens.  Make sure to
+    // enable macro expansion, in case the token after the # is an identifier
+    // that is expanded.
+    EnterTokenStream(Toks, 2, false, true);
+    return;
+  }
+
+  // If we reached here, the preprocessing token is not valid!
+  Diag(Result, diag::err_pp_invalid_directive);
+
+  // Read the rest of the PP line.
+  DiscardUntilEndOfDirective();
+
+  // Okay, we're done parsing the directive.
+}
+
+/// GetLineValue - Convert a numeric token into an unsigned value, emitting
+/// Diagnostic DiagID if it is invalid, and returning the value in Val.
+static bool GetLineValue(Token &DigitTok, unsigned &Val,
+                         unsigned DiagID, Preprocessor &PP) {
+  if (DigitTok.isNot(tok::numeric_constant)) {
+    PP.Diag(DigitTok, DiagID);
+
+    if (DigitTok.isNot(tok::eod))
+      PP.DiscardUntilEndOfDirective();
+    return true;
+  }
+
+  SmallString<64> IntegerBuffer;
+  IntegerBuffer.resize(DigitTok.getLength());
+  const char *DigitTokBegin = &IntegerBuffer[0];
+  bool Invalid = false;
+  unsigned ActualLength = PP.getSpelling(DigitTok, DigitTokBegin, &Invalid);
+  if (Invalid)
+    return true;
+  
+  // Verify that we have a simple digit-sequence, and compute the value.  This
+  // is always a simple digit string computed in decimal, so we do this manually
+  // here.
+  Val = 0;
+  for (unsigned i = 0; i != ActualLength; ++i) {
+    if (!isdigit(DigitTokBegin[i])) {
+      PP.Diag(PP.AdvanceToTokenCharacter(DigitTok.getLocation(), i),
+              diag::err_pp_line_digit_sequence);
+      PP.DiscardUntilEndOfDirective();
+      return true;
+    }
+
+    unsigned NextVal = Val*10+(DigitTokBegin[i]-'0');
+    if (NextVal < Val) { // overflow.
+      PP.Diag(DigitTok, DiagID);
+      PP.DiscardUntilEndOfDirective();
+      return true;
+    }
+    Val = NextVal;
+  }
+
+  // Reject 0, this is needed both by #line numbers and flags.
+  if (Val == 0) {
+    PP.Diag(DigitTok, DiagID);
+    PP.DiscardUntilEndOfDirective();
+    return true;
+  }
+
+  if (DigitTokBegin[0] == '0')
+    PP.Diag(DigitTok.getLocation(), diag::warn_pp_line_decimal);
+
+  return false;
+}
+
+/// HandleLineDirective - Handle #line directive: C99 6.10.4.  The two
+/// acceptable forms are:
+///   # line digit-sequence
+///   # line digit-sequence "s-char-sequence"
+void Preprocessor::HandleLineDirective(Token &Tok) {
+  // Read the line # and string argument.  Per C99 6.10.4p5, these tokens are
+  // expanded.
+  Token DigitTok;
+  Lex(DigitTok);
+
+  // Validate the number and convert it to an unsigned.
+  unsigned LineNo;
+  if (GetLineValue(DigitTok, LineNo, diag::err_pp_line_requires_integer,*this))
+    return;
+
+  // Enforce C99 6.10.4p3: "The digit sequence shall not specify ... a
+  // number greater than 2147483647".  C90 requires that the line # be <= 32767.
+  unsigned LineLimit = 32768U;
+  if (LangOpts.C99 || LangOpts.CPlusPlus0x)
+    LineLimit = 2147483648U;
+  if (LineNo >= LineLimit)
+    Diag(DigitTok, diag::ext_pp_line_too_big) << LineLimit;
+  else if (LangOpts.CPlusPlus0x && LineNo >= 32768U)
+    Diag(DigitTok, diag::warn_cxx98_compat_pp_line_too_big);
+
+  int FilenameID = -1;
+  Token StrTok;
+  Lex(StrTok);
+
+  // If the StrTok is "eod", then it wasn't present.  Otherwise, it must be a
+  // string followed by eod.
+  if (StrTok.is(tok::eod))
+    ; // ok
+  else if (StrTok.isNot(tok::string_literal)) {
+    Diag(StrTok, diag::err_pp_line_invalid_filename);
+    return DiscardUntilEndOfDirective();
+  } else if (StrTok.hasUDSuffix()) {
+    Diag(StrTok, diag::err_invalid_string_udl);
+    return DiscardUntilEndOfDirective();
+  } else {
+    // Parse and validate the string, converting it into a unique ID.
+    StringLiteralParser Literal(&StrTok, 1, *this);
+    assert(Literal.isAscii() && "Didn't allow wide strings in");
+    if (Literal.hadError)
+      return DiscardUntilEndOfDirective();
+    if (Literal.Pascal) {
+      Diag(StrTok, diag::err_pp_linemarker_invalid_filename);
+      return DiscardUntilEndOfDirective();
+    }
+    FilenameID = SourceMgr.getLineTableFilenameID(Literal.GetString());
+
+    // Verify that there is nothing after the string, other than EOD.  Because
+    // of C99 6.10.4p5, macros that expand to empty tokens are ok.
+    CheckEndOfDirective("line", true);
+  }
+
+  SourceMgr.AddLineNote(DigitTok.getLocation(), LineNo, FilenameID);
+
+  if (Callbacks)
+    Callbacks->FileChanged(CurPPLexer->getSourceLocation(),
+                           PPCallbacks::RenameFile,
+                           SrcMgr::C_User);
+}
+
+/// ReadLineMarkerFlags - Parse and validate any flags at the end of a GNU line
+/// marker directive.
+static bool ReadLineMarkerFlags(bool &IsFileEntry, bool &IsFileExit,
+                                bool &IsSystemHeader, bool &IsExternCHeader,
+                                Preprocessor &PP) {
+  unsigned FlagVal;
+  Token FlagTok;
+  PP.Lex(FlagTok);
+  if (FlagTok.is(tok::eod)) return false;
+  if (GetLineValue(FlagTok, FlagVal, diag::err_pp_linemarker_invalid_flag, PP))
+    return true;
+
+  if (FlagVal == 1) {
+    IsFileEntry = true;
+
+    PP.Lex(FlagTok);
+    if (FlagTok.is(tok::eod)) return false;
+    if (GetLineValue(FlagTok, FlagVal, diag::err_pp_linemarker_invalid_flag,PP))
+      return true;
+  } else if (FlagVal == 2) {
+    IsFileExit = true;
+
+    SourceManager &SM = PP.getSourceManager();
+    // If we are leaving the current presumed file, check to make sure the
+    // presumed include stack isn't empty!
+    FileID CurFileID =
+      SM.getDecomposedExpansionLoc(FlagTok.getLocation()).first;
+    PresumedLoc PLoc = SM.getPresumedLoc(FlagTok.getLocation());
+    if (PLoc.isInvalid())
+      return true;
+    
+    // If there is no include loc (main file) or if the include loc is in a
+    // different physical file, then we aren't in a "1" line marker flag region.
+    SourceLocation IncLoc = PLoc.getIncludeLoc();
+    if (IncLoc.isInvalid() ||
+        SM.getDecomposedExpansionLoc(IncLoc).first != CurFileID) {
+      PP.Diag(FlagTok, diag::err_pp_linemarker_invalid_pop);
+      PP.DiscardUntilEndOfDirective();
+      return true;
+    }
+
+    PP.Lex(FlagTok);
+    if (FlagTok.is(tok::eod)) return false;
+    if (GetLineValue(FlagTok, FlagVal, diag::err_pp_linemarker_invalid_flag,PP))
+      return true;
+  }
+
+  // We must have 3 if there are still flags.
+  if (FlagVal != 3) {
+    PP.Diag(FlagTok, diag::err_pp_linemarker_invalid_flag);
+    PP.DiscardUntilEndOfDirective();
+    return true;
+  }
+
+  IsSystemHeader = true;
+
+  PP.Lex(FlagTok);
+  if (FlagTok.is(tok::eod)) return false;
+  if (GetLineValue(FlagTok, FlagVal, diag::err_pp_linemarker_invalid_flag, PP))
+    return true;
+
+  // We must have 4 if there is yet another flag.
+  if (FlagVal != 4) {
+    PP.Diag(FlagTok, diag::err_pp_linemarker_invalid_flag);
+    PP.DiscardUntilEndOfDirective();
+    return true;
+  }
+
+  IsExternCHeader = true;
+
+  PP.Lex(FlagTok);
+  if (FlagTok.is(tok::eod)) return false;
+
+  // There are no more valid flags here.
+  PP.Diag(FlagTok, diag::err_pp_linemarker_invalid_flag);
+  PP.DiscardUntilEndOfDirective();
+  return true;
+}
+
+/// HandleDigitDirective - Handle a GNU line marker directive, whose syntax is
+/// one of the following forms:
+///
+///     # 42
+///     # 42 "file" ('1' | '2')?
+///     # 42 "file" ('1' | '2')? '3' '4'?
+///
+void Preprocessor::HandleDigitDirective(Token &DigitTok) {
+  // Validate the number and convert it to an unsigned.  GNU does not have a
+  // line # limit other than it fit in 32-bits.
+  unsigned LineNo;
+  if (GetLineValue(DigitTok, LineNo, diag::err_pp_linemarker_requires_integer,
+                   *this))
+    return;
+
+  Token StrTok;
+  Lex(StrTok);
+
+  bool IsFileEntry = false, IsFileExit = false;
+  bool IsSystemHeader = false, IsExternCHeader = false;
+  int FilenameID = -1;
+
+  // If the StrTok is "eod", then it wasn't present.  Otherwise, it must be a
+  // string followed by eod.
+  if (StrTok.is(tok::eod))
+    ; // ok
+  else if (StrTok.isNot(tok::string_literal)) {
+    Diag(StrTok, diag::err_pp_linemarker_invalid_filename);
+    return DiscardUntilEndOfDirective();
+  } else if (StrTok.hasUDSuffix()) {
+    Diag(StrTok, diag::err_invalid_string_udl);
+    return DiscardUntilEndOfDirective();
+  } else {
+    // Parse and validate the string, converting it into a unique ID.
+    StringLiteralParser Literal(&StrTok, 1, *this);
+    assert(Literal.isAscii() && "Didn't allow wide strings in");
+    if (Literal.hadError)
+      return DiscardUntilEndOfDirective();
+    if (Literal.Pascal) {
+      Diag(StrTok, diag::err_pp_linemarker_invalid_filename);
+      return DiscardUntilEndOfDirective();
+    }
+    FilenameID = SourceMgr.getLineTableFilenameID(Literal.GetString());
+
+    // If a filename was present, read any flags that are present.
+    if (ReadLineMarkerFlags(IsFileEntry, IsFileExit,
+                            IsSystemHeader, IsExternCHeader, *this))
+      return;
+  }
+
+  // Create a line note with this information.
+  SourceMgr.AddLineNote(DigitTok.getLocation(), LineNo, FilenameID,
+                        IsFileEntry, IsFileExit,
+                        IsSystemHeader, IsExternCHeader);
+
+  // If the preprocessor has callbacks installed, notify them of the #line
+  // change.  This is used so that the line marker comes out in -E mode for
+  // example.
+  if (Callbacks) {
+    PPCallbacks::FileChangeReason Reason = PPCallbacks::RenameFile;
+    if (IsFileEntry)
+      Reason = PPCallbacks::EnterFile;
+    else if (IsFileExit)
+      Reason = PPCallbacks::ExitFile;
+    SrcMgr::CharacteristicKind FileKind = SrcMgr::C_User;
+    if (IsExternCHeader)
+      FileKind = SrcMgr::C_ExternCSystem;
+    else if (IsSystemHeader)
+      FileKind = SrcMgr::C_System;
+
+    Callbacks->FileChanged(CurPPLexer->getSourceLocation(), Reason, FileKind);
+  }
+}
+
+
+/// HandleUserDiagnosticDirective - Handle a #warning or #error directive.
+///
+void Preprocessor::HandleUserDiagnosticDirective(Token &Tok,
+                                                 bool isWarning) {
+  // PTH doesn't emit #warning or #error directives.
+  if (CurPTHLexer)
+    return CurPTHLexer->DiscardToEndOfLine();
+
+  // Read the rest of the line raw.  We do this because we don't want macros
+  // to be expanded and we don't require that the tokens be valid preprocessing
+  // tokens.  For example, this is allowed: "#warning `   'foo".  GCC does
+  // collapse multiple consequtive white space between tokens, but this isn't
+  // specified by the standard.
+  std::string Message = CurLexer->ReadToEndOfLine();
+
+  // Find the first non-whitespace character, so that we can make the
+  // diagnostic more succinct.
+  StringRef Msg(Message);
+  size_t i = Msg.find_first_not_of(' ');
+  if (i < Msg.size())
+    Msg = Msg.substr(i);
+  
+  if (isWarning)
+    Diag(Tok, diag::pp_hash_warning) << Msg;
+  else
+    Diag(Tok, diag::err_pp_hash_error) << Msg;
+}
+
+/// HandleIdentSCCSDirective - Handle a #ident/#sccs directive.
+///
+void Preprocessor::HandleIdentSCCSDirective(Token &Tok) {
+  // Yes, this directive is an extension.
+  Diag(Tok, diag::ext_pp_ident_directive);
+
+  // Read the string argument.
+  Token StrTok;
+  Lex(StrTok);
+
+  // If the token kind isn't a string, it's a malformed directive.
+  if (StrTok.isNot(tok::string_literal) &&
+      StrTok.isNot(tok::wide_string_literal)) {
+    Diag(StrTok, diag::err_pp_malformed_ident);
+    if (StrTok.isNot(tok::eod))
+      DiscardUntilEndOfDirective();
+    return;
+  }
+
+  if (StrTok.hasUDSuffix()) {
+    Diag(StrTok, diag::err_invalid_string_udl);
+    return DiscardUntilEndOfDirective();
+  }
+
+  // Verify that there is nothing after the string, other than EOD.
+  CheckEndOfDirective("ident");
+
+  if (Callbacks) {
+    bool Invalid = false;
+    std::string Str = getSpelling(StrTok, &Invalid);
+    if (!Invalid)
+      Callbacks->Ident(Tok.getLocation(), Str);
+  }
+}
+
+/// \brief Handle a #public directive.
+void Preprocessor::HandleMacroPublicDirective(Token &Tok) {
+  Token MacroNameTok;
+  ReadMacroName(MacroNameTok, 2);
+  
+  // Error reading macro name?  If so, diagnostic already issued.
+  if (MacroNameTok.is(tok::eod))
+    return;
+
+  // Check to see if this is the last token on the #__public_macro line.
+  CheckEndOfDirective("__public_macro");
+
+  // Okay, we finally have a valid identifier to undef.
+  MacroInfo *MI = getMacroInfo(MacroNameTok.getIdentifierInfo());
+  
+  // If the macro is not defined, this is an error.
+  if (MI == 0) {
+    Diag(MacroNameTok, diag::err_pp_visibility_non_macro)
+      << MacroNameTok.getIdentifierInfo();
+    return;
+  }
+  
+  // Note that this macro has now been exported.
+  MI->setVisibility(/*IsPublic=*/true, MacroNameTok.getLocation());
+  
+  // If this macro definition came from a PCH file, mark it
+  // as having changed since serialization.
+  if (MI->isFromAST())
+    MI->setChangedAfterLoad();
+}
+
+/// \brief Handle a #private directive.
+void Preprocessor::HandleMacroPrivateDirective(Token &Tok) {
+  Token MacroNameTok;
+  ReadMacroName(MacroNameTok, 2);
+  
+  // Error reading macro name?  If so, diagnostic already issued.
+  if (MacroNameTok.is(tok::eod))
+    return;
+  
+  // Check to see if this is the last token on the #__private_macro line.
+  CheckEndOfDirective("__private_macro");
+  
+  // Okay, we finally have a valid identifier to undef.
+  MacroInfo *MI = getMacroInfo(MacroNameTok.getIdentifierInfo());
+  
+  // If the macro is not defined, this is an error.
+  if (MI == 0) {
+    Diag(MacroNameTok, diag::err_pp_visibility_non_macro)
+      << MacroNameTok.getIdentifierInfo();
+    return;
+  }
+  
+  // Note that this macro has now been marked private.
+  MI->setVisibility(/*IsPublic=*/false, MacroNameTok.getLocation());
+  
+  // If this macro definition came from a PCH file, mark it
+  // as having changed since serialization.
+  if (MI->isFromAST())
+    MI->setChangedAfterLoad();
+}
+
+//===----------------------------------------------------------------------===//
+// Preprocessor Include Directive Handling.
+//===----------------------------------------------------------------------===//
+
+/// GetIncludeFilenameSpelling - Turn the specified lexer token into a fully
+/// checked and spelled filename, e.g. as an operand of #include. This returns
+/// true if the input filename was in <>'s or false if it were in ""'s.  The
+/// caller is expected to provide a buffer that is large enough to hold the
+/// spelling of the filename, but is also expected to handle the case when
+/// this method decides to use a different buffer.
+bool Preprocessor::GetIncludeFilenameSpelling(SourceLocation Loc,
+                                              StringRef &Buffer) {
+  // Get the text form of the filename.
+  assert(!Buffer.empty() && "Can't have tokens with empty spellings!");
+
+  // Make sure the filename is <x> or "x".
+  bool isAngled;
+  if (Buffer[0] == '<') {
+    if (Buffer.back() != '>') {
+      Diag(Loc, diag::err_pp_expects_filename);
+      Buffer = StringRef();
+      return true;
+    }
+    isAngled = true;
+  } else if (Buffer[0] == '"') {
+    if (Buffer.back() != '"') {
+      Diag(Loc, diag::err_pp_expects_filename);
+      Buffer = StringRef();
+      return true;
+    }
+    isAngled = false;
+  } else {
+    Diag(Loc, diag::err_pp_expects_filename);
+    Buffer = StringRef();
+    return true;
+  }
+
+  // Diagnose #include "" as invalid.
+  if (Buffer.size() <= 2) {
+    Diag(Loc, diag::err_pp_empty_filename);
+    Buffer = StringRef();
+    return true;
+  }
+
+  // Skip the brackets.
+  Buffer = Buffer.substr(1, Buffer.size()-2);
+  return isAngled;
+}
+
+/// ConcatenateIncludeName - Handle cases where the #include name is expanded
+/// from a macro as multiple tokens, which need to be glued together.  This
+/// occurs for code like:
+///    #define FOO <a/b.h>
+///    #include FOO
+/// because in this case, "<a/b.h>" is returned as 7 tokens, not one.
+///
+/// This code concatenates and consumes tokens up to the '>' token.  It returns
+/// false if the > was found, otherwise it returns true if it finds and consumes
+/// the EOD marker.
+bool Preprocessor::ConcatenateIncludeName(
+                                        SmallString<128> &FilenameBuffer,
+                                          SourceLocation &End) {
+  Token CurTok;
+
+  Lex(CurTok);
+  while (CurTok.isNot(tok::eod)) {
+    End = CurTok.getLocation();
+    
+    // FIXME: Provide code completion for #includes.
+    if (CurTok.is(tok::code_completion)) {
+      setCodeCompletionReached();
+      Lex(CurTok);
+      continue;
+    }
+
+    // Append the spelling of this token to the buffer. If there was a space
+    // before it, add it now.
+    if (CurTok.hasLeadingSpace())
+      FilenameBuffer.push_back(' ');
+
+    // Get the spelling of the token, directly into FilenameBuffer if possible.
+    unsigned PreAppendSize = FilenameBuffer.size();
+    FilenameBuffer.resize(PreAppendSize+CurTok.getLength());
+
+    const char *BufPtr = &FilenameBuffer[PreAppendSize];
+    unsigned ActualLen = getSpelling(CurTok, BufPtr);
+
+    // If the token was spelled somewhere else, copy it into FilenameBuffer.
+    if (BufPtr != &FilenameBuffer[PreAppendSize])
+      memcpy(&FilenameBuffer[PreAppendSize], BufPtr, ActualLen);
+
+    // Resize FilenameBuffer to the correct size.
+    if (CurTok.getLength() != ActualLen)
+      FilenameBuffer.resize(PreAppendSize+ActualLen);
+
+    // If we found the '>' marker, return success.
+    if (CurTok.is(tok::greater))
+      return false;
+
+    Lex(CurTok);
+  }
+
+  // If we hit the eod marker, emit an error and return true so that the caller
+  // knows the EOD has been read.
+  Diag(CurTok.getLocation(), diag::err_pp_expects_filename);
+  return true;
+}
+
+/// HandleIncludeDirective - The "#include" tokens have just been read, read the
+/// file to be included from the lexer, then include it!  This is a common
+/// routine with functionality shared between #include, #include_next and
+/// #import.  LookupFrom is set when this is a #include_next directive, it
+/// specifies the file to start searching from.
+void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc, 
+                                          Token &IncludeTok,
+                                          const DirectoryLookup *LookupFrom,
+                                          bool isImport) {
+
+  Token FilenameTok;
+  CurPPLexer->LexIncludeFilename(FilenameTok);
+
+  // Reserve a buffer to get the spelling.
+  SmallString<128> FilenameBuffer;
+  StringRef Filename;
+  SourceLocation End;
+  SourceLocation CharEnd; // the end of this directive, in characters
+  
+  switch (FilenameTok.getKind()) {
+  case tok::eod:
+    // If the token kind is EOD, the error has already been diagnosed.
+    return;
+
+  case tok::angle_string_literal:
+  case tok::string_literal:
+    Filename = getSpelling(FilenameTok, FilenameBuffer);
+    End = FilenameTok.getLocation();
+    CharEnd = End.getLocWithOffset(Filename.size());
+    break;
+
+  case tok::less:
+    // This could be a <foo/bar.h> file coming from a macro expansion.  In this
+    // case, glue the tokens together into FilenameBuffer and interpret those.
+    FilenameBuffer.push_back('<');
+    if (ConcatenateIncludeName(FilenameBuffer, End))
+      return;   // Found <eod> but no ">"?  Diagnostic already emitted.
+    Filename = FilenameBuffer.str();
+    CharEnd = getLocForEndOfToken(End);
+    break;
+  default:
+    Diag(FilenameTok.getLocation(), diag::err_pp_expects_filename);
+    DiscardUntilEndOfDirective();
+    return;
+  }
+
+  StringRef OriginalFilename = Filename;
+  bool isAngled =
+    GetIncludeFilenameSpelling(FilenameTok.getLocation(), Filename);
+  // If GetIncludeFilenameSpelling set the start ptr to null, there was an
+  // error.
+  if (Filename.empty()) {
+    DiscardUntilEndOfDirective();
+    return;
+  }
+
+  // Verify that there is nothing after the filename, other than EOD.  Note that
+  // we allow macros that expand to nothing after the filename, because this
+  // falls into the category of "#include pp-tokens new-line" specified in
+  // C99 6.10.2p4.
+  CheckEndOfDirective(IncludeTok.getIdentifierInfo()->getNameStart(), true);
+
+  // Check that we don't have infinite #include recursion.
+  if (IncludeMacroStack.size() == MaxAllowedIncludeStackDepth-1) {
+    Diag(FilenameTok, diag::err_pp_include_too_deep);
+    return;
+  }
+
+  // Complain about attempts to #include files in an audit pragma.
+  if (PragmaARCCFCodeAuditedLoc.isValid()) {
+    Diag(HashLoc, diag::err_pp_include_in_arc_cf_code_audited);
+    Diag(PragmaARCCFCodeAuditedLoc, diag::note_pragma_entered_here);
+
+    // Immediately leave the pragma.
+    PragmaARCCFCodeAuditedLoc = SourceLocation();
+  }
+
+  if (HeaderInfo.HasIncludeAliasMap()) {
+    // Map the filename with the brackets still attached.  If the name doesn't 
+    // map to anything, fall back on the filename we've already gotten the 
+    // spelling for.
+    StringRef NewName = HeaderInfo.MapHeaderToIncludeAlias(OriginalFilename);
+    if (!NewName.empty())
+      Filename = NewName;
+  }
+
+  // Search include directories.
+  const DirectoryLookup *CurDir;
+  SmallString<1024> SearchPath;
+  SmallString<1024> RelativePath;
+  // We get the raw path only if we have 'Callbacks' to which we later pass
+  // the path.
+  Module *SuggestedModule = 0;
+  const FileEntry *File = LookupFile(
+      Filename, isAngled, LookupFrom, CurDir,
+      Callbacks ? &SearchPath : NULL, Callbacks ? &RelativePath : NULL,
+      getLangOpts().Modules? &SuggestedModule : 0);
+
+  if (Callbacks) {
+    if (!File) {
+      // Give the clients a chance to recover.
+      SmallString<128> RecoveryPath;
+      if (Callbacks->FileNotFound(Filename, RecoveryPath)) {
+        if (const DirectoryEntry *DE = FileMgr.getDirectory(RecoveryPath)) {
+          // Add the recovery path to the list of search paths.
+          DirectoryLookup DL(DE, SrcMgr::C_User, true, false);
+          HeaderInfo.AddSearchPath(DL, isAngled);
+          
+          // Try the lookup again, skipping the cache.
+          File = LookupFile(Filename, isAngled, LookupFrom, CurDir, 0, 0,
+                            getLangOpts().Modules? &SuggestedModule : 0,
+                            /*SkipCache*/true);
+        }
+      }
+    }
+    
+    // Notify the callback object that we've seen an inclusion directive.
+    Callbacks->InclusionDirective(HashLoc, IncludeTok, Filename, isAngled, File,
+                                  End, SearchPath, RelativePath);
+  }
+  
+  if (File == 0) {
+    if (!SuppressIncludeNotFoundError)
+      Diag(FilenameTok, diag::err_pp_file_not_found) << Filename;
+    return;
+  }
+
+  // If we are supposed to import a module rather than including the header,
+  // do so now.
+  if (SuggestedModule) {
+    // Compute the module access path corresponding to this module.
+    // FIXME: Should we have a second loadModule() overload to avoid this
+    // extra lookup step?
+    llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> Path;
+    for (Module *Mod = SuggestedModule; Mod; Mod = Mod->Parent)
+      Path.push_back(std::make_pair(getIdentifierInfo(Mod->Name),
+                                    FilenameTok.getLocation()));
+    std::reverse(Path.begin(), Path.end());
+
+    // Warn that we're replacing the include/import with a module import.
+    SmallString<128> PathString;
+    for (unsigned I = 0, N = Path.size(); I != N; ++I) {
+      if (I)
+        PathString += '.';
+      PathString += Path[I].first->getName();
+    }
+    int IncludeKind = 0;
+    
+    switch (IncludeTok.getIdentifierInfo()->getPPKeywordID()) {
+    case tok::pp_include:
+      IncludeKind = 0;
+      break;
+      
+    case tok::pp_import:
+      IncludeKind = 1;
+      break;        
+        
+    case tok::pp_include_next:
+      IncludeKind = 2;
+      break;
+        
+    case tok::pp___include_macros:
+      IncludeKind = 3;
+      break;
+        
+    default:
+      llvm_unreachable("unknown include directive kind");
+    }
+
+    // Determine whether we are actually building the module that this
+    // include directive maps to.
+    bool BuildingImportedModule
+      = Path[0].first->getName() == getLangOpts().CurrentModule;
+    
+    if (!BuildingImportedModule && getLangOpts().ObjC2) {
+      // If we're not building the imported module, warn that we're going
+      // to automatically turn this inclusion directive into a module import.
+      // We only do this in Objective-C, where we have a module-import syntax.
+      CharSourceRange ReplaceRange(SourceRange(HashLoc, CharEnd), 
+                                   /*IsTokenRange=*/false);
+      Diag(HashLoc, diag::warn_auto_module_import)
+        << IncludeKind << PathString 
+        << FixItHint::CreateReplacement(ReplaceRange,
+             "@__experimental_modules_import " + PathString.str().str() + ";");
+    }
+    
+    // Load the module.
+    // If this was an #__include_macros directive, only make macros visible.
+    Module::NameVisibilityKind Visibility 
+      = (IncludeKind == 3)? Module::MacrosVisible : Module::AllVisible;
+    Module *Imported
+      = TheModuleLoader.loadModule(IncludeTok.getLocation(), Path, Visibility,
+                                   /*IsIncludeDirective=*/true);
+    
+    // If this header isn't part of the module we're building, we're done.
+    if (!BuildingImportedModule && Imported)
+      return;
+  }
+  
+  // The #included file will be considered to be a system header if either it is
+  // in a system include directory, or if the #includer is a system include
+  // header.
+  SrcMgr::CharacteristicKind FileCharacter =
+    std::max(HeaderInfo.getFileDirFlavor(File),
+             SourceMgr.getFileCharacteristic(FilenameTok.getLocation()));
+
+  // Ask HeaderInfo if we should enter this #include file.  If not, #including
+  // this file will have no effect.
+  if (!HeaderInfo.ShouldEnterIncludeFile(File, isImport)) {
+    if (Callbacks)
+      Callbacks->FileSkipped(*File, FilenameTok, FileCharacter);
+    return;
+  }
+
+  // Look up the file, create a File ID for it.
+  SourceLocation IncludePos = End;
+  // If the filename string was the result of macro expansions, set the include
+  // position on the file where it will be included and after the expansions.
+  if (IncludePos.isMacroID())
+    IncludePos = SourceMgr.getExpansionRange(IncludePos).second;
+  FileID FID = SourceMgr.createFileID(File, IncludePos, FileCharacter);
+  assert(!FID.isInvalid() && "Expected valid file ID");
+
+  // Finally, if all is good, enter the new file!
+  EnterSourceFile(FID, CurDir, FilenameTok.getLocation());
+}
+
+/// HandleIncludeNextDirective - Implements #include_next.
+///
+void Preprocessor::HandleIncludeNextDirective(SourceLocation HashLoc,
+                                              Token &IncludeNextTok) {
+  Diag(IncludeNextTok, diag::ext_pp_include_next_directive);
+
+  // #include_next is like #include, except that we start searching after
+  // the current found directory.  If we can't do this, issue a
+  // diagnostic.
+  const DirectoryLookup *Lookup = CurDirLookup;
+  if (isInPrimaryFile()) {
+    Lookup = 0;
+    Diag(IncludeNextTok, diag::pp_include_next_in_primary);
+  } else if (Lookup == 0) {
+    Diag(IncludeNextTok, diag::pp_include_next_absolute_path);
+  } else {
+    // Start looking up in the next directory.
+    ++Lookup;
+  }
+
+  return HandleIncludeDirective(HashLoc, IncludeNextTok, Lookup);
+}
+
+/// HandleMicrosoftImportDirective - Implements #import for Microsoft Mode
+void Preprocessor::HandleMicrosoftImportDirective(Token &Tok) {
+  // The Microsoft #import directive takes a type library and generates header
+  // files from it, and includes those.  This is beyond the scope of what clang
+  // does, so we ignore it and error out.  However, #import can optionally have
+  // trailing attributes that span multiple lines.  We're going to eat those
+  // so we can continue processing from there.
+  Diag(Tok, diag::err_pp_import_directive_ms );
+
+  // Read tokens until we get to the end of the directive.  Note that the 
+  // directive can be split over multiple lines using the backslash character.
+  DiscardUntilEndOfDirective();
+}
+
+/// HandleImportDirective - Implements #import.
+///
+void Preprocessor::HandleImportDirective(SourceLocation HashLoc,
+                                         Token &ImportTok) {
+  if (!LangOpts.ObjC1) {  // #import is standard for ObjC.
+    if (LangOpts.MicrosoftMode)
+      return HandleMicrosoftImportDirective(ImportTok);
+    Diag(ImportTok, diag::ext_pp_import_directive);
+  }
+  return HandleIncludeDirective(HashLoc, ImportTok, 0, true);
+}
+
+/// HandleIncludeMacrosDirective - The -imacros command line option turns into a
+/// pseudo directive in the predefines buffer.  This handles it by sucking all
+/// tokens through the preprocessor and discarding them (only keeping the side
+/// effects on the preprocessor).
+void Preprocessor::HandleIncludeMacrosDirective(SourceLocation HashLoc,
+                                                Token &IncludeMacrosTok) {
+  // This directive should only occur in the predefines buffer.  If not, emit an
+  // error and reject it.
+  SourceLocation Loc = IncludeMacrosTok.getLocation();
+  if (strcmp(SourceMgr.getBufferName(Loc), "<built-in>") != 0) {
+    Diag(IncludeMacrosTok.getLocation(),
+         diag::pp_include_macros_out_of_predefines);
+    DiscardUntilEndOfDirective();
+    return;
+  }
+
+  // Treat this as a normal #include for checking purposes.  If this is
+  // successful, it will push a new lexer onto the include stack.
+  HandleIncludeDirective(HashLoc, IncludeMacrosTok, 0, false);
+
+  Token TmpTok;
+  do {
+    Lex(TmpTok);
+    assert(TmpTok.isNot(tok::eof) && "Didn't find end of -imacros!");
+  } while (TmpTok.isNot(tok::hashhash));
+}
+
+//===----------------------------------------------------------------------===//
+// Preprocessor Macro Directive Handling.
+//===----------------------------------------------------------------------===//
+
+/// ReadMacroDefinitionArgList - The ( starting an argument list of a macro
+/// definition has just been read.  Lex the rest of the arguments and the
+/// closing ), updating MI with what we learn.  Return true if an error occurs
+/// parsing the arg list.
+bool Preprocessor::ReadMacroDefinitionArgList(MacroInfo *MI, Token &Tok) {
+  SmallVector<IdentifierInfo*, 32> Arguments;
+
+  while (1) {
+    LexUnexpandedToken(Tok);
+    switch (Tok.getKind()) {
+    case tok::r_paren:
+      // Found the end of the argument list.
+      if (Arguments.empty())  // #define FOO()
+        return false;
+      // Otherwise we have #define FOO(A,)
+      Diag(Tok, diag::err_pp_expected_ident_in_arg_list);
+      return true;
+    case tok::ellipsis:  // #define X(... -> C99 varargs
+      if (!LangOpts.C99)
+        Diag(Tok, LangOpts.CPlusPlus0x ? 
+             diag::warn_cxx98_compat_variadic_macro :
+             diag::ext_variadic_macro);
+
+      // Lex the token after the identifier.
+      LexUnexpandedToken(Tok);
+      if (Tok.isNot(tok::r_paren)) {
+        Diag(Tok, diag::err_pp_missing_rparen_in_macro_def);
+        return true;
+      }
+      // Add the __VA_ARGS__ identifier as an argument.
+      Arguments.push_back(Ident__VA_ARGS__);
+      MI->setIsC99Varargs();
+      MI->setArgumentList(&Arguments[0], Arguments.size(), BP);
+      return false;
+    case tok::eod:  // #define X(
+      Diag(Tok, diag::err_pp_missing_rparen_in_macro_def);
+      return true;
+    default:
+      // Handle keywords and identifiers here to accept things like
+      // #define Foo(for) for.
+      IdentifierInfo *II = Tok.getIdentifierInfo();
+      if (II == 0) {
+        // #define X(1
+        Diag(Tok, diag::err_pp_invalid_tok_in_arg_list);
+        return true;
+      }
+
+      // If this is already used as an argument, it is used multiple times (e.g.
+      // #define X(A,A.
+      if (std::find(Arguments.begin(), Arguments.end(), II) !=
+          Arguments.end()) {  // C99 6.10.3p6
+        Diag(Tok, diag::err_pp_duplicate_name_in_arg_list) << II;
+        return true;
+      }
+
+      // Add the argument to the macro info.
+      Arguments.push_back(II);
+
+      // Lex the token after the identifier.
+      LexUnexpandedToken(Tok);
+
+      switch (Tok.getKind()) {
+      default:          // #define X(A B
+        Diag(Tok, diag::err_pp_expected_comma_in_arg_list);
+        return true;
+      case tok::r_paren: // #define X(A)
+        MI->setArgumentList(&Arguments[0], Arguments.size(), BP);
+        return false;
+      case tok::comma:  // #define X(A,
+        break;
+      case tok::ellipsis:  // #define X(A... -> GCC extension
+        // Diagnose extension.
+        Diag(Tok, diag::ext_named_variadic_macro);
+
+        // Lex the token after the identifier.
+        LexUnexpandedToken(Tok);
+        if (Tok.isNot(tok::r_paren)) {
+          Diag(Tok, diag::err_pp_missing_rparen_in_macro_def);
+          return true;
+        }
+
+        MI->setIsGNUVarargs();
+        MI->setArgumentList(&Arguments[0], Arguments.size(), BP);
+        return false;
+      }
+    }
+  }
+}
+
+/// HandleDefineDirective - Implements #define.  This consumes the entire macro
+/// line then lets the caller lex the next real token.
+void Preprocessor::HandleDefineDirective(Token &DefineTok) {
+  ++NumDefined;
+
+  Token MacroNameTok;
+  ReadMacroName(MacroNameTok, 1);
+
+  // Error reading macro name?  If so, diagnostic already issued.
+  if (MacroNameTok.is(tok::eod))
+    return;
+
+  Token LastTok = MacroNameTok;
+
+  // If we are supposed to keep comments in #defines, reenable comment saving
+  // mode.
+  if (CurLexer) CurLexer->SetCommentRetentionState(KeepMacroComments);
+
+  // Create the new macro.
+  MacroInfo *MI = AllocateMacroInfo(MacroNameTok.getLocation());
+
+  Token Tok;
+  LexUnexpandedToken(Tok);
+
+  // If this is a function-like macro definition, parse the argument list,
+  // marking each of the identifiers as being used as macro arguments.  Also,
+  // check other constraints on the first token of the macro body.
+  if (Tok.is(tok::eod)) {
+    // If there is no body to this macro, we have no special handling here.
+  } else if (Tok.hasLeadingSpace()) {
+    // This is a normal token with leading space.  Clear the leading space
+    // marker on the first token to get proper expansion.
+    Tok.clearFlag(Token::LeadingSpace);
+  } else if (Tok.is(tok::l_paren)) {
+    // This is a function-like macro definition.  Read the argument list.
+    MI->setIsFunctionLike();
+    if (ReadMacroDefinitionArgList(MI, LastTok)) {
+      // Forget about MI.
+      ReleaseMacroInfo(MI);
+      // Throw away the rest of the line.
+      if (CurPPLexer->ParsingPreprocessorDirective)
+        DiscardUntilEndOfDirective();
+      return;
+    }
+
+    // If this is a definition of a variadic C99 function-like macro, not using
+    // the GNU named varargs extension, enabled __VA_ARGS__.
+
+    // "Poison" __VA_ARGS__, which can only appear in the expansion of a macro.
+    // This gets unpoisoned where it is allowed.
+    assert(Ident__VA_ARGS__->isPoisoned() && "__VA_ARGS__ should be poisoned!");
+    if (MI->isC99Varargs())
+      Ident__VA_ARGS__->setIsPoisoned(false);
+
+    // Read the first token after the arg list for down below.
+    LexUnexpandedToken(Tok);
+  } else if (LangOpts.C99 || LangOpts.CPlusPlus0x) {
+    // C99 requires whitespace between the macro definition and the body.  Emit
+    // a diagnostic for something like "#define X+".
+    Diag(Tok, diag::ext_c99_whitespace_required_after_macro_name);
+  } else {
+    // C90 6.8 TC1 says: "In the definition of an object-like macro, if the
+    // first character of a replacement list is not a character required by
+    // subclause 5.2.1, then there shall be white-space separation between the
+    // identifier and the replacement list.".  5.2.1 lists this set:
+    //   "A-Za-z0-9!"#%&'()*+,_./:;<=>?[\]^_{|}~" as well as whitespace, which
+    // is irrelevant here.
+    bool isInvalid = false;
+    if (Tok.is(tok::at)) // @ is not in the list above.
+      isInvalid = true;
+    else if (Tok.is(tok::unknown)) {
+      // If we have an unknown token, it is something strange like "`".  Since
+      // all of valid characters would have lexed into a single character
+      // token of some sort, we know this is not a valid case.
+      isInvalid = true;
+    }
+    if (isInvalid)
+      Diag(Tok, diag::ext_missing_whitespace_after_macro_name);
+    else
+      Diag(Tok, diag::warn_missing_whitespace_after_macro_name);
+  }
+
+  if (!Tok.is(tok::eod))
+    LastTok = Tok;
+
+  // Read the rest of the macro body.
+  if (MI->isObjectLike()) {
+    // Object-like macros are very simple, just read their body.
+    while (Tok.isNot(tok::eod)) {
+      LastTok = Tok;
+      MI->AddTokenToBody(Tok);
+      // Get the next token of the macro.
+      LexUnexpandedToken(Tok);
+    }
+
+  } else {
+    // Otherwise, read the body of a function-like macro.  While we are at it,
+    // check C99 6.10.3.2p1: ensure that # operators are followed by macro
+    // parameters in function-like macro expansions.
+    while (Tok.isNot(tok::eod)) {
+      LastTok = Tok;
+
+      if (Tok.isNot(tok::hash)) {
+        MI->AddTokenToBody(Tok);
+
+        // Get the next token of the macro.
+        LexUnexpandedToken(Tok);
+        continue;
+      }
+
+      // Get the next token of the macro.
+      LexUnexpandedToken(Tok);
+
+      // Check for a valid macro arg identifier.
+      if (Tok.getIdentifierInfo() == 0 ||
+          MI->getArgumentNum(Tok.getIdentifierInfo()) == -1) {
+
+        // If this is assembler-with-cpp mode, we accept random gibberish after
+        // the '#' because '#' is often a comment character.  However, change
+        // the kind of the token to tok::unknown so that the preprocessor isn't
+        // confused.
+        if (getLangOpts().AsmPreprocessor && Tok.isNot(tok::eod)) {
+          LastTok.setKind(tok::unknown);
+        } else {
+          Diag(Tok, diag::err_pp_stringize_not_parameter);
+          ReleaseMacroInfo(MI);
+
+          // Disable __VA_ARGS__ again.
+          Ident__VA_ARGS__->setIsPoisoned(true);
+          return;
+        }
+      }
+
+      // Things look ok, add the '#' and param name tokens to the macro.
+      MI->AddTokenToBody(LastTok);
+      MI->AddTokenToBody(Tok);
+      LastTok = Tok;
+
+      // Get the next token of the macro.
+      LexUnexpandedToken(Tok);
+    }
+  }
+
+
+  // Disable __VA_ARGS__ again.
+  Ident__VA_ARGS__->setIsPoisoned(true);
+
+  // Check that there is no paste (##) operator at the beginning or end of the
+  // replacement list.
+  unsigned NumTokens = MI->getNumTokens();
+  if (NumTokens != 0) {
+    if (MI->getReplacementToken(0).is(tok::hashhash)) {
+      Diag(MI->getReplacementToken(0), diag::err_paste_at_start);
+      ReleaseMacroInfo(MI);
+      return;
+    }
+    if (MI->getReplacementToken(NumTokens-1).is(tok::hashhash)) {
+      Diag(MI->getReplacementToken(NumTokens-1), diag::err_paste_at_end);
+      ReleaseMacroInfo(MI);
+      return;
+    }
+  }
+
+  MI->setDefinitionEndLoc(LastTok.getLocation());
+
+  // Finally, if this identifier already had a macro defined for it, verify that
+  // the macro bodies are identical and free the old definition.
+  if (MacroInfo *OtherMI = getMacroInfo(MacroNameTok.getIdentifierInfo())) {
+    // It is very common for system headers to have tons of macro redefinitions
+    // and for warnings to be disabled in system headers.  If this is the case,
+    // then don't bother calling MacroInfo::isIdenticalTo.
+    if (!getDiagnostics().getSuppressSystemWarnings() ||
+        !SourceMgr.isInSystemHeader(DefineTok.getLocation())) {
+      if (!OtherMI->isUsed() && OtherMI->isWarnIfUnused())
+        Diag(OtherMI->getDefinitionLoc(), diag::pp_macro_not_used);
+
+      // Macros must be identical.  This means all tokens and whitespace
+      // separation must be the same.  C99 6.10.3.2.
+      if (!OtherMI->isAllowRedefinitionsWithoutWarning() &&
+          !MI->isIdenticalTo(*OtherMI, *this)) {
+        Diag(MI->getDefinitionLoc(), diag::ext_pp_macro_redef)
+          << MacroNameTok.getIdentifierInfo();
+        Diag(OtherMI->getDefinitionLoc(), diag::note_previous_definition);
+      }
+    }
+    if (OtherMI->isWarnIfUnused())
+      WarnUnusedMacroLocs.erase(OtherMI->getDefinitionLoc());
+    ReleaseMacroInfo(OtherMI);
+  }
+
+  setMacroInfo(MacroNameTok.getIdentifierInfo(), MI);
+
+  assert(!MI->isUsed());
+  // If we need warning for not using the macro, add its location in the
+  // warn-because-unused-macro set. If it gets used it will be removed from set.
+  if (isInPrimaryFile() && // don't warn for include'd macros.
+      Diags->getDiagnosticLevel(diag::pp_macro_not_used,
+          MI->getDefinitionLoc()) != DiagnosticsEngine::Ignored) {
+    MI->setIsWarnIfUnused(true);
+    WarnUnusedMacroLocs.insert(MI->getDefinitionLoc());
+  }
+
+  // If the callbacks want to know, tell them about the macro definition.
+  if (Callbacks)
+    Callbacks->MacroDefined(MacroNameTok, MI);
+}
+
+/// HandleUndefDirective - Implements #undef.
+///
+void Preprocessor::HandleUndefDirective(Token &UndefTok) {
+  ++NumUndefined;
+
+  Token MacroNameTok;
+  ReadMacroName(MacroNameTok, 2);
+
+  // Error reading macro name?  If so, diagnostic already issued.
+  if (MacroNameTok.is(tok::eod))
+    return;
+
+  // Check to see if this is the last token on the #undef line.
+  CheckEndOfDirective("undef");
+
+  // Okay, we finally have a valid identifier to undef.
+  MacroInfo *MI = getMacroInfo(MacroNameTok.getIdentifierInfo());
+
+  // If the macro is not defined, this is a noop undef, just return.
+  if (MI == 0) return;
+
+  if (!MI->isUsed() && MI->isWarnIfUnused())
+    Diag(MI->getDefinitionLoc(), diag::pp_macro_not_used);
+
+  // If the callbacks want to know, tell them about the macro #undef.
+  if (Callbacks)
+    Callbacks->MacroUndefined(MacroNameTok, MI);
+
+  if (MI->isWarnIfUnused())
+    WarnUnusedMacroLocs.erase(MI->getDefinitionLoc());
+
+  // Free macro definition.
+  ReleaseMacroInfo(MI);
+  setMacroInfo(MacroNameTok.getIdentifierInfo(), 0);
+}
+
+
+//===----------------------------------------------------------------------===//
+// Preprocessor Conditional Directive Handling.
+//===----------------------------------------------------------------------===//
+
+/// HandleIfdefDirective - Implements the #ifdef/#ifndef directive.  isIfndef is
+/// true when this is a #ifndef directive.  ReadAnyTokensBeforeDirective is true
+/// if any tokens have been returned or pp-directives activated before this
+/// #ifndef has been lexed.
+///
+void Preprocessor::HandleIfdefDirective(Token &Result, bool isIfndef,
+                                        bool ReadAnyTokensBeforeDirective) {
+  ++NumIf;
+  Token DirectiveTok = Result;
+
+  Token MacroNameTok;
+  ReadMacroName(MacroNameTok);
+
+  // Error reading macro name?  If so, diagnostic already issued.
+  if (MacroNameTok.is(tok::eod)) {
+    // Skip code until we get to #endif.  This helps with recovery by not
+    // emitting an error when the #endif is reached.
+    SkipExcludedConditionalBlock(DirectiveTok.getLocation(),
+                                 /*Foundnonskip*/false, /*FoundElse*/false);
+    return;
+  }
+
+  // Check to see if this is the last token on the #if[n]def line.
+  CheckEndOfDirective(isIfndef ? "ifndef" : "ifdef");
+
+  IdentifierInfo *MII = MacroNameTok.getIdentifierInfo();
+  MacroInfo *MI = getMacroInfo(MII);
+
+  if (CurPPLexer->getConditionalStackDepth() == 0) {
+    // If the start of a top-level #ifdef and if the macro is not defined,
+    // inform MIOpt that this might be the start of a proper include guard.
+    // Otherwise it is some other form of unknown conditional which we can't
+    // handle.
+    if (!ReadAnyTokensBeforeDirective && MI == 0) {
+      assert(isIfndef && "#ifdef shouldn't reach here");
+      CurPPLexer->MIOpt.EnterTopLevelIFNDEF(MII);
+    } else
+      CurPPLexer->MIOpt.EnterTopLevelConditional();
+  }
+
+  // If there is a macro, process it.
+  if (MI)  // Mark it used.
+    markMacroAsUsed(MI);
+
+  if (Callbacks) {
+    if (isIfndef)
+      Callbacks->Ifndef(DirectiveTok.getLocation(), MacroNameTok);
+    else
+      Callbacks->Ifdef(DirectiveTok.getLocation(), MacroNameTok);
+  }
+
+  // Should we include the stuff contained by this directive?
+  if (!MI == isIfndef) {
+    // Yes, remember that we are inside a conditional, then lex the next token.
+    CurPPLexer->pushConditionalLevel(DirectiveTok.getLocation(),
+                                     /*wasskip*/false, /*foundnonskip*/true,
+                                     /*foundelse*/false);
+  } else {
+    // No, skip the contents of this block.
+    SkipExcludedConditionalBlock(DirectiveTok.getLocation(),
+                                 /*Foundnonskip*/false,
+                                 /*FoundElse*/false);
+  }
+}
+
+/// HandleIfDirective - Implements the #if directive.
+///
+void Preprocessor::HandleIfDirective(Token &IfToken,
+                                     bool ReadAnyTokensBeforeDirective) {
+  ++NumIf;
+
+  // Parse and evaluate the conditional expression.
+  IdentifierInfo *IfNDefMacro = 0;
+  const SourceLocation ConditionalBegin = CurPPLexer->getSourceLocation();
+  const bool ConditionalTrue = EvaluateDirectiveExpression(IfNDefMacro);
+  const SourceLocation ConditionalEnd = CurPPLexer->getSourceLocation();
+
+  // If this condition is equivalent to #ifndef X, and if this is the first
+  // directive seen, handle it for the multiple-include optimization.
+  if (CurPPLexer->getConditionalStackDepth() == 0) {
+    if (!ReadAnyTokensBeforeDirective && IfNDefMacro && ConditionalTrue)
+      CurPPLexer->MIOpt.EnterTopLevelIFNDEF(IfNDefMacro);
+    else
+      CurPPLexer->MIOpt.EnterTopLevelConditional();
+  }
+
+  if (Callbacks)
+    Callbacks->If(IfToken.getLocation(),
+                  SourceRange(ConditionalBegin, ConditionalEnd));
+
+  // Should we include the stuff contained by this directive?
+  if (ConditionalTrue) {
+    // Yes, remember that we are inside a conditional, then lex the next token.
+    CurPPLexer->pushConditionalLevel(IfToken.getLocation(), /*wasskip*/false,
+                                   /*foundnonskip*/true, /*foundelse*/false);
+  } else {
+    // No, skip the contents of this block.
+    SkipExcludedConditionalBlock(IfToken.getLocation(), /*Foundnonskip*/false,
+                                 /*FoundElse*/false);
+  }
+}
+
+/// HandleEndifDirective - Implements the #endif directive.
+///
+void Preprocessor::HandleEndifDirective(Token &EndifToken) {
+  ++NumEndif;
+
+  // Check that this is the whole directive.
+  CheckEndOfDirective("endif");
+
+  PPConditionalInfo CondInfo;
+  if (CurPPLexer->popConditionalLevel(CondInfo)) {
+    // No conditionals on the stack: this is an #endif without an #if.
+    Diag(EndifToken, diag::err_pp_endif_without_if);
+    return;
+  }
+
+  // If this the end of a top-level #endif, inform MIOpt.
+  if (CurPPLexer->getConditionalStackDepth() == 0)
+    CurPPLexer->MIOpt.ExitTopLevelConditional();
+
+  assert(!CondInfo.WasSkipping && !CurPPLexer->LexingRawMode &&
+         "This code should only be reachable in the non-skipping case!");
+
+  if (Callbacks)
+    Callbacks->Endif(EndifToken.getLocation(), CondInfo.IfLoc);
+}
+
+/// HandleElseDirective - Implements the #else directive.
+///
+void Preprocessor::HandleElseDirective(Token &Result) {
+  ++NumElse;
+
+  // #else directive in a non-skipping conditional... start skipping.
+  CheckEndOfDirective("else");
+
+  PPConditionalInfo CI;
+  if (CurPPLexer->popConditionalLevel(CI)) {
+    Diag(Result, diag::pp_err_else_without_if);
+    return;
+  }
+
+  // If this is a top-level #else, inform the MIOpt.
+  if (CurPPLexer->getConditionalStackDepth() == 0)
+    CurPPLexer->MIOpt.EnterTopLevelConditional();
+
+  // If this is a #else with a #else before it, report the error.
+  if (CI.FoundElse) Diag(Result, diag::pp_err_else_after_else);
+
+  if (Callbacks)
+    Callbacks->Else(Result.getLocation(), CI.IfLoc);
+
+  // Finally, skip the rest of the contents of this block.
+  SkipExcludedConditionalBlock(CI.IfLoc, /*Foundnonskip*/true,
+                               /*FoundElse*/true, Result.getLocation());
+}
+
+/// HandleElifDirective - Implements the #elif directive.
+///
+void Preprocessor::HandleElifDirective(Token &ElifToken) {
+  ++NumElse;
+
+  // #elif directive in a non-skipping conditional... start skipping.
+  // We don't care what the condition is, because we will always skip it (since
+  // the block immediately before it was included).
+  const SourceLocation ConditionalBegin = CurPPLexer->getSourceLocation();
+  DiscardUntilEndOfDirective();
+  const SourceLocation ConditionalEnd = CurPPLexer->getSourceLocation();
+
+  PPConditionalInfo CI;
+  if (CurPPLexer->popConditionalLevel(CI)) {
+    Diag(ElifToken, diag::pp_err_elif_without_if);
+    return;
+  }
+
+  // If this is a top-level #elif, inform the MIOpt.
+  if (CurPPLexer->getConditionalStackDepth() == 0)
+    CurPPLexer->MIOpt.EnterTopLevelConditional();
+
+  // If this is a #elif with a #else before it, report the error.
+  if (CI.FoundElse) Diag(ElifToken, diag::pp_err_elif_after_else);
+  
+  if (Callbacks)
+    Callbacks->Elif(ElifToken.getLocation(),
+                    SourceRange(ConditionalBegin, ConditionalEnd), CI.IfLoc);
+
+  // Finally, skip the rest of the contents of this block.
+  SkipExcludedConditionalBlock(CI.IfLoc, /*Foundnonskip*/true,
+                               /*FoundElse*/CI.FoundElse,
+                               ElifToken.getLocation());
+}
diff --git a/clang/lib/Lex/PPExpressions.cpp b/clang/lib/Lex/PPExpressions.cpp
new file mode 100644
index 0000000..7cac63e
--- /dev/null
+++ b/clang/lib/Lex/PPExpressions.cpp
@@ -0,0 +1,786 @@
+//===--- PPExpressions.cpp - Preprocessor Expression Evaluation -----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the Preprocessor::EvaluateDirectiveExpression method,
+// which parses and evaluates integer constant expressions for #if directives.
+//
+//===----------------------------------------------------------------------===//
+//
+// FIXME: implement testing for #assert's.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/MacroInfo.h"
+#include "clang/Lex/LiteralSupport.h"
+#include "clang/Lex/CodeCompletionHandler.h"
+#include "clang/Basic/TargetInfo.h"
+#include "clang/Lex/LexDiagnostic.h"
+#include "llvm/ADT/APSInt.h"
+#include "llvm/Support/ErrorHandling.h"
+using namespace clang;
+
+namespace {
+
+/// PPValue - Represents the value of a subexpression of a preprocessor
+/// conditional and the source range covered by it.
+class PPValue {
+  SourceRange Range;
+public:
+  llvm::APSInt Val;
+
+  // Default ctor - Construct an 'invalid' PPValue.
+  PPValue(unsigned BitWidth) : Val(BitWidth) {}
+
+  unsigned getBitWidth() const { return Val.getBitWidth(); }
+  bool isUnsigned() const { return Val.isUnsigned(); }
+
+  const SourceRange &getRange() const { return Range; }
+
+  void setRange(SourceLocation L) { Range.setBegin(L); Range.setEnd(L); }
+  void setRange(SourceLocation B, SourceLocation E) {
+    Range.setBegin(B); Range.setEnd(E);
+  }
+  void setBegin(SourceLocation L) { Range.setBegin(L); }
+  void setEnd(SourceLocation L) { Range.setEnd(L); }
+};
+
+}
+
+static bool EvaluateDirectiveSubExpr(PPValue &LHS, unsigned MinPrec,
+                                     Token &PeekTok, bool ValueLive,
+                                     Preprocessor &PP);
+
+/// DefinedTracker - This struct is used while parsing expressions to keep track
+/// of whether !defined(X) has been seen.
+///
+/// With this simple scheme, we handle the basic forms:
+///    !defined(X)   and !defined X
+/// but we also trivially handle (silly) stuff like:
+///    !!!defined(X) and +!defined(X) and !+!+!defined(X) and !(defined(X)).
+struct DefinedTracker {
+  /// Each time a Value is evaluated, it returns information about whether the
+  /// parsed value is of the form defined(X), !defined(X) or is something else.
+  enum TrackerState {
+    DefinedMacro,        // defined(X)
+    NotDefinedMacro,     // !defined(X)
+    Unknown              // Something else.
+  } State;
+  /// TheMacro - When the state is DefinedMacro or NotDefinedMacro, this
+  /// indicates the macro that was checked.
+  IdentifierInfo *TheMacro;
+};
+
+/// EvaluateDefined - Process a 'defined(sym)' expression.
+static bool EvaluateDefined(PPValue &Result, Token &PeekTok, DefinedTracker &DT,
+                            bool ValueLive, Preprocessor &PP) {
+  IdentifierInfo *II;
+  Result.setBegin(PeekTok.getLocation());
+
+  // Get the next token, don't expand it.
+  PP.LexUnexpandedNonComment(PeekTok);
+
+  // Two options, it can either be a pp-identifier or a (.
+  SourceLocation LParenLoc;
+  if (PeekTok.is(tok::l_paren)) {
+    // Found a paren, remember we saw it and skip it.
+    LParenLoc = PeekTok.getLocation();
+    PP.LexUnexpandedNonComment(PeekTok);
+  }
+
+  if (PeekTok.is(tok::code_completion)) {
+    if (PP.getCodeCompletionHandler())
+      PP.getCodeCompletionHandler()->CodeCompleteMacroName(false);
+    PP.setCodeCompletionReached();
+    PP.LexUnexpandedNonComment(PeekTok);
+  }
+  
+  // If we don't have a pp-identifier now, this is an error.
+  if ((II = PeekTok.getIdentifierInfo()) == 0) {
+    PP.Diag(PeekTok, diag::err_pp_defined_requires_identifier);
+    return true;
+  }
+
+  // Otherwise, we got an identifier, is it defined to something?
+  Result.Val = II->hasMacroDefinition();
+  Result.Val.setIsUnsigned(false);  // Result is signed intmax_t.
+
+  // If there is a macro, mark it used.
+  if (Result.Val != 0 && ValueLive) {
+    MacroInfo *Macro = PP.getMacroInfo(II);
+    PP.markMacroAsUsed(Macro);
+  }
+
+  // Invoke the 'defined' callback.
+  if (PPCallbacks *Callbacks = PP.getPPCallbacks())
+    Callbacks->Defined(PeekTok);
+
+  // If we are in parens, ensure we have a trailing ).
+  if (LParenLoc.isValid()) {
+    // Consume identifier.
+    Result.setEnd(PeekTok.getLocation());
+    PP.LexUnexpandedNonComment(PeekTok);
+
+    if (PeekTok.isNot(tok::r_paren)) {
+      PP.Diag(PeekTok.getLocation(), diag::err_pp_missing_rparen) << "defined";
+      PP.Diag(LParenLoc, diag::note_matching) << "(";
+      return true;
+    }
+    // Consume the ).
+    Result.setEnd(PeekTok.getLocation());
+    PP.LexNonComment(PeekTok);
+  } else {
+    // Consume identifier.
+    Result.setEnd(PeekTok.getLocation());
+    PP.LexNonComment(PeekTok);
+  }
+
+  // Success, remember that we saw defined(X).
+  DT.State = DefinedTracker::DefinedMacro;
+  DT.TheMacro = II;
+  return false;
+}
+
+/// EvaluateValue - Evaluate the token PeekTok (and any others needed) and
+/// return the computed value in Result.  Return true if there was an error
+/// parsing.  This function also returns information about the form of the
+/// expression in DT.  See above for information on what DT means.
+///
+/// If ValueLive is false, then this value is being evaluated in a context where
+/// the result is not used.  As such, avoid diagnostics that relate to
+/// evaluation.
+static bool EvaluateValue(PPValue &Result, Token &PeekTok, DefinedTracker &DT,
+                          bool ValueLive, Preprocessor &PP) {
+  DT.State = DefinedTracker::Unknown;
+
+  if (PeekTok.is(tok::code_completion)) {
+    if (PP.getCodeCompletionHandler())
+      PP.getCodeCompletionHandler()->CodeCompletePreprocessorExpression();
+    PP.setCodeCompletionReached();
+    PP.LexNonComment(PeekTok);
+  }
+      
+  // If this token's spelling is a pp-identifier, check to see if it is
+  // 'defined' or if it is a macro.  Note that we check here because many
+  // keywords are pp-identifiers, so we can't check the kind.
+  if (IdentifierInfo *II = PeekTok.getIdentifierInfo()) {
+    // Handle "defined X" and "defined(X)".
+    if (II->isStr("defined"))
+      return(EvaluateDefined(Result, PeekTok, DT, ValueLive, PP));
+    
+    // If this identifier isn't 'defined' or one of the special
+    // preprocessor keywords and it wasn't macro expanded, it turns
+    // into a simple 0, unless it is the C++ keyword "true", in which case it
+    // turns into "1".
+    if (ValueLive)
+      PP.Diag(PeekTok, diag::warn_pp_undef_identifier) << II;
+    Result.Val = II->getTokenID() == tok::kw_true;
+    Result.Val.setIsUnsigned(false);  // "0" is signed intmax_t 0.
+    Result.setRange(PeekTok.getLocation());
+    PP.LexNonComment(PeekTok);
+    return false;
+  }
+
+  switch (PeekTok.getKind()) {
+  default:  // Non-value token.
+    PP.Diag(PeekTok, diag::err_pp_expr_bad_token_start_expr);
+    return true;
+  case tok::eod:
+  case tok::r_paren:
+    // If there is no expression, report and exit.
+    PP.Diag(PeekTok, diag::err_pp_expected_value_in_expr);
+    return true;
+  case tok::numeric_constant: {
+    SmallString<64> IntegerBuffer;
+    bool NumberInvalid = false;
+    StringRef Spelling = PP.getSpelling(PeekTok, IntegerBuffer, 
+                                              &NumberInvalid);
+    if (NumberInvalid)
+      return true; // a diagnostic was already reported
+
+    NumericLiteralParser Literal(Spelling.begin(), Spelling.end(),
+                                 PeekTok.getLocation(), PP);
+    if (Literal.hadError)
+      return true; // a diagnostic was already reported.
+
+    if (Literal.isFloatingLiteral() || Literal.isImaginary) {
+      PP.Diag(PeekTok, diag::err_pp_illegal_floating_literal);
+      return true;
+    }
+    assert(Literal.isIntegerLiteral() && "Unknown ppnumber");
+
+    // Complain about, and drop, any ud-suffix.
+    if (Literal.hasUDSuffix())
+      PP.Diag(PeekTok, diag::err_pp_invalid_udl) << /*integer*/1;
+
+    // long long is a C99 feature.
+    if (!PP.getLangOpts().C99 && Literal.isLongLong)
+      PP.Diag(PeekTok, PP.getLangOpts().CPlusPlus0x ?
+              diag::warn_cxx98_compat_longlong : diag::ext_longlong);
+
+    // Parse the integer literal into Result.
+    if (Literal.GetIntegerValue(Result.Val)) {
+      // Overflow parsing integer literal.
+      if (ValueLive) PP.Diag(PeekTok, diag::warn_integer_too_large);
+      Result.Val.setIsUnsigned(true);
+    } else {
+      // Set the signedness of the result to match whether there was a U suffix
+      // or not.
+      Result.Val.setIsUnsigned(Literal.isUnsigned);
+
+      // Detect overflow based on whether the value is signed.  If signed
+      // and if the value is too large, emit a warning "integer constant is so
+      // large that it is unsigned" e.g. on 12345678901234567890 where intmax_t
+      // is 64-bits.
+      if (!Literal.isUnsigned && Result.Val.isNegative()) {
+        // Don't warn for a hex literal: 0x8000..0 shouldn't warn.
+        if (ValueLive && Literal.getRadix() != 16)
+          PP.Diag(PeekTok, diag::warn_integer_too_large_for_signed);
+        Result.Val.setIsUnsigned(true);
+      }
+    }
+
+    // Consume the token.
+    Result.setRange(PeekTok.getLocation());
+    PP.LexNonComment(PeekTok);
+    return false;
+  }
+  case tok::char_constant:          // 'x'
+  case tok::wide_char_constant: {   // L'x'
+  case tok::utf16_char_constant:    // u'x'
+  case tok::utf32_char_constant:    // U'x'
+    // Complain about, and drop, any ud-suffix.
+    if (PeekTok.hasUDSuffix())
+      PP.Diag(PeekTok, diag::err_pp_invalid_udl) << /*character*/0;
+
+    SmallString<32> CharBuffer;
+    bool CharInvalid = false;
+    StringRef ThisTok = PP.getSpelling(PeekTok, CharBuffer, &CharInvalid);
+    if (CharInvalid)
+      return true;
+
+    CharLiteralParser Literal(ThisTok.begin(), ThisTok.end(),
+                              PeekTok.getLocation(), PP, PeekTok.getKind());
+    if (Literal.hadError())
+      return true;  // A diagnostic was already emitted.
+
+    // Character literals are always int or wchar_t, expand to intmax_t.
+    const TargetInfo &TI = PP.getTargetInfo();
+    unsigned NumBits;
+    if (Literal.isMultiChar())
+      NumBits = TI.getIntWidth();
+    else if (Literal.isWide())
+      NumBits = TI.getWCharWidth();
+    else if (Literal.isUTF16())
+      NumBits = TI.getChar16Width();
+    else if (Literal.isUTF32())
+      NumBits = TI.getChar32Width();
+    else
+      NumBits = TI.getCharWidth();
+
+    // Set the width.
+    llvm::APSInt Val(NumBits);
+    // Set the value.
+    Val = Literal.getValue();
+    // Set the signedness. UTF-16 and UTF-32 are always unsigned
+    if (!Literal.isUTF16() && !Literal.isUTF32())
+      Val.setIsUnsigned(!PP.getLangOpts().CharIsSigned);
+
+    if (Result.Val.getBitWidth() > Val.getBitWidth()) {
+      Result.Val = Val.extend(Result.Val.getBitWidth());
+    } else {
+      assert(Result.Val.getBitWidth() == Val.getBitWidth() &&
+             "intmax_t smaller than char/wchar_t?");
+      Result.Val = Val;
+    }
+
+    // Consume the token.
+    Result.setRange(PeekTok.getLocation());
+    PP.LexNonComment(PeekTok);
+    return false;
+  }
+  case tok::l_paren: {
+    SourceLocation Start = PeekTok.getLocation();
+    PP.LexNonComment(PeekTok);  // Eat the (.
+    // Parse the value and if there are any binary operators involved, parse
+    // them.
+    if (EvaluateValue(Result, PeekTok, DT, ValueLive, PP)) return true;
+
+    // If this is a silly value like (X), which doesn't need parens, check for
+    // !(defined X).
+    if (PeekTok.is(tok::r_paren)) {
+      // Just use DT unmodified as our result.
+    } else {
+      // Otherwise, we have something like (x+y), and we consumed '(x'.
+      if (EvaluateDirectiveSubExpr(Result, 1, PeekTok, ValueLive, PP))
+        return true;
+
+      if (PeekTok.isNot(tok::r_paren)) {
+        PP.Diag(PeekTok.getLocation(), diag::err_pp_expected_rparen)
+          << Result.getRange();
+        PP.Diag(Start, diag::note_matching) << "(";
+        return true;
+      }
+      DT.State = DefinedTracker::Unknown;
+    }
+    Result.setRange(Start, PeekTok.getLocation());
+    PP.LexNonComment(PeekTok);  // Eat the ).
+    return false;
+  }
+  case tok::plus: {
+    SourceLocation Start = PeekTok.getLocation();
+    // Unary plus doesn't modify the value.
+    PP.LexNonComment(PeekTok);
+    if (EvaluateValue(Result, PeekTok, DT, ValueLive, PP)) return true;
+    Result.setBegin(Start);
+    return false;
+  }
+  case tok::minus: {
+    SourceLocation Loc = PeekTok.getLocation();
+    PP.LexNonComment(PeekTok);
+    if (EvaluateValue(Result, PeekTok, DT, ValueLive, PP)) return true;
+    Result.setBegin(Loc);
+
+    // C99 6.5.3.3p3: The sign of the result matches the sign of the operand.
+    Result.Val = -Result.Val;
+
+    // -MININT is the only thing that overflows.  Unsigned never overflows.
+    bool Overflow = !Result.isUnsigned() && Result.Val.isMinSignedValue();
+
+    // If this operator is live and overflowed, report the issue.
+    if (Overflow && ValueLive)
+      PP.Diag(Loc, diag::warn_pp_expr_overflow) << Result.getRange();
+
+    DT.State = DefinedTracker::Unknown;
+    return false;
+  }
+
+  case tok::tilde: {
+    SourceLocation Start = PeekTok.getLocation();
+    PP.LexNonComment(PeekTok);
+    if (EvaluateValue(Result, PeekTok, DT, ValueLive, PP)) return true;
+    Result.setBegin(Start);
+
+    // C99 6.5.3.3p4: The sign of the result matches the sign of the operand.
+    Result.Val = ~Result.Val;
+    DT.State = DefinedTracker::Unknown;
+    return false;
+  }
+
+  case tok::exclaim: {
+    SourceLocation Start = PeekTok.getLocation();
+    PP.LexNonComment(PeekTok);
+    if (EvaluateValue(Result, PeekTok, DT, ValueLive, PP)) return true;
+    Result.setBegin(Start);
+    Result.Val = !Result.Val;
+    // C99 6.5.3.3p5: The sign of the result is 'int', aka it is signed.
+    Result.Val.setIsUnsigned(false);
+
+    if (DT.State == DefinedTracker::DefinedMacro)
+      DT.State = DefinedTracker::NotDefinedMacro;
+    else if (DT.State == DefinedTracker::NotDefinedMacro)
+      DT.State = DefinedTracker::DefinedMacro;
+    return false;
+  }
+
+  // FIXME: Handle #assert
+  }
+}
+
+
+
+/// getPrecedence - Return the precedence of the specified binary operator
+/// token.  This returns:
+///   ~0 - Invalid token.
+///   14 -> 3 - various operators.
+///    0 - 'eod' or ')'
+static unsigned getPrecedence(tok::TokenKind Kind) {
+  switch (Kind) {
+  default: return ~0U;
+  case tok::percent:
+  case tok::slash:
+  case tok::star:                 return 14;
+  case tok::plus:
+  case tok::minus:                return 13;
+  case tok::lessless:
+  case tok::greatergreater:       return 12;
+  case tok::lessequal:
+  case tok::less:
+  case tok::greaterequal:
+  case tok::greater:              return 11;
+  case tok::exclaimequal:
+  case tok::equalequal:           return 10;
+  case tok::amp:                  return 9;
+  case tok::caret:                return 8;
+  case tok::pipe:                 return 7;
+  case tok::ampamp:               return 6;
+  case tok::pipepipe:             return 5;
+  case tok::question:             return 4;
+  case tok::comma:                return 3;
+  case tok::colon:                return 2;
+  case tok::r_paren:              return 0;// Lowest priority, end of expr.
+  case tok::eod:                  return 0;// Lowest priority, end of directive.
+  }
+}
+
+
+/// EvaluateDirectiveSubExpr - Evaluate the subexpression whose first token is
+/// PeekTok, and whose precedence is PeekPrec.  This returns the result in LHS.
+///
+/// If ValueLive is false, then this value is being evaluated in a context where
+/// the result is not used.  As such, avoid diagnostics that relate to
+/// evaluation, such as division by zero warnings.
+static bool EvaluateDirectiveSubExpr(PPValue &LHS, unsigned MinPrec,
+                                     Token &PeekTok, bool ValueLive,
+                                     Preprocessor &PP) {
+  unsigned PeekPrec = getPrecedence(PeekTok.getKind());
+  // If this token isn't valid, report the error.
+  if (PeekPrec == ~0U) {
+    PP.Diag(PeekTok.getLocation(), diag::err_pp_expr_bad_token_binop)
+      << LHS.getRange();
+    return true;
+  }
+
+  while (1) {
+    // If this token has a lower precedence than we are allowed to parse, return
+    // it so that higher levels of the recursion can parse it.
+    if (PeekPrec < MinPrec)
+      return false;
+
+    tok::TokenKind Operator = PeekTok.getKind();
+
+    // If this is a short-circuiting operator, see if the RHS of the operator is
+    // dead.  Note that this cannot just clobber ValueLive.  Consider
+    // "0 && 1 ? 4 : 1 / 0", which is parsed as "(0 && 1) ? 4 : (1 / 0)".  In
+    // this example, the RHS of the && being dead does not make the rest of the
+    // expr dead.
+    bool RHSIsLive;
+    if (Operator == tok::ampamp && LHS.Val == 0)
+      RHSIsLive = false;   // RHS of "0 && x" is dead.
+    else if (Operator == tok::pipepipe && LHS.Val != 0)
+      RHSIsLive = false;   // RHS of "1 || x" is dead.
+    else if (Operator == tok::question && LHS.Val == 0)
+      RHSIsLive = false;   // RHS (x) of "0 ? x : y" is dead.
+    else
+      RHSIsLive = ValueLive;
+
+    // Consume the operator, remembering the operator's location for reporting.
+    SourceLocation OpLoc = PeekTok.getLocation();
+    PP.LexNonComment(PeekTok);
+
+    PPValue RHS(LHS.getBitWidth());
+    // Parse the RHS of the operator.
+    DefinedTracker DT;
+    if (EvaluateValue(RHS, PeekTok, DT, RHSIsLive, PP)) return true;
+
+    // Remember the precedence of this operator and get the precedence of the
+    // operator immediately to the right of the RHS.
+    unsigned ThisPrec = PeekPrec;
+    PeekPrec = getPrecedence(PeekTok.getKind());
+
+    // If this token isn't valid, report the error.
+    if (PeekPrec == ~0U) {
+      PP.Diag(PeekTok.getLocation(), diag::err_pp_expr_bad_token_binop)
+        << RHS.getRange();
+      return true;
+    }
+
+    // Decide whether to include the next binop in this subexpression.  For
+    // example, when parsing x+y*z and looking at '*', we want to recursively
+    // handle y*z as a single subexpression.  We do this because the precedence
+    // of * is higher than that of +.  The only strange case we have to handle
+    // here is for the ?: operator, where the precedence is actually lower than
+    // the LHS of the '?'.  The grammar rule is:
+    //
+    // conditional-expression ::=
+    //    logical-OR-expression ? expression : conditional-expression
+    // where 'expression' is actually comma-expression.
+    unsigned RHSPrec;
+    if (Operator == tok::question)
+      // The RHS of "?" should be maximally consumed as an expression.
+      RHSPrec = getPrecedence(tok::comma);
+    else  // All others should munch while higher precedence.
+      RHSPrec = ThisPrec+1;
+
+    if (PeekPrec >= RHSPrec) {
+      if (EvaluateDirectiveSubExpr(RHS, RHSPrec, PeekTok, RHSIsLive, PP))
+        return true;
+      PeekPrec = getPrecedence(PeekTok.getKind());
+    }
+    assert(PeekPrec <= ThisPrec && "Recursion didn't work!");
+
+    // Usual arithmetic conversions (C99 6.3.1.8p1): result is unsigned if
+    // either operand is unsigned.
+    llvm::APSInt Res(LHS.getBitWidth());
+    switch (Operator) {
+    case tok::question:       // No UAC for x and y in "x ? y : z".
+    case tok::lessless:       // Shift amount doesn't UAC with shift value.
+    case tok::greatergreater: // Shift amount doesn't UAC with shift value.
+    case tok::comma:          // Comma operands are not subject to UACs.
+    case tok::pipepipe:       // Logical || does not do UACs.
+    case tok::ampamp:         // Logical && does not do UACs.
+      break;                  // No UAC
+    default:
+      Res.setIsUnsigned(LHS.isUnsigned()|RHS.isUnsigned());
+      // If this just promoted something from signed to unsigned, and if the
+      // value was negative, warn about it.
+      if (ValueLive && Res.isUnsigned()) {
+        if (!LHS.isUnsigned() && LHS.Val.isNegative())
+          PP.Diag(OpLoc, diag::warn_pp_convert_lhs_to_positive)
+            << LHS.Val.toString(10, true) + " to " +
+               LHS.Val.toString(10, false)
+            << LHS.getRange() << RHS.getRange();
+        if (!RHS.isUnsigned() && RHS.Val.isNegative())
+          PP.Diag(OpLoc, diag::warn_pp_convert_rhs_to_positive)
+            << RHS.Val.toString(10, true) + " to " +
+               RHS.Val.toString(10, false)
+            << LHS.getRange() << RHS.getRange();
+      }
+      LHS.Val.setIsUnsigned(Res.isUnsigned());
+      RHS.Val.setIsUnsigned(Res.isUnsigned());
+    }
+
+    bool Overflow = false;
+    switch (Operator) {
+    default: llvm_unreachable("Unknown operator token!");
+    case tok::percent:
+      if (RHS.Val != 0)
+        Res = LHS.Val % RHS.Val;
+      else if (ValueLive) {
+        PP.Diag(OpLoc, diag::err_pp_remainder_by_zero)
+          << LHS.getRange() << RHS.getRange();
+        return true;
+      }
+      break;
+    case tok::slash:
+      if (RHS.Val != 0) {
+        if (LHS.Val.isSigned())
+          Res = llvm::APSInt(LHS.Val.sdiv_ov(RHS.Val, Overflow), false);
+        else
+          Res = LHS.Val / RHS.Val;
+      } else if (ValueLive) {
+        PP.Diag(OpLoc, diag::err_pp_division_by_zero)
+          << LHS.getRange() << RHS.getRange();
+        return true;
+      }
+      break;
+
+    case tok::star:
+      if (Res.isSigned())
+        Res = llvm::APSInt(LHS.Val.smul_ov(RHS.Val, Overflow), false);
+      else
+        Res = LHS.Val * RHS.Val;
+      break;
+    case tok::lessless: {
+      // Determine whether overflow is about to happen.
+      unsigned ShAmt = static_cast<unsigned>(RHS.Val.getLimitedValue());
+      if (LHS.isUnsigned()) {
+        Overflow = ShAmt >= LHS.Val.getBitWidth();
+        if (Overflow)
+          ShAmt = LHS.Val.getBitWidth()-1;
+        Res = LHS.Val << ShAmt;
+      } else {
+        Res = llvm::APSInt(LHS.Val.sshl_ov(ShAmt, Overflow), false);
+      }
+      break;
+    }
+    case tok::greatergreater: {
+      // Determine whether overflow is about to happen.
+      unsigned ShAmt = static_cast<unsigned>(RHS.Val.getLimitedValue());
+      if (ShAmt >= LHS.getBitWidth())
+        Overflow = true, ShAmt = LHS.getBitWidth()-1;
+      Res = LHS.Val >> ShAmt;
+      break;
+    }
+    case tok::plus:
+      if (LHS.isUnsigned())
+        Res = LHS.Val + RHS.Val;
+      else
+        Res = llvm::APSInt(LHS.Val.sadd_ov(RHS.Val, Overflow), false);
+      break;
+    case tok::minus:
+      if (LHS.isUnsigned())
+        Res = LHS.Val - RHS.Val;
+      else
+        Res = llvm::APSInt(LHS.Val.ssub_ov(RHS.Val, Overflow), false);
+      break;
+    case tok::lessequal:
+      Res = LHS.Val <= RHS.Val;
+      Res.setIsUnsigned(false);  // C99 6.5.8p6, result is always int (signed)
+      break;
+    case tok::less:
+      Res = LHS.Val < RHS.Val;
+      Res.setIsUnsigned(false);  // C99 6.5.8p6, result is always int (signed)
+      break;
+    case tok::greaterequal:
+      Res = LHS.Val >= RHS.Val;
+      Res.setIsUnsigned(false);  // C99 6.5.8p6, result is always int (signed)
+      break;
+    case tok::greater:
+      Res = LHS.Val > RHS.Val;
+      Res.setIsUnsigned(false);  // C99 6.5.8p6, result is always int (signed)
+      break;
+    case tok::exclaimequal:
+      Res = LHS.Val != RHS.Val;
+      Res.setIsUnsigned(false);  // C99 6.5.9p3, result is always int (signed)
+      break;
+    case tok::equalequal:
+      Res = LHS.Val == RHS.Val;
+      Res.setIsUnsigned(false);  // C99 6.5.9p3, result is always int (signed)
+      break;
+    case tok::amp:
+      Res = LHS.Val & RHS.Val;
+      break;
+    case tok::caret:
+      Res = LHS.Val ^ RHS.Val;
+      break;
+    case tok::pipe:
+      Res = LHS.Val | RHS.Val;
+      break;
+    case tok::ampamp:
+      Res = (LHS.Val != 0 && RHS.Val != 0);
+      Res.setIsUnsigned(false);  // C99 6.5.13p3, result is always int (signed)
+      break;
+    case tok::pipepipe:
+      Res = (LHS.Val != 0 || RHS.Val != 0);
+      Res.setIsUnsigned(false);  // C99 6.5.14p3, result is always int (signed)
+      break;
+    case tok::comma:
+      // Comma is invalid in pp expressions in c89/c++ mode, but is valid in C99
+      // if not being evaluated.
+      if (!PP.getLangOpts().C99 || ValueLive)
+        PP.Diag(OpLoc, diag::ext_pp_comma_expr)
+          << LHS.getRange() << RHS.getRange();
+      Res = RHS.Val; // LHS = LHS,RHS -> RHS.
+      break;
+    case tok::question: {
+      // Parse the : part of the expression.
+      if (PeekTok.isNot(tok::colon)) {
+        PP.Diag(PeekTok.getLocation(), diag::err_expected_colon)
+          << LHS.getRange(), RHS.getRange();
+        PP.Diag(OpLoc, diag::note_matching) << "?";
+        return true;
+      }
+      // Consume the :.
+      PP.LexNonComment(PeekTok);
+
+      // Evaluate the value after the :.
+      bool AfterColonLive = ValueLive && LHS.Val == 0;
+      PPValue AfterColonVal(LHS.getBitWidth());
+      DefinedTracker DT;
+      if (EvaluateValue(AfterColonVal, PeekTok, DT, AfterColonLive, PP))
+        return true;
+
+      // Parse anything after the : with the same precedence as ?.  We allow
+      // things of equal precedence because ?: is right associative.
+      if (EvaluateDirectiveSubExpr(AfterColonVal, ThisPrec,
+                                   PeekTok, AfterColonLive, PP))
+        return true;
+
+      // Now that we have the condition, the LHS and the RHS of the :, evaluate.
+      Res = LHS.Val != 0 ? RHS.Val : AfterColonVal.Val;
+      RHS.setEnd(AfterColonVal.getRange().getEnd());
+
+      // Usual arithmetic conversions (C99 6.3.1.8p1): result is unsigned if
+      // either operand is unsigned.
+      Res.setIsUnsigned(RHS.isUnsigned() | AfterColonVal.isUnsigned());
+
+      // Figure out the precedence of the token after the : part.
+      PeekPrec = getPrecedence(PeekTok.getKind());
+      break;
+    }
+    case tok::colon:
+      // Don't allow :'s to float around without being part of ?: exprs.
+      PP.Diag(OpLoc, diag::err_pp_colon_without_question)
+        << LHS.getRange() << RHS.getRange();
+      return true;
+    }
+
+    // If this operator is live and overflowed, report the issue.
+    if (Overflow && ValueLive)
+      PP.Diag(OpLoc, diag::warn_pp_expr_overflow)
+        << LHS.getRange() << RHS.getRange();
+
+    // Put the result back into 'LHS' for our next iteration.
+    LHS.Val = Res;
+    LHS.setEnd(RHS.getRange().getEnd());
+  }
+}
+
+/// EvaluateDirectiveExpression - Evaluate an integer constant expression that
+/// may occur after a #if or #elif directive.  If the expression is equivalent
+/// to "!defined(X)" return X in IfNDefMacro.
+bool Preprocessor::
+EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro) {
+  // Save the current state of 'DisableMacroExpansion' and reset it to false. If
+  // 'DisableMacroExpansion' is true, then we must be in a macro argument list
+  // in which case a directive is undefined behavior.  We want macros to be able
+  // to recursively expand in order to get more gcc-list behavior, so we force
+  // DisableMacroExpansion to false and restore it when we're done parsing the
+  // expression.
+  bool DisableMacroExpansionAtStartOfDirective = DisableMacroExpansion;
+  DisableMacroExpansion = false;
+  
+  // Peek ahead one token.
+  Token Tok;
+  LexNonComment(Tok);
+  
+  // C99 6.10.1p3 - All expressions are evaluated as intmax_t or uintmax_t.
+  unsigned BitWidth = getTargetInfo().getIntMaxTWidth();
+
+  PPValue ResVal(BitWidth);
+  DefinedTracker DT;
+  if (EvaluateValue(ResVal, Tok, DT, true, *this)) {
+    // Parse error, skip the rest of the macro line.
+    if (Tok.isNot(tok::eod))
+      DiscardUntilEndOfDirective();
+    
+    // Restore 'DisableMacroExpansion'.
+    DisableMacroExpansion = DisableMacroExpansionAtStartOfDirective;
+    return false;
+  }
+
+  // If we are at the end of the expression after just parsing a value, there
+  // must be no (unparenthesized) binary operators involved, so we can exit
+  // directly.
+  if (Tok.is(tok::eod)) {
+    // If the expression we parsed was of the form !defined(macro), return the
+    // macro in IfNDefMacro.
+    if (DT.State == DefinedTracker::NotDefinedMacro)
+      IfNDefMacro = DT.TheMacro;
+
+    // Restore 'DisableMacroExpansion'.
+    DisableMacroExpansion = DisableMacroExpansionAtStartOfDirective;
+    return ResVal.Val != 0;
+  }
+
+  // Otherwise, we must have a binary operator (e.g. "#if 1 < 2"), so parse the
+  // operator and the stuff after it.
+  if (EvaluateDirectiveSubExpr(ResVal, getPrecedence(tok::question),
+                               Tok, true, *this)) {
+    // Parse error, skip the rest of the macro line.
+    if (Tok.isNot(tok::eod))
+      DiscardUntilEndOfDirective();
+    
+    // Restore 'DisableMacroExpansion'.
+    DisableMacroExpansion = DisableMacroExpansionAtStartOfDirective;
+    return false;
+  }
+
+  // If we aren't at the tok::eod token, something bad happened, like an extra
+  // ')' token.
+  if (Tok.isNot(tok::eod)) {
+    Diag(Tok, diag::err_pp_expected_eol);
+    DiscardUntilEndOfDirective();
+  }
+
+  // Restore 'DisableMacroExpansion'.
+  DisableMacroExpansion = DisableMacroExpansionAtStartOfDirective;
+  return ResVal.Val != 0;
+}
diff --git a/clang/lib/Lex/PPLexerChange.cpp b/clang/lib/Lex/PPLexerChange.cpp
new file mode 100644
index 0000000..b6689df
--- /dev/null
+++ b/clang/lib/Lex/PPLexerChange.cpp
@@ -0,0 +1,494 @@
+//===--- PPLexerChange.cpp - Handle changing lexers in the preprocessor ---===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements pieces of the Preprocessor interface that manage the
+// current lexer stack.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/HeaderSearch.h"
+#include "clang/Lex/MacroInfo.h"
+#include "clang/Lex/LexDiagnostic.h"
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/SourceManager.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/PathV2.h"
+#include "llvm/ADT/StringSwitch.h"
+using namespace clang;
+
+PPCallbacks::~PPCallbacks() {}
+
+//===----------------------------------------------------------------------===//
+// Miscellaneous Methods.
+//===----------------------------------------------------------------------===//
+
+/// isInPrimaryFile - Return true if we're in the top-level file, not in a
+/// #include.  This looks through macro expansions and active _Pragma lexers.
+bool Preprocessor::isInPrimaryFile() const {
+  if (IsFileLexer())
+    return IncludeMacroStack.empty();
+
+  // If there are any stacked lexers, we're in a #include.
+  assert(IsFileLexer(IncludeMacroStack[0]) &&
+         "Top level include stack isn't our primary lexer?");
+  for (unsigned i = 1, e = IncludeMacroStack.size(); i != e; ++i)
+    if (IsFileLexer(IncludeMacroStack[i]))
+      return false;
+  return true;
+}
+
+/// getCurrentLexer - Return the current file lexer being lexed from.  Note
+/// that this ignores any potentially active macro expansions and _Pragma
+/// expansions going on at the time.
+PreprocessorLexer *Preprocessor::getCurrentFileLexer() const {
+  if (IsFileLexer())
+    return CurPPLexer;
+
+  // Look for a stacked lexer.
+  for (unsigned i = IncludeMacroStack.size(); i != 0; --i) {
+    const IncludeStackInfo& ISI = IncludeMacroStack[i-1];
+    if (IsFileLexer(ISI))
+      return ISI.ThePPLexer;
+  }
+  return 0;
+}
+
+
+//===----------------------------------------------------------------------===//
+// Methods for Entering and Callbacks for leaving various contexts
+//===----------------------------------------------------------------------===//
+
+/// EnterSourceFile - Add a source file to the top of the include stack and
+/// start lexing tokens from it instead of the current buffer.
+void Preprocessor::EnterSourceFile(FileID FID, const DirectoryLookup *CurDir,
+                                   SourceLocation Loc) {
+  assert(CurTokenLexer == 0 && "Cannot #include a file inside a macro!");
+  ++NumEnteredSourceFiles;
+
+  if (MaxIncludeStackDepth < IncludeMacroStack.size())
+    MaxIncludeStackDepth = IncludeMacroStack.size();
+
+  if (PTH) {
+    if (PTHLexer *PL = PTH->CreateLexer(FID)) {
+      EnterSourceFileWithPTH(PL, CurDir);
+      return;
+    }
+  }
+  
+  // Get the MemoryBuffer for this FID, if it fails, we fail.
+  bool Invalid = false;
+  const llvm::MemoryBuffer *InputFile = 
+    getSourceManager().getBuffer(FID, Loc, &Invalid);
+  if (Invalid) {
+    SourceLocation FileStart = SourceMgr.getLocForStartOfFile(FID);
+    Diag(Loc, diag::err_pp_error_opening_file)
+      << std::string(SourceMgr.getBufferName(FileStart)) << "";
+    return;
+  }
+
+  if (isCodeCompletionEnabled() &&
+      SourceMgr.getFileEntryForID(FID) == CodeCompletionFile) {
+    CodeCompletionFileLoc = SourceMgr.getLocForStartOfFile(FID);
+    CodeCompletionLoc =
+        CodeCompletionFileLoc.getLocWithOffset(CodeCompletionOffset);
+  }
+
+  EnterSourceFileWithLexer(new Lexer(FID, InputFile, *this), CurDir);
+  return;
+}
+
+/// EnterSourceFileWithLexer - Add a source file to the top of the include stack
+///  and start lexing tokens from it instead of the current buffer.
+void Preprocessor::EnterSourceFileWithLexer(Lexer *TheLexer,
+                                            const DirectoryLookup *CurDir) {
+
+  // Add the current lexer to the include stack.
+  if (CurPPLexer || CurTokenLexer)
+    PushIncludeMacroStack();
+
+  CurLexer.reset(TheLexer);
+  CurPPLexer = TheLexer;
+  CurDirLookup = CurDir;
+  if (CurLexerKind != CLK_LexAfterModuleImport)
+    CurLexerKind = CLK_Lexer;
+  
+  // Notify the client, if desired, that we are in a new source file.
+  if (Callbacks && !CurLexer->Is_PragmaLexer) {
+    SrcMgr::CharacteristicKind FileType =
+       SourceMgr.getFileCharacteristic(CurLexer->getFileLoc());
+
+    Callbacks->FileChanged(CurLexer->getFileLoc(),
+                           PPCallbacks::EnterFile, FileType);
+  }
+}
+
+/// EnterSourceFileWithPTH - Add a source file to the top of the include stack
+/// and start getting tokens from it using the PTH cache.
+void Preprocessor::EnterSourceFileWithPTH(PTHLexer *PL,
+                                          const DirectoryLookup *CurDir) {
+
+  if (CurPPLexer || CurTokenLexer)
+    PushIncludeMacroStack();
+
+  CurDirLookup = CurDir;
+  CurPTHLexer.reset(PL);
+  CurPPLexer = CurPTHLexer.get();
+  if (CurLexerKind != CLK_LexAfterModuleImport)
+    CurLexerKind = CLK_PTHLexer;
+  
+  // Notify the client, if desired, that we are in a new source file.
+  if (Callbacks) {
+    FileID FID = CurPPLexer->getFileID();
+    SourceLocation EnterLoc = SourceMgr.getLocForStartOfFile(FID);
+    SrcMgr::CharacteristicKind FileType =
+      SourceMgr.getFileCharacteristic(EnterLoc);
+    Callbacks->FileChanged(EnterLoc, PPCallbacks::EnterFile, FileType);
+  }
+}
+
+/// EnterMacro - Add a Macro to the top of the include stack and start lexing
+/// tokens from it instead of the current buffer.
+void Preprocessor::EnterMacro(Token &Tok, SourceLocation ILEnd,
+                              MacroArgs *Args) {
+  PushIncludeMacroStack();
+  CurDirLookup = 0;
+
+  if (NumCachedTokenLexers == 0) {
+    CurTokenLexer.reset(new TokenLexer(Tok, ILEnd, Args, *this));
+  } else {
+    CurTokenLexer.reset(TokenLexerCache[--NumCachedTokenLexers]);
+    CurTokenLexer->Init(Tok, ILEnd, Args);
+  }
+  if (CurLexerKind != CLK_LexAfterModuleImport)
+    CurLexerKind = CLK_TokenLexer;
+}
+
+/// EnterTokenStream - Add a "macro" context to the top of the include stack,
+/// which will cause the lexer to start returning the specified tokens.
+///
+/// If DisableMacroExpansion is true, tokens lexed from the token stream will
+/// not be subject to further macro expansion.  Otherwise, these tokens will
+/// be re-macro-expanded when/if expansion is enabled.
+///
+/// If OwnsTokens is false, this method assumes that the specified stream of
+/// tokens has a permanent owner somewhere, so they do not need to be copied.
+/// If it is true, it assumes the array of tokens is allocated with new[] and
+/// must be freed.
+///
+void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks,
+                                    bool DisableMacroExpansion,
+                                    bool OwnsTokens) {
+  // Save our current state.
+  PushIncludeMacroStack();
+  CurDirLookup = 0;
+
+  // Create a macro expander to expand from the specified token stream.
+  if (NumCachedTokenLexers == 0) {
+    CurTokenLexer.reset(new TokenLexer(Toks, NumToks, DisableMacroExpansion,
+                                       OwnsTokens, *this));
+  } else {
+    CurTokenLexer.reset(TokenLexerCache[--NumCachedTokenLexers]);
+    CurTokenLexer->Init(Toks, NumToks, DisableMacroExpansion, OwnsTokens);
+  }
+  if (CurLexerKind != CLK_LexAfterModuleImport)
+    CurLexerKind = CLK_TokenLexer;
+}
+
+/// \brief Compute the relative path that names the given file relative to
+/// the given directory.
+static void computeRelativePath(FileManager &FM, const DirectoryEntry *Dir,
+                                const FileEntry *File,
+                                SmallString<128> &Result) {
+  Result.clear();
+
+  StringRef FilePath = File->getDir()->getName();
+  StringRef Path = FilePath;
+  while (!Path.empty()) {
+    if (const DirectoryEntry *CurDir = FM.getDirectory(Path)) {
+      if (CurDir == Dir) {
+        Result = FilePath.substr(Path.size());
+        llvm::sys::path::append(Result, 
+                                llvm::sys::path::filename(File->getName()));
+        return;
+      }
+    }
+    
+    Path = llvm::sys::path::parent_path(Path);
+  }
+  
+  Result = File->getName();
+}
+
+/// HandleEndOfFile - This callback is invoked when the lexer hits the end of
+/// the current file.  This either returns the EOF token or pops a level off
+/// the include stack and keeps going.
+bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) {
+  assert(!CurTokenLexer &&
+         "Ending a file when currently in a macro!");
+
+  // See if this file had a controlling macro.
+  if (CurPPLexer) {  // Not ending a macro, ignore it.
+    if (const IdentifierInfo *ControllingMacro =
+          CurPPLexer->MIOpt.GetControllingMacroAtEndOfFile()) {
+      // Okay, this has a controlling macro, remember in HeaderFileInfo.
+      if (const FileEntry *FE =
+            SourceMgr.getFileEntryForID(CurPPLexer->getFileID()))
+        HeaderInfo.SetFileControllingMacro(FE, ControllingMacro);
+    }
+  }
+
+  // Complain about reaching a true EOF within arc_cf_code_audited.
+  // We don't want to complain about reaching the end of a macro
+  // instantiation or a _Pragma.
+  if (PragmaARCCFCodeAuditedLoc.isValid() &&
+      !isEndOfMacro && !(CurLexer && CurLexer->Is_PragmaLexer)) {
+    Diag(PragmaARCCFCodeAuditedLoc, diag::err_pp_eof_in_arc_cf_code_audited);
+
+    // Recover by leaving immediately.
+    PragmaARCCFCodeAuditedLoc = SourceLocation();
+  }
+
+  // If this is a #include'd file, pop it off the include stack and continue
+  // lexing the #includer file.
+  if (!IncludeMacroStack.empty()) {
+
+    // If we lexed the code-completion file, act as if we reached EOF.
+    if (isCodeCompletionEnabled() && CurPPLexer &&
+        SourceMgr.getLocForStartOfFile(CurPPLexer->getFileID()) ==
+            CodeCompletionFileLoc) {
+      if (CurLexer) {
+        Result.startToken();
+        CurLexer->FormTokenWithChars(Result, CurLexer->BufferEnd, tok::eof);
+        CurLexer.reset();
+      } else {
+        assert(CurPTHLexer && "Got EOF but no current lexer set!");
+        CurPTHLexer->getEOF(Result);
+        CurPTHLexer.reset();
+      }
+
+      CurPPLexer = 0;
+      return true;
+    }
+
+    if (!isEndOfMacro && CurPPLexer &&
+        SourceMgr.getIncludeLoc(CurPPLexer->getFileID()).isValid()) {
+      // Notify SourceManager to record the number of FileIDs that were created
+      // during lexing of the #include'd file.
+      unsigned NumFIDs =
+          SourceMgr.local_sloc_entry_size() -
+          CurPPLexer->getInitialNumSLocEntries() + 1/*#include'd file*/;
+      SourceMgr.setNumCreatedFIDsForFileID(CurPPLexer->getFileID(), NumFIDs);
+    }
+
+    FileID ExitedFID;
+    if (Callbacks && !isEndOfMacro && CurPPLexer)
+      ExitedFID = CurPPLexer->getFileID();
+    
+    // We're done with the #included file.
+    RemoveTopOfLexerStack();
+
+    // Notify the client, if desired, that we are in a new source file.
+    if (Callbacks && !isEndOfMacro && CurPPLexer) {
+      SrcMgr::CharacteristicKind FileType =
+        SourceMgr.getFileCharacteristic(CurPPLexer->getSourceLocation());
+      Callbacks->FileChanged(CurPPLexer->getSourceLocation(),
+                             PPCallbacks::ExitFile, FileType, ExitedFID);
+    }
+
+    // Client should lex another token.
+    return false;
+  }
+
+  // If the file ends with a newline, form the EOF token on the newline itself,
+  // rather than "on the line following it", which doesn't exist.  This makes
+  // diagnostics relating to the end of file include the last file that the user
+  // actually typed, which is goodness.
+  if (CurLexer) {
+    const char *EndPos = CurLexer->BufferEnd;
+    if (EndPos != CurLexer->BufferStart &&
+        (EndPos[-1] == '\n' || EndPos[-1] == '\r')) {
+      --EndPos;
+
+      // Handle \n\r and \r\n:
+      if (EndPos != CurLexer->BufferStart &&
+          (EndPos[-1] == '\n' || EndPos[-1] == '\r') &&
+          EndPos[-1] != EndPos[0])
+        --EndPos;
+    }
+
+    Result.startToken();
+    CurLexer->BufferPtr = EndPos;
+    CurLexer->FormTokenWithChars(Result, EndPos, tok::eof);
+
+    if (!isIncrementalProcessingEnabled())
+      // We're done with lexing.
+      CurLexer.reset();
+  } else {
+    assert(CurPTHLexer && "Got EOF but no current lexer set!");
+    CurPTHLexer->getEOF(Result);
+    CurPTHLexer.reset();
+  }
+  
+  if (!isIncrementalProcessingEnabled())
+    CurPPLexer = 0;
+
+  // This is the end of the top-level file. 'WarnUnusedMacroLocs' has collected
+  // all macro locations that we need to warn because they are not used.
+  for (WarnUnusedMacroLocsTy::iterator
+         I=WarnUnusedMacroLocs.begin(), E=WarnUnusedMacroLocs.end(); I!=E; ++I)
+    Diag(*I, diag::pp_macro_not_used);
+
+  // If we are building a module that has an umbrella header, make sure that
+  // each of the headers within the directory covered by the umbrella header
+  // was actually included by the umbrella header.
+  if (Module *Mod = getCurrentModule()) {
+    if (Mod->getUmbrellaHeader()) {
+      SourceLocation StartLoc
+        = SourceMgr.getLocForStartOfFile(SourceMgr.getMainFileID());
+
+      if (getDiagnostics().getDiagnosticLevel(
+            diag::warn_uncovered_module_header, 
+            StartLoc) != DiagnosticsEngine::Ignored) {
+        ModuleMap &ModMap = getHeaderSearchInfo().getModuleMap();
+        typedef llvm::sys::fs::recursive_directory_iterator
+          recursive_directory_iterator;
+        const DirectoryEntry *Dir = Mod->getUmbrellaDir();
+        llvm::error_code EC;
+        for (recursive_directory_iterator Entry(Dir->getName(), EC), End;
+             Entry != End && !EC; Entry.increment(EC)) {
+          using llvm::StringSwitch;
+          
+          // Check whether this entry has an extension typically associated with
+          // headers.
+          if (!StringSwitch<bool>(llvm::sys::path::extension(Entry->path()))
+                 .Cases(".h", ".H", ".hh", ".hpp", true)
+                 .Default(false))
+            continue;
+
+          if (const FileEntry *Header = getFileManager().getFile(Entry->path()))
+            if (!getSourceManager().hasFileInfo(Header)) {
+              if (!ModMap.isHeaderInUnavailableModule(Header)) {
+                // Find the relative path that would access this header.
+                SmallString<128> RelativePath;
+                computeRelativePath(FileMgr, Dir, Header, RelativePath);              
+                Diag(StartLoc, diag::warn_uncovered_module_header)
+                  << RelativePath;
+              }
+            }
+        }
+      }
+    }
+  }
+  
+  return true;
+}
+
+/// HandleEndOfTokenLexer - This callback is invoked when the current TokenLexer
+/// hits the end of its token stream.
+bool Preprocessor::HandleEndOfTokenLexer(Token &Result) {
+  assert(CurTokenLexer && !CurPPLexer &&
+         "Ending a macro when currently in a #include file!");
+
+  if (!MacroExpandingLexersStack.empty() &&
+      MacroExpandingLexersStack.back().first == CurTokenLexer.get())
+    removeCachedMacroExpandedTokensOfLastLexer();
+
+  // Delete or cache the now-dead macro expander.
+  if (NumCachedTokenLexers == TokenLexerCacheSize)
+    CurTokenLexer.reset();
+  else
+    TokenLexerCache[NumCachedTokenLexers++] = CurTokenLexer.take();
+
+  // Handle this like a #include file being popped off the stack.
+  return HandleEndOfFile(Result, true);
+}
+
+/// RemoveTopOfLexerStack - Pop the current lexer/macro exp off the top of the
+/// lexer stack.  This should only be used in situations where the current
+/// state of the top-of-stack lexer is unknown.
+void Preprocessor::RemoveTopOfLexerStack() {
+  assert(!IncludeMacroStack.empty() && "Ran out of stack entries to load");
+
+  if (CurTokenLexer) {
+    // Delete or cache the now-dead macro expander.
+    if (NumCachedTokenLexers == TokenLexerCacheSize)
+      CurTokenLexer.reset();
+    else
+      TokenLexerCache[NumCachedTokenLexers++] = CurTokenLexer.take();
+  }
+
+  PopIncludeMacroStack();
+}
+
+/// HandleMicrosoftCommentPaste - When the macro expander pastes together a
+/// comment (/##/) in microsoft mode, this method handles updating the current
+/// state, returning the token on the next source line.
+void Preprocessor::HandleMicrosoftCommentPaste(Token &Tok) {
+  assert(CurTokenLexer && !CurPPLexer &&
+         "Pasted comment can only be formed from macro");
+
+  // We handle this by scanning for the closest real lexer, switching it to
+  // raw mode and preprocessor mode.  This will cause it to return \n as an
+  // explicit EOD token.
+  PreprocessorLexer *FoundLexer = 0;
+  bool LexerWasInPPMode = false;
+  for (unsigned i = 0, e = IncludeMacroStack.size(); i != e; ++i) {
+    IncludeStackInfo &ISI = *(IncludeMacroStack.end()-i-1);
+    if (ISI.ThePPLexer == 0) continue;  // Scan for a real lexer.
+
+    // Once we find a real lexer, mark it as raw mode (disabling macro
+    // expansions) and preprocessor mode (return EOD).  We know that the lexer
+    // was *not* in raw mode before, because the macro that the comment came
+    // from was expanded.  However, it could have already been in preprocessor
+    // mode (#if COMMENT) in which case we have to return it to that mode and
+    // return EOD.
+    FoundLexer = ISI.ThePPLexer;
+    FoundLexer->LexingRawMode = true;
+    LexerWasInPPMode = FoundLexer->ParsingPreprocessorDirective;
+    FoundLexer->ParsingPreprocessorDirective = true;
+    break;
+  }
+
+  // Okay, we either found and switched over the lexer, or we didn't find a
+  // lexer.  In either case, finish off the macro the comment came from, getting
+  // the next token.
+  if (!HandleEndOfTokenLexer(Tok)) Lex(Tok);
+
+  // Discarding comments as long as we don't have EOF or EOD.  This 'comments
+  // out' the rest of the line, including any tokens that came from other macros
+  // that were active, as in:
+  //  #define submacro a COMMENT b
+  //    submacro c
+  // which should lex to 'a' only: 'b' and 'c' should be removed.
+  while (Tok.isNot(tok::eod) && Tok.isNot(tok::eof))
+    Lex(Tok);
+
+  // If we got an eod token, then we successfully found the end of the line.
+  if (Tok.is(tok::eod)) {
+    assert(FoundLexer && "Can't get end of line without an active lexer");
+    // Restore the lexer back to normal mode instead of raw mode.
+    FoundLexer->LexingRawMode = false;
+
+    // If the lexer was already in preprocessor mode, just return the EOD token
+    // to finish the preprocessor line.
+    if (LexerWasInPPMode) return;
+
+    // Otherwise, switch out of PP mode and return the next lexed token.
+    FoundLexer->ParsingPreprocessorDirective = false;
+    return Lex(Tok);
+  }
+
+  // If we got an EOF token, then we reached the end of the token stream but
+  // didn't find an explicit \n.  This can only happen if there was no lexer
+  // active (an active lexer would return EOD at EOF if there was no \n in
+  // preprocessor directive mode), so just return EOF as our token.
+  assert(!FoundLexer && "Lexer should return EOD before EOF in PP mode");
+}
diff --git a/clang/lib/Lex/PPMacroExpansion.cpp b/clang/lib/Lex/PPMacroExpansion.cpp
new file mode 100644
index 0000000..fe70585
--- /dev/null
+++ b/clang/lib/Lex/PPMacroExpansion.cpp
@@ -0,0 +1,1156 @@
+//===--- MacroExpansion.cpp - Top level Macro Expansion -------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the top level handling of macro expasion for the
+// preprocessor.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Lex/Preprocessor.h"
+#include "MacroArgs.h"
+#include "clang/Lex/MacroInfo.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/TargetInfo.h"
+#include "clang/Lex/LexDiagnostic.h"
+#include "clang/Lex/CodeCompletionHandler.h"
+#include "clang/Lex/ExternalPreprocessorSource.h"
+#include "clang/Lex/LiteralSupport.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Config/llvm-config.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/ErrorHandling.h"
+#include <cstdio>
+#include <ctime>
+using namespace clang;
+
+MacroInfo *Preprocessor::getInfoForMacro(IdentifierInfo *II) const {
+  assert(II->hasMacroDefinition() && "Identifier is not a macro!");
+  
+  llvm::DenseMap<IdentifierInfo*, MacroInfo*>::const_iterator Pos
+    = Macros.find(II);
+  if (Pos == Macros.end()) {
+    // Load this macro from the external source.
+    getExternalSource()->LoadMacroDefinition(II);
+    Pos = Macros.find(II);
+  }
+  assert(Pos != Macros.end() && "Identifier macro info is missing!");
+  return Pos->second;
+}
+
+/// setMacroInfo - Specify a macro for this identifier.
+///
+void Preprocessor::setMacroInfo(IdentifierInfo *II, MacroInfo *MI,
+                                bool LoadedFromAST) {
+  if (MI) {
+    Macros[II] = MI;
+    II->setHasMacroDefinition(true);
+    if (II->isFromAST() && !LoadedFromAST)
+      II->setChangedSinceDeserialization();
+  } else if (II->hasMacroDefinition()) {
+    Macros.erase(II);
+    II->setHasMacroDefinition(false);
+    if (II->isFromAST() && !LoadedFromAST)
+      II->setChangedSinceDeserialization();
+  }
+}
+
+/// RegisterBuiltinMacro - Register the specified identifier in the identifier
+/// table and mark it as a builtin macro to be expanded.
+static IdentifierInfo *RegisterBuiltinMacro(Preprocessor &PP, const char *Name){
+  // Get the identifier.
+  IdentifierInfo *Id = PP.getIdentifierInfo(Name);
+
+  // Mark it as being a macro that is builtin.
+  MacroInfo *MI = PP.AllocateMacroInfo(SourceLocation());
+  MI->setIsBuiltinMacro();
+  PP.setMacroInfo(Id, MI);
+  return Id;
+}
+
+
+/// RegisterBuiltinMacros - Register builtin macros, such as __LINE__ with the
+/// identifier table.
+void Preprocessor::RegisterBuiltinMacros() {
+  Ident__LINE__ = RegisterBuiltinMacro(*this, "__LINE__");
+  Ident__FILE__ = RegisterBuiltinMacro(*this, "__FILE__");
+  Ident__DATE__ = RegisterBuiltinMacro(*this, "__DATE__");
+  Ident__TIME__ = RegisterBuiltinMacro(*this, "__TIME__");
+  Ident__COUNTER__ = RegisterBuiltinMacro(*this, "__COUNTER__");
+  Ident_Pragma  = RegisterBuiltinMacro(*this, "_Pragma");
+
+  // GCC Extensions.
+  Ident__BASE_FILE__     = RegisterBuiltinMacro(*this, "__BASE_FILE__");
+  Ident__INCLUDE_LEVEL__ = RegisterBuiltinMacro(*this, "__INCLUDE_LEVEL__");
+  Ident__TIMESTAMP__     = RegisterBuiltinMacro(*this, "__TIMESTAMP__");
+
+  // Clang Extensions.
+  Ident__has_feature      = RegisterBuiltinMacro(*this, "__has_feature");
+  Ident__has_extension    = RegisterBuiltinMacro(*this, "__has_extension");
+  Ident__has_builtin      = RegisterBuiltinMacro(*this, "__has_builtin");
+  Ident__has_attribute    = RegisterBuiltinMacro(*this, "__has_attribute");
+  Ident__has_include      = RegisterBuiltinMacro(*this, "__has_include");
+  Ident__has_include_next = RegisterBuiltinMacro(*this, "__has_include_next");
+  Ident__has_warning      = RegisterBuiltinMacro(*this, "__has_warning");
+
+  // Microsoft Extensions.
+  if (LangOpts.MicrosoftExt) 
+    Ident__pragma = RegisterBuiltinMacro(*this, "__pragma");
+  else
+    Ident__pragma = 0;
+}
+
+/// isTrivialSingleTokenExpansion - Return true if MI, which has a single token
+/// in its expansion, currently expands to that token literally.
+static bool isTrivialSingleTokenExpansion(const MacroInfo *MI,
+                                          const IdentifierInfo *MacroIdent,
+                                          Preprocessor &PP) {
+  IdentifierInfo *II = MI->getReplacementToken(0).getIdentifierInfo();
+
+  // If the token isn't an identifier, it's always literally expanded.
+  if (II == 0) return true;
+
+  // If the information about this identifier is out of date, update it from
+  // the external source.
+  if (II->isOutOfDate())
+    PP.getExternalSource()->updateOutOfDateIdentifier(*II);
+
+  // If the identifier is a macro, and if that macro is enabled, it may be
+  // expanded so it's not a trivial expansion.
+  if (II->hasMacroDefinition() && PP.getMacroInfo(II)->isEnabled() &&
+      // Fast expanding "#define X X" is ok, because X would be disabled.
+      II != MacroIdent)
+    return false;
+
+  // If this is an object-like macro invocation, it is safe to trivially expand
+  // it.
+  if (MI->isObjectLike()) return true;
+
+  // If this is a function-like macro invocation, it's safe to trivially expand
+  // as long as the identifier is not a macro argument.
+  for (MacroInfo::arg_iterator I = MI->arg_begin(), E = MI->arg_end();
+       I != E; ++I)
+    if (*I == II)
+      return false;   // Identifier is a macro argument.
+
+  return true;
+}
+
+
+/// isNextPPTokenLParen - Determine whether the next preprocessor token to be
+/// lexed is a '('.  If so, consume the token and return true, if not, this
+/// method should have no observable side-effect on the lexed tokens.
+bool Preprocessor::isNextPPTokenLParen() {
+  // Do some quick tests for rejection cases.
+  unsigned Val;
+  if (CurLexer)
+    Val = CurLexer->isNextPPTokenLParen();
+  else if (CurPTHLexer)
+    Val = CurPTHLexer->isNextPPTokenLParen();
+  else
+    Val = CurTokenLexer->isNextTokenLParen();
+
+  if (Val == 2) {
+    // We have run off the end.  If it's a source file we don't
+    // examine enclosing ones (C99 5.1.1.2p4).  Otherwise walk up the
+    // macro stack.
+    if (CurPPLexer)
+      return false;
+    for (unsigned i = IncludeMacroStack.size(); i != 0; --i) {
+      IncludeStackInfo &Entry = IncludeMacroStack[i-1];
+      if (Entry.TheLexer)
+        Val = Entry.TheLexer->isNextPPTokenLParen();
+      else if (Entry.ThePTHLexer)
+        Val = Entry.ThePTHLexer->isNextPPTokenLParen();
+      else
+        Val = Entry.TheTokenLexer->isNextTokenLParen();
+
+      if (Val != 2)
+        break;
+
+      // Ran off the end of a source file?
+      if (Entry.ThePPLexer)
+        return false;
+    }
+  }
+
+  // Okay, if we know that the token is a '(', lex it and return.  Otherwise we
+  // have found something that isn't a '(' or we found the end of the
+  // translation unit.  In either case, return false.
+  return Val == 1;
+}
+
+/// HandleMacroExpandedIdentifier - If an identifier token is read that is to be
+/// expanded as a macro, handle it and return the next token as 'Identifier'.
+bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier,
+                                                 MacroInfo *MI) {
+  // If this is a macro expansion in the "#if !defined(x)" line for the file,
+  // then the macro could expand to different things in other contexts, we need
+  // to disable the optimization in this case.
+  if (CurPPLexer) CurPPLexer->MIOpt.ExpandedMacro();
+
+  // If this is a builtin macro, like __LINE__ or _Pragma, handle it specially.
+  if (MI->isBuiltinMacro()) {
+    if (Callbacks) Callbacks->MacroExpands(Identifier, MI,
+                                           Identifier.getLocation());
+    ExpandBuiltinMacro(Identifier);
+    return false;
+  }
+
+  /// Args - If this is a function-like macro expansion, this contains,
+  /// for each macro argument, the list of tokens that were provided to the
+  /// invocation.
+  MacroArgs *Args = 0;
+
+  // Remember where the end of the expansion occurred.  For an object-like
+  // macro, this is the identifier.  For a function-like macro, this is the ')'.
+  SourceLocation ExpansionEnd = Identifier.getLocation();
+
+  // If this is a function-like macro, read the arguments.
+  if (MI->isFunctionLike()) {
+    // C99 6.10.3p10: If the preprocessing token immediately after the the macro
+    // name isn't a '(', this macro should not be expanded.
+    if (!isNextPPTokenLParen())
+      return true;
+
+    // Remember that we are now parsing the arguments to a macro invocation.
+    // Preprocessor directives used inside macro arguments are not portable, and
+    // this enables the warning.
+    InMacroArgs = true;
+    Args = ReadFunctionLikeMacroArgs(Identifier, MI, ExpansionEnd);
+
+    // Finished parsing args.
+    InMacroArgs = false;
+
+    // If there was an error parsing the arguments, bail out.
+    if (Args == 0) return false;
+
+    ++NumFnMacroExpanded;
+  } else {
+    ++NumMacroExpanded;
+  }
+
+  // Notice that this macro has been used.
+  markMacroAsUsed(MI);
+
+  // Remember where the token is expanded.
+  SourceLocation ExpandLoc = Identifier.getLocation();
+
+  if (Callbacks) Callbacks->MacroExpands(Identifier, MI,
+                                         SourceRange(ExpandLoc, ExpansionEnd));
+  
+  // If we started lexing a macro, enter the macro expansion body.
+
+  // If this macro expands to no tokens, don't bother to push it onto the
+  // expansion stack, only to take it right back off.
+  if (MI->getNumTokens() == 0) {
+    // No need for arg info.
+    if (Args) Args->destroy(*this);
+
+    // Ignore this macro use, just return the next token in the current
+    // buffer.
+    bool HadLeadingSpace = Identifier.hasLeadingSpace();
+    bool IsAtStartOfLine = Identifier.isAtStartOfLine();
+
+    Lex(Identifier);
+
+    // If the identifier isn't on some OTHER line, inherit the leading
+    // whitespace/first-on-a-line property of this token.  This handles
+    // stuff like "! XX," -> "! ," and "   XX," -> "    ,", when XX is
+    // empty.
+    if (!Identifier.isAtStartOfLine()) {
+      if (IsAtStartOfLine) Identifier.setFlag(Token::StartOfLine);
+      if (HadLeadingSpace) Identifier.setFlag(Token::LeadingSpace);
+    }
+    Identifier.setFlag(Token::LeadingEmptyMacro);
+    ++NumFastMacroExpanded;
+    return false;
+
+  } else if (MI->getNumTokens() == 1 &&
+             isTrivialSingleTokenExpansion(MI, Identifier.getIdentifierInfo(),
+                                           *this)) {
+    // Otherwise, if this macro expands into a single trivially-expanded
+    // token: expand it now.  This handles common cases like
+    // "#define VAL 42".
+
+    // No need for arg info.
+    if (Args) Args->destroy(*this);
+
+    // Propagate the isAtStartOfLine/hasLeadingSpace markers of the macro
+    // identifier to the expanded token.
+    bool isAtStartOfLine = Identifier.isAtStartOfLine();
+    bool hasLeadingSpace = Identifier.hasLeadingSpace();
+
+    // Replace the result token.
+    Identifier = MI->getReplacementToken(0);
+
+    // Restore the StartOfLine/LeadingSpace markers.
+    Identifier.setFlagValue(Token::StartOfLine , isAtStartOfLine);
+    Identifier.setFlagValue(Token::LeadingSpace, hasLeadingSpace);
+
+    // Update the tokens location to include both its expansion and physical
+    // locations.
+    SourceLocation Loc =
+      SourceMgr.createExpansionLoc(Identifier.getLocation(), ExpandLoc,
+                                   ExpansionEnd,Identifier.getLength());
+    Identifier.setLocation(Loc);
+
+    // If this is a disabled macro or #define X X, we must mark the result as
+    // unexpandable.
+    if (IdentifierInfo *NewII = Identifier.getIdentifierInfo()) {
+      if (MacroInfo *NewMI = getMacroInfo(NewII))
+        if (!NewMI->isEnabled() || NewMI == MI) {
+          Identifier.setFlag(Token::DisableExpand);
+          Diag(Identifier, diag::pp_disabled_macro_expansion);
+        }
+    }
+
+    // Since this is not an identifier token, it can't be macro expanded, so
+    // we're done.
+    ++NumFastMacroExpanded;
+    return false;
+  }
+
+  // Start expanding the macro.
+  EnterMacro(Identifier, ExpansionEnd, Args);
+
+  // Now that the macro is at the top of the include stack, ask the
+  // preprocessor to read the next token from it.
+  Lex(Identifier);
+  return false;
+}
+
+/// ReadFunctionLikeMacroArgs - After reading "MACRO" and knowing that the next
+/// token is the '(' of the macro, this method is invoked to read all of the
+/// actual arguments specified for the macro invocation.  This returns null on
+/// error.
+MacroArgs *Preprocessor::ReadFunctionLikeMacroArgs(Token &MacroName,
+                                                   MacroInfo *MI,
+                                                   SourceLocation &MacroEnd) {
+  // The number of fixed arguments to parse.
+  unsigned NumFixedArgsLeft = MI->getNumArgs();
+  bool isVariadic = MI->isVariadic();
+
+  // Outer loop, while there are more arguments, keep reading them.
+  Token Tok;
+
+  // Read arguments as unexpanded tokens.  This avoids issues, e.g., where
+  // an argument value in a macro could expand to ',' or '(' or ')'.
+  LexUnexpandedToken(Tok);
+  assert(Tok.is(tok::l_paren) && "Error computing l-paren-ness?");
+
+  // ArgTokens - Build up a list of tokens that make up each argument.  Each
+  // argument is separated by an EOF token.  Use a SmallVector so we can avoid
+  // heap allocations in the common case.
+  SmallVector<Token, 64> ArgTokens;
+
+  unsigned NumActuals = 0;
+  while (Tok.isNot(tok::r_paren)) {
+    assert((Tok.is(tok::l_paren) || Tok.is(tok::comma)) &&
+           "only expect argument separators here");
+
+    unsigned ArgTokenStart = ArgTokens.size();
+    SourceLocation ArgStartLoc = Tok.getLocation();
+
+    // C99 6.10.3p11: Keep track of the number of l_parens we have seen.  Note
+    // that we already consumed the first one.
+    unsigned NumParens = 0;
+
+    while (1) {
+      // Read arguments as unexpanded tokens.  This avoids issues, e.g., where
+      // an argument value in a macro could expand to ',' or '(' or ')'.
+      LexUnexpandedToken(Tok);
+
+      if (Tok.is(tok::eof) || Tok.is(tok::eod)) { // "#if f(<eof>" & "#if f(\n"
+        Diag(MacroName, diag::err_unterm_macro_invoc);
+        // Do not lose the EOF/EOD.  Return it to the client.
+        MacroName = Tok;
+        return 0;
+      } else if (Tok.is(tok::r_paren)) {
+        // If we found the ) token, the macro arg list is done.
+        if (NumParens-- == 0) {
+          MacroEnd = Tok.getLocation();
+          break;
+        }
+      } else if (Tok.is(tok::l_paren)) {
+        ++NumParens;
+      } else if (Tok.is(tok::comma) && NumParens == 0) {
+        // Comma ends this argument if there are more fixed arguments expected.
+        // However, if this is a variadic macro, and this is part of the
+        // variadic part, then the comma is just an argument token.
+        if (!isVariadic) break;
+        if (NumFixedArgsLeft > 1)
+          break;
+      } else if (Tok.is(tok::comment) && !KeepMacroComments) {
+        // If this is a comment token in the argument list and we're just in
+        // -C mode (not -CC mode), discard the comment.
+        continue;
+      } else if (Tok.getIdentifierInfo() != 0) {
+        // Reading macro arguments can cause macros that we are currently
+        // expanding from to be popped off the expansion stack.  Doing so causes
+        // them to be reenabled for expansion.  Here we record whether any
+        // identifiers we lex as macro arguments correspond to disabled macros.
+        // If so, we mark the token as noexpand.  This is a subtle aspect of
+        // C99 6.10.3.4p2.
+        if (MacroInfo *MI = getMacroInfo(Tok.getIdentifierInfo()))
+          if (!MI->isEnabled())
+            Tok.setFlag(Token::DisableExpand);
+      } else if (Tok.is(tok::code_completion)) {
+        if (CodeComplete)
+          CodeComplete->CodeCompleteMacroArgument(MacroName.getIdentifierInfo(),
+                                                  MI, NumActuals);
+        // Don't mark that we reached the code-completion point because the
+        // parser is going to handle the token and there will be another
+        // code-completion callback.
+      }
+
+      ArgTokens.push_back(Tok);
+    }
+
+    // If this was an empty argument list foo(), don't add this as an empty
+    // argument.
+    if (ArgTokens.empty() && Tok.getKind() == tok::r_paren)
+      break;
+
+    // If this is not a variadic macro, and too many args were specified, emit
+    // an error.
+    if (!isVariadic && NumFixedArgsLeft == 0) {
+      if (ArgTokens.size() != ArgTokenStart)
+        ArgStartLoc = ArgTokens[ArgTokenStart].getLocation();
+
+      // Emit the diagnostic at the macro name in case there is a missing ).
+      // Emitting it at the , could be far away from the macro name.
+      Diag(ArgStartLoc, diag::err_too_many_args_in_macro_invoc);
+      return 0;
+    }
+
+    // Empty arguments are standard in C99 and C++0x, and are supported as an extension in
+    // other modes.
+    if (ArgTokens.size() == ArgTokenStart && !LangOpts.C99)
+      Diag(Tok, LangOpts.CPlusPlus0x ?
+           diag::warn_cxx98_compat_empty_fnmacro_arg :
+           diag::ext_empty_fnmacro_arg);
+
+    // Add a marker EOF token to the end of the token list for this argument.
+    Token EOFTok;
+    EOFTok.startToken();
+    EOFTok.setKind(tok::eof);
+    EOFTok.setLocation(Tok.getLocation());
+    EOFTok.setLength(0);
+    ArgTokens.push_back(EOFTok);
+    ++NumActuals;
+    assert(NumFixedArgsLeft != 0 && "Too many arguments parsed");
+    --NumFixedArgsLeft;
+  }
+
+  // Okay, we either found the r_paren.  Check to see if we parsed too few
+  // arguments.
+  unsigned MinArgsExpected = MI->getNumArgs();
+
+  // See MacroArgs instance var for description of this.
+  bool isVarargsElided = false;
+
+  if (NumActuals < MinArgsExpected) {
+    // There are several cases where too few arguments is ok, handle them now.
+    if (NumActuals == 0 && MinArgsExpected == 1) {
+      // #define A(X)  or  #define A(...)   ---> A()
+
+      // If there is exactly one argument, and that argument is missing,
+      // then we have an empty "()" argument empty list.  This is fine, even if
+      // the macro expects one argument (the argument is just empty).
+      isVarargsElided = MI->isVariadic();
+    } else if (MI->isVariadic() &&
+               (NumActuals+1 == MinArgsExpected ||  // A(x, ...) -> A(X)
+                (NumActuals == 0 && MinArgsExpected == 2))) {// A(x,...) -> A()
+      // Varargs where the named vararg parameter is missing: ok as extension.
+      // #define A(x, ...)
+      // A("blah")
+      Diag(Tok, diag::ext_missing_varargs_arg);
+
+      // Remember this occurred, allowing us to elide the comma when used for
+      // cases like:
+      //   #define A(x, foo...) blah(a, ## foo)
+      //   #define B(x, ...) blah(a, ## __VA_ARGS__)
+      //   #define C(...) blah(a, ## __VA_ARGS__)
+      //  A(x) B(x) C()
+      isVarargsElided = true;
+    } else {
+      // Otherwise, emit the error.
+      Diag(Tok, diag::err_too_few_args_in_macro_invoc);
+      return 0;
+    }
+
+    // Add a marker EOF token to the end of the token list for this argument.
+    SourceLocation EndLoc = Tok.getLocation();
+    Tok.startToken();
+    Tok.setKind(tok::eof);
+    Tok.setLocation(EndLoc);
+    Tok.setLength(0);
+    ArgTokens.push_back(Tok);
+
+    // If we expect two arguments, add both as empty.
+    if (NumActuals == 0 && MinArgsExpected == 2)
+      ArgTokens.push_back(Tok);
+
+  } else if (NumActuals > MinArgsExpected && !MI->isVariadic()) {
+    // Emit the diagnostic at the macro name in case there is a missing ).
+    // Emitting it at the , could be far away from the macro name.
+    Diag(MacroName, diag::err_too_many_args_in_macro_invoc);
+    return 0;
+  }
+
+  return MacroArgs::create(MI, ArgTokens, isVarargsElided, *this);
+}
+
+/// \brief Keeps macro expanded tokens for TokenLexers.
+//
+/// Works like a stack; a TokenLexer adds the macro expanded tokens that is
+/// going to lex in the cache and when it finishes the tokens are removed
+/// from the end of the cache.
+Token *Preprocessor::cacheMacroExpandedTokens(TokenLexer *tokLexer,
+                                              ArrayRef<Token> tokens) {
+  assert(tokLexer);
+  if (tokens.empty())
+    return 0;
+
+  size_t newIndex = MacroExpandedTokens.size();
+  bool cacheNeedsToGrow = tokens.size() >
+                      MacroExpandedTokens.capacity()-MacroExpandedTokens.size(); 
+  MacroExpandedTokens.append(tokens.begin(), tokens.end());
+
+  if (cacheNeedsToGrow) {
+    // Go through all the TokenLexers whose 'Tokens' pointer points in the
+    // buffer and update the pointers to the (potential) new buffer array.
+    for (unsigned i = 0, e = MacroExpandingLexersStack.size(); i != e; ++i) {
+      TokenLexer *prevLexer;
+      size_t tokIndex;
+      llvm::tie(prevLexer, tokIndex) = MacroExpandingLexersStack[i];
+      prevLexer->Tokens = MacroExpandedTokens.data() + tokIndex;
+    }
+  }
+
+  MacroExpandingLexersStack.push_back(std::make_pair(tokLexer, newIndex));
+  return MacroExpandedTokens.data() + newIndex;
+}
+
+void Preprocessor::removeCachedMacroExpandedTokensOfLastLexer() {
+  assert(!MacroExpandingLexersStack.empty());
+  size_t tokIndex = MacroExpandingLexersStack.back().second;
+  assert(tokIndex < MacroExpandedTokens.size());
+  // Pop the cached macro expanded tokens from the end.
+  MacroExpandedTokens.resize(tokIndex);
+  MacroExpandingLexersStack.pop_back();
+}
+
+/// ComputeDATE_TIME - Compute the current time, enter it into the specified
+/// scratch buffer, then return DATELoc/TIMELoc locations with the position of
+/// the identifier tokens inserted.
+static void ComputeDATE_TIME(SourceLocation &DATELoc, SourceLocation &TIMELoc,
+                             Preprocessor &PP) {
+  time_t TT = time(0);
+  struct tm *TM = localtime(&TT);
+
+  static const char * const Months[] = {
+    "Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"
+  };
+
+  char TmpBuffer[32];
+#ifdef LLVM_ON_WIN32
+  sprintf(TmpBuffer, "\"%s %2d %4d\"", Months[TM->tm_mon], TM->tm_mday,
+          TM->tm_year+1900);
+#else
+  snprintf(TmpBuffer, sizeof(TmpBuffer), "\"%s %2d %4d\"", Months[TM->tm_mon], TM->tm_mday,
+          TM->tm_year+1900);
+#endif
+
+  Token TmpTok;
+  TmpTok.startToken();
+  PP.CreateString(TmpBuffer, strlen(TmpBuffer), TmpTok);
+  DATELoc = TmpTok.getLocation();
+
+#ifdef LLVM_ON_WIN32
+  sprintf(TmpBuffer, "\"%02d:%02d:%02d\"", TM->tm_hour, TM->tm_min, TM->tm_sec);
+#else
+  snprintf(TmpBuffer, sizeof(TmpBuffer), "\"%02d:%02d:%02d\"", TM->tm_hour, TM->tm_min, TM->tm_sec);
+#endif
+  PP.CreateString(TmpBuffer, strlen(TmpBuffer), TmpTok);
+  TIMELoc = TmpTok.getLocation();
+}
+
+
+/// HasFeature - Return true if we recognize and implement the feature
+/// specified by the identifier as a standard language feature.
+static bool HasFeature(const Preprocessor &PP, const IdentifierInfo *II) {
+  const LangOptions &LangOpts = PP.getLangOpts();
+  StringRef Feature = II->getName();
+
+  // Normalize the feature name, __foo__ becomes foo.
+  if (Feature.startswith("__") && Feature.endswith("__") && Feature.size() >= 4)
+    Feature = Feature.substr(2, Feature.size() - 4);
+
+  return llvm::StringSwitch<bool>(Feature)
+           .Case("address_sanitizer", LangOpts.AddressSanitizer)
+           .Case("attribute_analyzer_noreturn", true)
+           .Case("attribute_availability", true)
+           .Case("attribute_cf_returns_not_retained", true)
+           .Case("attribute_cf_returns_retained", true)
+           .Case("attribute_deprecated_with_message", true)
+           .Case("attribute_ext_vector_type", true)
+           .Case("attribute_ns_returns_not_retained", true)
+           .Case("attribute_ns_returns_retained", true)
+           .Case("attribute_ns_consumes_self", true)
+           .Case("attribute_ns_consumed", true)
+           .Case("attribute_cf_consumed", true)
+           .Case("attribute_objc_ivar_unused", true)
+           .Case("attribute_objc_method_family", true)
+           .Case("attribute_overloadable", true)
+           .Case("attribute_unavailable_with_message", true)
+           .Case("blocks", LangOpts.Blocks)
+           .Case("cxx_exceptions", LangOpts.Exceptions)
+           .Case("cxx_rtti", LangOpts.RTTI)
+           .Case("enumerator_attributes", true)
+           // Objective-C features
+           .Case("objc_arr", LangOpts.ObjCAutoRefCount) // FIXME: REMOVE?
+           .Case("objc_arc", LangOpts.ObjCAutoRefCount)
+           .Case("objc_arc_weak", LangOpts.ObjCAutoRefCount && 
+                 LangOpts.ObjCRuntimeHasWeak)
+           .Case("objc_default_synthesize_properties", LangOpts.ObjC2)
+           .Case("objc_fixed_enum", LangOpts.ObjC2)
+           .Case("objc_instancetype", LangOpts.ObjC2)
+           .Case("objc_modules", LangOpts.ObjC2 && LangOpts.Modules)
+           .Case("objc_nonfragile_abi", LangOpts.ObjCNonFragileABI)
+           .Case("objc_weak_class", LangOpts.ObjCNonFragileABI)
+           .Case("ownership_holds", true)
+           .Case("ownership_returns", true)
+           .Case("ownership_takes", true)
+           .Case("objc_bool", true)
+           .Case("objc_subscripting", LangOpts.ObjCNonFragileABI)
+           .Case("objc_array_literals", LangOpts.ObjC2)
+           .Case("objc_dictionary_literals", LangOpts.ObjC2)
+           .Case("arc_cf_code_audited", true)
+           // C11 features
+           .Case("c_alignas", LangOpts.C11)
+           .Case("c_atomic", LangOpts.C11)
+           .Case("c_generic_selections", LangOpts.C11)
+           .Case("c_static_assert", LangOpts.C11)
+           // C++11 features
+           .Case("cxx_access_control_sfinae", LangOpts.CPlusPlus0x)
+           .Case("cxx_alias_templates", LangOpts.CPlusPlus0x)
+           .Case("cxx_alignas", LangOpts.CPlusPlus0x)
+           .Case("cxx_atomic", LangOpts.CPlusPlus0x)
+           .Case("cxx_attributes", LangOpts.CPlusPlus0x)
+           .Case("cxx_auto_type", LangOpts.CPlusPlus0x)
+           .Case("cxx_constexpr", LangOpts.CPlusPlus0x)
+           .Case("cxx_decltype", LangOpts.CPlusPlus0x)
+           .Case("cxx_decltype_incomplete_return_types", LangOpts.CPlusPlus0x)
+           .Case("cxx_default_function_template_args", LangOpts.CPlusPlus0x)
+           .Case("cxx_defaulted_functions", LangOpts.CPlusPlus0x)
+           .Case("cxx_delegating_constructors", LangOpts.CPlusPlus0x)
+           .Case("cxx_deleted_functions", LangOpts.CPlusPlus0x)
+           .Case("cxx_explicit_conversions", LangOpts.CPlusPlus0x)
+           .Case("cxx_generalized_initializers", LangOpts.CPlusPlus0x)
+           .Case("cxx_implicit_moves", LangOpts.CPlusPlus0x)
+         //.Case("cxx_inheriting_constructors", false)
+           .Case("cxx_inline_namespaces", LangOpts.CPlusPlus0x)
+           .Case("cxx_lambdas", LangOpts.CPlusPlus0x)
+           .Case("cxx_local_type_template_args", LangOpts.CPlusPlus0x)
+           .Case("cxx_nonstatic_member_init", LangOpts.CPlusPlus0x)
+           .Case("cxx_noexcept", LangOpts.CPlusPlus0x)
+           .Case("cxx_nullptr", LangOpts.CPlusPlus0x)
+           .Case("cxx_override_control", LangOpts.CPlusPlus0x)
+           .Case("cxx_range_for", LangOpts.CPlusPlus0x)
+           .Case("cxx_raw_string_literals", LangOpts.CPlusPlus0x)
+           .Case("cxx_reference_qualified_functions", LangOpts.CPlusPlus0x)
+           .Case("cxx_rvalue_references", LangOpts.CPlusPlus0x)
+           .Case("cxx_strong_enums", LangOpts.CPlusPlus0x)
+           .Case("cxx_static_assert", LangOpts.CPlusPlus0x)
+           .Case("cxx_trailing_return", LangOpts.CPlusPlus0x)
+           .Case("cxx_unicode_literals", LangOpts.CPlusPlus0x)
+           .Case("cxx_unrestricted_unions", LangOpts.CPlusPlus0x)
+           .Case("cxx_user_literals", LangOpts.CPlusPlus0x)
+           .Case("cxx_variadic_templates", LangOpts.CPlusPlus0x)
+           // Type traits
+           .Case("has_nothrow_assign", LangOpts.CPlusPlus)
+           .Case("has_nothrow_copy", LangOpts.CPlusPlus)
+           .Case("has_nothrow_constructor", LangOpts.CPlusPlus)
+           .Case("has_trivial_assign", LangOpts.CPlusPlus)
+           .Case("has_trivial_copy", LangOpts.CPlusPlus)
+           .Case("has_trivial_constructor", LangOpts.CPlusPlus)
+           .Case("has_trivial_destructor", LangOpts.CPlusPlus)
+           .Case("has_virtual_destructor", LangOpts.CPlusPlus)
+           .Case("is_abstract", LangOpts.CPlusPlus)
+           .Case("is_base_of", LangOpts.CPlusPlus)
+           .Case("is_class", LangOpts.CPlusPlus)
+           .Case("is_convertible_to", LangOpts.CPlusPlus)
+            // __is_empty is available only if the horrible
+            // "struct __is_empty" parsing hack hasn't been needed in this
+            // translation unit. If it has, __is_empty reverts to a normal
+            // identifier and __has_feature(is_empty) evaluates false.
+           .Case("is_empty", 
+                 LangOpts.CPlusPlus && 
+                 PP.getIdentifierInfo("__is_empty")->getTokenID()
+                                                            != tok::identifier)
+           .Case("is_enum", LangOpts.CPlusPlus)
+           .Case("is_final", LangOpts.CPlusPlus)
+           .Case("is_literal", LangOpts.CPlusPlus)
+           .Case("is_standard_layout", LangOpts.CPlusPlus)
+           // __is_pod is available only if the horrible
+           // "struct __is_pod" parsing hack hasn't been needed in this
+           // translation unit. If it has, __is_pod reverts to a normal
+           // identifier and __has_feature(is_pod) evaluates false.
+           .Case("is_pod", 
+                 LangOpts.CPlusPlus && 
+                 PP.getIdentifierInfo("__is_pod")->getTokenID()
+                                                            != tok::identifier)
+           .Case("is_polymorphic", LangOpts.CPlusPlus)
+           .Case("is_trivial", LangOpts.CPlusPlus)
+           .Case("is_trivially_assignable", LangOpts.CPlusPlus)
+           .Case("is_trivially_constructible", LangOpts.CPlusPlus)
+           .Case("is_trivially_copyable", LangOpts.CPlusPlus)
+           .Case("is_union", LangOpts.CPlusPlus)
+           .Case("modules", LangOpts.Modules)
+           .Case("tls", PP.getTargetInfo().isTLSSupported())
+           .Case("underlying_type", LangOpts.CPlusPlus)
+           .Default(false);
+}
+
+/// HasExtension - Return true if we recognize and implement the feature
+/// specified by the identifier, either as an extension or a standard language
+/// feature.
+static bool HasExtension(const Preprocessor &PP, const IdentifierInfo *II) {
+  if (HasFeature(PP, II))
+    return true;
+
+  // If the use of an extension results in an error diagnostic, extensions are
+  // effectively unavailable, so just return false here.
+  if (PP.getDiagnostics().getExtensionHandlingBehavior() ==
+      DiagnosticsEngine::Ext_Error)
+    return false;
+
+  const LangOptions &LangOpts = PP.getLangOpts();
+  StringRef Extension = II->getName();
+
+  // Normalize the extension name, __foo__ becomes foo.
+  if (Extension.startswith("__") && Extension.endswith("__") &&
+      Extension.size() >= 4)
+    Extension = Extension.substr(2, Extension.size() - 4);
+
+  // Because we inherit the feature list from HasFeature, this string switch
+  // must be less restrictive than HasFeature's.
+  return llvm::StringSwitch<bool>(Extension)
+           // C11 features supported by other languages as extensions.
+           .Case("c_alignas", true)
+           .Case("c_atomic", true)
+           .Case("c_generic_selections", true)
+           .Case("c_static_assert", true)
+           // C++0x features supported by other languages as extensions.
+           .Case("cxx_atomic", LangOpts.CPlusPlus)
+           .Case("cxx_deleted_functions", LangOpts.CPlusPlus)
+           .Case("cxx_explicit_conversions", LangOpts.CPlusPlus)
+           .Case("cxx_inline_namespaces", LangOpts.CPlusPlus)
+           .Case("cxx_local_type_template_args", LangOpts.CPlusPlus)
+           .Case("cxx_nonstatic_member_init", LangOpts.CPlusPlus)
+           .Case("cxx_override_control", LangOpts.CPlusPlus)
+           .Case("cxx_range_for", LangOpts.CPlusPlus)
+           .Case("cxx_reference_qualified_functions", LangOpts.CPlusPlus)
+           .Case("cxx_rvalue_references", LangOpts.CPlusPlus)
+           .Default(false);
+}
+
+/// HasAttribute -  Return true if we recognize and implement the attribute
+/// specified by the given identifier.
+static bool HasAttribute(const IdentifierInfo *II) {
+  StringRef Name = II->getName();
+  // Normalize the attribute name, __foo__ becomes foo.
+  if (Name.startswith("__") && Name.endswith("__") && Name.size() >= 4)
+    Name = Name.substr(2, Name.size() - 4);
+
+  return llvm::StringSwitch<bool>(Name)
+#include "clang/Lex/AttrSpellings.inc"
+        .Default(false);
+}
+
+/// EvaluateHasIncludeCommon - Process a '__has_include("path")'
+/// or '__has_include_next("path")' expression.
+/// Returns true if successful.
+static bool EvaluateHasIncludeCommon(Token &Tok,
+                                     IdentifierInfo *II, Preprocessor &PP,
+                                     const DirectoryLookup *LookupFrom) {
+  SourceLocation LParenLoc;
+
+  // Get '('.
+  PP.LexNonComment(Tok);
+
+  // Ensure we have a '('.
+  if (Tok.isNot(tok::l_paren)) {
+    PP.Diag(Tok.getLocation(), diag::err_pp_missing_lparen) << II->getName();
+    return false;
+  }
+
+  // Save '(' location for possible missing ')' message.
+  LParenLoc = Tok.getLocation();
+
+  // Get the file name.
+  PP.getCurrentLexer()->LexIncludeFilename(Tok);
+
+  // Reserve a buffer to get the spelling.
+  SmallString<128> FilenameBuffer;
+  StringRef Filename;
+  SourceLocation EndLoc;
+  
+  switch (Tok.getKind()) {
+  case tok::eod:
+    // If the token kind is EOD, the error has already been diagnosed.
+    return false;
+
+  case tok::angle_string_literal:
+  case tok::string_literal: {
+    bool Invalid = false;
+    Filename = PP.getSpelling(Tok, FilenameBuffer, &Invalid);
+    if (Invalid)
+      return false;
+    break;
+  }
+
+  case tok::less:
+    // This could be a <foo/bar.h> file coming from a macro expansion.  In this
+    // case, glue the tokens together into FilenameBuffer and interpret those.
+    FilenameBuffer.push_back('<');
+    if (PP.ConcatenateIncludeName(FilenameBuffer, EndLoc))
+      return false;   // Found <eod> but no ">"?  Diagnostic already emitted.
+    Filename = FilenameBuffer.str();
+    break;
+  default:
+    PP.Diag(Tok.getLocation(), diag::err_pp_expects_filename);
+    return false;
+  }
+
+  // Get ')'.
+  PP.LexNonComment(Tok);
+
+  // Ensure we have a trailing ).
+  if (Tok.isNot(tok::r_paren)) {
+    PP.Diag(Tok.getLocation(), diag::err_pp_missing_rparen) << II->getName();
+    PP.Diag(LParenLoc, diag::note_matching) << "(";
+    return false;
+  }
+
+  bool isAngled = PP.GetIncludeFilenameSpelling(Tok.getLocation(), Filename);
+  // If GetIncludeFilenameSpelling set the start ptr to null, there was an
+  // error.
+  if (Filename.empty())
+    return false;
+
+  // Search include directories.
+  const DirectoryLookup *CurDir;
+  const FileEntry *File =
+      PP.LookupFile(Filename, isAngled, LookupFrom, CurDir, NULL, NULL, NULL);
+
+  // Get the result value.  A result of true means the file exists.
+  return File != 0;
+}
+
+/// EvaluateHasInclude - Process a '__has_include("path")' expression.
+/// Returns true if successful.
+static bool EvaluateHasInclude(Token &Tok, IdentifierInfo *II,
+                               Preprocessor &PP) {
+  return EvaluateHasIncludeCommon(Tok, II, PP, NULL);
+}
+
+/// EvaluateHasIncludeNext - Process '__has_include_next("path")' expression.
+/// Returns true if successful.
+static bool EvaluateHasIncludeNext(Token &Tok,
+                                   IdentifierInfo *II, Preprocessor &PP) {
+  // __has_include_next is like __has_include, except that we start
+  // searching after the current found directory.  If we can't do this,
+  // issue a diagnostic.
+  const DirectoryLookup *Lookup = PP.GetCurDirLookup();
+  if (PP.isInPrimaryFile()) {
+    Lookup = 0;
+    PP.Diag(Tok, diag::pp_include_next_in_primary);
+  } else if (Lookup == 0) {
+    PP.Diag(Tok, diag::pp_include_next_absolute_path);
+  } else {
+    // Start looking up in the next directory.
+    ++Lookup;
+  }
+
+  return EvaluateHasIncludeCommon(Tok, II, PP, Lookup);
+}
+
+/// ExpandBuiltinMacro - If an identifier token is read that is to be expanded
+/// as a builtin macro, handle it and return the next token as 'Tok'.
+void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
+  // Figure out which token this is.
+  IdentifierInfo *II = Tok.getIdentifierInfo();
+  assert(II && "Can't be a macro without id info!");
+
+  // If this is an _Pragma or Microsoft __pragma directive, expand it,
+  // invoke the pragma handler, then lex the token after it.
+  if (II == Ident_Pragma)
+    return Handle_Pragma(Tok);
+  else if (II == Ident__pragma) // in non-MS mode this is null
+    return HandleMicrosoft__pragma(Tok);
+
+  ++NumBuiltinMacroExpanded;
+
+  SmallString<128> TmpBuffer;
+  llvm::raw_svector_ostream OS(TmpBuffer);
+
+  // Set up the return result.
+  Tok.setIdentifierInfo(0);
+  Tok.clearFlag(Token::NeedsCleaning);
+
+  if (II == Ident__LINE__) {
+    // C99 6.10.8: "__LINE__: The presumed line number (within the current
+    // source file) of the current source line (an integer constant)".  This can
+    // be affected by #line.
+    SourceLocation Loc = Tok.getLocation();
+
+    // Advance to the location of the first _, this might not be the first byte
+    // of the token if it starts with an escaped newline.
+    Loc = AdvanceToTokenCharacter(Loc, 0);
+
+    // One wrinkle here is that GCC expands __LINE__ to location of the *end* of
+    // a macro expansion.  This doesn't matter for object-like macros, but
+    // can matter for a function-like macro that expands to contain __LINE__.
+    // Skip down through expansion points until we find a file loc for the
+    // end of the expansion history.
+    Loc = SourceMgr.getExpansionRange(Loc).second;
+    PresumedLoc PLoc = SourceMgr.getPresumedLoc(Loc);
+
+    // __LINE__ expands to a simple numeric value.
+    OS << (PLoc.isValid()? PLoc.getLine() : 1);
+    Tok.setKind(tok::numeric_constant);
+  } else if (II == Ident__FILE__ || II == Ident__BASE_FILE__) {
+    // C99 6.10.8: "__FILE__: The presumed name of the current source file (a
+    // character string literal)". This can be affected by #line.
+    PresumedLoc PLoc = SourceMgr.getPresumedLoc(Tok.getLocation());
+
+    // __BASE_FILE__ is a GNU extension that returns the top of the presumed
+    // #include stack instead of the current file.
+    if (II == Ident__BASE_FILE__ && PLoc.isValid()) {
+      SourceLocation NextLoc = PLoc.getIncludeLoc();
+      while (NextLoc.isValid()) {
+        PLoc = SourceMgr.getPresumedLoc(NextLoc);
+        if (PLoc.isInvalid())
+          break;
+        
+        NextLoc = PLoc.getIncludeLoc();
+      }
+    }
+
+    // Escape this filename.  Turn '\' -> '\\' '"' -> '\"'
+    SmallString<128> FN;
+    if (PLoc.isValid()) {
+      FN += PLoc.getFilename();
+      Lexer::Stringify(FN);
+      OS << '"' << FN.str() << '"';
+    }
+    Tok.setKind(tok::string_literal);
+  } else if (II == Ident__DATE__) {
+    if (!DATELoc.isValid())
+      ComputeDATE_TIME(DATELoc, TIMELoc, *this);
+    Tok.setKind(tok::string_literal);
+    Tok.setLength(strlen("\"Mmm dd yyyy\""));
+    Tok.setLocation(SourceMgr.createExpansionLoc(DATELoc, Tok.getLocation(),
+                                                 Tok.getLocation(),
+                                                 Tok.getLength()));
+    return;
+  } else if (II == Ident__TIME__) {
+    if (!TIMELoc.isValid())
+      ComputeDATE_TIME(DATELoc, TIMELoc, *this);
+    Tok.setKind(tok::string_literal);
+    Tok.setLength(strlen("\"hh:mm:ss\""));
+    Tok.setLocation(SourceMgr.createExpansionLoc(TIMELoc, Tok.getLocation(),
+                                                 Tok.getLocation(),
+                                                 Tok.getLength()));
+    return;
+  } else if (II == Ident__INCLUDE_LEVEL__) {
+    // Compute the presumed include depth of this token.  This can be affected
+    // by GNU line markers.
+    unsigned Depth = 0;
+
+    PresumedLoc PLoc = SourceMgr.getPresumedLoc(Tok.getLocation());
+    if (PLoc.isValid()) {
+      PLoc = SourceMgr.getPresumedLoc(PLoc.getIncludeLoc());
+      for (; PLoc.isValid(); ++Depth)
+        PLoc = SourceMgr.getPresumedLoc(PLoc.getIncludeLoc());
+    }
+
+    // __INCLUDE_LEVEL__ expands to a simple numeric value.
+    OS << Depth;
+    Tok.setKind(tok::numeric_constant);
+  } else if (II == Ident__TIMESTAMP__) {
+    // MSVC, ICC, GCC, VisualAge C++ extension.  The generated string should be
+    // of the form "Ddd Mmm dd hh::mm::ss yyyy", which is returned by asctime.
+
+    // Get the file that we are lexing out of.  If we're currently lexing from
+    // a macro, dig into the include stack.
+    const FileEntry *CurFile = 0;
+    PreprocessorLexer *TheLexer = getCurrentFileLexer();
+
+    if (TheLexer)
+      CurFile = SourceMgr.getFileEntryForID(TheLexer->getFileID());
+
+    const char *Result;
+    if (CurFile) {
+      time_t TT = CurFile->getModificationTime();
+      struct tm *TM = localtime(&TT);
+      Result = asctime(TM);
+    } else {
+      Result = "??? ??? ?? ??:??:?? ????\n";
+    }
+    // Surround the string with " and strip the trailing newline.
+    OS << '"' << StringRef(Result, strlen(Result)-1) << '"';
+    Tok.setKind(tok::string_literal);
+  } else if (II == Ident__COUNTER__) {
+    // __COUNTER__ expands to a simple numeric value.
+    OS << CounterValue++;
+    Tok.setKind(tok::numeric_constant);
+  } else if (II == Ident__has_feature   ||
+             II == Ident__has_extension ||
+             II == Ident__has_builtin   ||
+             II == Ident__has_attribute) {
+    // The argument to these builtins should be a parenthesized identifier.
+    SourceLocation StartLoc = Tok.getLocation();
+
+    bool IsValid = false;
+    IdentifierInfo *FeatureII = 0;
+
+    // Read the '('.
+    Lex(Tok);
+    if (Tok.is(tok::l_paren)) {
+      // Read the identifier
+      Lex(Tok);
+      if (Tok.is(tok::identifier)) {
+        FeatureII = Tok.getIdentifierInfo();
+
+        // Read the ')'.
+        Lex(Tok);
+        if (Tok.is(tok::r_paren))
+          IsValid = true;
+      }
+    }
+
+    bool Value = false;
+    if (!IsValid)
+      Diag(StartLoc, diag::err_feature_check_malformed);
+    else if (II == Ident__has_builtin) {
+      // Check for a builtin is trivial.
+      Value = FeatureII->getBuiltinID() != 0;
+    } else if (II == Ident__has_attribute)
+      Value = HasAttribute(FeatureII);
+    else if (II == Ident__has_extension)
+      Value = HasExtension(*this, FeatureII);
+    else {
+      assert(II == Ident__has_feature && "Must be feature check");
+      Value = HasFeature(*this, FeatureII);
+    }
+
+    OS << (int)Value;
+    if (IsValid)
+      Tok.setKind(tok::numeric_constant);
+  } else if (II == Ident__has_include ||
+             II == Ident__has_include_next) {
+    // The argument to these two builtins should be a parenthesized
+    // file name string literal using angle brackets (<>) or
+    // double-quotes ("").
+    bool Value;
+    if (II == Ident__has_include)
+      Value = EvaluateHasInclude(Tok, II, *this);
+    else
+      Value = EvaluateHasIncludeNext(Tok, II, *this);
+    OS << (int)Value;
+    Tok.setKind(tok::numeric_constant);
+  } else if (II == Ident__has_warning) {
+    // The argument should be a parenthesized string literal.
+    // The argument to these builtins should be a parenthesized identifier.
+    SourceLocation StartLoc = Tok.getLocation();    
+    bool IsValid = false;
+    bool Value = false;
+    // Read the '('.
+    Lex(Tok);
+    do {
+      if (Tok.is(tok::l_paren)) {      
+        // Read the string.
+        Lex(Tok);
+      
+        // We need at least one string literal.
+        if (!Tok.is(tok::string_literal)) {
+          StartLoc = Tok.getLocation();
+          IsValid = false;
+          // Eat tokens until ')'.
+          do Lex(Tok); while (!(Tok.is(tok::r_paren) || Tok.is(tok::eod)));
+          break;
+        }
+        
+        // String concatenation allows multiple strings, which can even come
+        // from macro expansion.
+        SmallVector<Token, 4> StrToks;
+        while (Tok.is(tok::string_literal)) {
+          // Complain about, and drop, any ud-suffix.
+          if (Tok.hasUDSuffix())
+            Diag(Tok, diag::err_invalid_string_udl);
+          StrToks.push_back(Tok);
+          LexUnexpandedToken(Tok);
+        }
+        
+        // Is the end a ')'?
+        if (!(IsValid = Tok.is(tok::r_paren)))
+          break;
+        
+        // Concatenate and parse the strings.
+        StringLiteralParser Literal(&StrToks[0], StrToks.size(), *this);
+        assert(Literal.isAscii() && "Didn't allow wide strings in");
+        if (Literal.hadError)
+          break;
+        if (Literal.Pascal) {
+          Diag(Tok, diag::warn_pragma_diagnostic_invalid);
+          break;
+        }
+        
+        StringRef WarningName(Literal.GetString());
+        
+        if (WarningName.size() < 3 || WarningName[0] != '-' ||
+            WarningName[1] != 'W') {
+          Diag(StrToks[0].getLocation(), diag::warn_has_warning_invalid_option);
+          break;
+        }
+        
+        // Finally, check if the warning flags maps to a diagnostic group.
+        // We construct a SmallVector here to talk to getDiagnosticIDs().
+        // Although we don't use the result, this isn't a hot path, and not
+        // worth special casing.
+        llvm::SmallVector<diag::kind, 10> Diags;
+        Value = !getDiagnostics().getDiagnosticIDs()->
+          getDiagnosticsInGroup(WarningName.substr(2), Diags);
+      }
+    } while (false);
+    
+    if (!IsValid)
+      Diag(StartLoc, diag::err_warning_check_malformed);
+
+    OS << (int)Value;
+    Tok.setKind(tok::numeric_constant);
+  } else {
+    llvm_unreachable("Unknown identifier!");
+  }
+  CreateString(OS.str().data(), OS.str().size(), Tok,
+               Tok.getLocation(), Tok.getLocation());
+}
+
+void Preprocessor::markMacroAsUsed(MacroInfo *MI) {
+  // If the 'used' status changed, and the macro requires 'unused' warning,
+  // remove its SourceLocation from the warn-for-unused-macro locations.
+  if (MI->isWarnIfUnused() && !MI->isUsed())
+    WarnUnusedMacroLocs.erase(MI->getDefinitionLoc());
+  MI->setIsUsed(true);
+}
diff --git a/clang/lib/Lex/PTHLexer.cpp b/clang/lib/Lex/PTHLexer.cpp
new file mode 100644
index 0000000..f104f96
--- /dev/null
+++ b/clang/lib/Lex/PTHLexer.cpp
@@ -0,0 +1,710 @@
+//===--- PTHLexer.cpp - Lex from a token stream ---------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the PTHLexer interface.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Basic/TokenKinds.h"
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/FileSystemStatCache.h"
+#include "clang/Basic/IdentifierTable.h"
+#include "clang/Basic/OnDiskHashTable.h"
+#include "clang/Lex/LexDiagnostic.h"
+#include "clang/Lex/PTHLexer.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/PTHManager.h"
+#include "clang/Lex/Token.h"
+#include "clang/Lex/Preprocessor.h"
+#include "llvm/ADT/OwningPtr.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/system_error.h"
+using namespace clang;
+using namespace clang::io;
+
+#define DISK_TOKEN_SIZE (1+1+2+4+4)
+
+//===----------------------------------------------------------------------===//
+// PTHLexer methods.
+//===----------------------------------------------------------------------===//
+
+PTHLexer::PTHLexer(Preprocessor &PP, FileID FID, const unsigned char *D,
+                   const unsigned char *ppcond, PTHManager &PM)
+  : PreprocessorLexer(&PP, FID), TokBuf(D), CurPtr(D), LastHashTokPtr(0),
+    PPCond(ppcond), CurPPCondPtr(ppcond), PTHMgr(PM) {
+
+  FileStartLoc = PP.getSourceManager().getLocForStartOfFile(FID);
+}
+
+void PTHLexer::Lex(Token& Tok) {
+LexNextToken:
+
+  //===--------------------------------------==//
+  // Read the raw token data.
+  //===--------------------------------------==//
+
+  // Shadow CurPtr into an automatic variable.
+  const unsigned char *CurPtrShadow = CurPtr;
+
+  // Read in the data for the token.
+  unsigned Word0 = ReadLE32(CurPtrShadow);
+  uint32_t IdentifierID = ReadLE32(CurPtrShadow);
+  uint32_t FileOffset = ReadLE32(CurPtrShadow);
+
+  tok::TokenKind TKind = (tok::TokenKind) (Word0 & 0xFF);
+  Token::TokenFlags TFlags = (Token::TokenFlags) ((Word0 >> 8) & 0xFF);
+  uint32_t Len = Word0 >> 16;
+
+  CurPtr = CurPtrShadow;
+
+  //===--------------------------------------==//
+  // Construct the token itself.
+  //===--------------------------------------==//
+
+  Tok.startToken();
+  Tok.setKind(TKind);
+  Tok.setFlag(TFlags);
+  assert(!LexingRawMode);
+  Tok.setLocation(FileStartLoc.getLocWithOffset(FileOffset));
+  Tok.setLength(Len);
+
+  // Handle identifiers.
+  if (Tok.isLiteral()) {
+    Tok.setLiteralData((const char*) (PTHMgr.SpellingBase + IdentifierID));
+  }
+  else if (IdentifierID) {
+    MIOpt.ReadToken();
+    IdentifierInfo *II = PTHMgr.GetIdentifierInfo(IdentifierID-1);
+
+    Tok.setIdentifierInfo(II);
+
+    // Change the kind of this identifier to the appropriate token kind, e.g.
+    // turning "for" into a keyword.
+    Tok.setKind(II->getTokenID());
+
+    if (II->isHandleIdentifierCase())
+      PP->HandleIdentifier(Tok);
+    return;
+  }
+
+  //===--------------------------------------==//
+  // Process the token.
+  //===--------------------------------------==//
+  if (TKind == tok::eof) {
+    // Save the end-of-file token.
+    EofToken = Tok;
+
+    // Save 'PP' to 'PPCache' as LexEndOfFile can delete 'this'.
+    Preprocessor *PPCache = PP;
+
+    assert(!ParsingPreprocessorDirective);
+    assert(!LexingRawMode);
+    
+    if (LexEndOfFile(Tok))
+      return;
+
+    return PPCache->Lex(Tok);
+  }
+
+  if (TKind == tok::hash && Tok.isAtStartOfLine()) {
+    LastHashTokPtr = CurPtr - DISK_TOKEN_SIZE;
+    assert(!LexingRawMode);
+    PP->HandleDirective(Tok);
+
+    if (PP->isCurrentLexer(this))
+      goto LexNextToken;
+
+    return PP->Lex(Tok);
+  }
+
+  if (TKind == tok::eod) {
+    assert(ParsingPreprocessorDirective);
+    ParsingPreprocessorDirective = false;
+    return;
+  }
+
+  MIOpt.ReadToken();
+}
+
+bool PTHLexer::LexEndOfFile(Token &Result) {
+  // If we hit the end of the file while parsing a preprocessor directive,
+  // end the preprocessor directive first.  The next token returned will
+  // then be the end of file.
+  if (ParsingPreprocessorDirective) {
+    ParsingPreprocessorDirective = false; // Done parsing the "line".
+    return true;  // Have a token.
+  }
+  
+  assert(!LexingRawMode);
+
+  // If we are in a #if directive, emit an error.
+  while (!ConditionalStack.empty()) {
+    if (PP->getCodeCompletionFileLoc() != FileStartLoc)
+      PP->Diag(ConditionalStack.back().IfLoc,
+               diag::err_pp_unterminated_conditional);
+    ConditionalStack.pop_back();
+  }
+
+  // Finally, let the preprocessor handle this.
+  return PP->HandleEndOfFile(Result);
+}
+
+// FIXME: We can just grab the last token instead of storing a copy
+// into EofToken.
+void PTHLexer::getEOF(Token& Tok) {
+  assert(EofToken.is(tok::eof));
+  Tok = EofToken;
+}
+
+void PTHLexer::DiscardToEndOfLine() {
+  assert(ParsingPreprocessorDirective && ParsingFilename == false &&
+         "Must be in a preprocessing directive!");
+
+  // We assume that if the preprocessor wishes to discard to the end of
+  // the line that it also means to end the current preprocessor directive.
+  ParsingPreprocessorDirective = false;
+
+  // Skip tokens by only peeking at their token kind and the flags.
+  // We don't need to actually reconstruct full tokens from the token buffer.
+  // This saves some copies and it also reduces IdentifierInfo* lookup.
+  const unsigned char* p = CurPtr;
+  while (1) {
+    // Read the token kind.  Are we at the end of the file?
+    tok::TokenKind x = (tok::TokenKind) (uint8_t) *p;
+    if (x == tok::eof) break;
+
+    // Read the token flags.  Are we at the start of the next line?
+    Token::TokenFlags y = (Token::TokenFlags) (uint8_t) p[1];
+    if (y & Token::StartOfLine) break;
+
+    // Skip to the next token.
+    p += DISK_TOKEN_SIZE;
+  }
+
+  CurPtr = p;
+}
+
+/// SkipBlock - Used by Preprocessor to skip the current conditional block.
+bool PTHLexer::SkipBlock() {
+  assert(CurPPCondPtr && "No cached PP conditional information.");
+  assert(LastHashTokPtr && "No known '#' token.");
+
+  const unsigned char* HashEntryI = 0;
+  uint32_t Offset;
+  uint32_t TableIdx;
+
+  do {
+    // Read the token offset from the side-table.
+    Offset = ReadLE32(CurPPCondPtr);
+
+    // Read the target table index from the side-table.
+    TableIdx = ReadLE32(CurPPCondPtr);
+
+    // Compute the actual memory address of the '#' token data for this entry.
+    HashEntryI = TokBuf + Offset;
+
+    // Optmization: "Sibling jumping".  #if...#else...#endif blocks can
+    //  contain nested blocks.  In the side-table we can jump over these
+    //  nested blocks instead of doing a linear search if the next "sibling"
+    //  entry is not at a location greater than LastHashTokPtr.
+    if (HashEntryI < LastHashTokPtr && TableIdx) {
+      // In the side-table we are still at an entry for a '#' token that
+      // is earlier than the last one we saw.  Check if the location we would
+      // stride gets us closer.
+      const unsigned char* NextPPCondPtr =
+        PPCond + TableIdx*(sizeof(uint32_t)*2);
+      assert(NextPPCondPtr >= CurPPCondPtr);
+      // Read where we should jump to.
+      uint32_t TmpOffset = ReadLE32(NextPPCondPtr);
+      const unsigned char* HashEntryJ = TokBuf + TmpOffset;
+
+      if (HashEntryJ <= LastHashTokPtr) {
+        // Jump directly to the next entry in the side table.
+        HashEntryI = HashEntryJ;
+        Offset = TmpOffset;
+        TableIdx = ReadLE32(NextPPCondPtr);
+        CurPPCondPtr = NextPPCondPtr;
+      }
+    }
+  }
+  while (HashEntryI < LastHashTokPtr);
+  assert(HashEntryI == LastHashTokPtr && "No PP-cond entry found for '#'");
+  assert(TableIdx && "No jumping from #endifs.");
+
+  // Update our side-table iterator.
+  const unsigned char* NextPPCondPtr = PPCond + TableIdx*(sizeof(uint32_t)*2);
+  assert(NextPPCondPtr >= CurPPCondPtr);
+  CurPPCondPtr = NextPPCondPtr;
+
+  // Read where we should jump to.
+  HashEntryI = TokBuf + ReadLE32(NextPPCondPtr);
+  uint32_t NextIdx = ReadLE32(NextPPCondPtr);
+
+  // By construction NextIdx will be zero if this is a #endif.  This is useful
+  // to know to obviate lexing another token.
+  bool isEndif = NextIdx == 0;
+
+  // This case can occur when we see something like this:
+  //
+  //  #if ...
+  //   /* a comment or nothing */
+  //  #elif
+  //
+  // If we are skipping the first #if block it will be the case that CurPtr
+  // already points 'elif'.  Just return.
+
+  if (CurPtr > HashEntryI) {
+    assert(CurPtr == HashEntryI + DISK_TOKEN_SIZE);
+    // Did we reach a #endif?  If so, go ahead and consume that token as well.
+    if (isEndif)
+      CurPtr += DISK_TOKEN_SIZE*2;
+    else
+      LastHashTokPtr = HashEntryI;
+
+    return isEndif;
+  }
+
+  // Otherwise, we need to advance.  Update CurPtr to point to the '#' token.
+  CurPtr = HashEntryI;
+
+  // Update the location of the last observed '#'.  This is useful if we
+  // are skipping multiple blocks.
+  LastHashTokPtr = CurPtr;
+
+  // Skip the '#' token.
+  assert(((tok::TokenKind)*CurPtr) == tok::hash);
+  CurPtr += DISK_TOKEN_SIZE;
+
+  // Did we reach a #endif?  If so, go ahead and consume that token as well.
+  if (isEndif) { CurPtr += DISK_TOKEN_SIZE*2; }
+
+  return isEndif;
+}
+
+SourceLocation PTHLexer::getSourceLocation() {
+  // getSourceLocation is not on the hot path.  It is used to get the location
+  // of the next token when transitioning back to this lexer when done
+  // handling a #included file.  Just read the necessary data from the token
+  // data buffer to construct the SourceLocation object.
+  // NOTE: This is a virtual function; hence it is defined out-of-line.
+  const unsigned char *OffsetPtr = CurPtr + (DISK_TOKEN_SIZE - 4);
+  uint32_t Offset = ReadLE32(OffsetPtr);
+  return FileStartLoc.getLocWithOffset(Offset);
+}
+
+//===----------------------------------------------------------------------===//
+// PTH file lookup: map from strings to file data.
+//===----------------------------------------------------------------------===//
+
+/// PTHFileLookup - This internal data structure is used by the PTHManager
+///  to map from FileEntry objects managed by FileManager to offsets within
+///  the PTH file.
+namespace {
+class PTHFileData {
+  const uint32_t TokenOff;
+  const uint32_t PPCondOff;
+public:
+  PTHFileData(uint32_t tokenOff, uint32_t ppCondOff)
+    : TokenOff(tokenOff), PPCondOff(ppCondOff) {}
+
+  uint32_t getTokenOffset() const { return TokenOff; }
+  uint32_t getPPCondOffset() const { return PPCondOff; }
+};
+
+
+class PTHFileLookupCommonTrait {
+public:
+  typedef std::pair<unsigned char, const char*> internal_key_type;
+
+  static unsigned ComputeHash(internal_key_type x) {
+    return llvm::HashString(x.second);
+  }
+
+  static std::pair<unsigned, unsigned>
+  ReadKeyDataLength(const unsigned char*& d) {
+    unsigned keyLen = (unsigned) ReadUnalignedLE16(d);
+    unsigned dataLen = (unsigned) *(d++);
+    return std::make_pair(keyLen, dataLen);
+  }
+
+  static internal_key_type ReadKey(const unsigned char* d, unsigned) {
+    unsigned char k = *(d++); // Read the entry kind.
+    return std::make_pair(k, (const char*) d);
+  }
+};
+
+class PTHFileLookupTrait : public PTHFileLookupCommonTrait {
+public:
+  typedef const FileEntry* external_key_type;
+  typedef PTHFileData      data_type;
+
+  static internal_key_type GetInternalKey(const FileEntry* FE) {
+    return std::make_pair((unsigned char) 0x1, FE->getName());
+  }
+
+  static bool EqualKey(internal_key_type a, internal_key_type b) {
+    return a.first == b.first && strcmp(a.second, b.second) == 0;
+  }
+
+  static PTHFileData ReadData(const internal_key_type& k,
+                              const unsigned char* d, unsigned) {
+    assert(k.first == 0x1 && "Only file lookups can match!");
+    uint32_t x = ::ReadUnalignedLE32(d);
+    uint32_t y = ::ReadUnalignedLE32(d);
+    return PTHFileData(x, y);
+  }
+};
+
+class PTHStringLookupTrait {
+public:
+  typedef uint32_t
+          data_type;
+
+  typedef const std::pair<const char*, unsigned>
+          external_key_type;
+
+  typedef external_key_type internal_key_type;
+
+  static bool EqualKey(const internal_key_type& a,
+                       const internal_key_type& b) {
+    return (a.second == b.second) ? memcmp(a.first, b.first, a.second) == 0
+                                  : false;
+  }
+
+  static unsigned ComputeHash(const internal_key_type& a) {
+    return llvm::HashString(StringRef(a.first, a.second));
+  }
+
+  // This hopefully will just get inlined and removed by the optimizer.
+  static const internal_key_type&
+  GetInternalKey(const external_key_type& x) { return x; }
+
+  static std::pair<unsigned, unsigned>
+  ReadKeyDataLength(const unsigned char*& d) {
+    return std::make_pair((unsigned) ReadUnalignedLE16(d), sizeof(uint32_t));
+  }
+
+  static std::pair<const char*, unsigned>
+  ReadKey(const unsigned char* d, unsigned n) {
+      assert(n >= 2 && d[n-1] == '\0');
+      return std::make_pair((const char*) d, n-1);
+    }
+
+  static uint32_t ReadData(const internal_key_type& k, const unsigned char* d,
+                           unsigned) {
+    return ::ReadUnalignedLE32(d);
+  }
+};
+
+} // end anonymous namespace
+
+typedef OnDiskChainedHashTable<PTHFileLookupTrait>   PTHFileLookup;
+typedef OnDiskChainedHashTable<PTHStringLookupTrait> PTHStringIdLookup;
+
+//===----------------------------------------------------------------------===//
+// PTHManager methods.
+//===----------------------------------------------------------------------===//
+
+PTHManager::PTHManager(const llvm::MemoryBuffer* buf, void* fileLookup,
+                       const unsigned char* idDataTable,
+                       IdentifierInfo** perIDCache,
+                       void* stringIdLookup, unsigned numIds,
+                       const unsigned char* spellingBase,
+                       const char* originalSourceFile)
+: Buf(buf), PerIDCache(perIDCache), FileLookup(fileLookup),
+  IdDataTable(idDataTable), StringIdLookup(stringIdLookup),
+  NumIds(numIds), PP(0), SpellingBase(spellingBase),
+  OriginalSourceFile(originalSourceFile) {}
+
+PTHManager::~PTHManager() {
+  delete Buf;
+  delete (PTHFileLookup*) FileLookup;
+  delete (PTHStringIdLookup*) StringIdLookup;
+  free(PerIDCache);
+}
+
+static void InvalidPTH(DiagnosticsEngine &Diags, const char *Msg) {
+  Diags.Report(Diags.getCustomDiagID(DiagnosticsEngine::Error, Msg));
+}
+
+PTHManager *PTHManager::Create(const std::string &file,
+                               DiagnosticsEngine &Diags) {
+  // Memory map the PTH file.
+  OwningPtr<llvm::MemoryBuffer> File;
+
+  if (llvm::MemoryBuffer::getFile(file, File)) {
+    // FIXME: Add ec.message() to this diag.
+    Diags.Report(diag::err_invalid_pth_file) << file;
+    return 0;
+  }
+
+  // Get the buffer ranges and check if there are at least three 32-bit
+  // words at the end of the file.
+  const unsigned char *BufBeg = (unsigned char*)File->getBufferStart();
+  const unsigned char *BufEnd = (unsigned char*)File->getBufferEnd();
+
+  // Check the prologue of the file.
+  if ((BufEnd - BufBeg) < (signed)(sizeof("cfe-pth") + 3 + 4) ||
+      memcmp(BufBeg, "cfe-pth", sizeof("cfe-pth") - 1) != 0) {
+    Diags.Report(diag::err_invalid_pth_file) << file;
+    return 0;
+  }
+
+  // Read the PTH version.
+  const unsigned char *p = BufBeg + (sizeof("cfe-pth") - 1);
+  unsigned Version = ReadLE32(p);
+
+  if (Version < PTHManager::Version) {
+    InvalidPTH(Diags,
+        Version < PTHManager::Version
+        ? "PTH file uses an older PTH format that is no longer supported"
+        : "PTH file uses a newer PTH format that cannot be read");
+    return 0;
+  }
+
+  // Compute the address of the index table at the end of the PTH file.
+  const unsigned char *PrologueOffset = p;
+
+  if (PrologueOffset >= BufEnd) {
+    Diags.Report(diag::err_invalid_pth_file) << file;
+    return 0;
+  }
+
+  // Construct the file lookup table.  This will be used for mapping from
+  // FileEntry*'s to cached tokens.
+  const unsigned char* FileTableOffset = PrologueOffset + sizeof(uint32_t)*2;
+  const unsigned char* FileTable = BufBeg + ReadLE32(FileTableOffset);
+
+  if (!(FileTable > BufBeg && FileTable < BufEnd)) {
+    Diags.Report(diag::err_invalid_pth_file) << file;
+    return 0; // FIXME: Proper error diagnostic?
+  }
+
+  OwningPtr<PTHFileLookup> FL(PTHFileLookup::Create(FileTable, BufBeg));
+
+  // Warn if the PTH file is empty.  We still want to create a PTHManager
+  // as the PTH could be used with -include-pth.
+  if (FL->isEmpty())
+    InvalidPTH(Diags, "PTH file contains no cached source data");
+
+  // Get the location of the table mapping from persistent ids to the
+  // data needed to reconstruct identifiers.
+  const unsigned char* IDTableOffset = PrologueOffset + sizeof(uint32_t)*0;
+  const unsigned char* IData = BufBeg + ReadLE32(IDTableOffset);
+
+  if (!(IData >= BufBeg && IData < BufEnd)) {
+    Diags.Report(diag::err_invalid_pth_file) << file;
+    return 0;
+  }
+
+  // Get the location of the hashtable mapping between strings and
+  // persistent IDs.
+  const unsigned char* StringIdTableOffset = PrologueOffset + sizeof(uint32_t)*1;
+  const unsigned char* StringIdTable = BufBeg + ReadLE32(StringIdTableOffset);
+  if (!(StringIdTable >= BufBeg && StringIdTable < BufEnd)) {
+    Diags.Report(diag::err_invalid_pth_file) << file;
+    return 0;
+  }
+
+  OwningPtr<PTHStringIdLookup> SL(PTHStringIdLookup::Create(StringIdTable,
+                                                                  BufBeg));
+
+  // Get the location of the spelling cache.
+  const unsigned char* spellingBaseOffset = PrologueOffset + sizeof(uint32_t)*3;
+  const unsigned char* spellingBase = BufBeg + ReadLE32(spellingBaseOffset);
+  if (!(spellingBase >= BufBeg && spellingBase < BufEnd)) {
+    Diags.Report(diag::err_invalid_pth_file) << file;
+    return 0;
+  }
+
+  // Get the number of IdentifierInfos and pre-allocate the identifier cache.
+  uint32_t NumIds = ReadLE32(IData);
+
+  // Pre-allocate the persistent ID -> IdentifierInfo* cache.  We use calloc()
+  // so that we in the best case only zero out memory once when the OS returns
+  // us new pages.
+  IdentifierInfo** PerIDCache = 0;
+
+  if (NumIds) {
+    PerIDCache = (IdentifierInfo**)calloc(NumIds, sizeof(*PerIDCache));
+    if (!PerIDCache) {
+      InvalidPTH(Diags, "Could not allocate memory for processing PTH file");
+      return 0;
+    }
+  }
+
+  // Compute the address of the original source file.
+  const unsigned char* originalSourceBase = PrologueOffset + sizeof(uint32_t)*4;
+  unsigned len = ReadUnalignedLE16(originalSourceBase);
+  if (!len) originalSourceBase = 0;
+
+  // Create the new PTHManager.
+  return new PTHManager(File.take(), FL.take(), IData, PerIDCache,
+                        SL.take(), NumIds, spellingBase,
+                        (const char*) originalSourceBase);
+}
+
+IdentifierInfo* PTHManager::LazilyCreateIdentifierInfo(unsigned PersistentID) {
+  // Look in the PTH file for the string data for the IdentifierInfo object.
+  const unsigned char* TableEntry = IdDataTable + sizeof(uint32_t)*PersistentID;
+  const unsigned char* IDData =
+    (const unsigned char*)Buf->getBufferStart() + ReadLE32(TableEntry);
+  assert(IDData < (const unsigned char*)Buf->getBufferEnd());
+
+  // Allocate the object.
+  std::pair<IdentifierInfo,const unsigned char*> *Mem =
+    Alloc.Allocate<std::pair<IdentifierInfo,const unsigned char*> >();
+
+  Mem->second = IDData;
+  assert(IDData[0] != '\0');
+  IdentifierInfo *II = new ((void*) Mem) IdentifierInfo();
+
+  // Store the new IdentifierInfo in the cache.
+  PerIDCache[PersistentID] = II;
+  assert(II->getNameStart() && II->getNameStart()[0] != '\0');
+  return II;
+}
+
+IdentifierInfo* PTHManager::get(StringRef Name) {
+  PTHStringIdLookup& SL = *((PTHStringIdLookup*)StringIdLookup);
+  // Double check our assumption that the last character isn't '\0'.
+  assert(Name.empty() || Name.back() != '\0');
+  PTHStringIdLookup::iterator I = SL.find(std::make_pair(Name.data(),
+                                                         Name.size()));
+  if (I == SL.end()) // No identifier found?
+    return 0;
+
+  // Match found.  Return the identifier!
+  assert(*I > 0);
+  return GetIdentifierInfo(*I-1);
+}
+
+PTHLexer *PTHManager::CreateLexer(FileID FID) {
+  const FileEntry *FE = PP->getSourceManager().getFileEntryForID(FID);
+  if (!FE)
+    return 0;
+
+  // Lookup the FileEntry object in our file lookup data structure.  It will
+  // return a variant that indicates whether or not there is an offset within
+  // the PTH file that contains cached tokens.
+  PTHFileLookup& PFL = *((PTHFileLookup*)FileLookup);
+  PTHFileLookup::iterator I = PFL.find(FE);
+
+  if (I == PFL.end()) // No tokens available?
+    return 0;
+
+  const PTHFileData& FileData = *I;
+
+  const unsigned char *BufStart = (const unsigned char *)Buf->getBufferStart();
+  // Compute the offset of the token data within the buffer.
+  const unsigned char* data = BufStart + FileData.getTokenOffset();
+
+  // Get the location of pp-conditional table.
+  const unsigned char* ppcond = BufStart + FileData.getPPCondOffset();
+  uint32_t Len = ReadLE32(ppcond);
+  if (Len == 0) ppcond = 0;
+
+  assert(PP && "No preprocessor set yet!");
+  return new PTHLexer(*PP, FID, data, ppcond, *this);
+}
+
+//===----------------------------------------------------------------------===//
+// 'stat' caching.
+//===----------------------------------------------------------------------===//
+
+namespace {
+class PTHStatData {
+public:
+  const bool hasStat;
+  const ino_t ino;
+  const dev_t dev;
+  const mode_t mode;
+  const time_t mtime;
+  const off_t size;
+
+  PTHStatData(ino_t i, dev_t d, mode_t mo, time_t m, off_t s)
+  : hasStat(true), ino(i), dev(d), mode(mo), mtime(m), size(s) {}
+
+  PTHStatData()
+    : hasStat(false), ino(0), dev(0), mode(0), mtime(0), size(0) {}
+};
+
+class PTHStatLookupTrait : public PTHFileLookupCommonTrait {
+public:
+  typedef const char* external_key_type;  // const char*
+  typedef PTHStatData data_type;
+
+  static internal_key_type GetInternalKey(const char *path) {
+    // The key 'kind' doesn't matter here because it is ignored in EqualKey.
+    return std::make_pair((unsigned char) 0x0, path);
+  }
+
+  static bool EqualKey(internal_key_type a, internal_key_type b) {
+    // When doing 'stat' lookups we don't care about the kind of 'a' and 'b',
+    // just the paths.
+    return strcmp(a.second, b.second) == 0;
+  }
+
+  static data_type ReadData(const internal_key_type& k, const unsigned char* d,
+                            unsigned) {
+
+    if (k.first /* File or Directory */) {
+      if (k.first == 0x1 /* File */) d += 4 * 2; // Skip the first 2 words.
+      ino_t ino = (ino_t) ReadUnalignedLE32(d);
+      dev_t dev = (dev_t) ReadUnalignedLE32(d);
+      mode_t mode = (mode_t) ReadUnalignedLE16(d);
+      time_t mtime = (time_t) ReadUnalignedLE64(d);
+      return data_type(ino, dev, mode, mtime, (off_t) ReadUnalignedLE64(d));
+    }
+
+    // Negative stat.  Don't read anything.
+    return data_type();
+  }
+};
+
+class PTHStatCache : public FileSystemStatCache {
+  typedef OnDiskChainedHashTable<PTHStatLookupTrait> CacheTy;
+  CacheTy Cache;
+
+public:
+  PTHStatCache(PTHFileLookup &FL) :
+    Cache(FL.getNumBuckets(), FL.getNumEntries(), FL.getBuckets(),
+          FL.getBase()) {}
+
+  ~PTHStatCache() {}
+
+  LookupResult getStat(const char *Path, struct stat &StatBuf,
+                       int *FileDescriptor) {
+    // Do the lookup for the file's data in the PTH file.
+    CacheTy::iterator I = Cache.find(Path);
+
+    // If we don't get a hit in the PTH file just forward to 'stat'.
+    if (I == Cache.end())
+      return statChained(Path, StatBuf, FileDescriptor);
+
+    const PTHStatData &Data = *I;
+
+    if (!Data.hasStat)
+      return CacheMissing;
+
+    StatBuf.st_ino = Data.ino;
+    StatBuf.st_dev = Data.dev;
+    StatBuf.st_mtime = Data.mtime;
+    StatBuf.st_mode = Data.mode;
+    StatBuf.st_size = Data.size;
+    return CacheExists;
+  }
+};
+} // end anonymous namespace
+
+FileSystemStatCache *PTHManager::createStatCache() {
+  return new PTHStatCache(*((PTHFileLookup*) FileLookup));
+}
diff --git a/clang/lib/Lex/Pragma.cpp b/clang/lib/Lex/Pragma.cpp
new file mode 100644
index 0000000..e2a192b
--- /dev/null
+++ b/clang/lib/Lex/Pragma.cpp
@@ -0,0 +1,1292 @@
+//===--- Pragma.cpp - Pragma registration and handling --------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the PragmaHandler/PragmaTable interfaces and implements
+// pragma related methods of the Preprocessor class.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Lex/Pragma.h"
+#include "clang/Lex/HeaderSearch.h"
+#include "clang/Lex/LiteralSupport.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/MacroInfo.h"
+#include "clang/Lex/LexDiagnostic.h"
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/SourceManager.h"
+#include "llvm/Support/CrashRecoveryContext.h"
+#include "llvm/Support/ErrorHandling.h"
+#include <algorithm>
+using namespace clang;
+
+// Out-of-line destructor to provide a home for the class.
+PragmaHandler::~PragmaHandler() {
+}
+
+//===----------------------------------------------------------------------===//
+// EmptyPragmaHandler Implementation.
+//===----------------------------------------------------------------------===//
+
+EmptyPragmaHandler::EmptyPragmaHandler() {}
+
+void EmptyPragmaHandler::HandlePragma(Preprocessor &PP, 
+                                      PragmaIntroducerKind Introducer,
+                                      Token &FirstToken) {}
+
+//===----------------------------------------------------------------------===//
+// PragmaNamespace Implementation.
+//===----------------------------------------------------------------------===//
+
+
+PragmaNamespace::~PragmaNamespace() {
+  for (llvm::StringMap<PragmaHandler*>::iterator
+         I = Handlers.begin(), E = Handlers.end(); I != E; ++I)
+    delete I->second;
+}
+
+/// FindHandler - Check to see if there is already a handler for the
+/// specified name.  If not, return the handler for the null identifier if it
+/// exists, otherwise return null.  If IgnoreNull is true (the default) then
+/// the null handler isn't returned on failure to match.
+PragmaHandler *PragmaNamespace::FindHandler(StringRef Name,
+                                            bool IgnoreNull) const {
+  if (PragmaHandler *Handler = Handlers.lookup(Name))
+    return Handler;
+  return IgnoreNull ? 0 : Handlers.lookup(StringRef());
+}
+
+void PragmaNamespace::AddPragma(PragmaHandler *Handler) {
+  assert(!Handlers.lookup(Handler->getName()) &&
+         "A handler with this name is already registered in this namespace");
+  llvm::StringMapEntry<PragmaHandler *> &Entry =
+    Handlers.GetOrCreateValue(Handler->getName());
+  Entry.setValue(Handler);
+}
+
+void PragmaNamespace::RemovePragmaHandler(PragmaHandler *Handler) {
+  assert(Handlers.lookup(Handler->getName()) &&
+         "Handler not registered in this namespace");
+  Handlers.erase(Handler->getName());
+}
+
+void PragmaNamespace::HandlePragma(Preprocessor &PP, 
+                                   PragmaIntroducerKind Introducer,
+                                   Token &Tok) {
+  // Read the 'namespace' that the directive is in, e.g. STDC.  Do not macro
+  // expand it, the user can have a STDC #define, that should not affect this.
+  PP.LexUnexpandedToken(Tok);
+
+  // Get the handler for this token.  If there is no handler, ignore the pragma.
+  PragmaHandler *Handler
+    = FindHandler(Tok.getIdentifierInfo() ? Tok.getIdentifierInfo()->getName()
+                                          : StringRef(),
+                  /*IgnoreNull=*/false);
+  if (Handler == 0) {
+    PP.Diag(Tok, diag::warn_pragma_ignored);
+    return;
+  }
+
+  // Otherwise, pass it down.
+  Handler->HandlePragma(PP, Introducer, Tok);
+}
+
+//===----------------------------------------------------------------------===//
+// Preprocessor Pragma Directive Handling.
+//===----------------------------------------------------------------------===//
+
+/// HandlePragmaDirective - The "#pragma" directive has been parsed.  Lex the
+/// rest of the pragma, passing it to the registered pragma handlers.
+void Preprocessor::HandlePragmaDirective(unsigned Introducer) {
+  ++NumPragma;
+
+  // Invoke the first level of pragma handlers which reads the namespace id.
+  Token Tok;
+  PragmaHandlers->HandlePragma(*this, PragmaIntroducerKind(Introducer), Tok);
+
+  // If the pragma handler didn't read the rest of the line, consume it now.
+  if ((CurTokenLexer && CurTokenLexer->isParsingPreprocessorDirective()) 
+   || (CurPPLexer && CurPPLexer->ParsingPreprocessorDirective))
+    DiscardUntilEndOfDirective();
+}
+
+namespace {
+/// \brief Helper class for \see Preprocessor::Handle_Pragma.
+class LexingFor_PragmaRAII {
+  Preprocessor &PP;
+  bool InMacroArgPreExpansion;
+  bool Failed;
+  Token &OutTok;
+  Token PragmaTok;
+
+public:
+  LexingFor_PragmaRAII(Preprocessor &PP, bool InMacroArgPreExpansion,
+                       Token &Tok)
+    : PP(PP), InMacroArgPreExpansion(InMacroArgPreExpansion),
+      Failed(false), OutTok(Tok) {
+    if (InMacroArgPreExpansion) {
+      PragmaTok = OutTok;
+      PP.EnableBacktrackAtThisPos();
+    }
+  }
+
+  ~LexingFor_PragmaRAII() {
+    if (InMacroArgPreExpansion) {
+      if (Failed) {
+        PP.CommitBacktrackedTokens();
+      } else {
+        PP.Backtrack();
+        OutTok = PragmaTok;
+      }
+    }
+  }
+
+  void failed() {
+    Failed = true;
+  }
+};
+}
+
+/// Handle_Pragma - Read a _Pragma directive, slice it up, process it, then
+/// return the first token after the directive.  The _Pragma token has just
+/// been read into 'Tok'.
+void Preprocessor::Handle_Pragma(Token &Tok) {
+
+  // This works differently if we are pre-expanding a macro argument.
+  // In that case we don't actually "activate" the pragma now, we only lex it
+  // until we are sure it is lexically correct and then we backtrack so that
+  // we activate the pragma whenever we encounter the tokens again in the token
+  // stream. This ensures that we will activate it in the correct location
+  // or that we will ignore it if it never enters the token stream, e.g:
+  //
+  //     #define EMPTY(x)
+  //     #define INACTIVE(x) EMPTY(x)
+  //     INACTIVE(_Pragma("clang diagnostic ignored \"-Wconversion\""))
+
+  LexingFor_PragmaRAII _PragmaLexing(*this, InMacroArgPreExpansion, Tok);
+
+  // Remember the pragma token location.
+  SourceLocation PragmaLoc = Tok.getLocation();
+
+  // Read the '('.
+  Lex(Tok);
+  if (Tok.isNot(tok::l_paren)) {
+    Diag(PragmaLoc, diag::err__Pragma_malformed);
+    return _PragmaLexing.failed();
+  }
+
+  // Read the '"..."'.
+  Lex(Tok);
+  if (Tok.isNot(tok::string_literal) && Tok.isNot(tok::wide_string_literal)) {
+    Diag(PragmaLoc, diag::err__Pragma_malformed);
+    // Skip this token, and the ')', if present.
+    if (Tok.isNot(tok::r_paren))
+      Lex(Tok);
+    if (Tok.is(tok::r_paren))
+      Lex(Tok);
+    return _PragmaLexing.failed();
+  }
+
+  if (Tok.hasUDSuffix()) {
+    Diag(Tok, diag::err_invalid_string_udl);
+    // Skip this token, and the ')', if present.
+    Lex(Tok);
+    if (Tok.is(tok::r_paren))
+      Lex(Tok);
+    return _PragmaLexing.failed();
+  }
+
+  // Remember the string.
+  Token StrTok = Tok;
+
+  // Read the ')'.
+  Lex(Tok);
+  if (Tok.isNot(tok::r_paren)) {
+    Diag(PragmaLoc, diag::err__Pragma_malformed);
+    return _PragmaLexing.failed();
+  }
+
+  if (InMacroArgPreExpansion)
+    return;
+
+  SourceLocation RParenLoc = Tok.getLocation();
+  std::string StrVal = getSpelling(StrTok);
+
+  // The _Pragma is lexically sound.  Destringize according to C99 6.10.9.1:
+  // "The string literal is destringized by deleting the L prefix, if present,
+  // deleting the leading and trailing double-quotes, replacing each escape
+  // sequence \" by a double-quote, and replacing each escape sequence \\ by a
+  // single backslash."
+  if (StrVal[0] == 'L')  // Remove L prefix.
+    StrVal.erase(StrVal.begin());
+  assert(StrVal[0] == '"' && StrVal[StrVal.size()-1] == '"' &&
+         "Invalid string token!");
+
+  // Remove the front quote, replacing it with a space, so that the pragma
+  // contents appear to have a space before them.
+  StrVal[0] = ' ';
+
+  // Replace the terminating quote with a \n.
+  StrVal[StrVal.size()-1] = '\n';
+
+  // Remove escaped quotes and escapes.
+  for (unsigned i = 0, e = StrVal.size(); i != e-1; ++i) {
+    if (StrVal[i] == '\\' &&
+        (StrVal[i+1] == '\\' || StrVal[i+1] == '"')) {
+      // \\ -> '\' and \" -> '"'.
+      StrVal.erase(StrVal.begin()+i);
+      --e;
+    }
+  }
+  
+  // Plop the string (including the newline and trailing null) into a buffer
+  // where we can lex it.
+  Token TmpTok;
+  TmpTok.startToken();
+  CreateString(&StrVal[0], StrVal.size(), TmpTok);
+  SourceLocation TokLoc = TmpTok.getLocation();
+
+  // Make and enter a lexer object so that we lex and expand the tokens just
+  // like any others.
+  Lexer *TL = Lexer::Create_PragmaLexer(TokLoc, PragmaLoc, RParenLoc,
+                                        StrVal.size(), *this);
+
+  EnterSourceFileWithLexer(TL, 0);
+
+  // With everything set up, lex this as a #pragma directive.
+  HandlePragmaDirective(PIK__Pragma);
+
+  // Finally, return whatever came after the pragma directive.
+  return Lex(Tok);
+}
+
+/// HandleMicrosoft__pragma - Like Handle_Pragma except the pragma text
+/// is not enclosed within a string literal.
+void Preprocessor::HandleMicrosoft__pragma(Token &Tok) {
+  // Remember the pragma token location.
+  SourceLocation PragmaLoc = Tok.getLocation();
+
+  // Read the '('.
+  Lex(Tok);
+  if (Tok.isNot(tok::l_paren)) {
+    Diag(PragmaLoc, diag::err__Pragma_malformed);
+    return;
+  }
+
+  // Get the tokens enclosed within the __pragma(), as well as the final ')'.
+  SmallVector<Token, 32> PragmaToks;
+  int NumParens = 0;
+  Lex(Tok);
+  while (Tok.isNot(tok::eof)) {
+    PragmaToks.push_back(Tok);
+    if (Tok.is(tok::l_paren))
+      NumParens++;
+    else if (Tok.is(tok::r_paren) && NumParens-- == 0)
+      break;
+    Lex(Tok);
+  }
+
+  if (Tok.is(tok::eof)) {
+    Diag(PragmaLoc, diag::err_unterminated___pragma);
+    return;
+  }
+
+  PragmaToks.front().setFlag(Token::LeadingSpace);
+
+  // Replace the ')' with an EOD to mark the end of the pragma.
+  PragmaToks.back().setKind(tok::eod);
+
+  Token *TokArray = new Token[PragmaToks.size()];
+  std::copy(PragmaToks.begin(), PragmaToks.end(), TokArray);
+
+  // Push the tokens onto the stack.
+  EnterTokenStream(TokArray, PragmaToks.size(), true, true);
+
+  // With everything set up, lex this as a #pragma directive.
+  HandlePragmaDirective(PIK___pragma);
+
+  // Finally, return whatever came after the pragma directive.
+  return Lex(Tok);
+}
+
+/// HandlePragmaOnce - Handle #pragma once.  OnceTok is the 'once'.
+///
+void Preprocessor::HandlePragmaOnce(Token &OnceTok) {
+  if (isInPrimaryFile()) {
+    Diag(OnceTok, diag::pp_pragma_once_in_main_file);
+    return;
+  }
+
+  // Get the current file lexer we're looking at.  Ignore _Pragma 'files' etc.
+  // Mark the file as a once-only file now.
+  HeaderInfo.MarkFileIncludeOnce(getCurrentFileLexer()->getFileEntry());
+}
+
+void Preprocessor::HandlePragmaMark() {
+  assert(CurPPLexer && "No current lexer?");
+  if (CurLexer)
+    CurLexer->ReadToEndOfLine();
+  else
+    CurPTHLexer->DiscardToEndOfLine();
+}
+
+
+/// HandlePragmaPoison - Handle #pragma GCC poison.  PoisonTok is the 'poison'.
+///
+void Preprocessor::HandlePragmaPoison(Token &PoisonTok) {
+  Token Tok;
+
+  while (1) {
+    // Read the next token to poison.  While doing this, pretend that we are
+    // skipping while reading the identifier to poison.
+    // This avoids errors on code like:
+    //   #pragma GCC poison X
+    //   #pragma GCC poison X
+    if (CurPPLexer) CurPPLexer->LexingRawMode = true;
+    LexUnexpandedToken(Tok);
+    if (CurPPLexer) CurPPLexer->LexingRawMode = false;
+
+    // If we reached the end of line, we're done.
+    if (Tok.is(tok::eod)) return;
+
+    // Can only poison identifiers.
+    if (Tok.isNot(tok::raw_identifier)) {
+      Diag(Tok, diag::err_pp_invalid_poison);
+      return;
+    }
+
+    // Look up the identifier info for the token.  We disabled identifier lookup
+    // by saying we're skipping contents, so we need to do this manually.
+    IdentifierInfo *II = LookUpIdentifierInfo(Tok);
+
+    // Already poisoned.
+    if (II->isPoisoned()) continue;
+
+    // If this is a macro identifier, emit a warning.
+    if (II->hasMacroDefinition())
+      Diag(Tok, diag::pp_poisoning_existing_macro);
+
+    // Finally, poison it!
+    II->setIsPoisoned();
+    if (II->isFromAST())
+      II->setChangedSinceDeserialization();
+  }
+}
+
+/// HandlePragmaSystemHeader - Implement #pragma GCC system_header.  We know
+/// that the whole directive has been parsed.
+void Preprocessor::HandlePragmaSystemHeader(Token &SysHeaderTok) {
+  if (isInPrimaryFile()) {
+    Diag(SysHeaderTok, diag::pp_pragma_sysheader_in_main_file);
+    return;
+  }
+
+  // Get the current file lexer we're looking at.  Ignore _Pragma 'files' etc.
+  PreprocessorLexer *TheLexer = getCurrentFileLexer();
+
+  // Mark the file as a system header.
+  HeaderInfo.MarkFileSystemHeader(TheLexer->getFileEntry());
+
+
+  PresumedLoc PLoc = SourceMgr.getPresumedLoc(SysHeaderTok.getLocation());
+  if (PLoc.isInvalid())
+    return;
+  
+  unsigned FilenameID = SourceMgr.getLineTableFilenameID(PLoc.getFilename());
+
+  // Notify the client, if desired, that we are in a new source file.
+  if (Callbacks)
+    Callbacks->FileChanged(SysHeaderTok.getLocation(),
+                           PPCallbacks::SystemHeaderPragma, SrcMgr::C_System);
+
+  // Emit a line marker.  This will change any source locations from this point
+  // forward to realize they are in a system header.
+  // Create a line note with this information.
+  SourceMgr.AddLineNote(SysHeaderTok.getLocation(), PLoc.getLine(), FilenameID,
+                        false, false, true, false);
+}
+
+/// HandlePragmaDependency - Handle #pragma GCC dependency "foo" blah.
+///
+void Preprocessor::HandlePragmaDependency(Token &DependencyTok) {
+  Token FilenameTok;
+  CurPPLexer->LexIncludeFilename(FilenameTok);
+
+  // If the token kind is EOD, the error has already been diagnosed.
+  if (FilenameTok.is(tok::eod))
+    return;
+
+  // Reserve a buffer to get the spelling.
+  SmallString<128> FilenameBuffer;
+  bool Invalid = false;
+  StringRef Filename = getSpelling(FilenameTok, FilenameBuffer, &Invalid);
+  if (Invalid)
+    return;
+
+  bool isAngled =
+    GetIncludeFilenameSpelling(FilenameTok.getLocation(), Filename);
+  // If GetIncludeFilenameSpelling set the start ptr to null, there was an
+  // error.
+  if (Filename.empty())
+    return;
+
+  // Search include directories for this file.
+  const DirectoryLookup *CurDir;
+  const FileEntry *File = LookupFile(Filename, isAngled, 0, CurDir, NULL, NULL,
+                                     NULL);
+  if (File == 0) {
+    if (!SuppressIncludeNotFoundError)
+      Diag(FilenameTok, diag::err_pp_file_not_found) << Filename;
+    return;
+  }
+
+  const FileEntry *CurFile = getCurrentFileLexer()->getFileEntry();
+
+  // If this file is older than the file it depends on, emit a diagnostic.
+  if (CurFile && CurFile->getModificationTime() < File->getModificationTime()) {
+    // Lex tokens at the end of the message and include them in the message.
+    std::string Message;
+    Lex(DependencyTok);
+    while (DependencyTok.isNot(tok::eod)) {
+      Message += getSpelling(DependencyTok) + " ";
+      Lex(DependencyTok);
+    }
+
+    // Remove the trailing ' ' if present.
+    if (!Message.empty())
+      Message.erase(Message.end()-1);
+    Diag(FilenameTok, diag::pp_out_of_date_dependency) << Message;
+  }
+}
+
+/// HandlePragmaComment - Handle the microsoft #pragma comment extension.  The
+/// syntax is:
+///   #pragma comment(linker, "foo")
+/// 'linker' is one of five identifiers: compiler, exestr, lib, linker, user.
+/// "foo" is a string, which is fully macro expanded, and permits string
+/// concatenation, embedded escape characters etc.  See MSDN for more details.
+void Preprocessor::HandlePragmaComment(Token &Tok) {
+  SourceLocation CommentLoc = Tok.getLocation();
+  Lex(Tok);
+  if (Tok.isNot(tok::l_paren)) {
+    Diag(CommentLoc, diag::err_pragma_comment_malformed);
+    return;
+  }
+
+  // Read the identifier.
+  Lex(Tok);
+  if (Tok.isNot(tok::identifier)) {
+    Diag(CommentLoc, diag::err_pragma_comment_malformed);
+    return;
+  }
+
+  // Verify that this is one of the 5 whitelisted options.
+  // FIXME: warn that 'exestr' is deprecated.
+  const IdentifierInfo *II = Tok.getIdentifierInfo();
+  if (!II->isStr("compiler") && !II->isStr("exestr") && !II->isStr("lib") &&
+      !II->isStr("linker") && !II->isStr("user")) {
+    Diag(Tok.getLocation(), diag::err_pragma_comment_unknown_kind);
+    return;
+  }
+
+  // Read the optional string if present.
+  Lex(Tok);
+  std::string ArgumentString;
+  if (Tok.is(tok::comma)) {
+    Lex(Tok); // eat the comma.
+
+    // We need at least one string.
+    if (Tok.isNot(tok::string_literal)) {
+      Diag(Tok.getLocation(), diag::err_pragma_comment_malformed);
+      return;
+    }
+
+    // String concatenation allows multiple strings, which can even come from
+    // macro expansion.
+    // "foo " "bar" "Baz"
+    SmallVector<Token, 4> StrToks;
+    while (Tok.is(tok::string_literal)) {
+      if (Tok.hasUDSuffix())
+        Diag(Tok, diag::err_invalid_string_udl);
+      StrToks.push_back(Tok);
+      Lex(Tok);
+    }
+
+    // Concatenate and parse the strings.
+    StringLiteralParser Literal(&StrToks[0], StrToks.size(), *this);
+    assert(Literal.isAscii() && "Didn't allow wide strings in");
+    if (Literal.hadError)
+      return;
+    if (Literal.Pascal) {
+      Diag(StrToks[0].getLocation(), diag::err_pragma_comment_malformed);
+      return;
+    }
+
+    ArgumentString = Literal.GetString();
+  }
+
+  // FIXME: If the kind is "compiler" warn if the string is present (it is
+  // ignored).
+  // FIXME: 'lib' requires a comment string.
+  // FIXME: 'linker' requires a comment string, and has a specific list of
+  // things that are allowable.
+
+  if (Tok.isNot(tok::r_paren)) {
+    Diag(Tok.getLocation(), diag::err_pragma_comment_malformed);
+    return;
+  }
+  Lex(Tok);  // eat the r_paren.
+
+  if (Tok.isNot(tok::eod)) {
+    Diag(Tok.getLocation(), diag::err_pragma_comment_malformed);
+    return;
+  }
+
+  // If the pragma is lexically sound, notify any interested PPCallbacks.
+  if (Callbacks)
+    Callbacks->PragmaComment(CommentLoc, II, ArgumentString);
+}
+
+/// HandlePragmaMessage - Handle the microsoft and gcc #pragma message
+/// extension.  The syntax is:
+///   #pragma message(string)
+/// OR, in GCC mode:
+///   #pragma message string
+/// string is a string, which is fully macro expanded, and permits string
+/// concatenation, embedded escape characters, etc... See MSDN for more details.
+void Preprocessor::HandlePragmaMessage(Token &Tok) {
+  SourceLocation MessageLoc = Tok.getLocation();
+  Lex(Tok);
+  bool ExpectClosingParen = false;
+  switch (Tok.getKind()) {
+  case tok::l_paren:
+    // We have a MSVC style pragma message.
+    ExpectClosingParen = true;
+    // Read the string.
+    Lex(Tok);
+    break;
+  case tok::string_literal:
+    // We have a GCC style pragma message, and we just read the string.
+    break;
+  default:
+    Diag(MessageLoc, diag::err_pragma_message_malformed);
+    return;
+  }
+
+  // We need at least one string.
+  if (Tok.isNot(tok::string_literal)) {
+    Diag(Tok.getLocation(), diag::err_pragma_message_malformed);
+    return;
+  }
+
+  // String concatenation allows multiple strings, which can even come from
+  // macro expansion.
+  // "foo " "bar" "Baz"
+  SmallVector<Token, 4> StrToks;
+  while (Tok.is(tok::string_literal)) {
+    if (Tok.hasUDSuffix())
+      Diag(Tok, diag::err_invalid_string_udl);
+    StrToks.push_back(Tok);
+    Lex(Tok);
+  }
+
+  // Concatenate and parse the strings.
+  StringLiteralParser Literal(&StrToks[0], StrToks.size(), *this);
+  assert(Literal.isAscii() && "Didn't allow wide strings in");
+  if (Literal.hadError)
+    return;
+  if (Literal.Pascal) {
+    Diag(StrToks[0].getLocation(), diag::err_pragma_message_malformed);
+    return;
+  }
+
+  StringRef MessageString(Literal.GetString());
+
+  if (ExpectClosingParen) {
+    if (Tok.isNot(tok::r_paren)) {
+      Diag(Tok.getLocation(), diag::err_pragma_message_malformed);
+      return;
+    }
+    Lex(Tok);  // eat the r_paren.
+  }
+
+  if (Tok.isNot(tok::eod)) {
+    Diag(Tok.getLocation(), diag::err_pragma_message_malformed);
+    return;
+  }
+
+  // Output the message.
+  Diag(MessageLoc, diag::warn_pragma_message) << MessageString;
+
+  // If the pragma is lexically sound, notify any interested PPCallbacks.
+  if (Callbacks)
+    Callbacks->PragmaMessage(MessageLoc, MessageString);
+}
+
+/// ParsePragmaPushOrPopMacro - Handle parsing of pragma push_macro/pop_macro.  
+/// Return the IdentifierInfo* associated with the macro to push or pop.
+IdentifierInfo *Preprocessor::ParsePragmaPushOrPopMacro(Token &Tok) {
+  // Remember the pragma token location.
+  Token PragmaTok = Tok;
+
+  // Read the '('.
+  Lex(Tok);
+  if (Tok.isNot(tok::l_paren)) {
+    Diag(PragmaTok.getLocation(), diag::err_pragma_push_pop_macro_malformed)
+      << getSpelling(PragmaTok);
+    return 0;
+  }
+
+  // Read the macro name string.
+  Lex(Tok);
+  if (Tok.isNot(tok::string_literal)) {
+    Diag(PragmaTok.getLocation(), diag::err_pragma_push_pop_macro_malformed)
+      << getSpelling(PragmaTok);
+    return 0;
+  }
+
+  if (Tok.hasUDSuffix()) {
+    Diag(Tok, diag::err_invalid_string_udl);
+    return 0;
+  }
+
+  // Remember the macro string.
+  std::string StrVal = getSpelling(Tok);
+
+  // Read the ')'.
+  Lex(Tok);
+  if (Tok.isNot(tok::r_paren)) {
+    Diag(PragmaTok.getLocation(), diag::err_pragma_push_pop_macro_malformed)
+      << getSpelling(PragmaTok);
+    return 0;
+  }
+
+  assert(StrVal[0] == '"' && StrVal[StrVal.size()-1] == '"' &&
+         "Invalid string token!");
+
+  // Create a Token from the string.
+  Token MacroTok;
+  MacroTok.startToken();
+  MacroTok.setKind(tok::raw_identifier);
+  CreateString(&StrVal[1], StrVal.size() - 2, MacroTok);
+
+  // Get the IdentifierInfo of MacroToPushTok.
+  return LookUpIdentifierInfo(MacroTok);
+}
+
+/// HandlePragmaPushMacro - Handle #pragma push_macro.  
+/// The syntax is:
+///   #pragma push_macro("macro")
+void Preprocessor::HandlePragmaPushMacro(Token &PushMacroTok) {
+  // Parse the pragma directive and get the macro IdentifierInfo*.
+  IdentifierInfo *IdentInfo = ParsePragmaPushOrPopMacro(PushMacroTok);
+  if (!IdentInfo) return;
+
+  // Get the MacroInfo associated with IdentInfo.
+  MacroInfo *MI = getMacroInfo(IdentInfo);
+ 
+  MacroInfo *MacroCopyToPush = 0;
+  if (MI) {
+    // Make a clone of MI.
+    MacroCopyToPush = CloneMacroInfo(*MI);
+    
+    // Allow the original MacroInfo to be redefined later.
+    MI->setIsAllowRedefinitionsWithoutWarning(true);
+  }
+
+  // Push the cloned MacroInfo so we can retrieve it later.
+  PragmaPushMacroInfo[IdentInfo].push_back(MacroCopyToPush);
+}
+
+/// HandlePragmaPopMacro - Handle #pragma pop_macro.  
+/// The syntax is:
+///   #pragma pop_macro("macro")
+void Preprocessor::HandlePragmaPopMacro(Token &PopMacroTok) {
+  SourceLocation MessageLoc = PopMacroTok.getLocation();
+
+  // Parse the pragma directive and get the macro IdentifierInfo*.
+  IdentifierInfo *IdentInfo = ParsePragmaPushOrPopMacro(PopMacroTok);
+  if (!IdentInfo) return;
+
+  // Find the vector<MacroInfo*> associated with the macro.
+  llvm::DenseMap<IdentifierInfo*, std::vector<MacroInfo*> >::iterator iter =
+    PragmaPushMacroInfo.find(IdentInfo);
+  if (iter != PragmaPushMacroInfo.end()) {
+    // Release the MacroInfo currently associated with IdentInfo.
+    MacroInfo *CurrentMI = getMacroInfo(IdentInfo);
+    if (CurrentMI) {
+      if (CurrentMI->isWarnIfUnused())
+        WarnUnusedMacroLocs.erase(CurrentMI->getDefinitionLoc());
+      ReleaseMacroInfo(CurrentMI);
+    }
+
+    // Get the MacroInfo we want to reinstall.
+    MacroInfo *MacroToReInstall = iter->second.back();
+
+    // Reinstall the previously pushed macro.
+    setMacroInfo(IdentInfo, MacroToReInstall);
+
+    // Pop PragmaPushMacroInfo stack.
+    iter->second.pop_back();
+    if (iter->second.size() == 0)
+      PragmaPushMacroInfo.erase(iter);
+  } else {
+    Diag(MessageLoc, diag::warn_pragma_pop_macro_no_push)
+      << IdentInfo->getName();
+  }
+}
+
+void Preprocessor::HandlePragmaIncludeAlias(Token &Tok) {
+  // We will either get a quoted filename or a bracketed filename, and we 
+  // have to track which we got.  The first filename is the source name,
+  // and the second name is the mapped filename.  If the first is quoted,
+  // the second must be as well (cannot mix and match quotes and brackets).
+
+  // Get the open paren
+  Lex(Tok);
+  if (Tok.isNot(tok::l_paren)) {
+    Diag(Tok, diag::warn_pragma_include_alias_expected) << "(";
+    return;
+  }
+
+  // We expect either a quoted string literal, or a bracketed name
+  Token SourceFilenameTok;
+  CurPPLexer->LexIncludeFilename(SourceFilenameTok);
+  if (SourceFilenameTok.is(tok::eod)) {
+    // The diagnostic has already been handled
+    return;
+  }
+
+  StringRef SourceFileName;
+  SmallString<128> FileNameBuffer;
+  if (SourceFilenameTok.is(tok::string_literal) || 
+      SourceFilenameTok.is(tok::angle_string_literal)) {
+    SourceFileName = getSpelling(SourceFilenameTok, FileNameBuffer);
+  } else if (SourceFilenameTok.is(tok::less)) {
+    // This could be a path instead of just a name
+    FileNameBuffer.push_back('<');
+    SourceLocation End;
+    if (ConcatenateIncludeName(FileNameBuffer, End))
+      return; // Diagnostic already emitted
+    SourceFileName = FileNameBuffer.str();
+  } else {
+    Diag(Tok, diag::warn_pragma_include_alias_expected_filename);
+    return;
+  }
+  FileNameBuffer.clear();
+
+  // Now we expect a comma, followed by another include name
+  Lex(Tok);
+  if (Tok.isNot(tok::comma)) {
+    Diag(Tok, diag::warn_pragma_include_alias_expected) << ",";
+    return;
+  }
+
+  Token ReplaceFilenameTok;
+  CurPPLexer->LexIncludeFilename(ReplaceFilenameTok);
+  if (ReplaceFilenameTok.is(tok::eod)) {
+    // The diagnostic has already been handled
+    return;
+  }
+
+  StringRef ReplaceFileName;
+  if (ReplaceFilenameTok.is(tok::string_literal) || 
+      ReplaceFilenameTok.is(tok::angle_string_literal)) {
+    ReplaceFileName = getSpelling(ReplaceFilenameTok, FileNameBuffer);
+  } else if (ReplaceFilenameTok.is(tok::less)) {
+    // This could be a path instead of just a name
+    FileNameBuffer.push_back('<');
+    SourceLocation End;
+    if (ConcatenateIncludeName(FileNameBuffer, End))
+      return; // Diagnostic already emitted
+    ReplaceFileName = FileNameBuffer.str();
+  } else {
+    Diag(Tok, diag::warn_pragma_include_alias_expected_filename);
+    return;
+  }
+
+  // Finally, we expect the closing paren
+  Lex(Tok);
+  if (Tok.isNot(tok::r_paren)) {
+    Diag(Tok, diag::warn_pragma_include_alias_expected) << ")";
+    return;
+  }
+
+  // Now that we have the source and target filenames, we need to make sure
+  // they're both of the same type (angled vs non-angled)
+  StringRef OriginalSource = SourceFileName;
+
+  bool SourceIsAngled = 
+    GetIncludeFilenameSpelling(SourceFilenameTok.getLocation(), 
+                                SourceFileName);
+  bool ReplaceIsAngled =
+    GetIncludeFilenameSpelling(ReplaceFilenameTok.getLocation(),
+                                ReplaceFileName);
+  if (!SourceFileName.empty() && !ReplaceFileName.empty() &&
+      (SourceIsAngled != ReplaceIsAngled)) {
+    unsigned int DiagID;
+    if (SourceIsAngled)
+      DiagID = diag::warn_pragma_include_alias_mismatch_angle;
+    else
+      DiagID = diag::warn_pragma_include_alias_mismatch_quote;
+
+    Diag(SourceFilenameTok.getLocation(), DiagID)
+      << SourceFileName 
+      << ReplaceFileName;
+
+    return;
+  }
+
+  // Now we can let the include handler know about this mapping
+  getHeaderSearchInfo().AddIncludeAlias(OriginalSource, ReplaceFileName);
+}
+
+/// AddPragmaHandler - Add the specified pragma handler to the preprocessor.
+/// If 'Namespace' is non-null, then it is a token required to exist on the
+/// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
+void Preprocessor::AddPragmaHandler(StringRef Namespace,
+                                    PragmaHandler *Handler) {
+  PragmaNamespace *InsertNS = PragmaHandlers;
+
+  // If this is specified to be in a namespace, step down into it.
+  if (!Namespace.empty()) {
+    // If there is already a pragma handler with the name of this namespace,
+    // we either have an error (directive with the same name as a namespace) or
+    // we already have the namespace to insert into.
+    if (PragmaHandler *Existing = PragmaHandlers->FindHandler(Namespace)) {
+      InsertNS = Existing->getIfNamespace();
+      assert(InsertNS != 0 && "Cannot have a pragma namespace and pragma"
+             " handler with the same name!");
+    } else {
+      // Otherwise, this namespace doesn't exist yet, create and insert the
+      // handler for it.
+      InsertNS = new PragmaNamespace(Namespace);
+      PragmaHandlers->AddPragma(InsertNS);
+    }
+  }
+
+  // Check to make sure we don't already have a pragma for this identifier.
+  assert(!InsertNS->FindHandler(Handler->getName()) &&
+         "Pragma handler already exists for this identifier!");
+  InsertNS->AddPragma(Handler);
+}
+
+/// RemovePragmaHandler - Remove the specific pragma handler from the
+/// preprocessor. If \arg Namespace is non-null, then it should be the
+/// namespace that \arg Handler was added to. It is an error to remove
+/// a handler that has not been registered.
+void Preprocessor::RemovePragmaHandler(StringRef Namespace,
+                                       PragmaHandler *Handler) {
+  PragmaNamespace *NS = PragmaHandlers;
+
+  // If this is specified to be in a namespace, step down into it.
+  if (!Namespace.empty()) {
+    PragmaHandler *Existing = PragmaHandlers->FindHandler(Namespace);
+    assert(Existing && "Namespace containing handler does not exist!");
+
+    NS = Existing->getIfNamespace();
+    assert(NS && "Invalid namespace, registered as a regular pragma handler!");
+  }
+
+  NS->RemovePragmaHandler(Handler);
+
+  // If this is a non-default namespace and it is now empty, remove
+  // it.
+  if (NS != PragmaHandlers && NS->IsEmpty()) {
+    PragmaHandlers->RemovePragmaHandler(NS);
+    delete NS;
+  }
+}
+
+bool Preprocessor::LexOnOffSwitch(tok::OnOffSwitch &Result) {
+  Token Tok;
+  LexUnexpandedToken(Tok);
+
+  if (Tok.isNot(tok::identifier)) {
+    Diag(Tok, diag::ext_on_off_switch_syntax);
+    return true;
+  }
+  IdentifierInfo *II = Tok.getIdentifierInfo();
+  if (II->isStr("ON"))
+    Result = tok::OOS_ON;
+  else if (II->isStr("OFF"))
+    Result = tok::OOS_OFF;
+  else if (II->isStr("DEFAULT"))
+    Result = tok::OOS_DEFAULT;
+  else {
+    Diag(Tok, diag::ext_on_off_switch_syntax);
+    return true;
+  }
+
+  // Verify that this is followed by EOD.
+  LexUnexpandedToken(Tok);
+  if (Tok.isNot(tok::eod))
+    Diag(Tok, diag::ext_pragma_syntax_eod);
+  return false;
+}
+
+namespace {
+/// PragmaOnceHandler - "#pragma once" marks the file as atomically included.
+struct PragmaOnceHandler : public PragmaHandler {
+  PragmaOnceHandler() : PragmaHandler("once") {}
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &OnceTok) {
+    PP.CheckEndOfDirective("pragma once");
+    PP.HandlePragmaOnce(OnceTok);
+  }
+};
+
+/// PragmaMarkHandler - "#pragma mark ..." is ignored by the compiler, and the
+/// rest of the line is not lexed.
+struct PragmaMarkHandler : public PragmaHandler {
+  PragmaMarkHandler() : PragmaHandler("mark") {}
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &MarkTok) {
+    PP.HandlePragmaMark();
+  }
+};
+
+/// PragmaPoisonHandler - "#pragma poison x" marks x as not usable.
+struct PragmaPoisonHandler : public PragmaHandler {
+  PragmaPoisonHandler() : PragmaHandler("poison") {}
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &PoisonTok) {
+    PP.HandlePragmaPoison(PoisonTok);
+  }
+};
+
+/// PragmaSystemHeaderHandler - "#pragma system_header" marks the current file
+/// as a system header, which silences warnings in it.
+struct PragmaSystemHeaderHandler : public PragmaHandler {
+  PragmaSystemHeaderHandler() : PragmaHandler("system_header") {}
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &SHToken) {
+    PP.HandlePragmaSystemHeader(SHToken);
+    PP.CheckEndOfDirective("pragma");
+  }
+};
+struct PragmaDependencyHandler : public PragmaHandler {
+  PragmaDependencyHandler() : PragmaHandler("dependency") {}
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &DepToken) {
+    PP.HandlePragmaDependency(DepToken);
+  }
+};
+
+struct PragmaDebugHandler : public PragmaHandler {
+  PragmaDebugHandler() : PragmaHandler("__debug") {}
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &DepToken) {
+    Token Tok;
+    PP.LexUnexpandedToken(Tok);
+    if (Tok.isNot(tok::identifier)) {
+      PP.Diag(Tok, diag::warn_pragma_diagnostic_invalid);
+      return;
+    }
+    IdentifierInfo *II = Tok.getIdentifierInfo();
+
+    if (II->isStr("assert")) {
+      llvm_unreachable("This is an assertion!");
+    } else if (II->isStr("crash")) {
+      *(volatile int*) 0x11 = 0;
+    } else if (II->isStr("llvm_fatal_error")) {
+      llvm::report_fatal_error("#pragma clang __debug llvm_fatal_error");
+    } else if (II->isStr("llvm_unreachable")) {
+      llvm_unreachable("#pragma clang __debug llvm_unreachable");
+    } else if (II->isStr("overflow_stack")) {
+      DebugOverflowStack();
+    } else if (II->isStr("handle_crash")) {
+      llvm::CrashRecoveryContext *CRC =llvm::CrashRecoveryContext::GetCurrent();
+      if (CRC)
+        CRC->HandleCrash();
+    } else {
+      PP.Diag(Tok, diag::warn_pragma_debug_unexpected_command)
+        << II->getName();
+    }
+  }
+
+// Disable MSVC warning about runtime stack overflow.
+#ifdef _MSC_VER
+    #pragma warning(disable : 4717)
+#endif
+  void DebugOverflowStack() {
+    DebugOverflowStack();
+  }
+#ifdef _MSC_VER
+    #pragma warning(default : 4717)
+#endif
+
+};
+
+/// PragmaDiagnosticHandler - e.g. '#pragma GCC diagnostic ignored "-Wformat"'
+struct PragmaDiagnosticHandler : public PragmaHandler {
+private:
+  const char *Namespace;
+public:
+  explicit PragmaDiagnosticHandler(const char *NS) :
+    PragmaHandler("diagnostic"), Namespace(NS) {}
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &DiagToken) {
+    SourceLocation DiagLoc = DiagToken.getLocation();
+    Token Tok;
+    PP.LexUnexpandedToken(Tok);
+    if (Tok.isNot(tok::identifier)) {
+      PP.Diag(Tok, diag::warn_pragma_diagnostic_invalid);
+      return;
+    }
+    IdentifierInfo *II = Tok.getIdentifierInfo();
+    PPCallbacks *Callbacks = PP.getPPCallbacks();
+
+    diag::Mapping Map;
+    if (II->isStr("warning"))
+      Map = diag::MAP_WARNING;
+    else if (II->isStr("error"))
+      Map = diag::MAP_ERROR;
+    else if (II->isStr("ignored"))
+      Map = diag::MAP_IGNORE;
+    else if (II->isStr("fatal"))
+      Map = diag::MAP_FATAL;
+    else if (II->isStr("pop")) {
+      if (!PP.getDiagnostics().popMappings(DiagLoc))
+        PP.Diag(Tok, diag::warn_pragma_diagnostic_cannot_pop);
+      else if (Callbacks)
+        Callbacks->PragmaDiagnosticPop(DiagLoc, Namespace);
+      return;
+    } else if (II->isStr("push")) {
+      PP.getDiagnostics().pushMappings(DiagLoc);
+      if (Callbacks)
+        Callbacks->PragmaDiagnosticPush(DiagLoc, Namespace);
+      return;
+    } else {
+      PP.Diag(Tok, diag::warn_pragma_diagnostic_invalid);
+      return;
+    }
+
+    PP.LexUnexpandedToken(Tok);
+
+    // We need at least one string.
+    if (Tok.isNot(tok::string_literal)) {
+      PP.Diag(Tok.getLocation(), diag::warn_pragma_diagnostic_invalid_token);
+      return;
+    }
+
+    // String concatenation allows multiple strings, which can even come from
+    // macro expansion.
+    // "foo " "bar" "Baz"
+    SmallVector<Token, 4> StrToks;
+    while (Tok.is(tok::string_literal)) {
+      StrToks.push_back(Tok);
+      PP.LexUnexpandedToken(Tok);
+    }
+
+    if (Tok.isNot(tok::eod)) {
+      PP.Diag(Tok.getLocation(), diag::warn_pragma_diagnostic_invalid_token);
+      return;
+    }
+
+    // Concatenate and parse the strings.
+    StringLiteralParser Literal(&StrToks[0], StrToks.size(), PP);
+    assert(Literal.isAscii() && "Didn't allow wide strings in");
+    if (Literal.hadError)
+      return;
+    if (Literal.Pascal) {
+      PP.Diag(Tok, diag::warn_pragma_diagnostic_invalid);
+      return;
+    }
+
+    StringRef WarningName(Literal.GetString());
+
+    if (WarningName.size() < 3 || WarningName[0] != '-' ||
+        WarningName[1] != 'W') {
+      PP.Diag(StrToks[0].getLocation(),
+              diag::warn_pragma_diagnostic_invalid_option);
+      return;
+    }
+
+    if (PP.getDiagnostics().setDiagnosticGroupMapping(WarningName.substr(2),
+                                                      Map, DiagLoc))
+      PP.Diag(StrToks[0].getLocation(),
+              diag::warn_pragma_diagnostic_unknown_warning) << WarningName;
+    else if (Callbacks)
+      Callbacks->PragmaDiagnostic(DiagLoc, Namespace, Map, WarningName);
+  }
+};
+
+/// PragmaCommentHandler - "#pragma comment ...".
+struct PragmaCommentHandler : public PragmaHandler {
+  PragmaCommentHandler() : PragmaHandler("comment") {}
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &CommentTok) {
+    PP.HandlePragmaComment(CommentTok);
+  }
+};
+
+/// PragmaIncludeAliasHandler - "#pragma include_alias("...")".
+struct PragmaIncludeAliasHandler : public PragmaHandler {
+  PragmaIncludeAliasHandler() : PragmaHandler("include_alias") {}
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &IncludeAliasTok) {
+      PP.HandlePragmaIncludeAlias(IncludeAliasTok);
+  }
+};
+
+/// PragmaMessageHandler - "#pragma message("...")".
+struct PragmaMessageHandler : public PragmaHandler {
+  PragmaMessageHandler() : PragmaHandler("message") {}
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &CommentTok) {
+    PP.HandlePragmaMessage(CommentTok);
+  }
+};
+
+/// PragmaPushMacroHandler - "#pragma push_macro" saves the value of the
+/// macro on the top of the stack.
+struct PragmaPushMacroHandler : public PragmaHandler {
+  PragmaPushMacroHandler() : PragmaHandler("push_macro") {}
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &PushMacroTok) {
+    PP.HandlePragmaPushMacro(PushMacroTok);
+  }
+};
+
+
+/// PragmaPopMacroHandler - "#pragma pop_macro" sets the value of the
+/// macro to the value on the top of the stack.
+struct PragmaPopMacroHandler : public PragmaHandler {
+  PragmaPopMacroHandler() : PragmaHandler("pop_macro") {}
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &PopMacroTok) {
+    PP.HandlePragmaPopMacro(PopMacroTok);
+  }
+};
+
+// Pragma STDC implementations.
+
+/// PragmaSTDC_FENV_ACCESSHandler - "#pragma STDC FENV_ACCESS ...".
+struct PragmaSTDC_FENV_ACCESSHandler : public PragmaHandler {
+  PragmaSTDC_FENV_ACCESSHandler() : PragmaHandler("FENV_ACCESS") {}
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &Tok) {
+    tok::OnOffSwitch OOS;
+    if (PP.LexOnOffSwitch(OOS))
+     return;
+    if (OOS == tok::OOS_ON)
+      PP.Diag(Tok, diag::warn_stdc_fenv_access_not_supported);
+  }
+};
+
+/// PragmaSTDC_CX_LIMITED_RANGEHandler - "#pragma STDC CX_LIMITED_RANGE ...".
+struct PragmaSTDC_CX_LIMITED_RANGEHandler : public PragmaHandler {
+  PragmaSTDC_CX_LIMITED_RANGEHandler()
+    : PragmaHandler("CX_LIMITED_RANGE") {}
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &Tok) {
+    tok::OnOffSwitch OOS;
+    PP.LexOnOffSwitch(OOS);
+  }
+};
+
+/// PragmaSTDC_UnknownHandler - "#pragma STDC ...".
+struct PragmaSTDC_UnknownHandler : public PragmaHandler {
+  PragmaSTDC_UnknownHandler() {}
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &UnknownTok) {
+    // C99 6.10.6p2, unknown forms are not allowed.
+    PP.Diag(UnknownTok, diag::ext_stdc_pragma_ignored);
+  }
+};
+
+/// PragmaARCCFCodeAuditedHandler - 
+///   #pragma clang arc_cf_code_audited begin/end
+struct PragmaARCCFCodeAuditedHandler : public PragmaHandler {
+  PragmaARCCFCodeAuditedHandler() : PragmaHandler("arc_cf_code_audited") {}
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+                            Token &NameTok) {
+    SourceLocation Loc = NameTok.getLocation();
+    bool IsBegin;
+
+    Token Tok;
+
+    // Lex the 'begin' or 'end'.
+    PP.LexUnexpandedToken(Tok);
+    const IdentifierInfo *BeginEnd = Tok.getIdentifierInfo();
+    if (BeginEnd && BeginEnd->isStr("begin")) {
+      IsBegin = true;
+    } else if (BeginEnd && BeginEnd->isStr("end")) {
+      IsBegin = false;
+    } else {
+      PP.Diag(Tok.getLocation(), diag::err_pp_arc_cf_code_audited_syntax);
+      return;
+    }
+
+    // Verify that this is followed by EOD.
+    PP.LexUnexpandedToken(Tok);
+    if (Tok.isNot(tok::eod))
+      PP.Diag(Tok, diag::ext_pp_extra_tokens_at_eol) << "pragma";
+
+    // The start location of the active audit.
+    SourceLocation BeginLoc = PP.getPragmaARCCFCodeAuditedLoc();
+
+    // The start location we want after processing this.
+    SourceLocation NewLoc;
+
+    if (IsBegin) {
+      // Complain about attempts to re-enter an audit.
+      if (BeginLoc.isValid()) {
+        PP.Diag(Loc, diag::err_pp_double_begin_of_arc_cf_code_audited);
+        PP.Diag(BeginLoc, diag::note_pragma_entered_here);
+      }
+      NewLoc = Loc;
+    } else {
+      // Complain about attempts to leave an audit that doesn't exist.
+      if (!BeginLoc.isValid()) {
+        PP.Diag(Loc, diag::err_pp_unmatched_end_of_arc_cf_code_audited);
+        return;
+      }
+      NewLoc = SourceLocation();
+    }
+
+    PP.setPragmaARCCFCodeAuditedLoc(NewLoc);
+  }
+};
+
+}  // end anonymous namespace
+
+
+/// RegisterBuiltinPragmas - Install the standard preprocessor pragmas:
+/// #pragma GCC poison/system_header/dependency and #pragma once.
+void Preprocessor::RegisterBuiltinPragmas() {
+  AddPragmaHandler(new PragmaOnceHandler());
+  AddPragmaHandler(new PragmaMarkHandler());
+  AddPragmaHandler(new PragmaPushMacroHandler());
+  AddPragmaHandler(new PragmaPopMacroHandler());
+  AddPragmaHandler(new PragmaMessageHandler());
+
+  // #pragma GCC ...
+  AddPragmaHandler("GCC", new PragmaPoisonHandler());
+  AddPragmaHandler("GCC", new PragmaSystemHeaderHandler());
+  AddPragmaHandler("GCC", new PragmaDependencyHandler());
+  AddPragmaHandler("GCC", new PragmaDiagnosticHandler("GCC"));
+  // #pragma clang ...
+  AddPragmaHandler("clang", new PragmaPoisonHandler());
+  AddPragmaHandler("clang", new PragmaSystemHeaderHandler());
+  AddPragmaHandler("clang", new PragmaDebugHandler());
+  AddPragmaHandler("clang", new PragmaDependencyHandler());
+  AddPragmaHandler("clang", new PragmaDiagnosticHandler("clang"));
+  AddPragmaHandler("clang", new PragmaARCCFCodeAuditedHandler());
+
+  AddPragmaHandler("STDC", new PragmaSTDC_FENV_ACCESSHandler());
+  AddPragmaHandler("STDC", new PragmaSTDC_CX_LIMITED_RANGEHandler());
+  AddPragmaHandler("STDC", new PragmaSTDC_UnknownHandler());
+
+  // MS extensions.
+  if (LangOpts.MicrosoftExt) {
+    AddPragmaHandler(new PragmaCommentHandler());
+    AddPragmaHandler(new PragmaIncludeAliasHandler());
+  }
+}
diff --git a/clang/lib/Lex/PreprocessingRecord.cpp b/clang/lib/Lex/PreprocessingRecord.cpp
new file mode 100644
index 0000000..89d19fd
--- /dev/null
+++ b/clang/lib/Lex/PreprocessingRecord.cpp
@@ -0,0 +1,519 @@
+//===--- PreprocessingRecord.cpp - Record of Preprocessing ------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file implements the PreprocessingRecord class, which maintains a record
+//  of what occurred during preprocessing, and its helpers.
+//
+//===----------------------------------------------------------------------===//
+#include "clang/Lex/PreprocessingRecord.h"
+#include "clang/Lex/MacroInfo.h"
+#include "clang/Lex/Token.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/Capacity.h"
+
+using namespace clang;
+
+ExternalPreprocessingRecordSource::~ExternalPreprocessingRecordSource() { }
+
+
+InclusionDirective::InclusionDirective(PreprocessingRecord &PPRec,
+                                       InclusionKind Kind, 
+                                       StringRef FileName, 
+                                       bool InQuotes, const FileEntry *File, 
+                                       SourceRange Range)
+  : PreprocessingDirective(InclusionDirectiveKind, Range), 
+    InQuotes(InQuotes), Kind(Kind), File(File) 
+{ 
+  char *Memory 
+    = (char*)PPRec.Allocate(FileName.size() + 1, llvm::alignOf<char>());
+  memcpy(Memory, FileName.data(), FileName.size());
+  Memory[FileName.size()] = 0;
+  this->FileName = StringRef(Memory, FileName.size());
+}
+
+PreprocessingRecord::PreprocessingRecord(SourceManager &SM,
+                                         bool RecordConditionalDirectives)
+  : SourceMgr(SM),
+    RecordCondDirectives(RecordConditionalDirectives), CondDirectiveNextIdx(0),
+    ExternalSource(0)
+{
+  if (RecordCondDirectives)
+    CondDirectiveStack.push_back(CondDirectiveNextIdx++);
+}
+
+/// \brief Returns a pair of [Begin, End) iterators of preprocessed entities
+/// that source range \arg R encompasses.
+std::pair<PreprocessingRecord::iterator, PreprocessingRecord::iterator>
+PreprocessingRecord::getPreprocessedEntitiesInRange(SourceRange Range) {
+  if (Range.isInvalid())
+    return std::make_pair(iterator(), iterator());
+
+  if (CachedRangeQuery.Range == Range) {
+    return std::make_pair(iterator(this, CachedRangeQuery.Result.first),
+                          iterator(this, CachedRangeQuery.Result.second));
+  }
+
+  std::pair<PPEntityID, PPEntityID>
+    Res = getPreprocessedEntitiesInRangeSlow(Range);
+  
+  CachedRangeQuery.Range = Range;
+  CachedRangeQuery.Result = Res;
+  
+  return std::make_pair(iterator(this, Res.first), iterator(this, Res.second));
+}
+
+static bool isPreprocessedEntityIfInFileID(PreprocessedEntity *PPE, FileID FID,
+                                           SourceManager &SM) {
+  assert(!FID.isInvalid());
+  if (!PPE)
+    return false;
+
+  SourceLocation Loc = PPE->getSourceRange().getBegin();
+  if (Loc.isInvalid())
+    return false;
+  
+  if (SM.isInFileID(SM.getFileLoc(Loc), FID))
+    return true;
+  else
+    return false;
+}
+
+/// \brief Returns true if the preprocessed entity that \arg PPEI iterator
+/// points to is coming from the file \arg FID.
+///
+/// Can be used to avoid implicit deserializations of preallocated
+/// preprocessed entities if we only care about entities of a specific file
+/// and not from files #included in the range given at
+/// \see getPreprocessedEntitiesInRange.
+bool PreprocessingRecord::isEntityInFileID(iterator PPEI, FileID FID) {
+  if (FID.isInvalid())
+    return false;
+
+  PPEntityID PPID = PPEI.Position;
+  if (PPID < 0) {
+    assert(unsigned(-PPID-1) < LoadedPreprocessedEntities.size() &&
+           "Out-of bounds loaded preprocessed entity");
+    assert(ExternalSource && "No external source to load from");
+    unsigned LoadedIndex = LoadedPreprocessedEntities.size()+PPID;
+    if (PreprocessedEntity *PPE = LoadedPreprocessedEntities[LoadedIndex])
+      return isPreprocessedEntityIfInFileID(PPE, FID, SourceMgr);
+
+    // See if the external source can see if the entity is in the file without
+    // deserializing it.
+    llvm::Optional<bool>
+      IsInFile = ExternalSource->isPreprocessedEntityInFileID(LoadedIndex, FID);
+    if (IsInFile.hasValue())
+      return IsInFile.getValue();
+
+    // The external source did not provide a definite answer, go and deserialize
+    // the entity to check it.
+    return isPreprocessedEntityIfInFileID(
+                                       getLoadedPreprocessedEntity(LoadedIndex),
+                                          FID, SourceMgr);
+  }
+
+  assert(unsigned(PPID) < PreprocessedEntities.size() &&
+         "Out-of bounds local preprocessed entity");
+  return isPreprocessedEntityIfInFileID(PreprocessedEntities[PPID],
+                                        FID, SourceMgr);
+}
+
+/// \brief Returns a pair of [Begin, End) iterators of preprocessed entities
+/// that source range \arg R encompasses.
+std::pair<PreprocessingRecord::PPEntityID, PreprocessingRecord::PPEntityID>
+PreprocessingRecord::getPreprocessedEntitiesInRangeSlow(SourceRange Range) {
+  assert(Range.isValid());
+  assert(!SourceMgr.isBeforeInTranslationUnit(Range.getEnd(),Range.getBegin()));
+  
+  std::pair<unsigned, unsigned>
+    Local = findLocalPreprocessedEntitiesInRange(Range);
+  
+  // Check if range spans local entities.
+  if (!ExternalSource || SourceMgr.isLocalSourceLocation(Range.getBegin()))
+    return std::make_pair(Local.first, Local.second);
+  
+  std::pair<unsigned, unsigned>
+    Loaded = ExternalSource->findPreprocessedEntitiesInRange(Range);
+  
+  // Check if range spans local entities.
+  if (Loaded.first == Loaded.second)
+    return std::make_pair(Local.first, Local.second);
+  
+  unsigned TotalLoaded = LoadedPreprocessedEntities.size();
+  
+  // Check if range spans loaded entities.
+  if (Local.first == Local.second)
+    return std::make_pair(int(Loaded.first)-TotalLoaded,
+                          int(Loaded.second)-TotalLoaded);
+  
+  // Range spands loaded and local entities.
+  return std::make_pair(int(Loaded.first)-TotalLoaded, Local.second);
+}
+
+std::pair<unsigned, unsigned>
+PreprocessingRecord::findLocalPreprocessedEntitiesInRange(
+                                                      SourceRange Range) const {
+  if (Range.isInvalid())
+    return std::make_pair(0,0);
+  assert(!SourceMgr.isBeforeInTranslationUnit(Range.getEnd(),Range.getBegin()));
+
+  unsigned Begin = findBeginLocalPreprocessedEntity(Range.getBegin());
+  unsigned End = findEndLocalPreprocessedEntity(Range.getEnd());
+  return std::make_pair(Begin, End);
+}
+
+namespace {
+
+template <SourceLocation (SourceRange::*getRangeLoc)() const>
+struct PPEntityComp {
+  const SourceManager &SM;
+
+  explicit PPEntityComp(const SourceManager &SM) : SM(SM) { }
+
+  bool operator()(PreprocessedEntity *L, PreprocessedEntity *R) const {
+    SourceLocation LHS = getLoc(L);
+    SourceLocation RHS = getLoc(R);
+    return SM.isBeforeInTranslationUnit(LHS, RHS);
+  }
+
+  bool operator()(PreprocessedEntity *L, SourceLocation RHS) const {
+    SourceLocation LHS = getLoc(L);
+    return SM.isBeforeInTranslationUnit(LHS, RHS);
+  }
+
+  bool operator()(SourceLocation LHS, PreprocessedEntity *R) const {
+    SourceLocation RHS = getLoc(R);
+    return SM.isBeforeInTranslationUnit(LHS, RHS);
+  }
+
+  SourceLocation getLoc(PreprocessedEntity *PPE) const {
+    SourceRange Range = PPE->getSourceRange();
+    return (Range.*getRangeLoc)();
+  }
+};
+
+}
+
+unsigned PreprocessingRecord::findBeginLocalPreprocessedEntity(
+                                                     SourceLocation Loc) const {
+  if (SourceMgr.isLoadedSourceLocation(Loc))
+    return 0;
+
+  size_t Count = PreprocessedEntities.size();
+  size_t Half;
+  std::vector<PreprocessedEntity *>::const_iterator
+    First = PreprocessedEntities.begin();
+  std::vector<PreprocessedEntity *>::const_iterator I;
+
+  // Do a binary search manually instead of using std::lower_bound because
+  // The end locations of entities may be unordered (when a macro expansion
+  // is inside another macro argument), but for this case it is not important
+  // whether we get the first macro expansion or its containing macro.
+  while (Count > 0) {
+    Half = Count/2;
+    I = First;
+    std::advance(I, Half);
+    if (SourceMgr.isBeforeInTranslationUnit((*I)->getSourceRange().getEnd(),
+                                            Loc)){
+      First = I;
+      ++First;
+      Count = Count - Half - 1;
+    } else
+      Count = Half;
+  }
+
+  return First - PreprocessedEntities.begin();
+}
+
+unsigned PreprocessingRecord::findEndLocalPreprocessedEntity(
+                                                     SourceLocation Loc) const {
+  if (SourceMgr.isLoadedSourceLocation(Loc))
+    return 0;
+
+  std::vector<PreprocessedEntity *>::const_iterator
+  I = std::upper_bound(PreprocessedEntities.begin(),
+                       PreprocessedEntities.end(),
+                       Loc,
+                       PPEntityComp<&SourceRange::getBegin>(SourceMgr));
+  return I - PreprocessedEntities.begin();
+}
+
+PreprocessingRecord::PPEntityID
+PreprocessingRecord::addPreprocessedEntity(PreprocessedEntity *Entity) {
+  assert(Entity);
+  SourceLocation BeginLoc = Entity->getSourceRange().getBegin();
+
+  if (!isa<class InclusionDirective>(Entity)) {
+    assert((PreprocessedEntities.empty() ||
+            !SourceMgr.isBeforeInTranslationUnit(BeginLoc,
+                   PreprocessedEntities.back()->getSourceRange().getBegin())) &&
+           "a macro directive was encountered out-of-order");
+    PreprocessedEntities.push_back(Entity);
+    return getPPEntityID(PreprocessedEntities.size()-1, /*isLoaded=*/false);
+  }
+
+  // Check normal case, this entity begin location is after the previous one.
+  if (PreprocessedEntities.empty() ||
+      !SourceMgr.isBeforeInTranslationUnit(BeginLoc,
+                   PreprocessedEntities.back()->getSourceRange().getBegin())) {
+    PreprocessedEntities.push_back(Entity);
+    return getPPEntityID(PreprocessedEntities.size()-1, /*isLoaded=*/false);
+  }
+
+  // The entity's location is not after the previous one; this can happen with
+  // include directives that form the filename using macros, e.g:
+  // "#include MACRO(STUFF)".
+
+  typedef std::vector<PreprocessedEntity *>::iterator pp_iter;
+
+  // Usually there are few macro expansions when defining the filename, do a
+  // linear search for a few entities.
+  unsigned count = 0;
+  for (pp_iter RI    = PreprocessedEntities.end(),
+               Begin = PreprocessedEntities.begin();
+       RI != Begin && count < 4; --RI, ++count) {
+    pp_iter I = RI;
+    --I;
+    if (!SourceMgr.isBeforeInTranslationUnit(BeginLoc,
+                                           (*I)->getSourceRange().getBegin())) {
+      pp_iter insertI = PreprocessedEntities.insert(RI, Entity);
+      return getPPEntityID(insertI - PreprocessedEntities.begin(),
+                           /*isLoaded=*/false);
+    }
+  }
+
+  // Linear search unsuccessful. Do a binary search.
+  pp_iter I = std::upper_bound(PreprocessedEntities.begin(),
+                               PreprocessedEntities.end(),
+                               BeginLoc,
+                               PPEntityComp<&SourceRange::getBegin>(SourceMgr));
+  pp_iter insertI = PreprocessedEntities.insert(I, Entity);
+  return getPPEntityID(insertI - PreprocessedEntities.begin(),
+                       /*isLoaded=*/false);
+}
+
+void PreprocessingRecord::SetExternalSource(
+                                    ExternalPreprocessingRecordSource &Source) {
+  assert(!ExternalSource &&
+         "Preprocessing record already has an external source");
+  ExternalSource = &Source;
+}
+
+unsigned PreprocessingRecord::allocateLoadedEntities(unsigned NumEntities) {
+  unsigned Result = LoadedPreprocessedEntities.size();
+  LoadedPreprocessedEntities.resize(LoadedPreprocessedEntities.size() 
+                                    + NumEntities);
+  return Result;
+}
+
+void PreprocessingRecord::RegisterMacroDefinition(MacroInfo *Macro,
+                                                  PPEntityID PPID) {
+  MacroDefinitions[Macro] = PPID;
+}
+
+/// \brief Retrieve the preprocessed entity at the given ID.
+PreprocessedEntity *PreprocessingRecord::getPreprocessedEntity(PPEntityID PPID){
+  if (PPID < 0) {
+    assert(unsigned(-PPID-1) < LoadedPreprocessedEntities.size() &&
+           "Out-of bounds loaded preprocessed entity");
+    return getLoadedPreprocessedEntity(LoadedPreprocessedEntities.size()+PPID);
+  }
+  assert(unsigned(PPID) < PreprocessedEntities.size() &&
+         "Out-of bounds local preprocessed entity");
+  return PreprocessedEntities[PPID];
+}
+
+/// \brief Retrieve the loaded preprocessed entity at the given index.
+PreprocessedEntity *
+PreprocessingRecord::getLoadedPreprocessedEntity(unsigned Index) {
+  assert(Index < LoadedPreprocessedEntities.size() && 
+         "Out-of bounds loaded preprocessed entity");
+  assert(ExternalSource && "No external source to load from");
+  PreprocessedEntity *&Entity = LoadedPreprocessedEntities[Index];
+  if (!Entity) {
+    Entity = ExternalSource->ReadPreprocessedEntity(Index);
+    if (!Entity) // Failed to load.
+      Entity = new (*this)
+         PreprocessedEntity(PreprocessedEntity::InvalidKind, SourceRange());
+  }
+  return Entity;
+}
+
+MacroDefinition *PreprocessingRecord::findMacroDefinition(const MacroInfo *MI) {
+  llvm::DenseMap<const MacroInfo *, PPEntityID>::iterator Pos
+    = MacroDefinitions.find(MI);
+  if (Pos == MacroDefinitions.end())
+    return 0;
+  
+  PreprocessedEntity *Entity = getPreprocessedEntity(Pos->second);
+  if (Entity->isInvalid())
+    return 0;
+  return cast<MacroDefinition>(Entity);
+}
+
+void PreprocessingRecord::MacroExpands(const Token &Id, const MacroInfo* MI,
+                                       SourceRange Range) {
+  // We don't record nested macro expansions.
+  if (Id.getLocation().isMacroID())
+    return;
+
+  if (MI->isBuiltinMacro())
+    addPreprocessedEntity(
+                      new (*this) MacroExpansion(Id.getIdentifierInfo(),Range));
+  else if (MacroDefinition *Def = findMacroDefinition(MI))
+    addPreprocessedEntity(
+                       new (*this) MacroExpansion(Def, Range));
+}
+
+void PreprocessingRecord::MacroDefined(const Token &Id,
+                                       const MacroInfo *MI) {
+  SourceRange R(MI->getDefinitionLoc(), MI->getDefinitionEndLoc());
+  MacroDefinition *Def
+      = new (*this) MacroDefinition(Id.getIdentifierInfo(), R);
+  MacroDefinitions[MI] = addPreprocessedEntity(Def);
+}
+
+void PreprocessingRecord::MacroUndefined(const Token &Id,
+                                         const MacroInfo *MI) {
+  MacroDefinitions.erase(MI);
+}
+
+void PreprocessingRecord::InclusionDirective(
+    SourceLocation HashLoc,
+    const clang::Token &IncludeTok,
+    StringRef FileName,
+    bool IsAngled,
+    const FileEntry *File,
+    clang::SourceLocation EndLoc,
+    StringRef SearchPath,
+    StringRef RelativePath) {
+  InclusionDirective::InclusionKind Kind = InclusionDirective::Include;
+  
+  switch (IncludeTok.getIdentifierInfo()->getPPKeywordID()) {
+  case tok::pp_include: 
+    Kind = InclusionDirective::Include; 
+    break;
+    
+  case tok::pp_import: 
+    Kind = InclusionDirective::Import; 
+    break;
+    
+  case tok::pp_include_next: 
+    Kind = InclusionDirective::IncludeNext; 
+    break;
+    
+  case tok::pp___include_macros: 
+    Kind = InclusionDirective::IncludeMacros;
+    break;
+    
+  default:
+    llvm_unreachable("Unknown include directive kind");
+  }
+  
+  clang::InclusionDirective *ID
+    = new (*this) clang::InclusionDirective(*this, Kind, FileName, !IsAngled, 
+                                            File, SourceRange(HashLoc, EndLoc));
+  addPreprocessedEntity(ID);
+}
+
+bool PreprocessingRecord::rangeIntersectsConditionalDirective(
+                                                      SourceRange Range) const {
+  if (Range.isInvalid())
+    return false;
+
+  CondDirectiveLocsTy::const_iterator
+    low = std::lower_bound(CondDirectiveLocs.begin(), CondDirectiveLocs.end(),
+                           Range.getBegin(), CondDirectiveLoc::Comp(SourceMgr));
+  if (low == CondDirectiveLocs.end())
+    return false;
+
+  if (SourceMgr.isBeforeInTranslationUnit(Range.getEnd(), low->getLoc()))
+    return false;
+
+  CondDirectiveLocsTy::const_iterator
+    upp = std::upper_bound(low, CondDirectiveLocs.end(),
+                           Range.getEnd(), CondDirectiveLoc::Comp(SourceMgr));
+  unsigned uppIdx;
+  if (upp != CondDirectiveLocs.end())
+    uppIdx = upp->getIdx();
+  else
+    uppIdx = 0;
+
+  return low->getIdx() != uppIdx;
+}
+
+unsigned PreprocessingRecord::findCondDirectiveIdx(SourceLocation Loc) const {
+  if (Loc.isInvalid())
+    return 0;
+
+  CondDirectiveLocsTy::const_iterator
+    low = std::lower_bound(CondDirectiveLocs.begin(), CondDirectiveLocs.end(),
+                           Loc, CondDirectiveLoc::Comp(SourceMgr));
+  if (low == CondDirectiveLocs.end())
+    return 0;
+  return low->getIdx();
+}
+
+void PreprocessingRecord::addCondDirectiveLoc(CondDirectiveLoc DirLoc) {
+  // Ignore directives in system headers.
+  if (SourceMgr.isInSystemHeader(DirLoc.getLoc()))
+    return;
+
+  assert(CondDirectiveLocs.empty() ||
+         SourceMgr.isBeforeInTranslationUnit(CondDirectiveLocs.back().getLoc(),
+                                             DirLoc.getLoc()));
+  CondDirectiveLocs.push_back(DirLoc);
+}
+
+void PreprocessingRecord::If(SourceLocation Loc, SourceRange ConditionRange) {
+  if (RecordCondDirectives) {
+    addCondDirectiveLoc(CondDirectiveLoc(Loc, CondDirectiveStack.back()));
+    CondDirectiveStack.push_back(CondDirectiveNextIdx++);
+  }
+}
+
+void PreprocessingRecord::Ifdef(SourceLocation Loc, const Token &MacroNameTok) {
+  if (RecordCondDirectives) {
+    addCondDirectiveLoc(CondDirectiveLoc(Loc, CondDirectiveStack.back()));
+    CondDirectiveStack.push_back(CondDirectiveNextIdx++);
+  }
+}
+
+void PreprocessingRecord::Ifndef(SourceLocation Loc,const Token &MacroNameTok) {
+  if (RecordCondDirectives) {
+    addCondDirectiveLoc(CondDirectiveLoc(Loc, CondDirectiveStack.back()));
+    CondDirectiveStack.push_back(CondDirectiveNextIdx++);
+  }
+}
+
+void PreprocessingRecord::Elif(SourceLocation Loc, SourceRange ConditionRange,
+                               SourceLocation IfLoc) {
+  if (RecordCondDirectives)
+    addCondDirectiveLoc(CondDirectiveLoc(Loc, CondDirectiveStack.back()));
+}
+
+void PreprocessingRecord::Else(SourceLocation Loc, SourceLocation IfLoc) {
+  if (RecordCondDirectives)
+    addCondDirectiveLoc(CondDirectiveLoc(Loc, CondDirectiveStack.back()));
+}
+
+void PreprocessingRecord::Endif(SourceLocation Loc, SourceLocation IfLoc) {
+  if (RecordCondDirectives) {
+    addCondDirectiveLoc(CondDirectiveLoc(Loc, CondDirectiveStack.back()));
+    assert(!CondDirectiveStack.empty());
+    CondDirectiveStack.pop_back();
+  }
+}
+
+size_t PreprocessingRecord::getTotalMemory() const {
+  return BumpAlloc.getTotalMemory()
+    + llvm::capacity_in_bytes(MacroDefinitions)
+    + llvm::capacity_in_bytes(PreprocessedEntities)
+    + llvm::capacity_in_bytes(LoadedPreprocessedEntities);
+}
diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
new file mode 100644
index 0000000..06e5685
--- /dev/null
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -0,0 +1,666 @@
+//===--- Preprocess.cpp - C Language Family Preprocessor Implementation ---===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file implements the Preprocessor interface.
+//
+//===----------------------------------------------------------------------===//
+//
+// Options to support:
+//   -H       - Print the name of each header file used.
+//   -d[DNI] - Dump various things.
+//   -fworking-directory - #line's with preprocessor's working dir.
+//   -fpreprocessed
+//   -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD
+//   -W*
+//   -w
+//
+// Messages to emit:
+//   "Multiple include guards may be useful for:\n"
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Lex/Preprocessor.h"
+#include "MacroArgs.h"
+#include "clang/Lex/ExternalPreprocessorSource.h"
+#include "clang/Lex/HeaderSearch.h"
+#include "clang/Lex/MacroInfo.h"
+#include "clang/Lex/Pragma.h"
+#include "clang/Lex/PreprocessingRecord.h"
+#include "clang/Lex/ScratchBuffer.h"
+#include "clang/Lex/LexDiagnostic.h"
+#include "clang/Lex/CodeCompletionHandler.h"
+#include "clang/Lex/ModuleLoader.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/TargetInfo.h"
+#include "llvm/ADT/APFloat.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/Capacity.h"
+using namespace clang;
+
+//===----------------------------------------------------------------------===//
+ExternalPreprocessorSource::~ExternalPreprocessorSource() { }
+
+Preprocessor::Preprocessor(DiagnosticsEngine &diags, LangOptions &opts,
+                           const TargetInfo *target, SourceManager &SM,
+                           HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
+                           IdentifierInfoLookup* IILookup,
+                           bool OwnsHeaders,
+                           bool DelayInitialization,
+                           bool IncrProcessing)
+  : Diags(&diags), LangOpts(opts), Target(target),FileMgr(Headers.getFileMgr()),
+    SourceMgr(SM), HeaderInfo(Headers), TheModuleLoader(TheModuleLoader),
+    ExternalSource(0), Identifiers(opts, IILookup), 
+    IncrementalProcessing(IncrProcessing), CodeComplete(0), 
+    CodeCompletionFile(0), CodeCompletionOffset(0), CodeCompletionReached(0),
+    SkipMainFilePreamble(0, true), CurPPLexer(0), 
+    CurDirLookup(0), CurLexerKind(CLK_Lexer), Callbacks(0), MacroArgCache(0), 
+    Record(0), MIChainHead(0), MICache(0) 
+{
+  OwnsHeaderSearch = OwnsHeaders;
+
+  if (!DelayInitialization) {
+    assert(Target && "Must provide target information for PP initialization");
+    Initialize(*Target);
+  }
+}
+
+Preprocessor::~Preprocessor() {
+  assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!");
+
+  while (!IncludeMacroStack.empty()) {
+    delete IncludeMacroStack.back().TheLexer;
+    delete IncludeMacroStack.back().TheTokenLexer;
+    IncludeMacroStack.pop_back();
+  }
+
+  // Free any macro definitions.
+  for (MacroInfoChain *I = MIChainHead ; I ; I = I->Next)
+    I->MI.Destroy();
+
+  // Free any cached macro expanders.
+  for (unsigned i = 0, e = NumCachedTokenLexers; i != e; ++i)
+    delete TokenLexerCache[i];
+
+  // Free any cached MacroArgs.
+  for (MacroArgs *ArgList = MacroArgCache; ArgList; )
+    ArgList = ArgList->deallocate();
+
+  // Release pragma information.
+  delete PragmaHandlers;
+
+  // Delete the scratch buffer info.
+  delete ScratchBuf;
+
+  // Delete the header search info, if we own it.
+  if (OwnsHeaderSearch)
+    delete &HeaderInfo;
+
+  delete Callbacks;
+}
+
+void Preprocessor::Initialize(const TargetInfo &Target) {
+  assert((!this->Target || this->Target == &Target) &&
+         "Invalid override of target information");
+  this->Target = &Target;
+  
+  // Initialize information about built-ins.
+  BuiltinInfo.InitializeTarget(Target);
+  
+  ScratchBuf = new ScratchBuffer(SourceMgr);
+  CounterValue = 0; // __COUNTER__ starts at 0.
+  
+  // Clear stats.
+  NumDirectives = NumDefined = NumUndefined = NumPragma = 0;
+  NumIf = NumElse = NumEndif = 0;
+  NumEnteredSourceFiles = 0;
+  NumMacroExpanded = NumFnMacroExpanded = NumBuiltinMacroExpanded = 0;
+  NumFastMacroExpanded = NumTokenPaste = NumFastTokenPaste = 0;
+  MaxIncludeStackDepth = 0;
+  NumSkipped = 0;
+  
+  // Default to discarding comments.
+  KeepComments = false;
+  KeepMacroComments = false;
+  SuppressIncludeNotFoundError = false;
+  
+  // Macro expansion is enabled.
+  DisableMacroExpansion = false;
+  InMacroArgs = false;
+  InMacroArgPreExpansion = false;
+  NumCachedTokenLexers = 0;
+  
+  CachedLexPos = 0;
+  
+  // We haven't read anything from the external source.
+  ReadMacrosFromExternalSource = false;
+  
+  // "Poison" __VA_ARGS__, which can only appear in the expansion of a macro.
+  // This gets unpoisoned where it is allowed.
+  (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
+  SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use);
+  
+  // Initialize the pragma handlers.
+  PragmaHandlers = new PragmaNamespace(StringRef());
+  RegisterBuiltinPragmas();
+  
+  // Initialize builtin macros like __LINE__ and friends.
+  RegisterBuiltinMacros();
+  
+  if(LangOpts.Borland) {
+    Ident__exception_info        = getIdentifierInfo("_exception_info");
+    Ident___exception_info       = getIdentifierInfo("__exception_info");
+    Ident_GetExceptionInfo       = getIdentifierInfo("GetExceptionInformation");
+    Ident__exception_code        = getIdentifierInfo("_exception_code");
+    Ident___exception_code       = getIdentifierInfo("__exception_code");
+    Ident_GetExceptionCode       = getIdentifierInfo("GetExceptionCode");
+    Ident__abnormal_termination  = getIdentifierInfo("_abnormal_termination");
+    Ident___abnormal_termination = getIdentifierInfo("__abnormal_termination");
+    Ident_AbnormalTermination    = getIdentifierInfo("AbnormalTermination");
+  } else {
+    Ident__exception_info = Ident__exception_code = Ident__abnormal_termination = 0;
+    Ident___exception_info = Ident___exception_code = Ident___abnormal_termination = 0;
+    Ident_GetExceptionInfo = Ident_GetExceptionCode = Ident_AbnormalTermination = 0;
+  }
+  
+  HeaderInfo.setTarget(Target);
+}
+
+void Preprocessor::setPTHManager(PTHManager* pm) {
+  PTH.reset(pm);
+  FileMgr.addStatCache(PTH->createStatCache());
+}
+
+void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
+  llvm::errs() << tok::getTokenName(Tok.getKind()) << " '"
+               << getSpelling(Tok) << "'";
+
+  if (!DumpFlags) return;
+
+  llvm::errs() << "\t";
+  if (Tok.isAtStartOfLine())
+    llvm::errs() << " [StartOfLine]";
+  if (Tok.hasLeadingSpace())
+    llvm::errs() << " [LeadingSpace]";
+  if (Tok.isExpandDisabled())
+    llvm::errs() << " [ExpandDisabled]";
+  if (Tok.needsCleaning()) {
+    const char *Start = SourceMgr.getCharacterData(Tok.getLocation());
+    llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength())
+                 << "']";
+  }
+
+  llvm::errs() << "\tLoc=<";
+  DumpLocation(Tok.getLocation());
+  llvm::errs() << ">";
+}
+
+void Preprocessor::DumpLocation(SourceLocation Loc) const {
+  Loc.dump(SourceMgr);
+}
+
+void Preprocessor::DumpMacro(const MacroInfo &MI) const {
+  llvm::errs() << "MACRO: ";
+  for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
+    DumpToken(MI.getReplacementToken(i));
+    llvm::errs() << "  ";
+  }
+  llvm::errs() << "\n";
+}
+
+void Preprocessor::PrintStats() {
+  llvm::errs() << "\n*** Preprocessor Stats:\n";
+  llvm::errs() << NumDirectives << " directives found:\n";
+  llvm::errs() << "  " << NumDefined << " #define.\n";
+  llvm::errs() << "  " << NumUndefined << " #undef.\n";
+  llvm::errs() << "  #include/#include_next/#import:\n";
+  llvm::errs() << "    " << NumEnteredSourceFiles << " source files entered.\n";
+  llvm::errs() << "    " << MaxIncludeStackDepth << " max include stack depth\n";
+  llvm::errs() << "  " << NumIf << " #if/#ifndef/#ifdef.\n";
+  llvm::errs() << "  " << NumElse << " #else/#elif.\n";
+  llvm::errs() << "  " << NumEndif << " #endif.\n";
+  llvm::errs() << "  " << NumPragma << " #pragma.\n";
+  llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n";
+
+  llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/"
+             << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, "
+             << NumFastMacroExpanded << " on the fast path.\n";
+  llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
+             << " token paste (##) operations performed, "
+             << NumFastTokenPaste << " on the fast path.\n";
+}
+
+Preprocessor::macro_iterator
+Preprocessor::macro_begin(bool IncludeExternalMacros) const {
+  if (IncludeExternalMacros && ExternalSource &&
+      !ReadMacrosFromExternalSource) {
+    ReadMacrosFromExternalSource = true;
+    ExternalSource->ReadDefinedMacros();
+  }
+
+  return Macros.begin();
+}
+
+size_t Preprocessor::getTotalMemory() const {
+  return BP.getTotalMemory()
+    + llvm::capacity_in_bytes(MacroExpandedTokens)
+    + Predefines.capacity() /* Predefines buffer. */
+    + llvm::capacity_in_bytes(Macros)
+    + llvm::capacity_in_bytes(PragmaPushMacroInfo)
+    + llvm::capacity_in_bytes(PoisonReasons)
+    + llvm::capacity_in_bytes(CommentHandlers);
+}
+
+Preprocessor::macro_iterator
+Preprocessor::macro_end(bool IncludeExternalMacros) const {
+  if (IncludeExternalMacros && ExternalSource &&
+      !ReadMacrosFromExternalSource) {
+    ReadMacrosFromExternalSource = true;
+    ExternalSource->ReadDefinedMacros();
+  }
+
+  return Macros.end();
+}
+
+void Preprocessor::recomputeCurLexerKind() {
+  if (CurLexer)
+    CurLexerKind = CLK_Lexer;
+  else if (CurPTHLexer)
+    CurLexerKind = CLK_PTHLexer;
+  else if (CurTokenLexer)
+    CurLexerKind = CLK_TokenLexer;
+  else 
+    CurLexerKind = CLK_CachingLexer;
+}
+
+bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File,
+                                          unsigned CompleteLine,
+                                          unsigned CompleteColumn) {
+  assert(File);
+  assert(CompleteLine && CompleteColumn && "Starts from 1:1");
+  assert(!CodeCompletionFile && "Already set");
+
+  using llvm::MemoryBuffer;
+
+  // Load the actual file's contents.
+  bool Invalid = false;
+  const MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File, &Invalid);
+  if (Invalid)
+    return true;
+
+  // Find the byte position of the truncation point.
+  const char *Position = Buffer->getBufferStart();
+  for (unsigned Line = 1; Line < CompleteLine; ++Line) {
+    for (; *Position; ++Position) {
+      if (*Position != '\r' && *Position != '\n')
+        continue;
+
+      // Eat \r\n or \n\r as a single line.
+      if ((Position[1] == '\r' || Position[1] == '\n') &&
+          Position[0] != Position[1])
+        ++Position;
+      ++Position;
+      break;
+    }
+  }
+
+  Position += CompleteColumn - 1;
+
+  // Insert '\0' at the code-completion point.
+  if (Position < Buffer->getBufferEnd()) {
+    CodeCompletionFile = File;
+    CodeCompletionOffset = Position - Buffer->getBufferStart();
+
+    MemoryBuffer *NewBuffer =
+        MemoryBuffer::getNewUninitMemBuffer(Buffer->getBufferSize() + 1,
+                                            Buffer->getBufferIdentifier());
+    char *NewBuf = const_cast<char*>(NewBuffer->getBufferStart());
+    char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
+    *NewPos = '\0';
+    std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
+    SourceMgr.overrideFileContents(File, NewBuffer);
+  }
+
+  return false;
+}
+
+void Preprocessor::CodeCompleteNaturalLanguage() {
+  if (CodeComplete)
+    CodeComplete->CodeCompleteNaturalLanguage();
+  setCodeCompletionReached();
+}
+
+/// getSpelling - This method is used to get the spelling of a token into a
+/// SmallVector. Note that the returned StringRef may not point to the
+/// supplied buffer if a copy can be avoided.
+StringRef Preprocessor::getSpelling(const Token &Tok,
+                                          SmallVectorImpl<char> &Buffer,
+                                          bool *Invalid) const {
+  // NOTE: this has to be checked *before* testing for an IdentifierInfo.
+  if (Tok.isNot(tok::raw_identifier)) {
+    // Try the fast path.
+    if (const IdentifierInfo *II = Tok.getIdentifierInfo())
+      return II->getName();
+  }
+
+  // Resize the buffer if we need to copy into it.
+  if (Tok.needsCleaning())
+    Buffer.resize(Tok.getLength());
+
+  const char *Ptr = Buffer.data();
+  unsigned Len = getSpelling(Tok, Ptr, Invalid);
+  return StringRef(Ptr, Len);
+}
+
+/// CreateString - Plop the specified string into a scratch buffer and return a
+/// location for it.  If specified, the source location provides a source
+/// location for the token.
+void Preprocessor::CreateString(const char *Buf, unsigned Len, Token &Tok,
+                                SourceLocation ExpansionLocStart,
+                                SourceLocation ExpansionLocEnd) {
+  Tok.setLength(Len);
+
+  const char *DestPtr;
+  SourceLocation Loc = ScratchBuf->getToken(Buf, Len, DestPtr);
+
+  if (ExpansionLocStart.isValid())
+    Loc = SourceMgr.createExpansionLoc(Loc, ExpansionLocStart,
+                                       ExpansionLocEnd, Len);
+  Tok.setLocation(Loc);
+
+  // If this is a raw identifier or a literal token, set the pointer data.
+  if (Tok.is(tok::raw_identifier))
+    Tok.setRawIdentifierData(DestPtr);
+  else if (Tok.isLiteral())
+    Tok.setLiteralData(DestPtr);
+}
+
+Module *Preprocessor::getCurrentModule() {
+  if (getLangOpts().CurrentModule.empty())
+    return 0;
+  
+  return getHeaderSearchInfo().lookupModule(getLangOpts().CurrentModule);
+}
+
+//===----------------------------------------------------------------------===//
+// Preprocessor Initialization Methods
+//===----------------------------------------------------------------------===//
+
+
+/// EnterMainSourceFile - Enter the specified FileID as the main source file,
+/// which implicitly adds the builtin defines etc.
+void Preprocessor::EnterMainSourceFile() {
+  // We do not allow the preprocessor to reenter the main file.  Doing so will
+  // cause FileID's to accumulate information from both runs (e.g. #line
+  // information) and predefined macros aren't guaranteed to be set properly.
+  assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!");
+  FileID MainFileID = SourceMgr.getMainFileID();
+
+  // If MainFileID is loaded it means we loaded an AST file, no need to enter
+  // a main file.
+  if (!SourceMgr.isLoadedFileID(MainFileID)) {
+    // Enter the main file source buffer.
+    EnterSourceFile(MainFileID, 0, SourceLocation());
+  
+    // If we've been asked to skip bytes in the main file (e.g., as part of a
+    // precompiled preamble), do so now.
+    if (SkipMainFilePreamble.first > 0)
+      CurLexer->SkipBytes(SkipMainFilePreamble.first, 
+                          SkipMainFilePreamble.second);
+    
+    // Tell the header info that the main file was entered.  If the file is later
+    // #imported, it won't be re-entered.
+    if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID))
+      HeaderInfo.IncrementIncludeCount(FE);
+  }
+
+  // Preprocess Predefines to populate the initial preprocessor state.
+  llvm::MemoryBuffer *SB =
+    llvm::MemoryBuffer::getMemBufferCopy(Predefines, "<built-in>");
+  assert(SB && "Cannot create predefined source buffer");
+  FileID FID = SourceMgr.createFileIDForMemBuffer(SB);
+  assert(!FID.isInvalid() && "Could not create FileID for predefines?");
+
+  // Start parsing the predefines.
+  EnterSourceFile(FID, 0, SourceLocation());
+}
+
+void Preprocessor::EndSourceFile() {
+  // Notify the client that we reached the end of the source file.
+  if (Callbacks)
+    Callbacks->EndOfMainFile();
+}
+
+//===----------------------------------------------------------------------===//
+// Lexer Event Handling.
+//===----------------------------------------------------------------------===//
+
+/// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
+/// identifier information for the token and install it into the token,
+/// updating the token kind accordingly.
+IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier) const {
+  assert(Identifier.getRawIdentifierData() != 0 && "No raw identifier data!");
+
+  // Look up this token, see if it is a macro, or if it is a language keyword.
+  IdentifierInfo *II;
+  if (!Identifier.needsCleaning()) {
+    // No cleaning needed, just use the characters from the lexed buffer.
+    II = getIdentifierInfo(StringRef(Identifier.getRawIdentifierData(),
+                                           Identifier.getLength()));
+  } else {
+    // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
+    SmallString<64> IdentifierBuffer;
+    StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer);
+    II = getIdentifierInfo(CleanedStr);
+  }
+
+  // Update the token info (identifier info and appropriate token kind).
+  Identifier.setIdentifierInfo(II);
+  Identifier.setKind(II->getTokenID());
+
+  return II;
+}
+
+void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) {
+  PoisonReasons[II] = DiagID;
+}
+
+void Preprocessor::PoisonSEHIdentifiers(bool Poison) {
+  assert(Ident__exception_code && Ident__exception_info);
+  assert(Ident___exception_code && Ident___exception_info);
+  Ident__exception_code->setIsPoisoned(Poison);
+  Ident___exception_code->setIsPoisoned(Poison);
+  Ident_GetExceptionCode->setIsPoisoned(Poison);
+  Ident__exception_info->setIsPoisoned(Poison);
+  Ident___exception_info->setIsPoisoned(Poison);
+  Ident_GetExceptionInfo->setIsPoisoned(Poison);
+  Ident__abnormal_termination->setIsPoisoned(Poison);
+  Ident___abnormal_termination->setIsPoisoned(Poison);
+  Ident_AbnormalTermination->setIsPoisoned(Poison);
+}
+
+void Preprocessor::HandlePoisonedIdentifier(Token & Identifier) {
+  assert(Identifier.getIdentifierInfo() &&
+         "Can't handle identifiers without identifier info!");
+  llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
+    PoisonReasons.find(Identifier.getIdentifierInfo());
+  if(it == PoisonReasons.end())
+    Diag(Identifier, diag::err_pp_used_poisoned_id);
+  else
+    Diag(Identifier,it->second) << Identifier.getIdentifierInfo();
+}
+
+/// HandleIdentifier - This callback is invoked when the lexer reads an
+/// identifier.  This callback looks up the identifier in the map and/or
+/// potentially macro expands it or turns it into a named token (like 'for').
+///
+/// Note that callers of this method are guarded by checking the
+/// IdentifierInfo's 'isHandleIdentifierCase' bit.  If this method changes, the
+/// IdentifierInfo methods that compute these properties will need to change to
+/// match.
+void Preprocessor::HandleIdentifier(Token &Identifier) {
+  assert(Identifier.getIdentifierInfo() &&
+         "Can't handle identifiers without identifier info!");
+
+  IdentifierInfo &II = *Identifier.getIdentifierInfo();
+
+  // If the information about this identifier is out of date, update it from
+  // the external source.
+  if (II.isOutOfDate()) {
+    ExternalSource->updateOutOfDateIdentifier(II);
+    Identifier.setKind(II.getTokenID());
+  }
+  
+  // If this identifier was poisoned, and if it was not produced from a macro
+  // expansion, emit an error.
+  if (II.isPoisoned() && CurPPLexer) {
+    HandlePoisonedIdentifier(Identifier);
+  }
+
+  // If this is a macro to be expanded, do it.
+  if (MacroInfo *MI = getMacroInfo(&II)) {
+    if (!DisableMacroExpansion) {
+      if (Identifier.isExpandDisabled()) {
+        Diag(Identifier, diag::pp_disabled_macro_expansion);
+      } else if (MI->isEnabled()) {
+        if (!HandleMacroExpandedIdentifier(Identifier, MI))
+          return;
+      } else {
+        // C99 6.10.3.4p2 says that a disabled macro may never again be
+        // expanded, even if it's in a context where it could be expanded in the
+        // future.
+        Identifier.setFlag(Token::DisableExpand);
+        Diag(Identifier, diag::pp_disabled_macro_expansion);
+      }
+    }
+  }
+
+  // If this identifier is a keyword in C++11, produce a warning. Don't warn if
+  // we're not considering macro expansion, since this identifier might be the
+  // name of a macro.
+  // FIXME: This warning is disabled in cases where it shouldn't be, like
+  //   "#define constexpr constexpr", "int constexpr;"
+  if (II.isCXX11CompatKeyword() & !DisableMacroExpansion) {
+    Diag(Identifier, diag::warn_cxx11_keyword) << II.getName();
+    // Don't diagnose this keyword again in this translation unit.
+    II.setIsCXX11CompatKeyword(false);
+  }
+
+  // C++ 2.11p2: If this is an alternative representation of a C++ operator,
+  // then we act as if it is the actual operator and not the textual
+  // representation of it.
+  if (II.isCPlusPlusOperatorKeyword())
+    Identifier.setIdentifierInfo(0);
+
+  // If this is an extension token, diagnose its use.
+  // We avoid diagnosing tokens that originate from macro definitions.
+  // FIXME: This warning is disabled in cases where it shouldn't be,
+  // like "#define TY typeof", "TY(1) x".
+  if (II.isExtensionToken() && !DisableMacroExpansion)
+    Diag(Identifier, diag::ext_token_used);
+  
+  // If this is the '__experimental_modules_import' contextual keyword, note
+  // that the next token indicates a module name.
+  //
+  // Note that we do not treat '__experimental_modules_import' as a contextual
+  // keyword when we're in a caching lexer, because caching lexers only get
+  // used in contexts where import declarations are disallowed.
+  if (II.isModulesImport() && !InMacroArgs && !DisableMacroExpansion &&
+      getLangOpts().Modules && CurLexerKind != CLK_CachingLexer) {
+    ModuleImportLoc = Identifier.getLocation();
+    ModuleImportPath.clear();
+    ModuleImportExpectsIdentifier = true;
+    CurLexerKind = CLK_LexAfterModuleImport;
+  }
+}
+
+/// \brief Lex a token following the 'import' contextual keyword.
+///
+void Preprocessor::LexAfterModuleImport(Token &Result) {
+  // Figure out what kind of lexer we actually have.
+  recomputeCurLexerKind();
+  
+  // Lex the next token.
+  Lex(Result);
+
+  // The token sequence 
+  //
+  //   import identifier (. identifier)*
+  //
+  // indicates a module import directive. We already saw the 'import' 
+  // contextual keyword, so now we're looking for the identifiers.
+  if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) {
+    // We expected to see an identifier here, and we did; continue handling
+    // identifiers.
+    ModuleImportPath.push_back(std::make_pair(Result.getIdentifierInfo(),
+                                              Result.getLocation()));
+    ModuleImportExpectsIdentifier = false;
+    CurLexerKind = CLK_LexAfterModuleImport;
+    return;
+  }
+  
+  // If we're expecting a '.' or a ';', and we got a '.', then wait until we
+  // see the next identifier.
+  if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) {
+    ModuleImportExpectsIdentifier = true;
+    CurLexerKind = CLK_LexAfterModuleImport;
+    return;
+  }
+
+  // If we have a non-empty module path, load the named module.
+  if (!ModuleImportPath.empty())
+    (void)TheModuleLoader.loadModule(ModuleImportLoc, ModuleImportPath,
+                                     Module::MacrosVisible,
+                                     /*IsIncludeDirective=*/false);
+}
+
+void Preprocessor::AddCommentHandler(CommentHandler *Handler) {
+  assert(Handler && "NULL comment handler");
+  assert(std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler) ==
+         CommentHandlers.end() && "Comment handler already registered");
+  CommentHandlers.push_back(Handler);
+}
+
+void Preprocessor::RemoveCommentHandler(CommentHandler *Handler) {
+  std::vector<CommentHandler *>::iterator Pos
+  = std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler);
+  assert(Pos != CommentHandlers.end() && "Comment handler not registered");
+  CommentHandlers.erase(Pos);
+}
+
+bool Preprocessor::HandleComment(Token &result, SourceRange Comment) {
+  bool AnyPendingTokens = false;
+  for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
+       HEnd = CommentHandlers.end();
+       H != HEnd; ++H) {
+    if ((*H)->HandleComment(*this, Comment))
+      AnyPendingTokens = true;
+  }
+  if (!AnyPendingTokens || getCommentRetentionState())
+    return false;
+  Lex(result);
+  return true;
+}
+
+ModuleLoader::~ModuleLoader() { }
+
+CommentHandler::~CommentHandler() { }
+
+CodeCompletionHandler::~CodeCompletionHandler() { }
+
+void Preprocessor::createPreprocessingRecord(bool RecordConditionalDirectives) {
+  if (Record)
+    return;
+  
+  Record = new PreprocessingRecord(getSourceManager(),
+                                   RecordConditionalDirectives);
+  addPPCallbacks(Record);
+}
diff --git a/clang/lib/Lex/PreprocessorLexer.cpp b/clang/lib/Lex/PreprocessorLexer.cpp
new file mode 100644
index 0000000..a72bbca
--- /dev/null
+++ b/clang/lib/Lex/PreprocessorLexer.cpp
@@ -0,0 +1,55 @@
+//===--- PreprocessorLexer.cpp - C Language Family Lexer ------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file implements the PreprocessorLexer and Token interfaces.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Lex/PreprocessorLexer.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/LexDiagnostic.h"
+#include "clang/Basic/SourceManager.h"
+using namespace clang;
+
+void PreprocessorLexer::anchor() { }
+
+PreprocessorLexer::PreprocessorLexer(Preprocessor *pp, FileID fid)
+  : PP(pp), FID(fid), InitialNumSLocEntries(0),
+    ParsingPreprocessorDirective(false),
+    ParsingFilename(false), LexingRawMode(false) {
+  if (pp)
+    InitialNumSLocEntries = pp->getSourceManager().local_sloc_entry_size();
+}
+
+/// LexIncludeFilename - After the preprocessor has parsed a #include, lex and
+/// (potentially) macro expand the filename.
+void PreprocessorLexer::LexIncludeFilename(Token &FilenameTok) {
+  assert(ParsingPreprocessorDirective &&
+         ParsingFilename == false &&
+         "Must be in a preprocessing directive!");
+
+  // We are now parsing a filename!
+  ParsingFilename = true;
+
+  // Lex the filename.
+  IndirectLex(FilenameTok);
+
+  // We should have obtained the filename now.
+  ParsingFilename = false;
+
+  // No filename?
+  if (FilenameTok.is(tok::eod))
+    PP->Diag(FilenameTok.getLocation(), diag::err_pp_expects_filename);
+}
+
+/// getFileEntry - Return the FileEntry corresponding to this FileID.  Like
+/// getFileID(), this only works for lexers with attached preprocessors.
+const FileEntry *PreprocessorLexer::getFileEntry() const {
+  return PP->getSourceManager().getFileEntryForID(getFileID());
+}
diff --git a/clang/lib/Lex/ScratchBuffer.cpp b/clang/lib/Lex/ScratchBuffer.cpp
new file mode 100644
index 0000000..3d363fa
--- /dev/null
+++ b/clang/lib/Lex/ScratchBuffer.cpp
@@ -0,0 +1,73 @@
+//===--- ScratchBuffer.cpp - Scratch space for forming tokens -------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file implements the ScratchBuffer interface.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Lex/ScratchBuffer.h"
+#include "clang/Basic/SourceManager.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include <cstring>
+using namespace clang;
+
+// ScratchBufSize - The size of each chunk of scratch memory.  Slightly less
+//than a page, almost certainly enough for anything. :)
+static const unsigned ScratchBufSize = 4060;
+
+ScratchBuffer::ScratchBuffer(SourceManager &SM) : SourceMgr(SM), CurBuffer(0) {
+  // Set BytesUsed so that the first call to getToken will require an alloc.
+  BytesUsed = ScratchBufSize;
+}
+
+/// getToken - Splat the specified text into a temporary MemoryBuffer and
+/// return a SourceLocation that refers to the token.  This is just like the
+/// method below, but returns a location that indicates the physloc of the
+/// token.
+SourceLocation ScratchBuffer::getToken(const char *Buf, unsigned Len,
+                                       const char *&DestPtr) {
+  if (BytesUsed+Len+2 > ScratchBufSize)
+    AllocScratchBuffer(Len+2);
+
+  // Prefix the token with a \n, so that it looks like it is the first thing on
+  // its own virtual line in caret diagnostics.
+  CurBuffer[BytesUsed++] = '\n';
+
+  // Return a pointer to the character data.
+  DestPtr = CurBuffer+BytesUsed;
+
+  // Copy the token data into the buffer.
+  memcpy(CurBuffer+BytesUsed, Buf, Len);
+
+  // Remember that we used these bytes.
+  BytesUsed += Len+1;
+
+  // Add a NUL terminator to the token.  This keeps the tokens separated, in
+  // case they get relexed, and puts them on their own virtual lines in case a
+  // diagnostic points to one.
+  CurBuffer[BytesUsed-1] = '\0';
+
+  return BufferStartLoc.getLocWithOffset(BytesUsed-Len-1);
+}
+
+void ScratchBuffer::AllocScratchBuffer(unsigned RequestLen) {
+  // Only pay attention to the requested length if it is larger than our default
+  // page size.  If it is, we allocate an entire chunk for it.  This is to
+  // support gigantic tokens, which almost certainly won't happen. :)
+  if (RequestLen < ScratchBufSize)
+    RequestLen = ScratchBufSize;
+
+  llvm::MemoryBuffer *Buf =
+    llvm::MemoryBuffer::getNewMemBuffer(RequestLen, "<scratch space>");
+  FileID FID = SourceMgr.createFileIDForMemBuffer(Buf);
+  BufferStartLoc = SourceMgr.getLocForStartOfFile(FID);
+  CurBuffer = const_cast<char*>(Buf->getBufferStart());
+  BytesUsed = 1;
+  CurBuffer[0] = '0';  // Start out with a \0 for cleanliness.
+}
diff --git a/clang/lib/Lex/TokenConcatenation.cpp b/clang/lib/Lex/TokenConcatenation.cpp
new file mode 100644
index 0000000..84a46ed
--- /dev/null
+++ b/clang/lib/Lex/TokenConcatenation.cpp
@@ -0,0 +1,272 @@
+//===--- TokenConcatenation.cpp - Token Concatenation Avoidance -----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the TokenConcatenation class.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Lex/TokenConcatenation.h"
+#include "clang/Lex/Preprocessor.h"
+#include "llvm/Support/ErrorHandling.h"
+using namespace clang;
+
+
+/// IsStringPrefix - Return true if Str is a string prefix.
+/// 'L', 'u', 'U', or 'u8'. Including raw versions.
+static bool IsStringPrefix(StringRef Str, bool CPlusPlus0x) {
+
+  if (Str[0] == 'L' ||
+      (CPlusPlus0x && (Str[0] == 'u' || Str[0] == 'U' || Str[0] == 'R'))) {
+
+    if (Str.size() == 1)
+      return true; // "L", "u", "U", and "R"
+
+    // Check for raw flavors. Need to make sure the first character wasn't
+    // already R. Need CPlusPlus0x check for "LR".
+    if (Str[1] == 'R' && Str[0] != 'R' && Str.size() == 2 && CPlusPlus0x)
+      return true; // "LR", "uR", "UR"
+
+    // Check for "u8" and "u8R"
+    if (Str[0] == 'u' && Str[1] == '8') {
+      if (Str.size() == 2) return true; // "u8"
+      if (Str.size() == 3 && Str[2] == 'R') return true; // "u8R"
+    }
+  }
+
+  return false;
+}
+
+/// IsIdentifierStringPrefix - Return true if the spelling of the token
+/// is literally 'L', 'u', 'U', or 'u8'. Including raw versions.
+bool TokenConcatenation::IsIdentifierStringPrefix(const Token &Tok) const {
+  const LangOptions &LangOpts = PP.getLangOpts();
+
+  if (!Tok.needsCleaning()) {
+    if (Tok.getLength() < 1 || Tok.getLength() > 3)
+      return false;
+    SourceManager &SM = PP.getSourceManager();
+    const char *Ptr = SM.getCharacterData(SM.getSpellingLoc(Tok.getLocation()));
+    return IsStringPrefix(StringRef(Ptr, Tok.getLength()),
+                          LangOpts.CPlusPlus0x);
+  }
+
+  if (Tok.getLength() < 256) {
+    char Buffer[256];
+    const char *TokPtr = Buffer;
+    unsigned length = PP.getSpelling(Tok, TokPtr);
+    return IsStringPrefix(StringRef(TokPtr, length), LangOpts.CPlusPlus0x);
+  }
+
+  return IsStringPrefix(StringRef(PP.getSpelling(Tok)), LangOpts.CPlusPlus0x);
+}
+
+TokenConcatenation::TokenConcatenation(Preprocessor &pp) : PP(pp) {
+  memset(TokenInfo, 0, sizeof(TokenInfo));
+
+  // These tokens have custom code in AvoidConcat.
+  TokenInfo[tok::identifier      ] |= aci_custom;
+  TokenInfo[tok::numeric_constant] |= aci_custom_firstchar;
+  TokenInfo[tok::period          ] |= aci_custom_firstchar;
+  TokenInfo[tok::amp             ] |= aci_custom_firstchar;
+  TokenInfo[tok::plus            ] |= aci_custom_firstchar;
+  TokenInfo[tok::minus           ] |= aci_custom_firstchar;
+  TokenInfo[tok::slash           ] |= aci_custom_firstchar;
+  TokenInfo[tok::less            ] |= aci_custom_firstchar;
+  TokenInfo[tok::greater         ] |= aci_custom_firstchar;
+  TokenInfo[tok::pipe            ] |= aci_custom_firstchar;
+  TokenInfo[tok::percent         ] |= aci_custom_firstchar;
+  TokenInfo[tok::colon           ] |= aci_custom_firstchar;
+  TokenInfo[tok::hash            ] |= aci_custom_firstchar;
+  TokenInfo[tok::arrow           ] |= aci_custom_firstchar;
+
+  // These tokens have custom code in C++11 mode.
+  if (PP.getLangOpts().CPlusPlus0x) {
+    TokenInfo[tok::string_literal      ] |= aci_custom;
+    TokenInfo[tok::wide_string_literal ] |= aci_custom;
+    TokenInfo[tok::utf8_string_literal ] |= aci_custom;
+    TokenInfo[tok::utf16_string_literal] |= aci_custom;
+    TokenInfo[tok::utf32_string_literal] |= aci_custom;
+    TokenInfo[tok::char_constant       ] |= aci_custom;
+    TokenInfo[tok::wide_char_constant  ] |= aci_custom;
+    TokenInfo[tok::utf16_char_constant ] |= aci_custom;
+    TokenInfo[tok::utf32_char_constant ] |= aci_custom;
+  }
+
+  // These tokens change behavior if followed by an '='.
+  TokenInfo[tok::amp         ] |= aci_avoid_equal;           // &=
+  TokenInfo[tok::plus        ] |= aci_avoid_equal;           // +=
+  TokenInfo[tok::minus       ] |= aci_avoid_equal;           // -=
+  TokenInfo[tok::slash       ] |= aci_avoid_equal;           // /=
+  TokenInfo[tok::less        ] |= aci_avoid_equal;           // <=
+  TokenInfo[tok::greater     ] |= aci_avoid_equal;           // >=
+  TokenInfo[tok::pipe        ] |= aci_avoid_equal;           // |=
+  TokenInfo[tok::percent     ] |= aci_avoid_equal;           // %=
+  TokenInfo[tok::star        ] |= aci_avoid_equal;           // *=
+  TokenInfo[tok::exclaim     ] |= aci_avoid_equal;           // !=
+  TokenInfo[tok::lessless    ] |= aci_avoid_equal;           // <<=
+  TokenInfo[tok::greatergreater] |= aci_avoid_equal;         // >>=
+  TokenInfo[tok::caret       ] |= aci_avoid_equal;           // ^=
+  TokenInfo[tok::equal       ] |= aci_avoid_equal;           // ==
+}
+
+/// GetFirstChar - Get the first character of the token \arg Tok,
+/// avoiding calls to getSpelling where possible.
+static char GetFirstChar(Preprocessor &PP, const Token &Tok) {
+  if (IdentifierInfo *II = Tok.getIdentifierInfo()) {
+    // Avoid spelling identifiers, the most common form of token.
+    return II->getNameStart()[0];
+  } else if (!Tok.needsCleaning()) {
+    if (Tok.isLiteral() && Tok.getLiteralData()) {
+      return *Tok.getLiteralData();
+    } else {
+      SourceManager &SM = PP.getSourceManager();
+      return *SM.getCharacterData(SM.getSpellingLoc(Tok.getLocation()));
+    }
+  } else if (Tok.getLength() < 256) {
+    char Buffer[256];
+    const char *TokPtr = Buffer;
+    PP.getSpelling(Tok, TokPtr);
+    return TokPtr[0];
+  } else {
+    return PP.getSpelling(Tok)[0];
+  }
+}
+
+/// AvoidConcat - If printing PrevTok immediately followed by Tok would cause
+/// the two individual tokens to be lexed as a single token, return true
+/// (which causes a space to be printed between them).  This allows the output
+/// of -E mode to be lexed to the same token stream as lexing the input
+/// directly would.
+///
+/// This code must conservatively return true if it doesn't want to be 100%
+/// accurate.  This will cause the output to include extra space characters,
+/// but the resulting output won't have incorrect concatenations going on.
+/// Examples include "..", which we print with a space between, because we
+/// don't want to track enough to tell "x.." from "...".
+bool TokenConcatenation::AvoidConcat(const Token &PrevPrevTok,
+                                     const Token &PrevTok,
+                                     const Token &Tok) const {
+  // First, check to see if the tokens were directly adjacent in the original
+  // source.  If they were, it must be okay to stick them together: if there
+  // were an issue, the tokens would have been lexed differently.
+  if (PrevTok.getLocation().isFileID() && Tok.getLocation().isFileID() &&
+      PrevTok.getLocation().getLocWithOffset(PrevTok.getLength()) ==
+        Tok.getLocation())
+    return false;
+
+  tok::TokenKind PrevKind = PrevTok.getKind();
+  if (PrevTok.getIdentifierInfo())  // Language keyword or named operator.
+    PrevKind = tok::identifier;
+
+  // Look up information on when we should avoid concatenation with prevtok.
+  unsigned ConcatInfo = TokenInfo[PrevKind];
+
+  // If prevtok never causes a problem for anything after it, return quickly.
+  if (ConcatInfo == 0) return false;
+
+  if (ConcatInfo & aci_avoid_equal) {
+    // If the next token is '=' or '==', avoid concatenation.
+    if (Tok.is(tok::equal) || Tok.is(tok::equalequal))
+      return true;
+    ConcatInfo &= ~aci_avoid_equal;
+  }
+
+  if (ConcatInfo == 0) return false;
+
+  // Basic algorithm: we look at the first character of the second token, and
+  // determine whether it, if appended to the first token, would form (or
+  // would contribute) to a larger token if concatenated.
+  char FirstChar = 0;
+  if (ConcatInfo & aci_custom) {
+    // If the token does not need to know the first character, don't get it.
+  } else {
+    FirstChar = GetFirstChar(PP, Tok);
+  }
+
+  switch (PrevKind) {
+  default:
+    llvm_unreachable("InitAvoidConcatTokenInfo built wrong");
+
+  case tok::raw_identifier:
+    llvm_unreachable("tok::raw_identifier in non-raw lexing mode!");
+
+  case tok::string_literal:
+  case tok::wide_string_literal:
+  case tok::utf8_string_literal:
+  case tok::utf16_string_literal:
+  case tok::utf32_string_literal:
+  case tok::char_constant:
+  case tok::wide_char_constant:
+  case tok::utf16_char_constant:
+  case tok::utf32_char_constant:
+    if (!PP.getLangOpts().CPlusPlus0x)
+      return false;
+
+    // In C++11, a string or character literal followed by an identifier is a
+    // single token.
+    if (Tok.getIdentifierInfo())
+      return true;
+
+    // A ud-suffix is an identifier. If the previous token ends with one, treat
+    // it as an identifier.
+    if (!PrevTok.hasUDSuffix())
+      return false;
+    // FALL THROUGH.
+  case tok::identifier:   // id+id or id+number or id+L"foo".
+    // id+'.'... will not append.
+    if (Tok.is(tok::numeric_constant))
+      return GetFirstChar(PP, Tok) != '.';
+
+    if (Tok.getIdentifierInfo() || Tok.is(tok::wide_string_literal) ||
+        Tok.is(tok::utf8_string_literal) || Tok.is(tok::utf16_string_literal) ||
+        Tok.is(tok::utf32_string_literal) || Tok.is(tok::wide_char_constant) ||
+        Tok.is(tok::utf16_char_constant) || Tok.is(tok::utf32_char_constant))
+      return true;
+
+    // If this isn't identifier + string, we're done.
+    if (Tok.isNot(tok::char_constant) && Tok.isNot(tok::string_literal))
+      return false;
+
+    // Otherwise, this is a narrow character or string.  If the *identifier*
+    // is a literal 'L', 'u8', 'u' or 'U', avoid pasting L "foo" -> L"foo".
+    return IsIdentifierStringPrefix(PrevTok);
+
+  case tok::numeric_constant:
+    return isalnum(FirstChar) || Tok.is(tok::numeric_constant) ||
+           FirstChar == '+' || FirstChar == '-' || FirstChar == '.' ||
+           (PP.getLangOpts().CPlusPlus0x && FirstChar == '_');
+  case tok::period:          // ..., .*, .1234
+    return (FirstChar == '.' && PrevPrevTok.is(tok::period)) ||
+    isdigit(FirstChar) ||
+    (PP.getLangOpts().CPlusPlus && FirstChar == '*');
+  case tok::amp:             // &&
+    return FirstChar == '&';
+  case tok::plus:            // ++
+    return FirstChar == '+';
+  case tok::minus:           // --, ->, ->*
+    return FirstChar == '-' || FirstChar == '>';
+  case tok::slash:           //, /*, //
+    return FirstChar == '*' || FirstChar == '/';
+  case tok::less:            // <<, <<=, <:, <%
+    return FirstChar == '<' || FirstChar == ':' || FirstChar == '%';
+  case tok::greater:         // >>, >>=
+    return FirstChar == '>';
+  case tok::pipe:            // ||
+    return FirstChar == '|';
+  case tok::percent:         // %>, %:
+    return FirstChar == '>' || FirstChar == ':';
+  case tok::colon:           // ::, :>
+    return FirstChar == '>' ||
+    (PP.getLangOpts().CPlusPlus && FirstChar == ':');
+  case tok::hash:            // ##, #@, %:%:
+    return FirstChar == '#' || FirstChar == '@' || FirstChar == '%';
+  case tok::arrow:           // ->*
+    return PP.getLangOpts().CPlusPlus && FirstChar == '*';
+  }
+}
diff --git a/clang/lib/Lex/TokenLexer.cpp b/clang/lib/Lex/TokenLexer.cpp
new file mode 100644
index 0000000..696754c
--- /dev/null
+++ b/clang/lib/Lex/TokenLexer.cpp
@@ -0,0 +1,756 @@
+//===--- TokenLexer.cpp - Lex from a token stream -------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the TokenLexer interface.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Lex/TokenLexer.h"
+#include "MacroArgs.h"
+#include "clang/Lex/MacroInfo.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Lex/LexDiagnostic.h"
+#include "llvm/ADT/SmallString.h"
+using namespace clang;
+
+
+/// Create a TokenLexer for the specified macro with the specified actual
+/// arguments.  Note that this ctor takes ownership of the ActualArgs pointer.
+void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroArgs *Actuals) {
+  // If the client is reusing a TokenLexer, make sure to free any memory
+  // associated with it.
+  destroy();
+
+  Macro = PP.getMacroInfo(Tok.getIdentifierInfo());
+  ActualArgs = Actuals;
+  CurToken = 0;
+
+  ExpandLocStart = Tok.getLocation();
+  ExpandLocEnd = ELEnd;
+  AtStartOfLine = Tok.isAtStartOfLine();
+  HasLeadingSpace = Tok.hasLeadingSpace();
+  Tokens = &*Macro->tokens_begin();
+  OwnsTokens = false;
+  DisableMacroExpansion = false;
+  NumTokens = Macro->tokens_end()-Macro->tokens_begin();
+  MacroExpansionStart = SourceLocation();
+
+  SourceManager &SM = PP.getSourceManager();
+  MacroStartSLocOffset = SM.getNextLocalOffset();
+
+  if (NumTokens > 0) {
+    assert(Tokens[0].getLocation().isValid());
+    assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) &&
+           "Macro defined in macro?");
+    assert(ExpandLocStart.isValid());
+
+    // Reserve a source location entry chunk for the length of the macro
+    // definition. Tokens that get lexed directly from the definition will
+    // have their locations pointing inside this chunk. This is to avoid
+    // creating separate source location entries for each token.
+    MacroDefStart = SM.getExpansionLoc(Tokens[0].getLocation());
+    MacroDefLength = Macro->getDefinitionLength(SM);
+    MacroExpansionStart = SM.createExpansionLoc(MacroDefStart,
+                                                ExpandLocStart,
+                                                ExpandLocEnd,
+                                                MacroDefLength);
+  }
+
+  // If this is a function-like macro, expand the arguments and change
+  // Tokens to point to the expanded tokens.
+  if (Macro->isFunctionLike() && Macro->getNumArgs())
+    ExpandFunctionArguments();
+
+  // Mark the macro as currently disabled, so that it is not recursively
+  // expanded.  The macro must be disabled only after argument pre-expansion of
+  // function-like macro arguments occurs.
+  Macro->DisableMacro();
+}
+
+
+
+/// Create a TokenLexer for the specified token stream.  This does not
+/// take ownership of the specified token vector.
+void TokenLexer::Init(const Token *TokArray, unsigned NumToks,
+                      bool disableMacroExpansion, bool ownsTokens) {
+  // If the client is reusing a TokenLexer, make sure to free any memory
+  // associated with it.
+  destroy();
+
+  Macro = 0;
+  ActualArgs = 0;
+  Tokens = TokArray;
+  OwnsTokens = ownsTokens;
+  DisableMacroExpansion = disableMacroExpansion;
+  NumTokens = NumToks;
+  CurToken = 0;
+  ExpandLocStart = ExpandLocEnd = SourceLocation();
+  AtStartOfLine = false;
+  HasLeadingSpace = false;
+  MacroExpansionStart = SourceLocation();
+
+  // Set HasLeadingSpace/AtStartOfLine so that the first token will be
+  // returned unmodified.
+  if (NumToks != 0) {
+    AtStartOfLine   = TokArray[0].isAtStartOfLine();
+    HasLeadingSpace = TokArray[0].hasLeadingSpace();
+  }
+}
+
+
+void TokenLexer::destroy() {
+  // If this was a function-like macro that actually uses its arguments, delete
+  // the expanded tokens.
+  if (OwnsTokens) {
+    delete [] Tokens;
+    Tokens = 0;
+    OwnsTokens = false;
+  }
+
+  // TokenLexer owns its formal arguments.
+  if (ActualArgs) ActualArgs->destroy(PP);
+}
+
+/// Expand the arguments of a function-like macro so that we can quickly
+/// return preexpanded tokens from Tokens.
+void TokenLexer::ExpandFunctionArguments() {
+
+  SmallVector<Token, 128> ResultToks;
+
+  // Loop through 'Tokens', expanding them into ResultToks.  Keep
+  // track of whether we change anything.  If not, no need to keep them.  If so,
+  // we install the newly expanded sequence as the new 'Tokens' list.
+  bool MadeChange = false;
+
+  // NextTokGetsSpace - When this is true, the next token appended to the
+  // output list will get a leading space, regardless of whether it had one to
+  // begin with or not.  This is used for placemarker support.
+  bool NextTokGetsSpace = false;
+
+  for (unsigned i = 0, e = NumTokens; i != e; ++i) {
+    // If we found the stringify operator, get the argument stringified.  The
+    // preprocessor already verified that the following token is a macro name
+    // when the #define was parsed.
+    const Token &CurTok = Tokens[i];
+    if (CurTok.is(tok::hash) || CurTok.is(tok::hashat)) {
+      int ArgNo = Macro->getArgumentNum(Tokens[i+1].getIdentifierInfo());
+      assert(ArgNo != -1 && "Token following # is not an argument?");
+
+      SourceLocation ExpansionLocStart =
+          getExpansionLocForMacroDefLoc(CurTok.getLocation());
+      SourceLocation ExpansionLocEnd =
+          getExpansionLocForMacroDefLoc(Tokens[i+1].getLocation());
+
+      Token Res;
+      if (CurTok.is(tok::hash))  // Stringify
+        Res = ActualArgs->getStringifiedArgument(ArgNo, PP,
+                                                 ExpansionLocStart,
+                                                 ExpansionLocEnd);
+      else {
+        // 'charify': don't bother caching these.
+        Res = MacroArgs::StringifyArgument(ActualArgs->getUnexpArgument(ArgNo),
+                                           PP, true,
+                                           ExpansionLocStart,
+                                           ExpansionLocEnd);
+      }
+
+      // The stringified/charified string leading space flag gets set to match
+      // the #/#@ operator.
+      if (CurTok.hasLeadingSpace() || NextTokGetsSpace)
+        Res.setFlag(Token::LeadingSpace);
+
+      ResultToks.push_back(Res);
+      MadeChange = true;
+      ++i;  // Skip arg name.
+      NextTokGetsSpace = false;
+      continue;
+    }
+
+    // Otherwise, if this is not an argument token, just add the token to the
+    // output buffer.
+    IdentifierInfo *II = CurTok.getIdentifierInfo();
+    int ArgNo = II ? Macro->getArgumentNum(II) : -1;
+    if (ArgNo == -1) {
+      // This isn't an argument, just add it.
+      ResultToks.push_back(CurTok);
+
+      if (NextTokGetsSpace) {
+        ResultToks.back().setFlag(Token::LeadingSpace);
+        NextTokGetsSpace = false;
+      }
+      continue;
+    }
+
+    // An argument is expanded somehow, the result is different than the
+    // input.
+    MadeChange = true;
+
+    // Otherwise, this is a use of the argument.  Find out if there is a paste
+    // (##) operator before or after the argument.
+    bool PasteBefore =
+      !ResultToks.empty() && ResultToks.back().is(tok::hashhash);
+    bool PasteAfter = i+1 != e && Tokens[i+1].is(tok::hashhash);
+
+    // If it is not the LHS/RHS of a ## operator, we must pre-expand the
+    // argument and substitute the expanded tokens into the result.  This is
+    // C99 6.10.3.1p1.
+    if (!PasteBefore && !PasteAfter) {
+      const Token *ResultArgToks;
+
+      // Only preexpand the argument if it could possibly need it.  This
+      // avoids some work in common cases.
+      const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo);
+      if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP))
+        ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, Macro, PP)[0];
+      else
+        ResultArgToks = ArgTok;  // Use non-preexpanded tokens.
+
+      // If the arg token expanded into anything, append it.
+      if (ResultArgToks->isNot(tok::eof)) {
+        unsigned FirstResult = ResultToks.size();
+        unsigned NumToks = MacroArgs::getArgLength(ResultArgToks);
+        ResultToks.append(ResultArgToks, ResultArgToks+NumToks);
+
+        // If the '##' came from expanding an argument, turn it into 'unknown'
+        // to avoid pasting.
+        for (unsigned i = FirstResult, e = ResultToks.size(); i != e; ++i) {
+          Token &Tok = ResultToks[i];
+          if (Tok.is(tok::hashhash))
+            Tok.setKind(tok::unknown);
+        }
+
+        if(ExpandLocStart.isValid()) {
+          updateLocForMacroArgTokens(CurTok.getLocation(),
+                                     ResultToks.begin()+FirstResult,
+                                     ResultToks.end());
+        }
+
+        // If any tokens were substituted from the argument, the whitespace
+        // before the first token should match the whitespace of the arg
+        // identifier.
+        ResultToks[FirstResult].setFlagValue(Token::LeadingSpace,
+                                             CurTok.hasLeadingSpace() ||
+                                             NextTokGetsSpace);
+        NextTokGetsSpace = false;
+      } else {
+        // If this is an empty argument, and if there was whitespace before the
+        // formal token, make sure the next token gets whitespace before it.
+        NextTokGetsSpace = CurTok.hasLeadingSpace();
+      }
+      continue;
+    }
+
+    // Okay, we have a token that is either the LHS or RHS of a paste (##)
+    // argument.  It gets substituted as its non-pre-expanded tokens.
+    const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo);
+    unsigned NumToks = MacroArgs::getArgLength(ArgToks);
+    if (NumToks) {  // Not an empty argument?
+      // If this is the GNU ", ## __VA_ARG__" extension, and we just learned
+      // that __VA_ARG__ expands to multiple tokens, avoid a pasting error when
+      // the expander trys to paste ',' with the first token of the __VA_ARG__
+      // expansion.
+      if (PasteBefore && ResultToks.size() >= 2 &&
+          ResultToks[ResultToks.size()-2].is(tok::comma) &&
+          (unsigned)ArgNo == Macro->getNumArgs()-1 &&
+          Macro->isVariadic()) {
+        // Remove the paste operator, report use of the extension.
+        PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);
+        ResultToks.pop_back();
+      }
+
+      ResultToks.append(ArgToks, ArgToks+NumToks);
+
+      // If the '##' came from expanding an argument, turn it into 'unknown'
+      // to avoid pasting.
+      for (unsigned i = ResultToks.size() - NumToks, e = ResultToks.size();
+             i != e; ++i) {
+        Token &Tok = ResultToks[i];
+        if (Tok.is(tok::hashhash))
+          Tok.setKind(tok::unknown);
+      }
+
+      if (ExpandLocStart.isValid()) {
+        updateLocForMacroArgTokens(CurTok.getLocation(),
+                                   ResultToks.end()-NumToks, ResultToks.end());
+      }
+
+      // If this token (the macro argument) was supposed to get leading
+      // whitespace, transfer this information onto the first token of the
+      // expansion.
+      //
+      // Do not do this if the paste operator occurs before the macro argument,
+      // as in "A ## MACROARG".  In valid code, the first token will get
+      // smooshed onto the preceding one anyway (forming AMACROARG).  In
+      // assembler-with-cpp mode, invalid pastes are allowed through: in this
+      // case, we do not want the extra whitespace to be added.  For example,
+      // we want ". ## foo" -> ".foo" not ". foo".
+      if ((CurTok.hasLeadingSpace() || NextTokGetsSpace) &&
+          !PasteBefore)
+        ResultToks[ResultToks.size()-NumToks].setFlag(Token::LeadingSpace);
+
+      NextTokGetsSpace = false;
+      continue;
+    }
+
+    // If an empty argument is on the LHS or RHS of a paste, the standard (C99
+    // 6.10.3.3p2,3) calls for a bunch of placemarker stuff to occur.  We
+    // implement this by eating ## operators when a LHS or RHS expands to
+    // empty.
+    NextTokGetsSpace |= CurTok.hasLeadingSpace();
+    if (PasteAfter) {
+      // Discard the argument token and skip (don't copy to the expansion
+      // buffer) the paste operator after it.
+      NextTokGetsSpace |= Tokens[i+1].hasLeadingSpace();
+      ++i;
+      continue;
+    }
+
+    // If this is on the RHS of a paste operator, we've already copied the
+    // paste operator to the ResultToks list.  Remove it.
+    assert(PasteBefore && ResultToks.back().is(tok::hashhash));
+    NextTokGetsSpace |= ResultToks.back().hasLeadingSpace();
+    ResultToks.pop_back();
+
+    // If this is the __VA_ARGS__ token, and if the argument wasn't provided,
+    // and if the macro had at least one real argument, and if the token before
+    // the ## was a comma, remove the comma.
+    if ((unsigned)ArgNo == Macro->getNumArgs()-1 && // is __VA_ARGS__
+        ActualArgs->isVarargsElidedUse() &&       // Argument elided.
+        !ResultToks.empty() && ResultToks.back().is(tok::comma)) {
+      // Never add a space, even if the comma, ##, or arg had a space.
+      NextTokGetsSpace = false;
+      // Remove the paste operator, report use of the extension.
+      PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);
+      ResultToks.pop_back();
+      
+      // If the comma was right after another paste (e.g. "X##,##__VA_ARGS__"),
+      // then removal of the comma should produce a placemarker token (in C99
+      // terms) which we model by popping off the previous ##, giving us a plain
+      // "X" when __VA_ARGS__ is empty.
+      if (!ResultToks.empty() && ResultToks.back().is(tok::hashhash))
+        ResultToks.pop_back();
+    }
+    continue;
+  }
+
+  // If anything changed, install this as the new Tokens list.
+  if (MadeChange) {
+    assert(!OwnsTokens && "This would leak if we already own the token list");
+    // This is deleted in the dtor.
+    NumTokens = ResultToks.size();
+    // The tokens will be added to Preprocessor's cache and will be removed
+    // when this TokenLexer finishes lexing them.
+    Tokens = PP.cacheMacroExpandedTokens(this, ResultToks);
+
+    // The preprocessor cache of macro expanded tokens owns these tokens,not us.
+    OwnsTokens = false;
+  }
+}
+
+/// Lex - Lex and return a token from this macro stream.
+///
+void TokenLexer::Lex(Token &Tok) {
+  // Lexing off the end of the macro, pop this macro off the expansion stack.
+  if (isAtEnd()) {
+    // If this is a macro (not a token stream), mark the macro enabled now
+    // that it is no longer being expanded.
+    if (Macro) Macro->EnableMacro();
+
+    // Pop this context off the preprocessors lexer stack and get the next
+    // token.  This will delete "this" so remember the PP instance var.
+    Preprocessor &PPCache = PP;
+    if (PP.HandleEndOfTokenLexer(Tok))
+      return;
+
+    // HandleEndOfTokenLexer may not return a token.  If it doesn't, lex
+    // whatever is next.
+    return PPCache.Lex(Tok);
+  }
+
+  SourceManager &SM = PP.getSourceManager();
+
+  // If this is the first token of the expanded result, we inherit spacing
+  // properties later.
+  bool isFirstToken = CurToken == 0;
+
+  // Get the next token to return.
+  Tok = Tokens[CurToken++];
+
+  bool TokenIsFromPaste = false;
+
+  // If this token is followed by a token paste (##) operator, paste the tokens!
+  // Note that ## is a normal token when not expanding a macro.
+  if (!isAtEnd() && Tokens[CurToken].is(tok::hashhash) && Macro) {
+    // When handling the microsoft /##/ extension, the final token is
+    // returned by PasteTokens, not the pasted token.
+    if (PasteTokens(Tok))
+      return;
+
+    TokenIsFromPaste = true;
+  }
+
+  // The token's current location indicate where the token was lexed from.  We
+  // need this information to compute the spelling of the token, but any
+  // diagnostics for the expanded token should appear as if they came from
+  // ExpansionLoc.  Pull this information together into a new SourceLocation
+  // that captures all of this.
+  if (ExpandLocStart.isValid() &&   // Don't do this for token streams.
+      // Check that the token's location was not already set properly.
+      SM.isBeforeInSLocAddrSpace(Tok.getLocation(), MacroStartSLocOffset)) {
+    SourceLocation instLoc;
+    if (Tok.is(tok::comment)) {
+      instLoc = SM.createExpansionLoc(Tok.getLocation(),
+                                      ExpandLocStart,
+                                      ExpandLocEnd,
+                                      Tok.getLength());
+    } else {
+      instLoc = getExpansionLocForMacroDefLoc(Tok.getLocation());
+    }
+
+    Tok.setLocation(instLoc);
+  }
+
+  // If this is the first token, set the lexical properties of the token to
+  // match the lexical properties of the macro identifier.
+  if (isFirstToken) {
+    Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
+    Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
+  }
+
+  // Handle recursive expansion!
+  if (!Tok.isAnnotation() && Tok.getIdentifierInfo() != 0) {
+    // Change the kind of this identifier to the appropriate token kind, e.g.
+    // turning "for" into a keyword.
+    IdentifierInfo *II = Tok.getIdentifierInfo();
+    Tok.setKind(II->getTokenID());
+
+    // If this identifier was poisoned and from a paste, emit an error.  This
+    // won't be handled by Preprocessor::HandleIdentifier because this is coming
+    // from a macro expansion.
+    if (II->isPoisoned() && TokenIsFromPaste) {
+      PP.HandlePoisonedIdentifier(Tok);
+    }
+
+    if (!DisableMacroExpansion && II->isHandleIdentifierCase())
+      PP.HandleIdentifier(Tok);
+  }
+
+  // Otherwise, return a normal token.
+}
+
+/// PasteTokens - Tok is the LHS of a ## operator, and CurToken is the ##
+/// operator.  Read the ## and RHS, and paste the LHS/RHS together.  If there
+/// are more ## after it, chomp them iteratively.  Return the result as Tok.
+/// If this returns true, the caller should immediately return the token.
+bool TokenLexer::PasteTokens(Token &Tok) {
+  SmallString<128> Buffer;
+  const char *ResultTokStrPtr = 0;
+  SourceLocation StartLoc = Tok.getLocation();
+  SourceLocation PasteOpLoc;
+  do {
+    // Consume the ## operator.
+    PasteOpLoc = Tokens[CurToken].getLocation();
+    ++CurToken;
+    assert(!isAtEnd() && "No token on the RHS of a paste operator!");
+
+    // Get the RHS token.
+    const Token &RHS = Tokens[CurToken];
+
+    // Allocate space for the result token.  This is guaranteed to be enough for
+    // the two tokens.
+    Buffer.resize(Tok.getLength() + RHS.getLength());
+
+    // Get the spelling of the LHS token in Buffer.
+    const char *BufPtr = &Buffer[0];
+    bool Invalid = false;
+    unsigned LHSLen = PP.getSpelling(Tok, BufPtr, &Invalid);
+    if (BufPtr != &Buffer[0])   // Really, we want the chars in Buffer!
+      memcpy(&Buffer[0], BufPtr, LHSLen);
+    if (Invalid)
+      return true;
+    
+    BufPtr = &Buffer[LHSLen];
+    unsigned RHSLen = PP.getSpelling(RHS, BufPtr, &Invalid);
+    if (Invalid)
+      return true;
+    if (BufPtr != &Buffer[LHSLen])   // Really, we want the chars in Buffer!
+      memcpy(&Buffer[LHSLen], BufPtr, RHSLen);
+
+    // Trim excess space.
+    Buffer.resize(LHSLen+RHSLen);
+
+    // Plop the pasted result (including the trailing newline and null) into a
+    // scratch buffer where we can lex it.
+    Token ResultTokTmp;
+    ResultTokTmp.startToken();
+
+    // Claim that the tmp token is a string_literal so that we can get the
+    // character pointer back from CreateString in getLiteralData().
+    ResultTokTmp.setKind(tok::string_literal);
+    PP.CreateString(&Buffer[0], Buffer.size(), ResultTokTmp);
+    SourceLocation ResultTokLoc = ResultTokTmp.getLocation();
+    ResultTokStrPtr = ResultTokTmp.getLiteralData();
+
+    // Lex the resultant pasted token into Result.
+    Token Result;
+
+    if (Tok.isAnyIdentifier() && RHS.isAnyIdentifier()) {
+      // Common paste case: identifier+identifier = identifier.  Avoid creating
+      // a lexer and other overhead.
+      PP.IncrementPasteCounter(true);
+      Result.startToken();
+      Result.setKind(tok::raw_identifier);
+      Result.setRawIdentifierData(ResultTokStrPtr);
+      Result.setLocation(ResultTokLoc);
+      Result.setLength(LHSLen+RHSLen);
+    } else {
+      PP.IncrementPasteCounter(false);
+
+      assert(ResultTokLoc.isFileID() &&
+             "Should be a raw location into scratch buffer");
+      SourceManager &SourceMgr = PP.getSourceManager();
+      FileID LocFileID = SourceMgr.getFileID(ResultTokLoc);
+
+      bool Invalid = false;
+      const char *ScratchBufStart
+        = SourceMgr.getBufferData(LocFileID, &Invalid).data();
+      if (Invalid)
+        return false;
+
+      // Make a lexer to lex this string from.  Lex just this one token.
+      // Make a lexer object so that we lex and expand the paste result.
+      Lexer TL(SourceMgr.getLocForStartOfFile(LocFileID),
+               PP.getLangOpts(), ScratchBufStart,
+               ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen);
+
+      // Lex a token in raw mode.  This way it won't look up identifiers
+      // automatically, lexing off the end will return an eof token, and
+      // warnings are disabled.  This returns true if the result token is the
+      // entire buffer.
+      bool isInvalid = !TL.LexFromRawLexer(Result);
+
+      // If we got an EOF token, we didn't form even ONE token.  For example, we
+      // did "/ ## /" to get "//".
+      isInvalid |= Result.is(tok::eof);
+
+      // If pasting the two tokens didn't form a full new token, this is an
+      // error.  This occurs with "x ## +"  and other stuff.  Return with Tok
+      // unmodified and with RHS as the next token to lex.
+      if (isInvalid) {
+        // Test for the Microsoft extension of /##/ turning into // here on the
+        // error path.
+        if (PP.getLangOpts().MicrosoftExt && Tok.is(tok::slash) &&
+            RHS.is(tok::slash)) {
+          HandleMicrosoftCommentPaste(Tok);
+          return true;
+        }
+
+        // Do not emit the error when preprocessing assembler code.
+        if (!PP.getLangOpts().AsmPreprocessor) {
+          // Explicitly convert the token location to have proper expansion
+          // information so that the user knows where it came from.
+          SourceManager &SM = PP.getSourceManager();
+          SourceLocation Loc =
+            SM.createExpansionLoc(PasteOpLoc, ExpandLocStart, ExpandLocEnd, 2);
+          // If we're in microsoft extensions mode, downgrade this from a hard
+          // error to a warning that defaults to an error.  This allows
+          // disabling it.
+          PP.Diag(Loc,
+                  PP.getLangOpts().MicrosoftExt ? diag::err_pp_bad_paste_ms 
+                                                   : diag::err_pp_bad_paste)
+            << Buffer.str();
+        }
+
+        // Do not consume the RHS.
+        --CurToken;
+      }
+
+      // Turn ## into 'unknown' to avoid # ## # from looking like a paste
+      // operator.
+      if (Result.is(tok::hashhash))
+        Result.setKind(tok::unknown);
+    }
+
+    // Transfer properties of the LHS over the the Result.
+    Result.setFlagValue(Token::StartOfLine , Tok.isAtStartOfLine());
+    Result.setFlagValue(Token::LeadingSpace, Tok.hasLeadingSpace());
+    
+    // Finally, replace LHS with the result, consume the RHS, and iterate.
+    ++CurToken;
+    Tok = Result;
+  } while (!isAtEnd() && Tokens[CurToken].is(tok::hashhash));
+
+  SourceLocation EndLoc = Tokens[CurToken - 1].getLocation();
+
+  // The token's current location indicate where the token was lexed from.  We
+  // need this information to compute the spelling of the token, but any
+  // diagnostics for the expanded token should appear as if the token was
+  // expanded from the full ## expression. Pull this information together into
+  // a new SourceLocation that captures all of this.
+  SourceManager &SM = PP.getSourceManager();
+  if (StartLoc.isFileID())
+    StartLoc = getExpansionLocForMacroDefLoc(StartLoc);
+  if (EndLoc.isFileID())
+    EndLoc = getExpansionLocForMacroDefLoc(EndLoc);
+  Tok.setLocation(SM.createExpansionLoc(Tok.getLocation(), StartLoc, EndLoc,
+                                        Tok.getLength()));
+
+  // Now that we got the result token, it will be subject to expansion.  Since
+  // token pasting re-lexes the result token in raw mode, identifier information
+  // isn't looked up.  As such, if the result is an identifier, look up id info.
+  if (Tok.is(tok::raw_identifier)) {
+    // Look up the identifier info for the token.  We disabled identifier lookup
+    // by saying we're skipping contents, so we need to do this manually.
+    PP.LookUpIdentifierInfo(Tok);
+  }
+  return false;
+}
+
+/// isNextTokenLParen - If the next token lexed will pop this macro off the
+/// expansion stack, return 2.  If the next unexpanded token is a '(', return
+/// 1, otherwise return 0.
+unsigned TokenLexer::isNextTokenLParen() const {
+  // Out of tokens?
+  if (isAtEnd())
+    return 2;
+  return Tokens[CurToken].is(tok::l_paren);
+}
+
+/// isParsingPreprocessorDirective - Return true if we are in the middle of a
+/// preprocessor directive.
+bool TokenLexer::isParsingPreprocessorDirective() const {
+  return Tokens[NumTokens-1].is(tok::eod) && !isAtEnd();
+}
+
+/// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes
+/// together to form a comment that comments out everything in the current
+/// macro, other active macros, and anything left on the current physical
+/// source line of the expanded buffer.  Handle this by returning the
+/// first token on the next line.
+void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok) {
+  // We 'comment out' the rest of this macro by just ignoring the rest of the
+  // tokens that have not been lexed yet, if any.
+
+  // Since this must be a macro, mark the macro enabled now that it is no longer
+  // being expanded.
+  assert(Macro && "Token streams can't paste comments");
+  Macro->EnableMacro();
+
+  PP.HandleMicrosoftCommentPaste(Tok);
+}
+
+/// \brief If \arg loc is a file ID and points inside the current macro
+/// definition, returns the appropriate source location pointing at the
+/// macro expansion source location entry, otherwise it returns an invalid
+/// SourceLocation.
+SourceLocation
+TokenLexer::getExpansionLocForMacroDefLoc(SourceLocation loc) const {
+  assert(ExpandLocStart.isValid() && MacroExpansionStart.isValid() &&
+         "Not appropriate for token streams");
+  assert(loc.isValid() && loc.isFileID());
+  
+  SourceManager &SM = PP.getSourceManager();
+  assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) &&
+         "Expected loc to come from the macro definition");
+
+  unsigned relativeOffset = 0;
+  SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength, &relativeOffset);
+  return MacroExpansionStart.getLocWithOffset(relativeOffset);
+}
+
+/// \brief Finds the tokens that are consecutive (from the same FileID)
+/// creates a single SLocEntry, and assigns SourceLocations to each token that
+/// point to that SLocEntry. e.g for
+///   assert(foo == bar);
+/// There will be a single SLocEntry for the "foo == bar" chunk and locations
+/// for the 'foo', '==', 'bar' tokens will point inside that chunk.
+///
+/// \arg begin_tokens will be updated to a position past all the found
+/// consecutive tokens.
+static void updateConsecutiveMacroArgTokens(SourceManager &SM,
+                                            SourceLocation InstLoc,
+                                            Token *&begin_tokens,
+                                            Token * end_tokens) {
+  assert(begin_tokens < end_tokens);
+
+  SourceLocation FirstLoc = begin_tokens->getLocation();
+  SourceLocation CurLoc = FirstLoc;
+
+  // Compare the source location offset of tokens and group together tokens that
+  // are close, even if their locations point to different FileIDs. e.g.
+  //
+  //  |bar    |  foo | cake   |  (3 tokens from 3 consecutive FileIDs)
+  //  ^                    ^
+  //  |bar       foo   cake|     (one SLocEntry chunk for all tokens)
+  //
+  // we can perform this "merge" since the token's spelling location depends
+  // on the relative offset.
+
+  Token *NextTok = begin_tokens + 1;
+  for (; NextTok < end_tokens; ++NextTok) {
+    int RelOffs;
+    if (!SM.isInSameSLocAddrSpace(CurLoc, NextTok->getLocation(), &RelOffs))
+      break; // Token from different local/loaded location.
+    // Check that token is not before the previous token or more than 50
+    // "characters" away.
+    if (RelOffs < 0 || RelOffs > 50)
+      break;
+    CurLoc = NextTok->getLocation();
+  }
+
+  // For the consecutive tokens, find the length of the SLocEntry to contain
+  // all of them.
+  Token &LastConsecutiveTok = *(NextTok-1);
+  int LastRelOffs = 0;
+  SM.isInSameSLocAddrSpace(FirstLoc, LastConsecutiveTok.getLocation(),
+                           &LastRelOffs);
+  unsigned FullLength = LastRelOffs + LastConsecutiveTok.getLength();
+
+  // Create a macro expansion SLocEntry that will "contain" all of the tokens.
+  SourceLocation Expansion =
+      SM.createMacroArgExpansionLoc(FirstLoc, InstLoc,FullLength);
+
+  // Change the location of the tokens from the spelling location to the new
+  // expanded location.
+  for (; begin_tokens < NextTok; ++begin_tokens) {
+    Token &Tok = *begin_tokens;
+    int RelOffs = 0;
+    SM.isInSameSLocAddrSpace(FirstLoc, Tok.getLocation(), &RelOffs);
+    Tok.setLocation(Expansion.getLocWithOffset(RelOffs));
+  }
+}
+
+/// \brief Creates SLocEntries and updates the locations of macro argument
+/// tokens to their new expanded locations.
+///
+/// \param ArgIdDefLoc the location of the macro argument id inside the macro
+/// definition.
+/// \param Tokens the macro argument tokens to update.
+void TokenLexer::updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc,
+                                            Token *begin_tokens,
+                                            Token *end_tokens) {
+  SourceManager &SM = PP.getSourceManager();
+
+  SourceLocation InstLoc =
+      getExpansionLocForMacroDefLoc(ArgIdSpellLoc);
+  
+  while (begin_tokens < end_tokens) {
+    // If there's only one token just create a SLocEntry for it.
+    if (end_tokens - begin_tokens == 1) {
+      Token &Tok = *begin_tokens;
+      Tok.setLocation(SM.createMacroArgExpansionLoc(Tok.getLocation(),
+                                                    InstLoc,
+                                                    Tok.getLength()));
+      return;
+    }
+
+    updateConsecutiveMacroArgTokens(SM, InstLoc, begin_tokens, end_tokens);
+  }
+}