From 222e2a7620e6520ffaf4fc4e69d79c18da31542e Mon Sep 17 00:00:00 2001 From: "Zancanaro; Carlo" Date: Mon, 24 Sep 2012 09:58:17 +1000 Subject: Add the clang library to the repo (with some of my changes, too). --- clang/lib/Tooling/CMakeLists.txt | 7 + clang/lib/Tooling/CompilationDatabase.cpp | 266 +++++++++++++++++++++++++++ clang/lib/Tooling/Makefile | 13 ++ clang/lib/Tooling/Tooling.cpp | 296 ++++++++++++++++++++++++++++++ 4 files changed, 582 insertions(+) create mode 100644 clang/lib/Tooling/CMakeLists.txt create mode 100644 clang/lib/Tooling/CompilationDatabase.cpp create mode 100644 clang/lib/Tooling/Makefile create mode 100644 clang/lib/Tooling/Tooling.cpp (limited to 'clang/lib/Tooling') diff --git a/clang/lib/Tooling/CMakeLists.txt b/clang/lib/Tooling/CMakeLists.txt new file mode 100644 index 0000000..b84b211 --- /dev/null +++ b/clang/lib/Tooling/CMakeLists.txt @@ -0,0 +1,7 @@ +set(LLVM_LINK_COMPONENTS support) +SET(LLVM_USED_LIBS clangBasic clangFrontend clangAST) + +add_clang_library(clangTooling + CompilationDatabase.cpp + Tooling.cpp + ) diff --git a/clang/lib/Tooling/CompilationDatabase.cpp b/clang/lib/Tooling/CompilationDatabase.cpp new file mode 100644 index 0000000..dd9ccc0 --- /dev/null +++ b/clang/lib/Tooling/CompilationDatabase.cpp @@ -0,0 +1,266 @@ +//===--- CompilationDatabase.cpp - ----------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains multiple implementations for CompilationDatabases. +// +//===----------------------------------------------------------------------===// + +#include "clang/Tooling/CompilationDatabase.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/Support/YAMLParser.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/system_error.h" + +namespace clang { +namespace tooling { + +namespace { + +/// \brief A parser for escaped strings of command line arguments. +/// +/// Assumes \-escaping for quoted arguments (see the documentation of +/// unescapeCommandLine(...)). +class CommandLineArgumentParser { + public: + CommandLineArgumentParser(StringRef CommandLine) + : Input(CommandLine), Position(Input.begin()-1) {} + + std::vector parse() { + bool HasMoreInput = true; + while (HasMoreInput && nextNonWhitespace()) { + std::string Argument; + HasMoreInput = parseStringInto(Argument); + CommandLine.push_back(Argument); + } + return CommandLine; + } + + private: + // All private methods return true if there is more input available. + + bool parseStringInto(std::string &String) { + do { + if (*Position == '"') { + if (!parseQuotedStringInto(String)) return false; + } else { + if (!parseFreeStringInto(String)) return false; + } + } while (*Position != ' '); + return true; + } + + bool parseQuotedStringInto(std::string &String) { + if (!next()) return false; + while (*Position != '"') { + if (!skipEscapeCharacter()) return false; + String.push_back(*Position); + if (!next()) return false; + } + return next(); + } + + bool parseFreeStringInto(std::string &String) { + do { + if (!skipEscapeCharacter()) return false; + String.push_back(*Position); + if (!next()) return false; + } while (*Position != ' ' && *Position != '"'); + return true; + } + + bool skipEscapeCharacter() { + if (*Position == '\\') { + return next(); + } + return true; + } + + bool nextNonWhitespace() { + do { + if (!next()) return false; + } while (*Position == ' '); + return true; + } + + bool next() { + ++Position; + return Position != Input.end(); + } + + const StringRef Input; + StringRef::iterator Position; + std::vector CommandLine; +}; + +std::vector unescapeCommandLine( + StringRef EscapedCommandLine) { + CommandLineArgumentParser parser(EscapedCommandLine); + return parser.parse(); +} + +} // end namespace + +CompilationDatabase::~CompilationDatabase() {} + +CompilationDatabase * +CompilationDatabase::loadFromDirectory(StringRef BuildDirectory, + std::string &ErrorMessage) { + llvm::SmallString<1024> JSONDatabasePath(BuildDirectory); + llvm::sys::path::append(JSONDatabasePath, "compile_commands.json"); + llvm::OwningPtr Database( + JSONCompilationDatabase::loadFromFile(JSONDatabasePath, ErrorMessage)); + if (!Database) { + return NULL; + } + return Database.take(); +} + +FixedCompilationDatabase * +FixedCompilationDatabase::loadFromCommandLine(int &Argc, + const char **Argv, + Twine Directory) { + const char **DoubleDash = std::find(Argv, Argv + Argc, StringRef("--")); + if (DoubleDash == Argv + Argc) + return NULL; + std::vector CommandLine(DoubleDash + 1, Argv + Argc); + Argc = DoubleDash - Argv; + return new FixedCompilationDatabase(Directory, CommandLine); +} + +FixedCompilationDatabase:: +FixedCompilationDatabase(Twine Directory, ArrayRef CommandLine) { + std::vector ToolCommandLine(1, "clang-tool"); + ToolCommandLine.insert(ToolCommandLine.end(), + CommandLine.begin(), CommandLine.end()); + CompileCommands.push_back(CompileCommand(Directory, ToolCommandLine)); +} + +std::vector +FixedCompilationDatabase::getCompileCommands(StringRef FilePath) const { + std::vector Result(CompileCommands); + Result[0].CommandLine.push_back(FilePath); + return Result; +} + +JSONCompilationDatabase * +JSONCompilationDatabase::loadFromFile(StringRef FilePath, + std::string &ErrorMessage) { + llvm::OwningPtr DatabaseBuffer; + llvm::error_code Result = + llvm::MemoryBuffer::getFile(FilePath, DatabaseBuffer); + if (Result != 0) { + ErrorMessage = "Error while opening JSON database: " + Result.message(); + return NULL; + } + llvm::OwningPtr Database( + new JSONCompilationDatabase(DatabaseBuffer.take())); + if (!Database->parse(ErrorMessage)) + return NULL; + return Database.take(); +} + +JSONCompilationDatabase * +JSONCompilationDatabase::loadFromBuffer(StringRef DatabaseString, + std::string &ErrorMessage) { + llvm::OwningPtr DatabaseBuffer( + llvm::MemoryBuffer::getMemBuffer(DatabaseString)); + llvm::OwningPtr Database( + new JSONCompilationDatabase(DatabaseBuffer.take())); + if (!Database->parse(ErrorMessage)) + return NULL; + return Database.take(); +} + +std::vector +JSONCompilationDatabase::getCompileCommands(StringRef FilePath) const { + llvm::StringMap< std::vector >::const_iterator + CommandsRefI = IndexByFile.find(FilePath); + if (CommandsRefI == IndexByFile.end()) + return std::vector(); + const std::vector &CommandsRef = CommandsRefI->getValue(); + std::vector Commands; + for (int I = 0, E = CommandsRef.size(); I != E; ++I) { + llvm::SmallString<8> DirectoryStorage; + llvm::SmallString<1024> CommandStorage; + Commands.push_back(CompileCommand( + // FIXME: Escape correctly: + CommandsRef[I].first->getValue(DirectoryStorage), + unescapeCommandLine(CommandsRef[I].second->getValue(CommandStorage)))); + } + return Commands; +} + +bool JSONCompilationDatabase::parse(std::string &ErrorMessage) { + llvm::yaml::document_iterator I = YAMLStream.begin(); + if (I == YAMLStream.end()) { + ErrorMessage = "Error while parsing YAML."; + return false; + } + llvm::yaml::Node *Root = I->getRoot(); + if (Root == NULL) { + ErrorMessage = "Error while parsing YAML."; + return false; + } + llvm::yaml::SequenceNode *Array = + llvm::dyn_cast(Root); + if (Array == NULL) { + ErrorMessage = "Expected array."; + return false; + } + for (llvm::yaml::SequenceNode::iterator AI = Array->begin(), + AE = Array->end(); + AI != AE; ++AI) { + llvm::yaml::MappingNode *Object = + llvm::dyn_cast(&*AI); + if (Object == NULL) { + ErrorMessage = "Expected object."; + return false; + } + llvm::yaml::ScalarNode *Directory; + llvm::yaml::ScalarNode *Command; + llvm::SmallString<8> FileStorage; + llvm::StringRef File; + for (llvm::yaml::MappingNode::iterator KVI = Object->begin(), + KVE = Object->end(); + KVI != KVE; ++KVI) { + llvm::yaml::Node *Value = (*KVI).getValue(); + if (Value == NULL) { + ErrorMessage = "Expected value."; + return false; + } + llvm::yaml::ScalarNode *ValueString = + llvm::dyn_cast(Value); + if (ValueString == NULL) { + ErrorMessage = "Expected string as value."; + return false; + } + llvm::yaml::ScalarNode *KeyString = + llvm::dyn_cast((*KVI).getKey()); + llvm::SmallString<8> KeyStorage; + if (KeyString->getValue(KeyStorage) == "directory") { + Directory = ValueString; + } else if (KeyString->getValue(KeyStorage) == "command") { + Command = ValueString; + } else if (KeyString->getValue(KeyStorage) == "file") { + File = ValueString->getValue(FileStorage); + } else { + ErrorMessage = ("Unknown key: \"" + + KeyString->getRawValue() + "\"").str(); + return false; + } + } + IndexByFile[File].push_back( + CompileCommandRef(Directory, Command)); + } + return true; +} + +} // end namespace tooling +} // end namespace clang + diff --git a/clang/lib/Tooling/Makefile b/clang/lib/Tooling/Makefile new file mode 100644 index 0000000..0d2e7a2 --- /dev/null +++ b/clang/lib/Tooling/Makefile @@ -0,0 +1,13 @@ +##===- clang/lib/Tooling/Makefile ---------------------------*- Makefile -*-===## +# +# The LLVM Compiler Infrastructure +# +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# +##===----------------------------------------------------------------------===## + +CLANG_LEVEL := ../.. +LIBRARYNAME := clangTooling + +include $(CLANG_LEVEL)/Makefile diff --git a/clang/lib/Tooling/Tooling.cpp b/clang/lib/Tooling/Tooling.cpp new file mode 100644 index 0000000..fa2374f --- /dev/null +++ b/clang/lib/Tooling/Tooling.cpp @@ -0,0 +1,296 @@ +//===--- Tooling.cpp - Running clang standalone tools ---------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements functions to run clang tools standalone instead +// of running them as a plugin. +// +//===----------------------------------------------------------------------===// + +#include "clang/Tooling/Tooling.h" +#include "clang/Tooling/CompilationDatabase.h" +#include "clang/Driver/Compilation.h" +#include "clang/Driver/Driver.h" +#include "clang/Driver/Tool.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/FrontendAction.h" +#include "clang/Frontend/FrontendDiagnostic.h" +#include "clang/Frontend/TextDiagnosticPrinter.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Host.h" +#include "llvm/Support/raw_ostream.h" + +namespace clang { +namespace tooling { + +FrontendActionFactory::~FrontendActionFactory() {} + +// FIXME: This file contains structural duplication with other parts of the +// code that sets up a compiler to run tools on it, and we should refactor +// it to be based on the same framework. + +/// \brief Builds a clang driver initialized for running clang tools. +static clang::driver::Driver *newDriver(clang::DiagnosticsEngine *Diagnostics, + const char *BinaryName) { + const std::string DefaultOutputName = "a.out"; + clang::driver::Driver *CompilerDriver = new clang::driver::Driver( + BinaryName, llvm::sys::getDefaultTargetTriple(), + DefaultOutputName, false, *Diagnostics); + CompilerDriver->setTitle("clang_based_tool"); + return CompilerDriver; +} + +/// \brief Retrieves the clang CC1 specific flags out of the compilation's jobs. +/// +/// Returns NULL on error. +static const clang::driver::ArgStringList *getCC1Arguments( + clang::DiagnosticsEngine *Diagnostics, + clang::driver::Compilation *Compilation) { + // We expect to get back exactly one Command job, if we didn't something + // failed. Extract that job from the Compilation. + const clang::driver::JobList &Jobs = Compilation->getJobs(); + if (Jobs.size() != 1 || !isa(*Jobs.begin())) { + llvm::SmallString<256> error_msg; + llvm::raw_svector_ostream error_stream(error_msg); + Compilation->PrintJob(error_stream, Compilation->getJobs(), "; ", true); + Diagnostics->Report(clang::diag::err_fe_expected_compiler_job) + << error_stream.str(); + return NULL; + } + + // The one job we find should be to invoke clang again. + const clang::driver::Command *Cmd = + cast(*Jobs.begin()); + if (StringRef(Cmd->getCreator().getName()) != "clang") { + Diagnostics->Report(clang::diag::err_fe_expected_clang_command); + return NULL; + } + + return &Cmd->getArguments(); +} + +/// \brief Returns a clang build invocation initialized from the CC1 flags. +static clang::CompilerInvocation *newInvocation( + clang::DiagnosticsEngine *Diagnostics, + const clang::driver::ArgStringList &CC1Args) { + assert(!CC1Args.empty() && "Must at least contain the program name!"); + clang::CompilerInvocation *Invocation = new clang::CompilerInvocation; + clang::CompilerInvocation::CreateFromArgs( + *Invocation, CC1Args.data() + 1, CC1Args.data() + CC1Args.size(), + *Diagnostics); + Invocation->getFrontendOpts().DisableFree = false; + return Invocation; +} + +bool runToolOnCode(clang::FrontendAction *ToolAction, const Twine &Code, + const Twine &FileName) { + SmallString<16> FileNameStorage; + StringRef FileNameRef = FileName.toNullTerminatedStringRef(FileNameStorage); + const char *const CommandLine[] = { + "clang-tool", "-fsyntax-only", FileNameRef.data() + }; + FileManager Files((FileSystemOptions())); + ToolInvocation Invocation( + std::vector( + CommandLine, + CommandLine + llvm::array_lengthof(CommandLine)), + ToolAction, &Files); + + SmallString<1024> CodeStorage; + Invocation.mapVirtualFile(FileNameRef, + Code.toNullTerminatedStringRef(CodeStorage)); + return Invocation.run(); +} + +/// \brief Returns the absolute path of 'File', by prepending it with +/// 'BaseDirectory' if 'File' is not absolute. +/// +/// Otherwise returns 'File'. +/// If 'File' starts with "./", the returned path will not contain the "./". +/// Otherwise, the returned path will contain the literal path-concatenation of +/// 'BaseDirectory' and 'File'. +/// +/// \param File Either an absolute or relative path. +/// \param BaseDirectory An absolute path. +static std::string getAbsolutePath( + StringRef File, StringRef BaseDirectory) { + assert(llvm::sys::path::is_absolute(BaseDirectory)); + if (llvm::sys::path::is_absolute(File)) { + return File; + } + StringRef RelativePath(File); + if (RelativePath.startswith("./")) { + RelativePath = RelativePath.substr(strlen("./")); + } + llvm::SmallString<1024> AbsolutePath(BaseDirectory); + llvm::sys::path::append(AbsolutePath, RelativePath); + return AbsolutePath.str(); +} + +ToolInvocation::ToolInvocation( + ArrayRef CommandLine, FrontendAction *ToolAction, + FileManager *Files) + : CommandLine(CommandLine.vec()), ToolAction(ToolAction), Files(Files) { +} + +void ToolInvocation::mapVirtualFile(StringRef FilePath, StringRef Content) { + MappedFileContents[FilePath] = Content; +} + +bool ToolInvocation::run() { + std::vector Argv; + for (int I = 0, E = CommandLine.size(); I != E; ++I) + Argv.push_back(CommandLine[I].c_str()); + const char *const BinaryName = Argv[0]; + DiagnosticOptions DefaultDiagnosticOptions; + TextDiagnosticPrinter DiagnosticPrinter( + llvm::errs(), DefaultDiagnosticOptions); + DiagnosticsEngine Diagnostics(llvm::IntrusiveRefCntPtr( + new DiagnosticIDs()), &DiagnosticPrinter, false); + + const llvm::OwningPtr Driver( + newDriver(&Diagnostics, BinaryName)); + // Since the input might only be virtual, don't check whether it exists. + Driver->setCheckInputsExist(false); + const llvm::OwningPtr Compilation( + Driver->BuildCompilation(llvm::makeArrayRef(Argv))); + const clang::driver::ArgStringList *const CC1Args = getCC1Arguments( + &Diagnostics, Compilation.get()); + if (CC1Args == NULL) { + return false; + } + llvm::OwningPtr Invocation( + newInvocation(&Diagnostics, *CC1Args)); + return runInvocation(BinaryName, Compilation.get(), + Invocation.take(), *CC1Args, ToolAction.take()); +} + +// Exists solely for the purpose of lookup of the resource path. +static int StaticSymbol; + +bool ToolInvocation::runInvocation( + const char *BinaryName, + clang::driver::Compilation *Compilation, + clang::CompilerInvocation *Invocation, + const clang::driver::ArgStringList &CC1Args, + clang::FrontendAction *ToolAction) { + llvm::OwningPtr ScopedToolAction(ToolAction); + // Show the invocation, with -v. + if (Invocation->getHeaderSearchOpts().Verbose) { + llvm::errs() << "clang Invocation:\n"; + Compilation->PrintJob(llvm::errs(), Compilation->getJobs(), "\n", true); + llvm::errs() << "\n"; + } + + // Create a compiler instance to handle the actual work. + clang::CompilerInstance Compiler; + Compiler.setInvocation(Invocation); + Compiler.setFileManager(Files); + // FIXME: What about LangOpts? + + // Create the compilers actual diagnostics engine. + Compiler.createDiagnostics(CC1Args.size(), + const_cast(CC1Args.data())); + if (!Compiler.hasDiagnostics()) + return false; + + Compiler.createSourceManager(*Files); + addFileMappingsTo(Compiler.getSourceManager()); + + // Infer the builtin include path if unspecified. + if (Compiler.getHeaderSearchOpts().UseBuiltinIncludes && + Compiler.getHeaderSearchOpts().ResourceDir.empty()) { + // This just needs to be some symbol in the binary. + void *const SymbolAddr = &StaticSymbol; + Compiler.getHeaderSearchOpts().ResourceDir = + clang::CompilerInvocation::GetResourcesPath(BinaryName, SymbolAddr); + } + + const bool Success = Compiler.ExecuteAction(*ToolAction); + + Compiler.resetAndLeakFileManager(); + return Success; +} + +void ToolInvocation::addFileMappingsTo(SourceManager &Sources) { + for (llvm::StringMap::const_iterator + It = MappedFileContents.begin(), End = MappedFileContents.end(); + It != End; ++It) { + // Inject the code as the given file name into the preprocessor options. + const llvm::MemoryBuffer *Input = + llvm::MemoryBuffer::getMemBuffer(It->getValue()); + // FIXME: figure out what '0' stands for. + const FileEntry *FromFile = Files->getVirtualFile( + It->getKey(), Input->getBufferSize(), 0); + // FIXME: figure out memory management ('true'). + Sources.overrideFileContents(FromFile, Input, true); + } +} + +ClangTool::ClangTool(const CompilationDatabase &Compilations, + ArrayRef SourcePaths) + : Files((FileSystemOptions())) { + llvm::SmallString<1024> BaseDirectory; + if (const char *PWD = ::getenv("PWD")) + BaseDirectory = PWD; + else + llvm::sys::fs::current_path(BaseDirectory); + for (unsigned I = 0, E = SourcePaths.size(); I != E; ++I) { + llvm::SmallString<1024> File(getAbsolutePath( + SourcePaths[I], BaseDirectory)); + + std::vector CompileCommands = + Compilations.getCompileCommands(File.str()); + if (!CompileCommands.empty()) { + for (int I = 0, E = CompileCommands.size(); I != E; ++I) { + CompileCommand &Command = CompileCommands[I]; + if (!Command.Directory.empty()) { + // FIXME: What should happen if CommandLine includes -working-directory + // as well? + Command.CommandLine.push_back( + "-working-directory=" + Command.Directory); + } + CommandLines.push_back(std::make_pair(File.str(), Command.CommandLine)); + } + } else { + // FIXME: There are two use cases here: doing a fuzzy + // "find . -name '*.cc' |xargs tool" match, where as a user I don't care + // about the .cc files that were not found, and the use case where I + // specify all files I want to run over explicitly, where this should + // be an error. We'll want to add an option for this. + llvm::outs() << "Skipping " << File << ". Command line not found.\n"; + } + } +} + +void ClangTool::mapVirtualFile(StringRef FilePath, StringRef Content) { + MappedFileContents.push_back(std::make_pair(FilePath, Content)); +} + +int ClangTool::run(FrontendActionFactory *ActionFactory) { + bool ProcessingFailed = false; + for (unsigned I = 0; I < CommandLines.size(); ++I) { + std::string File = CommandLines[I].first; + std::vector &CommandLine = CommandLines[I].second; + llvm::outs() << "Processing: " << File << ".\n"; + ToolInvocation Invocation(CommandLine, ActionFactory->create(), &Files); + for (int I = 0, E = MappedFileContents.size(); I != E; ++I) { + Invocation.mapVirtualFile(MappedFileContents[I].first, + MappedFileContents[I].second); + } + if (!Invocation.run()) { + llvm::outs() << "Error while processing " << File << ".\n"; + ProcessingFailed = true; + } + } + return ProcessingFailed ? 1 : 0; +} + +} // end namespace tooling +} // end namespace clang -- cgit v1.2.3