clang-tools  9.0.0
QueryDriverDatabase.cpp
Go to the documentation of this file.
1 //===--- QueryDriverDatabase.cpp ---------------------------------*- C++-*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 // Some compiler drivers have implicit search mechanism for system headers.
9 // This compilation database implementation tries to extract that information by
10 // executing the driver in verbose mode. gcc-compatible drivers print something
11 // like:
12 // ....
13 // ....
14 // #include <...> search starts here:
15 // /usr/lib/gcc/x86_64-linux-gnu/7/include
16 // /usr/local/include
17 // /usr/lib/gcc/x86_64-linux-gnu/7/include-fixed
18 // /usr/include/x86_64-linux-gnu
19 // /usr/include
20 // End of search list.
21 // ....
22 // ....
23 // This component parses that output and adds each path to command line args
24 // provided by Base, after prepending them with -isystem. Therefore current
25 // implementation would not work with a driver that is not gcc-compatible.
26 //
27 // First argument of the command line received from underlying compilation
28 // database is used as compiler driver path. Due to this arbitrary binary
29 // execution, this mechanism is not used by default and only executes binaries
30 // in the paths that are explicitly whitelisted by the user.
31 
33 #include "Logger.h"
34 #include "Path.h"
35 #include "Trace.h"
36 #include "clang/Driver/Types.h"
37 #include "clang/Tooling/CompilationDatabase.h"
38 #include "llvm/ADT/DenseMap.h"
39 #include "llvm/ADT/ScopeExit.h"
40 #include "llvm/ADT/SmallString.h"
41 #include "llvm/ADT/StringExtras.h"
42 #include "llvm/ADT/StringRef.h"
43 #include "llvm/ADT/iterator_range.h"
44 #include "llvm/Support/FileSystem.h"
45 #include "llvm/Support/MemoryBuffer.h"
46 #include "llvm/Support/Path.h"
47 #include "llvm/Support/Program.h"
48 #include "llvm/Support/Regex.h"
49 #include "llvm/Support/ScopedPrinter.h"
50 #include <algorithm>
51 #include <map>
52 #include <string>
53 #include <vector>
54 
55 namespace clang {
56 namespace clangd {
57 namespace {
58 
59 std::vector<std::string> parseDriverOutput(llvm::StringRef Output) {
60  std::vector<std::string> SystemIncludes;
61  const char SIS[] = "#include <...> search starts here:";
62  constexpr char const *SIE = "End of search list.";
63  llvm::SmallVector<llvm::StringRef, 8> Lines;
64  Output.split(Lines, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
65 
66  auto StartIt = llvm::find_if(
67  Lines, [SIS](llvm::StringRef Line) { return Line.trim() == SIS; });
68  if (StartIt == Lines.end()) {
69  elog("System include extraction: start marker not found: {0}", Output);
70  return {};
71  }
72  ++StartIt;
73  const auto EndIt = std::find(StartIt, Lines.end(), SIE);
74  if (EndIt == Lines.end()) {
75  elog("System include extraction: end marker missing: {0}", Output);
76  return {};
77  }
78 
79  for (llvm::StringRef Line : llvm::make_range(StartIt, EndIt)) {
80  SystemIncludes.push_back(Line.trim().str());
81  vlog("System include extraction: adding {0}", Line);
82  }
83  return SystemIncludes;
84 }
85 
86 std::vector<std::string> extractSystemIncludes(PathRef Driver,
87  llvm::StringRef Lang,
88  llvm::Regex &QueryDriverRegex) {
89  trace::Span Tracer("Extract system includes");
90  SPAN_ATTACH(Tracer, "driver", Driver);
91  SPAN_ATTACH(Tracer, "lang", Lang);
92 
93  if (!QueryDriverRegex.match(Driver)) {
94  vlog("System include extraction: not whitelisted driver {0}", Driver);
95  return {};
96  }
97 
98  if (!llvm::sys::fs::exists(Driver)) {
99  elog("System include extraction: {0} does not exist.", Driver);
100  return {};
101  }
102  if (!llvm::sys::fs::can_execute(Driver)) {
103  elog("System include extraction: {0} is not executable.", Driver);
104  return {};
105  }
106 
107  llvm::SmallString<128> StdErrPath;
108  if (auto EC = llvm::sys::fs::createTemporaryFile("system-includes", "clangd",
109  StdErrPath)) {
110  elog("System include extraction: failed to create temporary file with "
111  "error {0}",
112  EC.message());
113  return {};
114  }
115  auto CleanUp = llvm::make_scope_exit(
116  [&StdErrPath]() { llvm::sys::fs::remove(StdErrPath); });
117 
118  llvm::Optional<llvm::StringRef> Redirects[] = {
119  {""}, {""}, llvm::StringRef(StdErrPath)};
120 
121  // Should we also preserve flags like "-sysroot", "-nostdinc" ?
122  const llvm::StringRef Args[] = {Driver, "-E", "-x", Lang, "-", "-v"};
123 
124  if (int RC = llvm::sys::ExecuteAndWait(Driver, Args, /*Env=*/llvm::None,
125  Redirects)) {
126  elog("System include extraction: driver execution failed with return code: "
127  "{0}",
128  llvm::to_string(RC));
129  return {};
130  }
131 
132  auto BufOrError = llvm::MemoryBuffer::getFile(StdErrPath);
133  if (!BufOrError) {
134  elog("System include extraction: failed to read {0} with error {1}",
135  StdErrPath, BufOrError.getError().message());
136  return {};
137  }
138 
139  auto Includes = parseDriverOutput(BufOrError->get()->getBuffer());
140  log("System include extractor: succesfully executed {0}, got includes: "
141  "\"{1}\"",
142  Driver, llvm::join(Includes, ", "));
143  return Includes;
144 }
145 
146 tooling::CompileCommand &
147 addSystemIncludes(tooling::CompileCommand &Cmd,
148  llvm::ArrayRef<std::string> SystemIncludes) {
149  for (llvm::StringRef Include : SystemIncludes) {
150  // FIXME(kadircet): This doesn't work when we have "--driver-mode=cl"
151  Cmd.CommandLine.push_back("-isystem");
152  Cmd.CommandLine.push_back(Include.str());
153  }
154  return Cmd;
155 }
156 
157 /// Converts a glob containing only ** or * into a regex.
158 std::string convertGlobToRegex(llvm::StringRef Glob) {
159  std::string RegText;
160  llvm::raw_string_ostream RegStream(RegText);
161  RegStream << '^';
162  for (size_t I = 0, E = Glob.size(); I < E; ++I) {
163  if (Glob[I] == '*') {
164  if (I + 1 < E && Glob[I + 1] == '*') {
165  // Double star, accept any sequence.
166  RegStream << ".*";
167  // Also skip the second star.
168  ++I;
169  } else {
170  // Single star, accept any sequence without a slash.
171  RegStream << "[^/]*";
172  }
173  } else {
174  RegStream << llvm::Regex::escape(Glob.substr(I, 1));
175  }
176  }
177  RegStream << '$';
178  RegStream.flush();
179  return RegText;
180 }
181 
182 /// Converts a glob containing only ** or * into a regex.
183 llvm::Regex convertGlobsToRegex(llvm::ArrayRef<std::string> Globs) {
184  assert(!Globs.empty() && "Globs cannot be empty!");
185  std::vector<std::string> RegTexts;
186  RegTexts.reserve(Globs.size());
187  for (llvm::StringRef Glob : Globs)
188  RegTexts.push_back(convertGlobToRegex(Glob));
189 
190  llvm::Regex Reg(llvm::join(RegTexts, "|"));
191  assert(Reg.isValid(RegTexts.front()) &&
192  "Created an invalid regex from globs");
193  return Reg;
194 }
195 
196 /// Extracts system includes from a trusted driver by parsing the output of
197 /// include search path and appends them to the commands coming from underlying
198 /// compilation database.
199 class QueryDriverDatabase : public GlobalCompilationDatabase {
200 public:
201  QueryDriverDatabase(llvm::ArrayRef<std::string> QueryDriverGlobs,
202  std::unique_ptr<GlobalCompilationDatabase> Base)
203  : QueryDriverRegex(convertGlobsToRegex(QueryDriverGlobs)),
204  Base(std::move(Base)) {
205  assert(this->Base);
206  BaseChanged =
207  this->Base->watch([this](const std::vector<std::string> &Changes) {
208  OnCommandChanged.broadcast(Changes);
209  });
210  }
211 
212  llvm::Optional<tooling::CompileCommand>
213  getCompileCommand(PathRef File) const override {
214  auto Cmd = Base->getCompileCommand(File);
215  if (!Cmd || Cmd->CommandLine.empty())
216  return Cmd;
217 
218  llvm::StringRef Lang;
219  for (size_t I = 0, E = Cmd->CommandLine.size(); I < E; ++I) {
220  llvm::StringRef Arg = Cmd->CommandLine[I];
221  if (Arg == "-x" && I + 1 < E)
222  Lang = Cmd->CommandLine[I + 1];
223  else if (Arg.startswith("-x"))
224  Lang = Arg.drop_front(2).trim();
225  }
226  if (Lang.empty()) {
227  llvm::StringRef Ext = llvm::sys::path::extension(File).trim('.');
228  auto Type = driver::types::lookupTypeForExtension(Ext);
229  if (Type == driver::types::TY_INVALID) {
230  elog("System include extraction: invalid file type for {0}", Ext);
231  return {};
232  }
233  Lang = driver::types::getTypeName(Type);
234  }
235 
236  llvm::SmallString<128> Driver(Cmd->CommandLine.front());
237  llvm::sys::fs::make_absolute(Cmd->Directory, Driver);
238  auto Key = std::make_pair(Driver.str(), Lang);
239 
240  std::vector<std::string> SystemIncludes;
241  {
242  std::lock_guard<std::mutex> Lock(Mu);
243 
244  auto It = DriverToIncludesCache.find(Key);
245  if (It != DriverToIncludesCache.end())
246  SystemIncludes = It->second;
247  else
248  DriverToIncludesCache[Key] = SystemIncludes =
249  extractSystemIncludes(Key.first, Key.second, QueryDriverRegex);
250  }
251 
252  return addSystemIncludes(*Cmd, SystemIncludes);
253  }
254 
255  llvm::Optional<ProjectInfo> getProjectInfo(PathRef File) const override {
256  return Base->getProjectInfo(File);
257  }
258 
259 private:
260  mutable std::mutex Mu;
261  // Caches includes extracted from a driver.
262  mutable std::map<std::pair<std::string, std::string>,
263  std::vector<std::string>>
264  DriverToIncludesCache;
265  mutable llvm::Regex QueryDriverRegex;
266 
267  std::unique_ptr<GlobalCompilationDatabase> Base;
268  CommandChanged::Subscription BaseChanged;
269 };
270 } // namespace
271 
272 std::unique_ptr<GlobalCompilationDatabase>
273 getQueryDriverDatabase(llvm::ArrayRef<std::string> QueryDriverGlobs,
274  std::unique_ptr<GlobalCompilationDatabase> Base) {
275  assert(Base && "Null base to SystemIncludeExtractor");
276  if (QueryDriverGlobs.empty())
277  return Base;
278  return llvm::make_unique<QueryDriverDatabase>(QueryDriverGlobs,
279  std::move(Base));
280 }
281 
282 } // namespace clangd
283 } // namespace clang
std::unique_ptr< GlobalCompilationDatabase > getQueryDriverDatabase(llvm::ArrayRef< std::string > QueryDriverGlobs, std::unique_ptr< GlobalCompilationDatabase > Base)
Extracts system include search path from drivers matching QueryDriverGlobs and adds them to the compi...
tooling::Replacements Changes
Definition: Format.cpp:108
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:23
def make_absolute(f, directory)
Documents should not be synced at all.
void vlog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:67
void elog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:56
void log(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:62
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
static std::string join(ArrayRef< SpecialMemberFunctionsCheck::SpecialMemberFunctionKind > SMFS, llvm::StringRef AndOr)
std::unique_ptr< GlobalCompilationDatabase > Base
#define SPAN_ATTACH(S, Name, Expr)
Attach a key-value pair to a Span event.
Definition: Trace.h:98
NodeType Type
unsigned Lines