clang-tools  11.0.0
QueryDriverDatabase.cpp
Go to the documentation of this file.
1 //===--- QueryDriverDatabase.cpp ---------------------------------*- C++-*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 // Some compiler drivers have implicit search mechanism for system headers.
9 // This compilation database implementation tries to extract that information by
10 // executing the driver in verbose mode. gcc-compatible drivers print something
11 // like:
12 // ....
13 // ....
14 // #include <...> search starts here:
15 // /usr/lib/gcc/x86_64-linux-gnu/7/include
16 // /usr/local/include
17 // /usr/lib/gcc/x86_64-linux-gnu/7/include-fixed
18 // /usr/include/x86_64-linux-gnu
19 // /usr/include
20 // End of search list.
21 // ....
22 // ....
23 // This component parses that output and adds each path to command line args
24 // provided by Base, after prepending them with -isystem. Therefore current
25 // implementation would not work with a driver that is not gcc-compatible.
26 //
27 // First argument of the command line received from underlying compilation
28 // database is used as compiler driver path. Due to this arbitrary binary
29 // execution, this mechanism is not used by default and only executes binaries
30 // in the paths that are explicitly included by the user.
31 
33 #include "support/Logger.h"
34 #include "support/Path.h"
35 #include "support/Trace.h"
36 #include "clang/Driver/Types.h"
37 #include "clang/Tooling/CompilationDatabase.h"
38 #include "llvm/ADT/DenseMap.h"
39 #include "llvm/ADT/ScopeExit.h"
40 #include "llvm/ADT/SmallString.h"
41 #include "llvm/ADT/StringExtras.h"
42 #include "llvm/ADT/StringRef.h"
43 #include "llvm/ADT/iterator_range.h"
44 #include "llvm/Support/FileSystem.h"
45 #include "llvm/Support/MemoryBuffer.h"
46 #include "llvm/Support/Path.h"
47 #include "llvm/Support/Program.h"
48 #include "llvm/Support/Regex.h"
49 #include "llvm/Support/ScopedPrinter.h"
50 #include <algorithm>
51 #include <map>
52 #include <string>
53 #include <vector>
54 
55 namespace clang {
56 namespace clangd {
57 namespace {
58 
59 std::vector<std::string> parseDriverOutput(llvm::StringRef Output) {
60  std::vector<std::string> SystemIncludes;
61  const char SIS[] = "#include <...> search starts here:";
62  const char SIE[] = "End of search list.";
63  llvm::SmallVector<llvm::StringRef, 8> Lines;
64  Output.split(Lines, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
65 
66  auto StartIt = llvm::find_if(
67  Lines, [SIS](llvm::StringRef Line) { return Line.trim() == SIS; });
68  if (StartIt == Lines.end()) {
69  elog("System include extraction: start marker not found: {0}", Output);
70  return {};
71  }
72  ++StartIt;
73  const auto EndIt =
74  llvm::find_if(llvm::make_range(StartIt, Lines.end()),
75  [SIE](llvm::StringRef Line) { return Line.trim() == SIE; });
76  if (EndIt == Lines.end()) {
77  elog("System include extraction: end marker missing: {0}", Output);
78  return {};
79  }
80 
81  for (llvm::StringRef Line : llvm::make_range(StartIt, EndIt)) {
82  SystemIncludes.push_back(Line.trim().str());
83  vlog("System include extraction: adding {0}", Line);
84  }
85  return SystemIncludes;
86 }
87 
88 std::vector<std::string>
89 extractSystemIncludes(PathRef Driver, llvm::StringRef Lang,
90  llvm::ArrayRef<std::string> CommandLine,
91  const llvm::Regex &QueryDriverRegex) {
92  trace::Span Tracer("Extract system includes");
93  SPAN_ATTACH(Tracer, "driver", Driver);
94  SPAN_ATTACH(Tracer, "lang", Lang);
95 
96  if (!QueryDriverRegex.match(Driver)) {
97  vlog("System include extraction: not allowed driver {0}", Driver);
98  return {};
99  }
100 
101  if (!llvm::sys::fs::exists(Driver)) {
102  elog("System include extraction: {0} does not exist.", Driver);
103  return {};
104  }
105  if (!llvm::sys::fs::can_execute(Driver)) {
106  elog("System include extraction: {0} is not executable.", Driver);
107  return {};
108  }
109 
110  llvm::SmallString<128> StdErrPath;
111  if (auto EC = llvm::sys::fs::createTemporaryFile("system-includes", "clangd",
112  StdErrPath)) {
113  elog("System include extraction: failed to create temporary file with "
114  "error {0}",
115  EC.message());
116  return {};
117  }
118  auto CleanUp = llvm::make_scope_exit(
119  [&StdErrPath]() { llvm::sys::fs::remove(StdErrPath); });
120 
121  llvm::Optional<llvm::StringRef> Redirects[] = {
122  {""}, {""}, llvm::StringRef(StdErrPath)};
123 
124  llvm::SmallVector<llvm::StringRef, 12> Args = {Driver, "-E", "-x",
125  Lang, "-", "-v"};
126 
127  // These flags will be preserved
128  const llvm::StringRef FlagsToPreserve[] = {
129  "-nostdinc", "--no-standard-includes", "-nostdinc++", "-nobuiltininc"};
130  // Preserves these flags and their values, either as separate args or with an
131  // equalsbetween them
132  const llvm::StringRef ArgsToPreserve[] = {"--sysroot", "-isysroot"};
133 
134  for (size_t I = 0, E = CommandLine.size(); I < E; ++I) {
135  llvm::StringRef Arg = CommandLine[I];
136  if (llvm::any_of(FlagsToPreserve,
137  [&Arg](llvm::StringRef S) { return S == Arg; })) {
138  Args.push_back(Arg);
139  } else {
140  const auto *Found =
141  llvm::find_if(ArgsToPreserve, [&Arg](llvm::StringRef S) {
142  return Arg.startswith(S);
143  });
144  if (Found == std::end(ArgsToPreserve))
145  continue;
146  Arg = Arg.drop_front(Found->size());
147  if (Arg.empty() && I + 1 < E) {
148  Args.push_back(CommandLine[I]);
149  Args.push_back(CommandLine[++I]);
150  } else if (Arg.startswith("=")) {
151  Args.push_back(CommandLine[I]);
152  }
153  }
154  }
155 
156  if (int RC = llvm::sys::ExecuteAndWait(Driver, Args, /*Env=*/llvm::None,
157  Redirects)) {
158  elog("System include extraction: driver execution failed with return code: "
159  "{0}. Args: ['{1}']",
160  llvm::to_string(RC), llvm::join(Args, "', '"));
161  return {};
162  }
163 
164  auto BufOrError = llvm::MemoryBuffer::getFile(StdErrPath);
165  if (!BufOrError) {
166  elog("System include extraction: failed to read {0} with error {1}",
167  StdErrPath, BufOrError.getError().message());
168  return {};
169  }
170 
171  auto Includes = parseDriverOutput(BufOrError->get()->getBuffer());
172  log("System include extractor: successfully executed {0}, got includes: "
173  "\"{1}\"",
174  Driver, llvm::join(Includes, ", "));
175  return Includes;
176 }
177 
178 tooling::CompileCommand &
179 addSystemIncludes(tooling::CompileCommand &Cmd,
180  llvm::ArrayRef<std::string> SystemIncludes) {
181  for (llvm::StringRef Include : SystemIncludes) {
182  // FIXME(kadircet): This doesn't work when we have "--driver-mode=cl"
183  Cmd.CommandLine.push_back("-isystem");
184  Cmd.CommandLine.push_back(Include.str());
185  }
186  return Cmd;
187 }
188 
189 /// Converts a glob containing only ** or * into a regex.
190 std::string convertGlobToRegex(llvm::StringRef Glob) {
191  std::string RegText;
192  llvm::raw_string_ostream RegStream(RegText);
193  RegStream << '^';
194  for (size_t I = 0, E = Glob.size(); I < E; ++I) {
195  if (Glob[I] == '*') {
196  if (I + 1 < E && Glob[I + 1] == '*') {
197  // Double star, accept any sequence.
198  RegStream << ".*";
199  // Also skip the second star.
200  ++I;
201  } else {
202  // Single star, accept any sequence without a slash.
203  RegStream << "[^/]*";
204  }
205  } else {
206  RegStream << llvm::Regex::escape(Glob.substr(I, 1));
207  }
208  }
209  RegStream << '$';
210  RegStream.flush();
211  return RegText;
212 }
213 
214 /// Converts a glob containing only ** or * into a regex.
215 llvm::Regex convertGlobsToRegex(llvm::ArrayRef<std::string> Globs) {
216  assert(!Globs.empty() && "Globs cannot be empty!");
217  std::vector<std::string> RegTexts;
218  RegTexts.reserve(Globs.size());
219  for (llvm::StringRef Glob : Globs)
220  RegTexts.push_back(convertGlobToRegex(Glob));
221 
222  llvm::Regex Reg(llvm::join(RegTexts, "|"));
223  assert(Reg.isValid(RegTexts.front()) &&
224  "Created an invalid regex from globs");
225  return Reg;
226 }
227 
228 /// Extracts system includes from a trusted driver by parsing the output of
229 /// include search path and appends them to the commands coming from underlying
230 /// compilation database.
231 class QueryDriverDatabase : public GlobalCompilationDatabase {
232 public:
233  QueryDriverDatabase(llvm::ArrayRef<std::string> QueryDriverGlobs,
234  std::unique_ptr<GlobalCompilationDatabase> Base)
235  : QueryDriverRegex(convertGlobsToRegex(QueryDriverGlobs)),
236  Base(std::move(Base)) {
237  assert(this->Base);
238  BaseChanged =
239  this->Base->watch([this](const std::vector<std::string> &Changes) {
240  OnCommandChanged.broadcast(Changes);
241  });
242  }
243 
244  llvm::Optional<tooling::CompileCommand>
245  getCompileCommand(PathRef File) const override {
246  auto Cmd = Base->getCompileCommand(File);
247  if (!Cmd || Cmd->CommandLine.empty())
248  return Cmd;
249 
250  llvm::StringRef Lang;
251  for (size_t I = 0, E = Cmd->CommandLine.size(); I < E; ++I) {
252  llvm::StringRef Arg = Cmd->CommandLine[I];
253  if (Arg == "-x" && I + 1 < E)
254  Lang = Cmd->CommandLine[I + 1];
255  else if (Arg.startswith("-x"))
256  Lang = Arg.drop_front(2).trim();
257  }
258  if (Lang.empty()) {
259  llvm::StringRef Ext = llvm::sys::path::extension(File).trim('.');
260  auto Type = driver::types::lookupTypeForExtension(Ext);
261  if (Type == driver::types::TY_INVALID) {
262  elog("System include extraction: invalid file type for {0}", Ext);
263  return {};
264  }
265  Lang = driver::types::getTypeName(Type);
266  }
267 
268  llvm::SmallString<128> Driver(Cmd->CommandLine.front());
269  llvm::sys::fs::make_absolute(Cmd->Directory, Driver);
270 
271  std::vector<std::string> SystemIncludes =
272  QueriedDrivers.get(/*Key=*/(Driver + ":" + Lang).str(), [&] {
273  return extractSystemIncludes(Driver, Lang, Cmd->CommandLine,
274  QueryDriverRegex);
275  });
276 
277  return addSystemIncludes(*Cmd, SystemIncludes);
278  }
279 
280  llvm::Optional<ProjectInfo> getProjectInfo(PathRef File) const override {
281  return Base->getProjectInfo(File);
282  }
283 
284 private:
285  // Caches includes extracted from a driver. Key is driver:lang.
286  Memoize<llvm::StringMap<std::vector<std::string>>> QueriedDrivers;
287  llvm::Regex QueryDriverRegex;
288 
289  std::unique_ptr<GlobalCompilationDatabase> Base;
290  CommandChanged::Subscription BaseChanged;
291 };
292 } // namespace
293 
294 std::unique_ptr<GlobalCompilationDatabase>
295 getQueryDriverDatabase(llvm::ArrayRef<std::string> QueryDriverGlobs,
296  std::unique_ptr<GlobalCompilationDatabase> Base) {
297  assert(Base && "Null base to SystemIncludeExtractor");
298  if (QueryDriverGlobs.empty())
299  return Base;
300  return std::make_unique<QueryDriverDatabase>(QueryDriverGlobs,
301  std::move(Base));
302 }
303 
304 } // namespace clangd
305 } // namespace clang
Base
std::unique_ptr< GlobalCompilationDatabase > Base
Definition: GlobalCompilationDatabaseTests.cpp:85
E
const Expr * E
Definition: AvoidBindCheck.cpp:88
Type
NodeType Type
Definition: HTMLGenerator.cpp:73
Tracer
std::unique_ptr< trace::EventTracer > Tracer
Definition: TraceTests.cpp:163
Path.h
Trace.h
clang::tidy::cppcoreguidelines::join
static std::string join(ArrayRef< SpecialMemberFunctionsCheck::SpecialMemberFunctionKind > SMFS, llvm::StringRef AndOr)
Definition: SpecialMemberFunctionsCheck.cpp:83
clang::clangd::TextDocumentSyncKind::None
Documents should not be synced at all.
Changes
tooling::Replacements Changes
Definition: Format.cpp:109
run-clang-tidy.make_absolute
def make_absolute(f, directory)
Definition: run-clang-tidy.py:76
Line
int Line
Definition: PreprocessorTracker.cpp:513
GlobalCompilationDatabase.h
Logger.h
CommandLine
std::vector< llvm::StringRef > CommandLine
Definition: Serialization.cpp:388
SPAN_ATTACH
#define SPAN_ATTACH(S, Name, Expr)
Attach a key-value pair to a Span event.
Definition: Trace.h:154
clang::clangd::vlog
void vlog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:67
Output
std::string Output
Definition: TraceTests.cpp:161
clang::clangd::log
void log(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:62
clang::clangd::CompletionItemKind::File
clang::clangd::getQueryDriverDatabase
std::unique_ptr< GlobalCompilationDatabase > getQueryDriverDatabase(llvm::ArrayRef< std::string > QueryDriverGlobs, std::unique_ptr< GlobalCompilationDatabase > Base)
Extracts system include search path from drivers matching QueryDriverGlobs and adds them to the compi...
Definition: QueryDriverDatabase.cpp:295
clang::clangd::PathRef
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:23
clang
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
Definition: ApplyReplacements.h:27
clang::clangd::elog
void elog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:56
Lines
unsigned Lines
Definition: FunctionSizeCheck.cpp:113