clang-tools  9.0.0
ClangdUnit.cpp
Go to the documentation of this file.
1 //===--- ClangdUnit.cpp ------------------------------------------*- C++-*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "ClangdUnit.h"
10 #include "../clang-tidy/ClangTidyDiagnosticConsumer.h"
11 #include "../clang-tidy/ClangTidyModuleRegistry.h"
12 #include "Compiler.h"
13 #include "Diagnostics.h"
14 #include "Headers.h"
15 #include "IncludeFixer.h"
16 #include "Logger.h"
17 #include "SourceCode.h"
18 #include "Trace.h"
20 #include "index/Index.h"
21 #include "clang/AST/ASTContext.h"
22 #include "clang/Basic/LangOptions.h"
23 #include "clang/Basic/SourceManager.h"
24 #include "clang/Basic/TokenKinds.h"
25 #include "clang/Frontend/CompilerInstance.h"
26 #include "clang/Frontend/CompilerInvocation.h"
27 #include "clang/Frontend/FrontendActions.h"
28 #include "clang/Frontend/Utils.h"
29 #include "clang/Index/IndexDataConsumer.h"
30 #include "clang/Index/IndexingAction.h"
31 #include "clang/Lex/Lexer.h"
32 #include "clang/Lex/MacroInfo.h"
33 #include "clang/Lex/Preprocessor.h"
34 #include "clang/Lex/PreprocessorOptions.h"
35 #include "clang/Sema/Sema.h"
36 #include "clang/Serialization/ASTWriter.h"
37 #include "clang/Serialization/PCHContainerOperations.h"
38 #include "clang/Tooling/CompilationDatabase.h"
39 #include "clang/Tooling/Syntax/Tokens.h"
40 #include "llvm/ADT/ArrayRef.h"
41 #include "llvm/ADT/STLExtras.h"
42 #include "llvm/ADT/SmallString.h"
43 #include "llvm/ADT/SmallVector.h"
44 #include "llvm/Support/raw_ostream.h"
45 #include <algorithm>
46 #include <memory>
47 
48 namespace clang {
49 namespace clangd {
50 namespace {
51 
52 bool compileCommandsAreEqual(const tooling::CompileCommand &LHS,
53  const tooling::CompileCommand &RHS) {
54  // We don't check for Output, it should not matter to clangd.
55  return LHS.Directory == RHS.Directory && LHS.Filename == RHS.Filename &&
56  llvm::makeArrayRef(LHS.CommandLine).equals(RHS.CommandLine);
57 }
58 
59 template <class T> std::size_t getUsedBytes(const std::vector<T> &Vec) {
60  return Vec.capacity() * sizeof(T);
61 }
62 
63 class DeclTrackingASTConsumer : public ASTConsumer {
64 public:
65  DeclTrackingASTConsumer(std::vector<Decl *> &TopLevelDecls)
66  : TopLevelDecls(TopLevelDecls) {}
67 
68  bool HandleTopLevelDecl(DeclGroupRef DG) override {
69  for (Decl *D : DG) {
70  auto &SM = D->getASTContext().getSourceManager();
71  if (!SM.isWrittenInMainFile(SM.getExpansionLoc(D->getLocation())))
72  continue;
73 
74  // ObjCMethodDecl are not actually top-level decls.
75  if (isa<ObjCMethodDecl>(D))
76  continue;
77 
78  TopLevelDecls.push_back(D);
79  }
80  return true;
81  }
82 
83 private:
84  std::vector<Decl *> &TopLevelDecls;
85 };
86 
87 class ClangdFrontendAction : public SyntaxOnlyAction {
88 public:
89  std::vector<Decl *> takeTopLevelDecls() { return std::move(TopLevelDecls); }
90 
91 protected:
92  std::unique_ptr<ASTConsumer>
93  CreateASTConsumer(CompilerInstance &CI, llvm::StringRef InFile) override {
94  return llvm::make_unique<DeclTrackingASTConsumer>(/*ref*/ TopLevelDecls);
95  }
96 
97 private:
98  std::vector<Decl *> TopLevelDecls;
99 };
100 
101 class CollectMainFileMacros : public PPCallbacks {
102 public:
103  explicit CollectMainFileMacros(const SourceManager &SM,
104  std::vector<std::string> *Out)
105  : SM(SM), Out(Out) {}
106 
107  void FileChanged(SourceLocation Loc, FileChangeReason,
108  SrcMgr::CharacteristicKind, FileID Prev) {
109  InMainFile = SM.isWrittenInMainFile(Loc);
110  }
111 
112  void MacroDefined(const Token &MacroName, const MacroDirective *MD) {
113  if (InMainFile)
114  MainFileMacros.insert(MacroName.getIdentifierInfo()->getName());
115  }
116 
117  void EndOfMainFile() {
118  for (const auto &Entry : MainFileMacros)
119  Out->push_back(Entry.getKey());
120  llvm::sort(*Out);
121  }
122 
123 private:
124  const SourceManager &SM;
125  bool InMainFile = true;
126  llvm::StringSet<> MainFileMacros;
127  std::vector<std::string> *Out;
128 };
129 
130 class CppFilePreambleCallbacks : public PreambleCallbacks {
131 public:
132  CppFilePreambleCallbacks(PathRef File, PreambleParsedCallback ParsedCallback)
133  : File(File), ParsedCallback(ParsedCallback) {
134  }
135 
136  IncludeStructure takeIncludes() { return std::move(Includes); }
137 
138  std::vector<std::string> takeMainFileMacros() {
139  return std::move(MainFileMacros);
140  }
141 
142  CanonicalIncludes takeCanonicalIncludes() { return std::move(CanonIncludes); }
143 
144  void AfterExecute(CompilerInstance &CI) override {
145  if (!ParsedCallback)
146  return;
147  trace::Span Tracer("Running PreambleCallback");
148  ParsedCallback(CI.getASTContext(), CI.getPreprocessorPtr(), CanonIncludes);
149  }
150 
151  void BeforeExecute(CompilerInstance &CI) override {
152  addSystemHeadersMapping(&CanonIncludes, CI.getLangOpts());
153  SourceMgr = &CI.getSourceManager();
154  }
155 
156  std::unique_ptr<PPCallbacks> createPPCallbacks() override {
157  assert(SourceMgr && "SourceMgr must be set at this point");
158  return llvm::make_unique<PPChainedCallbacks>(
159  collectIncludeStructureCallback(*SourceMgr, &Includes),
160  llvm::make_unique<CollectMainFileMacros>(*SourceMgr, &MainFileMacros));
161  }
162 
163  CommentHandler *getCommentHandler() override {
164  IWYUHandler = collectIWYUHeaderMaps(&CanonIncludes);
165  return IWYUHandler.get();
166  }
167 
168 private:
169  PathRef File;
170  PreambleParsedCallback ParsedCallback;
171  IncludeStructure Includes;
172  CanonicalIncludes CanonIncludes;
173  std::vector<std::string> MainFileMacros;
174  std::unique_ptr<CommentHandler> IWYUHandler = nullptr;
175  SourceManager *SourceMgr = nullptr;
176 };
177 
178 // When using a preamble, only preprocessor events outside its bounds are seen.
179 // This is almost what we want: replaying transitive preprocessing wastes time.
180 // However this confuses clang-tidy checks: they don't see any #includes!
181 // So we replay the *non-transitive* #includes that appear in the main-file.
182 // It would be nice to replay other events (macro definitions, ifdefs etc) but
183 // this addresses the most common cases fairly cheaply.
184 class ReplayPreamble : private PPCallbacks {
185 public:
186  // Attach preprocessor hooks such that preamble events will be injected at
187  // the appropriate time.
188  // Events will be delivered to the *currently registered* PP callbacks.
189  static void attach(const IncludeStructure &Includes,
190  CompilerInstance &Clang) {
191  auto &PP = Clang.getPreprocessor();
192  auto *ExistingCallbacks = PP.getPPCallbacks();
193  // No need to replay events if nobody is listening.
194  if (!ExistingCallbacks)
195  return;
196  PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(
197  new ReplayPreamble(Includes, ExistingCallbacks,
198  Clang.getSourceManager(), PP, Clang.getLangOpts())));
199  // We're relying on the fact that addPPCallbacks keeps the old PPCallbacks
200  // around, creating a chaining wrapper. Guard against other implementations.
201  assert(PP.getPPCallbacks() != ExistingCallbacks &&
202  "Expected chaining implementation");
203  }
204 
205 private:
206  ReplayPreamble(const IncludeStructure &Includes, PPCallbacks *Delegate,
207  const SourceManager &SM, Preprocessor &PP,
208  const LangOptions &LangOpts)
209  : Includes(Includes), Delegate(Delegate), SM(SM), PP(PP),
210  LangOpts(LangOpts) {}
211 
212  // In a normal compile, the preamble traverses the following structure:
213  //
214  // mainfile.cpp
215  // <built-in>
216  // ... macro definitions like __cplusplus ...
217  // <command-line>
218  // ... macro definitions for args like -Dfoo=bar ...
219  // "header1.h"
220  // ... header file contents ...
221  // "header2.h"
222  // ... header file contents ...
223  // ... main file contents ...
224  //
225  // When using a preamble, the "header1" and "header2" subtrees get skipped.
226  // We insert them right after the built-in header, which still appears.
227  void FileChanged(SourceLocation Loc, FileChangeReason Reason,
228  SrcMgr::CharacteristicKind Kind, FileID PrevFID) override {
229  // It'd be nice if there was a better way to identify built-in headers...
230  if (Reason == FileChangeReason::ExitFile &&
231  SM.getBuffer(PrevFID)->getBufferIdentifier() == "<built-in>")
232  replay();
233  }
234 
235  void replay() {
236  for (const auto &Inc : Includes.MainFileIncludes) {
237  const FileEntry *File = nullptr;
238  if (Inc.Resolved != "")
239  File = SM.getFileManager().getFile(Inc.Resolved);
240 
241  llvm::StringRef WrittenFilename =
242  llvm::StringRef(Inc.Written).drop_front().drop_back();
243  bool Angled = llvm::StringRef(Inc.Written).startswith("<");
244 
245  // Re-lex the #include directive to find its interesting parts.
246  llvm::StringRef Src = SM.getBufferData(SM.getMainFileID());
247  Lexer RawLexer(SM.getLocForStartOfFile(SM.getMainFileID()), LangOpts,
248  Src.begin(), Src.begin() + Inc.HashOffset, Src.end());
249  Token HashTok, IncludeTok, FilenameTok;
250  RawLexer.LexFromRawLexer(HashTok);
251  assert(HashTok.getKind() == tok::hash);
252  RawLexer.setParsingPreprocessorDirective(true);
253  RawLexer.LexFromRawLexer(IncludeTok);
254  IdentifierInfo *II = PP.getIdentifierInfo(IncludeTok.getRawIdentifier());
255  IncludeTok.setIdentifierInfo(II);
256  IncludeTok.setKind(II->getTokenID());
257  RawLexer.LexIncludeFilename(FilenameTok);
258 
259  Delegate->InclusionDirective(
260  HashTok.getLocation(), IncludeTok, WrittenFilename, Angled,
261  CharSourceRange::getCharRange(FilenameTok.getLocation(),
262  FilenameTok.getEndLoc()),
263  File, "SearchPath", "RelPath", /*Imported=*/nullptr, Inc.FileKind);
264  if (File)
265  Delegate->FileSkipped(*File, FilenameTok, Inc.FileKind);
266  else {
267  llvm::SmallString<1> UnusedRecovery;
268  Delegate->FileNotFound(WrittenFilename, UnusedRecovery);
269  }
270  }
271  }
272 
273  const IncludeStructure &Includes;
274  PPCallbacks *Delegate;
275  const SourceManager &SM;
276  Preprocessor &PP;
277  const LangOptions &LangOpts;
278 };
279 
280 } // namespace
281 
282 void dumpAST(ParsedAST &AST, llvm::raw_ostream &OS) {
283  AST.getASTContext().getTranslationUnitDecl()->dump(OS, true);
284 }
285 
286 llvm::Optional<ParsedAST>
287 ParsedAST::build(std::unique_ptr<CompilerInvocation> CI,
288  std::shared_ptr<const PreambleData> Preamble,
289  std::unique_ptr<llvm::MemoryBuffer> Buffer,
290  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,
291  const SymbolIndex *Index, const ParseOptions &Opts) {
292  assert(CI);
293  // Command-line parsing sets DisableFree to true by default, but we don't want
294  // to leak memory in clangd.
295  CI->getFrontendOpts().DisableFree = false;
296  const PrecompiledPreamble *PreamblePCH =
297  Preamble ? &Preamble->Preamble : nullptr;
298 
299  StoreDiags ASTDiags;
300  std::string Content = Buffer->getBuffer();
301 
302  auto Clang = prepareCompilerInstance(std::move(CI), PreamblePCH,
303  std::move(Buffer), VFS, ASTDiags);
304  if (!Clang)
305  return None;
306 
307  auto Action = llvm::make_unique<ClangdFrontendAction>();
308  const FrontendInputFile &MainInput = Clang->getFrontendOpts().Inputs[0];
309  if (!Action->BeginSourceFile(*Clang, MainInput)) {
310  log("BeginSourceFile() failed when building AST for {0}",
311  MainInput.getFile());
312  return None;
313  }
314 
315  // Set up ClangTidy. Must happen after BeginSourceFile() so ASTContext exists.
316  // Clang-tidy has some limitiations to ensure reasonable performance:
317  // - checks don't see all preprocessor events in the preamble
318  // - matchers run only over the main-file top-level decls (and can't see
319  // ancestors outside this scope).
320  // In practice almost all checks work well without modifications.
321  std::vector<std::unique_ptr<tidy::ClangTidyCheck>> CTChecks;
322  ast_matchers::MatchFinder CTFinder;
323  llvm::Optional<tidy::ClangTidyContext> CTContext;
324  {
325  trace::Span Tracer("ClangTidyInit");
326  dlog("ClangTidy configuration for file {0}: {1}", MainInput.getFile(),
328  tidy::ClangTidyCheckFactories CTFactories;
329  for (const auto &E : tidy::ClangTidyModuleRegistry::entries())
330  E.instantiate()->addCheckFactories(CTFactories);
331  CTContext.emplace(llvm::make_unique<tidy::DefaultOptionsProvider>(
333  CTContext->setDiagnosticsEngine(&Clang->getDiagnostics());
334  CTContext->setASTContext(&Clang->getASTContext());
335  CTContext->setCurrentFile(MainInput.getFile());
336  CTFactories.createChecks(CTContext.getPointer(), CTChecks);
337  ASTDiags.setLevelAdjuster([&CTContext](DiagnosticsEngine::Level DiagLevel,
338  const clang::Diagnostic &Info) {
339  if (CTContext) {
340  std::string CheckName = CTContext->getCheckName(Info.getID());
341  bool IsClangTidyDiag = !CheckName.empty();
342  if (IsClangTidyDiag) {
343  // Check for warning-as-error.
344  // We deliberately let this take precedence over suppression comments
345  // to match clang-tidy's behaviour.
346  if (DiagLevel == DiagnosticsEngine::Warning &&
347  CTContext->treatAsError(CheckName)) {
349  }
350 
351  // Check for suppression comment. Skip the check for diagnostics not
352  // in the main file, because we don't want that function to query the
353  // source buffer for preamble files. For the same reason, we ask
354  // ShouldSuppressDiagnostic not to follow macro expansions, since
355  // those might take us into a preamble file as well.
356  bool IsInsideMainFile =
357  Info.hasSourceManager() &&
358  Info.getSourceManager().isWrittenInMainFile(
359  Info.getSourceManager().getFileLoc(Info.getLocation()));
360  if (IsInsideMainFile && tidy::ShouldSuppressDiagnostic(
361  DiagLevel, Info, *CTContext,
362  /* CheckMacroExpansion = */ false)) {
363  return DiagnosticsEngine::Ignored;
364  }
365  }
366  }
367  return DiagLevel;
368  });
369  Preprocessor *PP = &Clang->getPreprocessor();
370  for (const auto &Check : CTChecks) {
371  // FIXME: the PP callbacks skip the entire preamble.
372  // Checks that want to see #includes in the main file do not see them.
373  Check->registerPPCallbacks(Clang->getSourceManager(), PP, PP);
374  Check->registerMatchers(&CTFinder);
375  }
376  }
377 
378  // Add IncludeFixer which can recorver diagnostics caused by missing includes
379  // (e.g. incomplete type) and attach include insertion fixes to diagnostics.
380  llvm::Optional<IncludeFixer> FixIncludes;
381  auto BuildDir = VFS->getCurrentWorkingDirectory();
382  if (Opts.SuggestMissingIncludes && Index && !BuildDir.getError()) {
383  auto Style = getFormatStyleForFile(MainInput.getFile(), Content, VFS.get());
384  auto Inserter = std::make_shared<IncludeInserter>(
385  MainInput.getFile(), Content, Style, BuildDir.get(),
386  &Clang->getPreprocessor().getHeaderSearchInfo());
387  if (Preamble) {
388  for (const auto &Inc : Preamble->Includes.MainFileIncludes)
389  Inserter->addExisting(Inc);
390  }
391  FixIncludes.emplace(MainInput.getFile(), Inserter, *Index,
392  /*IndexRequestLimit=*/5);
393  ASTDiags.contributeFixes([&FixIncludes](DiagnosticsEngine::Level DiagLevl,
394  const clang::Diagnostic &Info) {
395  return FixIncludes->fix(DiagLevl, Info);
396  });
397  Clang->setExternalSemaSource(FixIncludes->unresolvedNameRecorder());
398  }
399 
400  // Copy over the includes from the preamble, then combine with the
401  // non-preamble includes below.
402  auto Includes = Preamble ? Preamble->Includes : IncludeStructure{};
403  // Replay the preamble includes so that clang-tidy checks can see them.
404  if (Preamble)
405  ReplayPreamble::attach(Includes, *Clang);
406  // Important: collectIncludeStructure is registered *after* ReplayPreamble!
407  // Otherwise we would collect the replayed includes again...
408  // (We can't *just* use the replayed includes, they don't have Resolved path).
409  Clang->getPreprocessor().addPPCallbacks(
410  collectIncludeStructureCallback(Clang->getSourceManager(), &Includes));
411 
412  // Copy over the includes from the preamble, then combine with the
413  // non-preamble includes below.
414  CanonicalIncludes CanonIncludes;
415  if (Preamble)
416  CanonIncludes = Preamble->CanonIncludes;
417  else
418  addSystemHeadersMapping(&CanonIncludes, Clang->getLangOpts());
419  std::unique_ptr<CommentHandler> IWYUHandler =
420  collectIWYUHeaderMaps(&CanonIncludes);
421  Clang->getPreprocessor().addCommentHandler(IWYUHandler.get());
422 
423  // Collect tokens of the main file.
424  syntax::TokenCollector CollectTokens(Clang->getPreprocessor());
425 
426  if (llvm::Error Err = Action->Execute())
427  log("Execute() failed when building AST for {0}: {1}", MainInput.getFile(),
428  toString(std::move(Err)));
429 
430  // We have to consume the tokens before running clang-tidy to avoid collecting
431  // tokens from running the preprocessor inside the checks (only
432  // modernize-use-trailing-return-type does that today).
433  syntax::TokenBuffer Tokens = std::move(CollectTokens).consume();
434  std::vector<Decl *> ParsedDecls = Action->takeTopLevelDecls();
435  // AST traversals should exclude the preamble, to avoid performance cliffs.
436  Clang->getASTContext().setTraversalScope(ParsedDecls);
437  {
438  // Run the AST-dependent part of the clang-tidy checks.
439  // (The preprocessor part ran already, via PPCallbacks).
440  trace::Span Tracer("ClangTidyMatch");
441  CTFinder.matchAST(Clang->getASTContext());
442  }
443 
444  // UnitDiagsConsumer is local, we can not store it in CompilerInstance that
445  // has a longer lifetime.
446  Clang->getDiagnostics().setClient(new IgnoreDiagnostics);
447  // CompilerInstance won't run this callback, do it directly.
448  ASTDiags.EndSourceFile();
449  // XXX: This is messy: clang-tidy checks flush some diagnostics at EOF.
450  // However Action->EndSourceFile() would destroy the ASTContext!
451  // So just inform the preprocessor of EOF, while keeping everything alive.
452  Clang->getPreprocessor().EndSourceFile();
453 
454  std::vector<Diag> Diags = ASTDiags.take(CTContext.getPointer());
455  // Add diagnostics from the preamble, if any.
456  if (Preamble)
457  Diags.insert(Diags.begin(), Preamble->Diags.begin(), Preamble->Diags.end());
458  return ParsedAST(std::move(Preamble), std::move(Clang), std::move(Action),
459  std::move(Tokens), std::move(ParsedDecls), std::move(Diags),
460  std::move(Includes), std::move(CanonIncludes));
461 }
462 
463 ParsedAST::ParsedAST(ParsedAST &&Other) = default;
464 
465 ParsedAST &ParsedAST::operator=(ParsedAST &&Other) = default;
466 
468  if (Action) {
469  // We already notified the PP of end-of-file earlier, so detach it first.
470  // We must keep it alive until after EndSourceFile(), Sema relies on this.
471  auto PP = Clang->getPreprocessorPtr(); // Keep PP alive for now.
472  Clang->setPreprocessor(nullptr); // Detach so we don't send EOF again.
473  Action->EndSourceFile(); // Destroy ASTContext and Sema.
474  // Now Sema is gone, it's safe for PP to go out of scope.
475  }
476 }
477 
478 ASTContext &ParsedAST::getASTContext() { return Clang->getASTContext(); }
479 
480 const ASTContext &ParsedAST::getASTContext() const {
481  return Clang->getASTContext();
482 }
483 
484 Preprocessor &ParsedAST::getPreprocessor() { return Clang->getPreprocessor(); }
485 
486 std::shared_ptr<Preprocessor> ParsedAST::getPreprocessorPtr() {
487  return Clang->getPreprocessorPtr();
488 }
489 
490 const Preprocessor &ParsedAST::getPreprocessor() const {
491  return Clang->getPreprocessor();
492 }
493 
494 llvm::ArrayRef<Decl *> ParsedAST::getLocalTopLevelDecls() {
495  return LocalTopLevelDecls;
496 }
497 
498 const std::vector<Diag> &ParsedAST::getDiagnostics() const { return Diags; }
499 
500 std::size_t ParsedAST::getUsedBytes() const {
501  auto &AST = getASTContext();
502  // FIXME(ibiryukov): we do not account for the dynamically allocated part of
503  // Message and Fixes inside each diagnostic.
504  std::size_t Total =
505  clangd::getUsedBytes(LocalTopLevelDecls) + clangd::getUsedBytes(Diags);
506 
507  // FIXME: the rest of the function is almost a direct copy-paste from
508  // libclang's clang_getCXTUResourceUsage. We could share the implementation.
509 
510  // Sum up variaous allocators inside the ast context and the preprocessor.
511  Total += AST.getASTAllocatedMemory();
512  Total += AST.getSideTableAllocatedMemory();
513  Total += AST.Idents.getAllocator().getTotalMemory();
514  Total += AST.Selectors.getTotalMemory();
515 
516  Total += AST.getSourceManager().getContentCacheSize();
517  Total += AST.getSourceManager().getDataStructureSizes();
518  Total += AST.getSourceManager().getMemoryBufferSizes().malloc_bytes;
519 
520  if (ExternalASTSource *Ext = AST.getExternalSource())
521  Total += Ext->getMemoryBufferSizes().malloc_bytes;
522 
523  const Preprocessor &PP = getPreprocessor();
524  Total += PP.getTotalMemory();
525  if (PreprocessingRecord *PRec = PP.getPreprocessingRecord())
526  Total += PRec->getTotalMemory();
527  Total += PP.getHeaderSearchInfo().getTotalMemory();
528 
529  return Total;
530 }
531 
533  return Includes;
534 }
535 
537  return CanonIncludes;
538 }
539 
541  std::vector<Diag> Diags, IncludeStructure Includes,
542  std::vector<std::string> MainFileMacros,
543  std::unique_ptr<PreambleFileStatusCache> StatCache,
544  CanonicalIncludes CanonIncludes)
545  : Preamble(std::move(Preamble)), Diags(std::move(Diags)),
546  Includes(std::move(Includes)), MainFileMacros(std::move(MainFileMacros)),
547  StatCache(std::move(StatCache)), CanonIncludes(std::move(CanonIncludes)) {
548 }
549 
550 ParsedAST::ParsedAST(std::shared_ptr<const PreambleData> Preamble,
551  std::unique_ptr<CompilerInstance> Clang,
552  std::unique_ptr<FrontendAction> Action,
553  syntax::TokenBuffer Tokens,
554  std::vector<Decl *> LocalTopLevelDecls,
555  std::vector<Diag> Diags, IncludeStructure Includes,
557  : Preamble(std::move(Preamble)), Clang(std::move(Clang)),
558  Action(std::move(Action)), Tokens(std::move(Tokens)),
559  Diags(std::move(Diags)),
560  LocalTopLevelDecls(std::move(LocalTopLevelDecls)),
561  Includes(std::move(Includes)), CanonIncludes(std::move(CanonIncludes)) {
562  assert(this->Clang);
563  assert(this->Action);
564 }
565 
566 std::shared_ptr<const PreambleData>
567 buildPreamble(PathRef FileName, CompilerInvocation &CI,
568  std::shared_ptr<const PreambleData> OldPreamble,
569  const tooling::CompileCommand &OldCompileCommand,
570  const ParseInputs &Inputs, bool StoreInMemory,
571  PreambleParsedCallback PreambleCallback) {
572  // Note that we don't need to copy the input contents, preamble can live
573  // without those.
574  auto ContentsBuffer = llvm::MemoryBuffer::getMemBuffer(Inputs.Contents);
575  auto Bounds =
576  ComputePreambleBounds(*CI.getLangOpts(), ContentsBuffer.get(), 0);
577 
578  if (OldPreamble &&
579  compileCommandsAreEqual(Inputs.CompileCommand, OldCompileCommand) &&
580  OldPreamble->Preamble.CanReuse(CI, ContentsBuffer.get(), Bounds,
581  Inputs.FS.get())) {
582  vlog("Reusing preamble for file {0}", llvm::Twine(FileName));
583  return OldPreamble;
584  }
585  vlog("Preamble for file {0} cannot be reused. Attempting to rebuild it.",
586  FileName);
587 
588  trace::Span Tracer("BuildPreamble");
589  SPAN_ATTACH(Tracer, "File", FileName);
590  StoreDiags PreambleDiagnostics;
591  llvm::IntrusiveRefCntPtr<DiagnosticsEngine> PreambleDiagsEngine =
592  CompilerInstance::createDiagnostics(&CI.getDiagnosticOpts(),
593  &PreambleDiagnostics, false);
594 
595  // Skip function bodies when building the preamble to speed up building
596  // the preamble and make it smaller.
597  assert(!CI.getFrontendOpts().SkipFunctionBodies);
598  CI.getFrontendOpts().SkipFunctionBodies = true;
599  // We don't want to write comment locations into PCH. They are racy and slow
600  // to read back. We rely on dynamic index for the comments instead.
601  CI.getPreprocessorOpts().WriteCommentListToPCH = false;
602 
603  CppFilePreambleCallbacks SerializedDeclsCollector(FileName, PreambleCallback);
604  if (Inputs.FS->setCurrentWorkingDirectory(Inputs.CompileCommand.Directory)) {
605  log("Couldn't set working directory when building the preamble.");
606  // We proceed anyway, our lit-tests rely on results for non-existing working
607  // dirs.
608  }
609 
610  llvm::SmallString<32> AbsFileName(FileName);
611  Inputs.FS->makeAbsolute(AbsFileName);
612  auto StatCache = llvm::make_unique<PreambleFileStatusCache>(AbsFileName);
613  auto BuiltPreamble = PrecompiledPreamble::Build(
614  CI, ContentsBuffer.get(), Bounds, *PreambleDiagsEngine,
615  StatCache->getProducingFS(Inputs.FS),
616  std::make_shared<PCHContainerOperations>(), StoreInMemory,
617  SerializedDeclsCollector);
618 
619  // When building the AST for the main file, we do want the function
620  // bodies.
621  CI.getFrontendOpts().SkipFunctionBodies = false;
622 
623  if (BuiltPreamble) {
624  vlog("Built preamble of size {0} for file {1}", BuiltPreamble->getSize(),
625  FileName);
626  std::vector<Diag> Diags = PreambleDiagnostics.take();
627  return std::make_shared<PreambleData>(
628  std::move(*BuiltPreamble), std::move(Diags),
629  SerializedDeclsCollector.takeIncludes(),
630  SerializedDeclsCollector.takeMainFileMacros(), std::move(StatCache),
631  SerializedDeclsCollector.takeCanonicalIncludes());
632  } else {
633  elog("Could not build a preamble for file {0}", FileName);
634  return nullptr;
635  }
636 }
637 
638 llvm::Optional<ParsedAST>
639 buildAST(PathRef FileName, std::unique_ptr<CompilerInvocation> Invocation,
640  const ParseInputs &Inputs,
641  std::shared_ptr<const PreambleData> Preamble) {
642  trace::Span Tracer("BuildAST");
643  SPAN_ATTACH(Tracer, "File", FileName);
644 
645  auto VFS = Inputs.FS;
646  if (Preamble && Preamble->StatCache)
647  VFS = Preamble->StatCache->getConsumingFS(std::move(VFS));
648  if (VFS->setCurrentWorkingDirectory(Inputs.CompileCommand.Directory)) {
649  log("Couldn't set working directory when building the preamble.");
650  // We proceed anyway, our lit-tests rely on results for non-existing working
651  // dirs.
652  }
653 
654  return ParsedAST::build(llvm::make_unique<CompilerInvocation>(*Invocation),
655  Preamble,
656  llvm::MemoryBuffer::getMemBufferCopy(Inputs.Contents),
657  std::move(VFS), Inputs.Index, Inputs.Opts);
658 }
659 
661  const Position &Pos, const FileID FID) {
662  const ASTContext &AST = Unit.getASTContext();
663  const SourceManager &SourceMgr = AST.getSourceManager();
664  auto Offset = positionToOffset(SourceMgr.getBufferData(FID), Pos);
665  if (!Offset) {
666  log("getBeginningOfIdentifier: {0}", Offset.takeError());
667  return SourceLocation();
668  }
669 
670  // GetBeginningOfToken(pos) is almost what we want, but does the wrong thing
671  // if the cursor is at the end of the identifier.
672  // Instead, we lex at GetBeginningOfToken(pos - 1). The cases are:
673  // 1) at the beginning of an identifier, we'll be looking at something
674  // that isn't an identifier.
675  // 2) at the middle or end of an identifier, we get the identifier.
676  // 3) anywhere outside an identifier, we'll get some non-identifier thing.
677  // We can't actually distinguish cases 1 and 3, but returning the original
678  // location is correct for both!
679  SourceLocation InputLoc = SourceMgr.getComposedLoc(FID, *Offset);
680  if (*Offset == 0) // Case 1 or 3.
681  return SourceMgr.getMacroArgExpandedLocation(InputLoc);
682  SourceLocation Before = SourceMgr.getComposedLoc(FID, *Offset - 1);
683 
684  Before = Lexer::GetBeginningOfToken(Before, SourceMgr, AST.getLangOpts());
685  Token Tok;
686  if (Before.isValid() &&
687  !Lexer::getRawToken(Before, Tok, SourceMgr, AST.getLangOpts(), false) &&
688  Tok.is(tok::raw_identifier))
689  return SourceMgr.getMacroArgExpandedLocation(Before); // Case 2.
690  return SourceMgr.getMacroArgExpandedLocation(InputLoc); // Case 1 or 3.
691 }
692 
693 } // namespace clangd
694 namespace tidy {
695 // Force the linker to link in Clang-tidy modules.
696 #define LINK_TIDY_MODULE(X) \
697  extern volatile int X##ModuleAnchorSource; \
698  static int LLVM_ATTRIBUTE_UNUSED X##ModuleAnchorDestination = \
699  X##ModuleAnchorSource
700 LINK_TIDY_MODULE(CERT);
701 LINK_TIDY_MODULE(Abseil);
702 LINK_TIDY_MODULE(Boost);
703 LINK_TIDY_MODULE(Bugprone);
704 LINK_TIDY_MODULE(LLVM);
705 LINK_TIDY_MODULE(CppCoreGuidelines);
706 LINK_TIDY_MODULE(Fuchsia);
707 LINK_TIDY_MODULE(Google);
708 LINK_TIDY_MODULE(Android);
709 LINK_TIDY_MODULE(Misc);
710 LINK_TIDY_MODULE(Modernize);
711 LINK_TIDY_MODULE(Performance);
712 LINK_TIDY_MODULE(Portability);
713 LINK_TIDY_MODULE(Readability);
714 LINK_TIDY_MODULE(ObjC);
715 LINK_TIDY_MODULE(HICPP);
716 LINK_TIDY_MODULE(Zircon);
717 #undef LINK_TIDY_MODULE
718 } // namespace tidy
719 } // namespace clang
std::unique_ptr< CommentHandler > collectIWYUHeaderMaps(CanonicalIncludes *Includes)
Returns a CommentHandler that parses pragma comment on include files to determine when we should incl...
SourceLocation Loc
&#39;#&#39; location in the include directive
bool ShouldSuppressDiagnostic(DiagnosticsEngine::Level DiagLevel, const Diagnostic &Info, ClangTidyContext &Context, bool CheckMacroExpansion)
Check whether a given diagnostic should be suppressed due to the presence of a "NOLINT" suppression c...
StoreDiags collects the diagnostics that can later be reported by clangd.
Definition: Diagnostics.h:119
ParsedAST & operator=(ParsedAST &&Other)
Preprocessor & getPreprocessor()
Definition: ClangdUnit.cpp:484
llvm::IntrusiveRefCntPtr< llvm::vfs::FileSystem > VFS
void dumpAST(ParsedAST &AST, llvm::raw_ostream &OS)
For testing/debugging purposes.
Definition: ClangdUnit.cpp:282
static llvm::Optional< ParsedAST > build(std::unique_ptr< clang::CompilerInvocation > CI, std::shared_ptr< const PreambleData > Preamble, std::unique_ptr< llvm::MemoryBuffer > Buffer, IntrusiveRefCntPtr< llvm::vfs::FileSystem > VFS, const SymbolIndex *Index, const ParseOptions &Opts)
Attempts to run Clang and store parsed AST.
Definition: ClangdUnit.cpp:287
void contributeFixes(DiagFixer Fixer)
If set, possibly adds fixes for diagnostics using Fixer.
Definition: Diagnostics.h:134
Interface for symbol indexes that can be used for searching or matching symbols among a set of symbol...
Definition: Index.h:85
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:23
ArrayRef< Decl * > getLocalTopLevelDecls()
This function returns top-level decls present in the main file of the AST.
Definition: ClangdUnit.cpp:494
void addSystemHeadersMapping(CanonicalIncludes *Includes, const LangOptions &Language)
Adds mapping for system headers and some special symbols (e.g.
std::unique_ptr< CompilerInstance > prepareCompilerInstance(std::unique_ptr< clang::CompilerInvocation > CI, const PrecompiledPreamble *Preamble, std::unique_ptr< llvm::MemoryBuffer > Buffer, llvm::IntrusiveRefCntPtr< llvm::vfs::FileSystem > VFS, DiagnosticConsumer &DiagsClient)
Definition: Compiler.cpp:72
Documents should not be synced at all.
tidy::ClangTidyOptions ClangTidyOpts
Definition: Compiler.h:39
A collection of ClangTidyCheckFactory instances.
void vlog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:67
void elog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:56
ASTContext & getASTContext()
Note that the returned ast will not contain decls from the preamble that were not deserialized during...
Definition: ClangdUnit.cpp:478
llvm::StringRef Src
std::size_t getUsedBytes() const
Returns the esitmated size of the AST and the accessory structures, in bytes.
Definition: ClangdUnit.cpp:500
SourceLocation getBeginningOfIdentifier(const ParsedAST &Unit, const Position &Pos, const FileID FID)
Get the beginning SourceLocation at a specified Pos.
Definition: ClangdUnit.cpp:660
BindArgumentKind Kind
std::shared_ptr< const PreambleData > buildPreamble(PathRef FileName, CompilerInvocation &CI, std::shared_ptr< const PreambleData > OldPreamble, const tooling::CompileCommand &OldCompileCommand, const ParseInputs &Inputs, bool StoreInMemory, PreambleParsedCallback PreambleCallback)
Rebuild the preamble for the new inputs unless the old one can be reused.
Definition: ClangdUnit.cpp:567
Maps a definition location onto an #include file, based on a set of filename rules.
const CanonicalIncludes & getCanonicalIncludes() const
Definition: ClangdUnit.cpp:536
CanonicalIncludes CanonIncludes
Definition: ClangdUnit.h:69
const IncludeStructure & getIncludeStructure() const
Definition: ClangdUnit.cpp:532
std::unique_ptr< PreambleFileStatusCache > StatCache
Definition: ClangdUnit.h:68
#define LINK_TIDY_MODULE(X)
Definition: ClangdUnit.cpp:696
llvm::Expected< size_t > positionToOffset(llvm::StringRef Code, Position P, bool AllowColumnsBeyondLineLength)
Turn a [line, column] pair into an offset in Code.
Definition: SourceCode.cpp:141
llvm::unique_function< void()> Action
void createChecks(ClangTidyContext *Context, std::vector< std::unique_ptr< ClangTidyCheck >> &Checks)
Create instances of all checks matching CheckRegexString and store them in Checks.
std::string configurationAsText(const ClangTidyOptions &Options)
Serializes configuration to a YAML-encoded string.
void log(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:62
static const char * toString(OffsetEncoding OE)
Definition: Protocol.cpp:1017
std::shared_ptr< Preprocessor > getPreprocessorPtr()
Definition: ClangdUnit.cpp:486
llvm::Optional< ParsedAST > buildAST(PathRef FileName, std::unique_ptr< CompilerInvocation > Invocation, const ParseInputs &Inputs, std::shared_ptr< const PreambleData > Preamble)
Build an AST from provided user inputs.
Definition: ClangdUnit.cpp:639
tooling::CompileCommand CompileCommand
Definition: Compiler.h:45
#define dlog(...)
Definition: Logger.h:72
StringRef Tokens
const Decl * D
Definition: XRefs.cpp:868
PathRef FileName
An information message.
format::FormatStyle getFormatStyleForFile(llvm::StringRef File, llvm::StringRef Content, llvm::vfs::FileSystem *FS)
Choose the clang-format style we should apply to a certain file.
Definition: SourceCode.cpp:485
Stores and provides access to parsed AST.
Definition: ClangdUnit.h:73
std::function< void(ASTContext &, std::shared_ptr< clang::Preprocessor >, const CanonicalIncludes &)> PreambleParsedCallback
Definition: ClangdUnit.h:157
const SymbolIndex * Index
Definition: Compiler.h:49
size_t Offset
Information required to run clang, e.g. to parse AST or do code completion.
Definition: Compiler.h:44
const PreambleData * Preamble
std::unique_ptr< PPCallbacks > collectIncludeStructureCallback(const SourceManager &SM, IncludeStructure *Out)
Returns a PPCallback that visits all inclusions in the main file.
Definition: Headers.cpp:113
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
static GeneratorRegistry::Add< MDGenerator > MD(MDGenerator::Format, "Generator for MD output.")
void EndSourceFile() override
std::vector< Diag > Diags
Definition: ClangdUnit.h:58
IntrusiveRefCntPtr< llvm::vfs::FileSystem > FS
Definition: Compiler.h:46
PrecompiledPreamble Preamble
Definition: ClangdUnit.h:57
IncludeStructure Includes
Definition: ClangdUnit.h:61
void setLevelAdjuster(LevelAdjuster Adjuster)
If set, this allows the client of this class to adjust the level of diagnostics, such as promoting wa...
Definition: Diagnostics.h:138
Records an event whose duration is the lifetime of the Span object.
Definition: Trace.h:82
PreambleData(PrecompiledPreamble Preamble, std::vector< Diag > Diags, IncludeStructure Includes, std::vector< std::string > MainFileMacros, std::unique_ptr< PreambleFileStatusCache > StatCache, CanonicalIncludes CanonIncludes)
Definition: ClangdUnit.cpp:540
#define SPAN_ATTACH(S, Name, Expr)
Attach a key-value pair to a Span event.
Definition: Trace.h:98
ParsedAST(ParsedAST &&Other)
std::vector< Diag > take(const clang::tidy::ClangTidyContext *Tidy=nullptr)
const std::vector< Diag > & getDiagnostics() const
Definition: ClangdUnit.cpp:498
const SymbolIndex * Index
Definition: Dexp.cpp:84