clang-tools  7.0.0
ClangdUnit.cpp
Go to the documentation of this file.
1 //===--- ClangdUnit.cpp -----------------------------------------*- C++-*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===---------------------------------------------------------------------===//
9 
10 #include "ClangdUnit.h"
11 #include "Compiler.h"
12 #include "Diagnostics.h"
13 #include "Logger.h"
14 #include "SourceCode.h"
15 #include "Trace.h"
16 #include "clang/AST/ASTContext.h"
17 #include "clang/Basic/LangOptions.h"
18 #include "clang/Frontend/CompilerInstance.h"
19 #include "clang/Frontend/CompilerInvocation.h"
20 #include "clang/Frontend/FrontendActions.h"
21 #include "clang/Frontend/Utils.h"
22 #include "clang/Index/IndexDataConsumer.h"
23 #include "clang/Index/IndexingAction.h"
24 #include "clang/Lex/Lexer.h"
25 #include "clang/Lex/MacroInfo.h"
26 #include "clang/Lex/Preprocessor.h"
27 #include "clang/Lex/PreprocessorOptions.h"
28 #include "clang/Sema/Sema.h"
29 #include "clang/Serialization/ASTWriter.h"
30 #include "clang/Tooling/CompilationDatabase.h"
31 #include "llvm/ADT/ArrayRef.h"
32 #include "llvm/ADT/SmallVector.h"
33 #include "llvm/Support/CrashRecoveryContext.h"
34 #include "llvm/Support/raw_ostream.h"
35 #include <algorithm>
36 
37 using namespace clang::clangd;
38 using namespace clang;
39 
40 namespace {
41 
42 bool compileCommandsAreEqual(const tooling::CompileCommand &LHS,
43  const tooling::CompileCommand &RHS) {
44  // We don't check for Output, it should not matter to clangd.
45  return LHS.Directory == RHS.Directory && LHS.Filename == RHS.Filename &&
46  llvm::makeArrayRef(LHS.CommandLine).equals(RHS.CommandLine);
47 }
48 
49 template <class T> std::size_t getUsedBytes(const std::vector<T> &Vec) {
50  return Vec.capacity() * sizeof(T);
51 }
52 
53 class DeclTrackingASTConsumer : public ASTConsumer {
54 public:
55  DeclTrackingASTConsumer(std::vector<Decl *> &TopLevelDecls)
56  : TopLevelDecls(TopLevelDecls) {}
57 
58  bool HandleTopLevelDecl(DeclGroupRef DG) override {
59  for (Decl *D : DG) {
60  // ObjCMethodDecl are not actually top-level decls.
61  if (isa<ObjCMethodDecl>(D))
62  continue;
63 
64  TopLevelDecls.push_back(D);
65  }
66  return true;
67  }
68 
69 private:
70  std::vector<Decl *> &TopLevelDecls;
71 };
72 
73 class ClangdFrontendAction : public SyntaxOnlyAction {
74 public:
75  std::vector<Decl *> takeTopLevelDecls() { return std::move(TopLevelDecls); }
76 
77 protected:
78  std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
79  StringRef InFile) override {
80  return llvm::make_unique<DeclTrackingASTConsumer>(/*ref*/ TopLevelDecls);
81  }
82 
83 private:
84  std::vector<Decl *> TopLevelDecls;
85 };
86 
87 class CppFilePreambleCallbacks : public PreambleCallbacks {
88 public:
89  CppFilePreambleCallbacks(PathRef File, PreambleParsedCallback ParsedCallback)
90  : File(File), ParsedCallback(ParsedCallback) {}
91 
92  IncludeStructure takeIncludes() { return std::move(Includes); }
93 
94  void AfterExecute(CompilerInstance &CI) override {
95  if (!ParsedCallback)
96  return;
97  trace::Span Tracer("Running PreambleCallback");
98  ParsedCallback(File, CI.getASTContext(), CI.getPreprocessorPtr());
99  }
100 
101  void BeforeExecute(CompilerInstance &CI) override {
102  SourceMgr = &CI.getSourceManager();
103  }
104 
105  std::unique_ptr<PPCallbacks> createPPCallbacks() override {
106  assert(SourceMgr && "SourceMgr must be set at this point");
107  return collectIncludeStructureCallback(*SourceMgr, &Includes);
108  }
109 
110 private:
111  PathRef File;
112  PreambleParsedCallback ParsedCallback;
113  IncludeStructure Includes;
114  SourceManager *SourceMgr = nullptr;
115 };
116 
117 } // namespace
118 
119 void clangd::dumpAST(ParsedAST &AST, llvm::raw_ostream &OS) {
120  AST.getASTContext().getTranslationUnitDecl()->dump(OS, true);
121 }
122 
123 llvm::Optional<ParsedAST>
124 ParsedAST::build(std::unique_ptr<clang::CompilerInvocation> CI,
125  std::shared_ptr<const PreambleData> Preamble,
126  std::unique_ptr<llvm::MemoryBuffer> Buffer,
127  std::shared_ptr<PCHContainerOperations> PCHs,
128  IntrusiveRefCntPtr<vfs::FileSystem> VFS) {
129  assert(CI);
130  // Command-line parsing sets DisableFree to true by default, but we don't want
131  // to leak memory in clangd.
132  CI->getFrontendOpts().DisableFree = false;
133  const PrecompiledPreamble *PreamblePCH =
134  Preamble ? &Preamble->Preamble : nullptr;
135 
136  StoreDiags ASTDiags;
137  auto Clang =
138  prepareCompilerInstance(std::move(CI), PreamblePCH, std::move(Buffer),
139  std::move(PCHs), std::move(VFS), ASTDiags);
140  if (!Clang)
141  return llvm::None;
142 
143  // Recover resources if we crash before exiting this method.
144  llvm::CrashRecoveryContextCleanupRegistrar<CompilerInstance> CICleanup(
145  Clang.get());
146 
147  auto Action = llvm::make_unique<ClangdFrontendAction>();
148  const FrontendInputFile &MainInput = Clang->getFrontendOpts().Inputs[0];
149  if (!Action->BeginSourceFile(*Clang, MainInput)) {
150  log("BeginSourceFile() failed when building AST for {0}",
151  MainInput.getFile());
152  return llvm::None;
153  }
154 
155  // Copy over the includes from the preamble, then combine with the
156  // non-preamble includes below.
157  auto Includes = Preamble ? Preamble->Includes : IncludeStructure{};
158  Clang->getPreprocessor().addPPCallbacks(
159  collectIncludeStructureCallback(Clang->getSourceManager(), &Includes));
160 
161  if (!Action->Execute())
162  log("Execute() failed when building AST for {0}", MainInput.getFile());
163 
164  // UnitDiagsConsumer is local, we can not store it in CompilerInstance that
165  // has a longer lifetime.
166  Clang->getDiagnostics().setClient(new IgnoreDiagnostics);
167  // CompilerInstance won't run this callback, do it directly.
168  ASTDiags.EndSourceFile();
169 
170  std::vector<Decl *> ParsedDecls = Action->takeTopLevelDecls();
171  std::vector<Diag> Diags = ASTDiags.take();
172  // Add diagnostics from the preamble, if any.
173  if (Preamble)
174  Diags.insert(Diags.begin(), Preamble->Diags.begin(), Preamble->Diags.end());
175  return ParsedAST(std::move(Preamble), std::move(Clang), std::move(Action),
176  std::move(ParsedDecls), std::move(Diags),
177  std::move(Includes));
178 }
179 
180 ParsedAST::ParsedAST(ParsedAST &&Other) = default;
181 
182 ParsedAST &ParsedAST::operator=(ParsedAST &&Other) = default;
183 
185  if (Action) {
186  Action->EndSourceFile();
187  }
188 }
189 
190 ASTContext &ParsedAST::getASTContext() { return Clang->getASTContext(); }
191 
192 const ASTContext &ParsedAST::getASTContext() const {
193  return Clang->getASTContext();
194 }
195 
196 Preprocessor &ParsedAST::getPreprocessor() { return Clang->getPreprocessor(); }
197 
198 std::shared_ptr<Preprocessor> ParsedAST::getPreprocessorPtr() {
199  return Clang->getPreprocessorPtr();
200 }
201 
202 const Preprocessor &ParsedAST::getPreprocessor() const {
203  return Clang->getPreprocessor();
204 }
205 
207  return LocalTopLevelDecls;
208 }
209 
210 const std::vector<Diag> &ParsedAST::getDiagnostics() const { return Diags; }
211 
212 std::size_t ParsedAST::getUsedBytes() const {
213  auto &AST = getASTContext();
214  // FIXME(ibiryukov): we do not account for the dynamically allocated part of
215  // Message and Fixes inside each diagnostic.
216  std::size_t Total =
217  ::getUsedBytes(LocalTopLevelDecls) + ::getUsedBytes(Diags);
218 
219  // FIXME: the rest of the function is almost a direct copy-paste from
220  // libclang's clang_getCXTUResourceUsage. We could share the implementation.
221 
222  // Sum up variaous allocators inside the ast context and the preprocessor.
223  Total += AST.getASTAllocatedMemory();
224  Total += AST.getSideTableAllocatedMemory();
225  Total += AST.Idents.getAllocator().getTotalMemory();
226  Total += AST.Selectors.getTotalMemory();
227 
228  Total += AST.getSourceManager().getContentCacheSize();
229  Total += AST.getSourceManager().getDataStructureSizes();
230  Total += AST.getSourceManager().getMemoryBufferSizes().malloc_bytes;
231 
232  if (ExternalASTSource *Ext = AST.getExternalSource())
233  Total += Ext->getMemoryBufferSizes().malloc_bytes;
234 
235  const Preprocessor &PP = getPreprocessor();
236  Total += PP.getTotalMemory();
237  if (PreprocessingRecord *PRec = PP.getPreprocessingRecord())
238  Total += PRec->getTotalMemory();
239  Total += PP.getHeaderSearchInfo().getTotalMemory();
240 
241  return Total;
242 }
243 
245  return Includes;
246 }
247 
249  std::vector<Diag> Diags, IncludeStructure Includes)
250  : Preamble(std::move(Preamble)), Diags(std::move(Diags)),
251  Includes(std::move(Includes)) {}
252 
253 ParsedAST::ParsedAST(std::shared_ptr<const PreambleData> Preamble,
254  std::unique_ptr<CompilerInstance> Clang,
255  std::unique_ptr<FrontendAction> Action,
256  std::vector<Decl *> LocalTopLevelDecls,
257  std::vector<Diag> Diags, IncludeStructure Includes)
258  : Preamble(std::move(Preamble)), Clang(std::move(Clang)),
259  Action(std::move(Action)), Diags(std::move(Diags)),
260  LocalTopLevelDecls(std::move(LocalTopLevelDecls)),
261  Includes(std::move(Includes)) {
262  assert(this->Clang);
263  assert(this->Action);
264 }
265 
266 std::unique_ptr<CompilerInvocation>
268  std::vector<const char *> ArgStrs;
269  for (const auto &S : Inputs.CompileCommand.CommandLine)
270  ArgStrs.push_back(S.c_str());
271 
272  if (Inputs.FS->setCurrentWorkingDirectory(Inputs.CompileCommand.Directory)) {
273  log("Couldn't set working directory when creating compiler invocation.");
274  // We proceed anyway, our lit-tests rely on results for non-existing working
275  // dirs.
276  }
277 
278  // FIXME(ibiryukov): store diagnostics from CommandLine when we start
279  // reporting them.
281  IntrusiveRefCntPtr<DiagnosticsEngine> CommandLineDiagsEngine =
282  CompilerInstance::createDiagnostics(new DiagnosticOptions,
283  &IgnoreDiagnostics, false);
284  std::unique_ptr<CompilerInvocation> CI = createInvocationFromCommandLine(
285  ArgStrs, CommandLineDiagsEngine, Inputs.FS);
286  if (!CI)
287  return nullptr;
288  // createInvocationFromCommandLine sets DisableFree.
289  CI->getFrontendOpts().DisableFree = false;
290  CI->getLangOpts()->CommentOpts.ParseAllComments = true;
291  return CI;
292 }
293 
294 std::shared_ptr<const PreambleData> clangd::buildPreamble(
295  PathRef FileName, CompilerInvocation &CI,
296  std::shared_ptr<const PreambleData> OldPreamble,
297  const tooling::CompileCommand &OldCompileCommand, const ParseInputs &Inputs,
298  std::shared_ptr<PCHContainerOperations> PCHs, bool StoreInMemory,
299  PreambleParsedCallback PreambleCallback) {
300  // Note that we don't need to copy the input contents, preamble can live
301  // without those.
302  auto ContentsBuffer = llvm::MemoryBuffer::getMemBuffer(Inputs.Contents);
303  auto Bounds =
304  ComputePreambleBounds(*CI.getLangOpts(), ContentsBuffer.get(), 0);
305 
306  if (OldPreamble &&
307  compileCommandsAreEqual(Inputs.CompileCommand, OldCompileCommand) &&
308  OldPreamble->Preamble.CanReuse(CI, ContentsBuffer.get(), Bounds,
309  Inputs.FS.get())) {
310  vlog("Reusing preamble for file {0}", Twine(FileName));
311  return OldPreamble;
312  }
313  vlog("Preamble for file {0} cannot be reused. Attempting to rebuild it.",
314  FileName);
315 
316  trace::Span Tracer("BuildPreamble");
317  SPAN_ATTACH(Tracer, "File", FileName);
318  StoreDiags PreambleDiagnostics;
319  IntrusiveRefCntPtr<DiagnosticsEngine> PreambleDiagsEngine =
320  CompilerInstance::createDiagnostics(&CI.getDiagnosticOpts(),
321  &PreambleDiagnostics, false);
322 
323  // Skip function bodies when building the preamble to speed up building
324  // the preamble and make it smaller.
325  assert(!CI.getFrontendOpts().SkipFunctionBodies);
326  CI.getFrontendOpts().SkipFunctionBodies = true;
327  // We don't want to write comment locations into PCH. They are racy and slow
328  // to read back. We rely on dynamic index for the comments instead.
329  CI.getPreprocessorOpts().WriteCommentListToPCH = false;
330 
331  CppFilePreambleCallbacks SerializedDeclsCollector(FileName, PreambleCallback);
332  if (Inputs.FS->setCurrentWorkingDirectory(Inputs.CompileCommand.Directory)) {
333  log("Couldn't set working directory when building the preamble.");
334  // We proceed anyway, our lit-tests rely on results for non-existing working
335  // dirs.
336  }
337  auto BuiltPreamble = PrecompiledPreamble::Build(
338  CI, ContentsBuffer.get(), Bounds, *PreambleDiagsEngine, Inputs.FS, PCHs,
339  StoreInMemory, SerializedDeclsCollector);
340 
341  // When building the AST for the main file, we do want the function
342  // bodies.
343  CI.getFrontendOpts().SkipFunctionBodies = false;
344 
345  if (BuiltPreamble) {
346  vlog("Built preamble of size {0} for file {1}", BuiltPreamble->getSize(),
347  FileName);
348  return std::make_shared<PreambleData>(
349  std::move(*BuiltPreamble), PreambleDiagnostics.take(),
350  SerializedDeclsCollector.takeIncludes());
351  } else {
352  elog("Could not build a preamble for file {0}", FileName);
353  return nullptr;
354  }
355 }
356 
357 llvm::Optional<ParsedAST> clangd::buildAST(
358  PathRef FileName, std::unique_ptr<CompilerInvocation> Invocation,
359  const ParseInputs &Inputs, std::shared_ptr<const PreambleData> Preamble,
360  std::shared_ptr<PCHContainerOperations> PCHs) {
361  trace::Span Tracer("BuildAST");
362  SPAN_ATTACH(Tracer, "File", FileName);
363 
364  if (Inputs.FS->setCurrentWorkingDirectory(Inputs.CompileCommand.Directory)) {
365  log("Couldn't set working directory when building the preamble.");
366  // We proceed anyway, our lit-tests rely on results for non-existing working
367  // dirs.
368  }
369 
370  return ParsedAST::build(
371  llvm::make_unique<CompilerInvocation>(*Invocation), Preamble,
372  llvm::MemoryBuffer::getMemBufferCopy(Inputs.Contents), PCHs, Inputs.FS);
373 }
374 
376  const Position &Pos,
377  const FileID FID) {
378  const ASTContext &AST = Unit.getASTContext();
379  const SourceManager &SourceMgr = AST.getSourceManager();
380  auto Offset = positionToOffset(SourceMgr.getBufferData(FID), Pos);
381  if (!Offset) {
382  log("getBeginningOfIdentifier: {0}", Offset.takeError());
383  return SourceLocation();
384  }
385  SourceLocation InputLoc = SourceMgr.getComposedLoc(FID, *Offset);
386 
387  // GetBeginningOfToken(pos) is almost what we want, but does the wrong thing
388  // if the cursor is at the end of the identifier.
389  // Instead, we lex at GetBeginningOfToken(pos - 1). The cases are:
390  // 1) at the beginning of an identifier, we'll be looking at something
391  // that isn't an identifier.
392  // 2) at the middle or end of an identifier, we get the identifier.
393  // 3) anywhere outside an identifier, we'll get some non-identifier thing.
394  // We can't actually distinguish cases 1 and 3, but returning the original
395  // location is correct for both!
396  if (*Offset == 0) // Case 1 or 3.
397  return SourceMgr.getMacroArgExpandedLocation(InputLoc);
398  SourceLocation Before =
399  SourceMgr.getMacroArgExpandedLocation(InputLoc.getLocWithOffset(-1));
400  Before = Lexer::GetBeginningOfToken(Before, SourceMgr, AST.getLangOpts());
401  Token Tok;
402  if (Before.isValid() &&
403  !Lexer::getRawToken(Before, Tok, SourceMgr, AST.getLangOpts(), false) &&
404  Tok.is(tok::raw_identifier))
405  return Before; // Case 2.
406  return SourceMgr.getMacroArgExpandedLocation(InputLoc); // Case 1 or 3.
407 }
StoreDiags collects the diagnostics that can later be reported by clangd.
Definition: Diagnostics.h:78
Preprocessor & getPreprocessor()
Definition: ClangdUnit.cpp:196
std::vector< Diag > take()
std::function< void(PathRef Path, ASTContext &, std::shared_ptr< clang::Preprocessor >)> PreambleParsedCallback
Definition: ClangdUnit.h:131
llvm::Optional< ParsedAST > buildAST(PathRef FileName, std::unique_ptr< CompilerInvocation > Invocation, const ParseInputs &Inputs, std::shared_ptr< const PreambleData > Preamble, std::shared_ptr< PCHContainerOperations > PCHs)
Build an AST from provided user inputs.
Definition: ClangdUnit.cpp:357
void dumpAST(ParsedAST &AST, llvm::raw_ostream &OS)
For testing/debugging purposes.
Definition: ClangdUnit.cpp:119
const std::vector< Diag > & getDiagnostics() const
Definition: ClangdUnit.cpp:210
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:24
Documents should not be synced at all.
const IncludeStructure & getIncludeStructure() const
Definition: ClangdUnit.cpp:244
void vlog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:67
ASTContext & getASTContext()
Note that the returned ast will not contain decls from the preamble that were not deserialized during...
Definition: ClangdUnit.cpp:190
void elog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:56
llvm::Expected< size_t > positionToOffset(StringRef Code, Position P, bool AllowColumnsBeyondLineLength)
Definition: SourceCode.cpp:83
ParsedAST(ParsedAST &&Other)
ArrayRef< Decl * > getLocalTopLevelDecls()
This function returns top-level decls present in the main file of the AST.
Definition: ClangdUnit.cpp:206
std::size_t getUsedBytes() const
Returns the esitmated size of the AST and the accessory structures, in bytes.
Definition: ClangdUnit.cpp:212
llvm::unique_function< void()> Action
void log(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:62
PreambleData(PrecompiledPreamble Preamble, std::vector< Diag > Diags, IncludeStructure Includes)
Definition: ClangdUnit.cpp:248
ParsedAST & operator=(ParsedAST &&Other)
std::unique_ptr< CompilerInvocation > buildCompilerInvocation(const ParseInputs &Inputs)
Builds compiler invocation that could be used to build AST or preamble.
Definition: ClangdUnit.cpp:267
tooling::CompileCommand CompileCommand
Definition: ClangdUnit.h:60
std::shared_ptr< Preprocessor > getPreprocessorPtr()
Definition: ClangdUnit.cpp:198
std::shared_ptr< const PreambleData > buildPreamble(PathRef FileName, CompilerInvocation &CI, std::shared_ptr< const PreambleData > OldPreamble, const tooling::CompileCommand &OldCompileCommand, const ParseInputs &Inputs, std::shared_ptr< PCHContainerOperations > PCHs, bool StoreInMemory, PreambleParsedCallback PreambleCallback)
Rebuild the preamble for the new inputs unless the old one can be reused.
Definition: ClangdUnit.cpp:294
PathRef FileName
PrecompiledPreamble const * Preamble
Position Pos
IntrusiveRefCntPtr< vfs::FileSystem > VFS
IntrusiveRefCntPtr< vfs::FileSystem > FS
Definition: ClangdUnit.h:61
Stores and provides access to parsed AST.
Definition: ClangdUnit.h:66
Information required to run clang, e.g. to parse AST or do code completion.
Definition: ClangdUnit.h:59
SourceLocation getBeginningOfIdentifier(ParsedAST &Unit, const Position &Pos, const FileID FID)
Get the beginning SourceLocation at a specified Pos.
Definition: ClangdUnit.cpp:375
std::unique_ptr< PPCallbacks > collectIncludeStructureCallback(const SourceManager &SM, IncludeStructure *Out)
Returns a PPCallback that visits all inclusions in the main file.
Definition: Headers.cpp:74
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
std::unique_ptr< CompilerInstance > prepareCompilerInstance(std::unique_ptr< clang::CompilerInvocation > CI, const PrecompiledPreamble *Preamble, std::unique_ptr< llvm::MemoryBuffer > Buffer, std::shared_ptr< PCHContainerOperations > PCHs, IntrusiveRefCntPtr< vfs::FileSystem > VFS, DiagnosticConsumer &DiagsClient)
Creates a compiler instance, configured so that:
Definition: Compiler.cpp:43
std::shared_ptr< PCHContainerOperations > PCHs
void EndSourceFile() override
static llvm::Optional< ParsedAST > build(std::unique_ptr< clang::CompilerInvocation > CI, std::shared_ptr< const PreambleData > Preamble, std::unique_ptr< llvm::MemoryBuffer > Buffer, std::shared_ptr< PCHContainerOperations > PCHs, IntrusiveRefCntPtr< vfs::FileSystem > VFS)
Attempts to run Clang and store parsed AST.
Definition: ClangdUnit.cpp:124
std::vector< Diag > Diags
Definition: ClangdUnit.h:52
PrecompiledPreamble Preamble
Definition: ClangdUnit.h:51
IncludeStructure Includes
Definition: ClangdUnit.h:55
Records an event whose duration is the lifetime of the Span object.
Definition: Trace.h:83
#define SPAN_ATTACH(S, Name, Expr)
Attach a key-value pair to a Span event.
Definition: Trace.h:98