clang-tools  7.0.0
CodeComplete.cpp
Go to the documentation of this file.
1 //===--- CodeComplete.cpp ---------------------------------------*- C++-*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===---------------------------------------------------------------------===//
9 //
10 // Code completion has several moving parts:
11 // - AST-based completions are provided using the completion hooks in Sema.
12 // - external completions are retrieved from the index (using hints from Sema)
13 // - the two sources overlap, and must be merged and overloads bundled
14 // - results must be scored and ranked (see Quality.h) before rendering
15 //
16 // Signature help works in a similar way as code completion, but it is simpler:
17 // it's purely AST-based, and there are few candidates.
18 //
19 //===---------------------------------------------------------------------===//
20 
21 #include "CodeComplete.h"
22 #include "AST.h"
23 #include "CodeCompletionStrings.h"
24 #include "Compiler.h"
25 #include "FileDistance.h"
26 #include "FuzzyMatch.h"
27 #include "Headers.h"
28 #include "Logger.h"
29 #include "Quality.h"
30 #include "SourceCode.h"
31 #include "Trace.h"
32 #include "URI.h"
33 #include "index/Index.h"
34 #include "clang/ASTMatchers/ASTMatchFinder.h"
35 #include "clang/Basic/LangOptions.h"
36 #include "clang/Format/Format.h"
37 #include "clang/Frontend/CompilerInstance.h"
38 #include "clang/Frontend/FrontendActions.h"
39 #include "clang/Index/USRGeneration.h"
40 #include "clang/Sema/CodeCompleteConsumer.h"
41 #include "clang/Sema/Sema.h"
42 #include "clang/Tooling/Core/Replacement.h"
43 #include "llvm/Support/Format.h"
44 #include "llvm/Support/FormatVariadic.h"
45 #include "llvm/Support/ScopedPrinter.h"
46 #include <queue>
47 
48 // We log detailed candidate here if you run with -debug-only=codecomplete.
49 #define DEBUG_TYPE "CodeComplete"
50 
51 namespace clang {
52 namespace clangd {
53 namespace {
54 
55 CompletionItemKind toCompletionItemKind(index::SymbolKind Kind) {
56  using SK = index::SymbolKind;
57  switch (Kind) {
58  case SK::Unknown:
60  case SK::Module:
61  case SK::Namespace:
62  case SK::NamespaceAlias:
64  case SK::Macro:
66  case SK::Enum:
68  // FIXME(ioeric): use LSP struct instead of class when it is suppoted in the
69  // protocol.
70  case SK::Struct:
71  case SK::Class:
72  case SK::Protocol:
73  case SK::Extension:
74  case SK::Union:
76  // FIXME(ioeric): figure out whether reference is the right type for aliases.
77  case SK::TypeAlias:
78  case SK::Using:
80  case SK::Function:
81  // FIXME(ioeric): this should probably be an operator. This should be fixed
82  // when `Operator` is support type in the protocol.
83  case SK::ConversionFunction:
85  case SK::Variable:
86  case SK::Parameter:
88  case SK::Field:
90  // FIXME(ioeric): use LSP enum constant when it is supported in the protocol.
91  case SK::EnumConstant:
93  case SK::InstanceMethod:
94  case SK::ClassMethod:
95  case SK::StaticMethod:
96  case SK::Destructor:
98  case SK::InstanceProperty:
99  case SK::ClassProperty:
100  case SK::StaticProperty:
102  case SK::Constructor:
104  }
105  llvm_unreachable("Unhandled clang::index::SymbolKind.");
106 }
107 
109 toCompletionItemKind(CodeCompletionResult::ResultKind ResKind,
110  const NamedDecl *Decl) {
111  if (Decl)
112  return toCompletionItemKind(index::getSymbolInfo(Decl).Kind);
113  switch (ResKind) {
114  case CodeCompletionResult::RK_Declaration:
115  llvm_unreachable("RK_Declaration without Decl");
116  case CodeCompletionResult::RK_Keyword:
118  case CodeCompletionResult::RK_Macro:
119  return CompletionItemKind::Text; // unfortunately, there's no 'Macro'
120  // completion items in LSP.
121  case CodeCompletionResult::RK_Pattern:
123  }
124  llvm_unreachable("Unhandled CodeCompletionResult::ResultKind.");
125 }
126 
127 /// Get the optional chunk as a string. This function is possibly recursive.
128 ///
129 /// The parameter info for each parameter is appended to the Parameters.
130 std::string
131 getOptionalParameters(const CodeCompletionString &CCS,
132  std::vector<ParameterInformation> &Parameters) {
133  std::string Result;
134  for (const auto &Chunk : CCS) {
135  switch (Chunk.Kind) {
136  case CodeCompletionString::CK_Optional:
137  assert(Chunk.Optional &&
138  "Expected the optional code completion string to be non-null.");
139  Result += getOptionalParameters(*Chunk.Optional, Parameters);
140  break;
141  case CodeCompletionString::CK_VerticalSpace:
142  break;
143  case CodeCompletionString::CK_Placeholder:
144  // A string that acts as a placeholder for, e.g., a function call
145  // argument.
146  // Intentional fallthrough here.
147  case CodeCompletionString::CK_CurrentParameter: {
148  // A piece of text that describes the parameter that corresponds to
149  // the code-completion location within a function call, message send,
150  // macro invocation, etc.
151  Result += Chunk.Text;
152  ParameterInformation Info;
153  Info.label = Chunk.Text;
154  Parameters.push_back(std::move(Info));
155  break;
156  }
157  default:
158  Result += Chunk.Text;
159  break;
160  }
161  }
162  return Result;
163 }
164 
165 /// Creates a `HeaderFile` from \p Header which can be either a URI or a literal
166 /// include.
167 static llvm::Expected<HeaderFile> toHeaderFile(StringRef Header,
168  llvm::StringRef HintPath) {
169  if (isLiteralInclude(Header))
170  return HeaderFile{Header.str(), /*Verbatim=*/true};
171  auto U = URI::parse(Header);
172  if (!U)
173  return U.takeError();
174 
175  auto IncludePath = URI::includeSpelling(*U);
176  if (!IncludePath)
177  return IncludePath.takeError();
178  if (!IncludePath->empty())
179  return HeaderFile{std::move(*IncludePath), /*Verbatim=*/true};
180 
181  auto Resolved = URI::resolve(*U, HintPath);
182  if (!Resolved)
183  return Resolved.takeError();
184  return HeaderFile{std::move(*Resolved), /*Verbatim=*/false};
185 }
186 
187 /// A code completion result, in clang-native form.
188 /// It may be promoted to a CompletionItem if it's among the top-ranked results.
189 struct CompletionCandidate {
190  llvm::StringRef Name; // Used for filtering and sorting.
191  // We may have a result from Sema, from the index, or both.
192  const CodeCompletionResult *SemaResult = nullptr;
193  const Symbol *IndexResult = nullptr;
194 
195  // Returns a token identifying the overload set this is part of.
196  // 0 indicates it's not part of any overload set.
197  size_t overloadSet() const {
198  SmallString<256> Scratch;
199  if (IndexResult) {
200  switch (IndexResult->SymInfo.Kind) {
201  case index::SymbolKind::ClassMethod:
202  case index::SymbolKind::InstanceMethod:
203  case index::SymbolKind::StaticMethod:
204  assert(false && "Don't expect members from index in code completion");
205  // fall through
206  case index::SymbolKind::Function:
207  // We can't group overloads together that need different #includes.
208  // This could break #include insertion.
209  return hash_combine(
210  (IndexResult->Scope + IndexResult->Name).toStringRef(Scratch),
211  headerToInsertIfNotPresent().getValueOr(""));
212  default:
213  return 0;
214  }
215  }
216  assert(SemaResult);
217  // We need to make sure we're consistent with the IndexResult case!
218  const NamedDecl *D = SemaResult->Declaration;
219  if (!D || !D->isFunctionOrFunctionTemplate())
220  return 0;
221  {
222  llvm::raw_svector_ostream OS(Scratch);
223  D->printQualifiedName(OS);
224  }
225  return hash_combine(Scratch, headerToInsertIfNotPresent().getValueOr(""));
226  }
227 
228  llvm::Optional<llvm::StringRef> headerToInsertIfNotPresent() const {
229  if (!IndexResult || !IndexResult->Detail ||
230  IndexResult->Detail->IncludeHeader.empty())
231  return llvm::None;
232  if (SemaResult && SemaResult->Declaration) {
233  // Avoid inserting new #include if the declaration is found in the current
234  // file e.g. the symbol is forward declared.
235  auto &SM = SemaResult->Declaration->getASTContext().getSourceManager();
236  for (const Decl *RD : SemaResult->Declaration->redecls())
237  if (SM.isInMainFile(SM.getExpansionLoc(RD->getLocStart())))
238  return llvm::None;
239  }
240  return IndexResult->Detail->IncludeHeader;
241  }
242 
243  using Bundle = llvm::SmallVector<CompletionCandidate, 4>;
244 };
245 using ScoredBundle =
246  std::pair<CompletionCandidate::Bundle, CodeCompletion::Scores>;
247 struct ScoredBundleGreater {
248  bool operator()(const ScoredBundle &L, const ScoredBundle &R) {
249  if (L.second.Total != R.second.Total)
250  return L.second.Total > R.second.Total;
251  return L.first.front().Name <
252  R.first.front().Name; // Earlier name is better.
253  }
254 };
255 
256 // Assembles a code completion out of a bundle of >=1 completion candidates.
257 // Many of the expensive strings are only computed at this point, once we know
258 // the candidate bundle is going to be returned.
259 //
260 // Many fields are the same for all candidates in a bundle (e.g. name), and are
261 // computed from the first candidate, in the constructor.
262 // Others vary per candidate, so add() must be called for remaining candidates.
263 struct CodeCompletionBuilder {
264  CodeCompletionBuilder(ASTContext &ASTCtx, const CompletionCandidate &C,
265  CodeCompletionString *SemaCCS,
266  const IncludeInserter &Includes, StringRef FileName,
267  const CodeCompleteOptions &Opts)
268  : ASTCtx(ASTCtx), ExtractDocumentation(Opts.IncludeComments) {
269  add(C, SemaCCS);
270  if (C.SemaResult) {
271  Completion.Origin |= SymbolOrigin::AST;
272  Completion.Name = llvm::StringRef(SemaCCS->getTypedText());
273  if (Completion.Scope.empty()) {
274  if ((C.SemaResult->Kind == CodeCompletionResult::RK_Declaration) ||
275  (C.SemaResult->Kind == CodeCompletionResult::RK_Pattern))
276  if (const auto *D = C.SemaResult->getDeclaration())
277  if (const auto *ND = llvm::dyn_cast<NamedDecl>(D))
278  Completion.Scope =
280  }
281  Completion.Kind =
282  toCompletionItemKind(C.SemaResult->Kind, C.SemaResult->Declaration);
283  }
284  if (C.IndexResult) {
285  Completion.Origin |= C.IndexResult->Origin;
286  if (Completion.Scope.empty())
287  Completion.Scope = C.IndexResult->Scope;
288  if (Completion.Kind == CompletionItemKind::Missing)
289  Completion.Kind = toCompletionItemKind(C.IndexResult->SymInfo.Kind);
290  if (Completion.Name.empty())
291  Completion.Name = C.IndexResult->Name;
292  }
293  if (auto Inserted = C.headerToInsertIfNotPresent()) {
294  // Turn absolute path into a literal string that can be #included.
295  auto Include = [&]() -> Expected<std::pair<std::string, bool>> {
296  auto ResolvedDeclaring =
297  toHeaderFile(C.IndexResult->CanonicalDeclaration.FileURI, FileName);
298  if (!ResolvedDeclaring)
299  return ResolvedDeclaring.takeError();
300  auto ResolvedInserted = toHeaderFile(*Inserted, FileName);
301  if (!ResolvedInserted)
302  return ResolvedInserted.takeError();
303  return std::make_pair(Includes.calculateIncludePath(*ResolvedDeclaring,
304  *ResolvedInserted),
305  Includes.shouldInsertInclude(*ResolvedDeclaring,
306  *ResolvedInserted));
307  }();
308  if (Include) {
309  Completion.Header = Include->first;
310  if (Include->second)
311  Completion.HeaderInsertion = Includes.insert(Include->first);
312  } else
313  log("Failed to generate include insertion edits for adding header "
314  "(FileURI='{0}', IncludeHeader='{1}') into {2}",
315  C.IndexResult->CanonicalDeclaration.FileURI,
316  C.IndexResult->Detail->IncludeHeader, FileName);
317  }
318  }
319 
320  void add(const CompletionCandidate &C, CodeCompletionString *SemaCCS) {
321  assert(bool(C.SemaResult) == bool(SemaCCS));
322  Bundled.emplace_back();
323  BundledEntry &S = Bundled.back();
324  if (C.SemaResult) {
325  getSignature(*SemaCCS, &S.Signature, &S.SnippetSuffix,
326  &Completion.RequiredQualifier);
327  S.ReturnType = getReturnType(*SemaCCS);
328  } else if (C.IndexResult) {
329  S.Signature = C.IndexResult->Signature;
330  S.SnippetSuffix = C.IndexResult->CompletionSnippetSuffix;
331  if (auto *D = C.IndexResult->Detail)
332  S.ReturnType = D->ReturnType;
333  }
334  if (ExtractDocumentation && Completion.Documentation.empty()) {
335  if (C.IndexResult && C.IndexResult->Detail)
336  Completion.Documentation = C.IndexResult->Detail->Documentation;
337  else if (C.SemaResult)
338  Completion.Documentation = getDocComment(ASTCtx, *C.SemaResult,
339  /*CommentsFromHeader=*/false);
340  }
341  }
342 
343  CodeCompletion build() {
344  Completion.ReturnType = summarizeReturnType();
345  Completion.Signature = summarizeSignature();
346  Completion.SnippetSuffix = summarizeSnippet();
347  Completion.BundleSize = Bundled.size();
348  return std::move(Completion);
349  }
350 
351 private:
352  struct BundledEntry {
353  std::string SnippetSuffix;
354  std::string Signature;
355  std::string ReturnType;
356  };
357 
358  // If all BundledEntrys have the same value for a property, return it.
359  template <std::string BundledEntry::*Member>
360  const std::string *onlyValue() const {
361  auto B = Bundled.begin(), E = Bundled.end();
362  for (auto I = B + 1; I != E; ++I)
363  if (I->*Member != B->*Member)
364  return nullptr;
365  return &(B->*Member);
366  }
367 
368  std::string summarizeReturnType() const {
369  if (auto *RT = onlyValue<&BundledEntry::ReturnType>())
370  return *RT;
371  return "";
372  }
373 
374  std::string summarizeSnippet() const {
375  if (auto *Snippet = onlyValue<&BundledEntry::SnippetSuffix>())
376  return *Snippet;
377  // All bundles are function calls.
378  return "(${0})";
379  }
380 
381  std::string summarizeSignature() const {
382  if (auto *Signature = onlyValue<&BundledEntry::Signature>())
383  return *Signature;
384  // All bundles are function calls.
385  return "(…)";
386  }
387 
388  ASTContext &ASTCtx;
389  CodeCompletion Completion;
390  SmallVector<BundledEntry, 1> Bundled;
391  bool ExtractDocumentation;
392 };
393 
394 // Determine the symbol ID for a Sema code completion result, if possible.
395 llvm::Optional<SymbolID> getSymbolID(const CodeCompletionResult &R) {
396  switch (R.Kind) {
397  case CodeCompletionResult::RK_Declaration:
398  case CodeCompletionResult::RK_Pattern: {
399  llvm::SmallString<128> USR;
400  if (/*Ignore=*/clang::index::generateUSRForDecl(R.Declaration, USR))
401  return None;
402  return SymbolID(USR);
403  }
404  case CodeCompletionResult::RK_Macro:
405  // FIXME: Macros do have USRs, but the CCR doesn't contain enough info.
406  case CodeCompletionResult::RK_Keyword:
407  return None;
408  }
409  llvm_unreachable("unknown CodeCompletionResult kind");
410 }
411 
412 // Scopes of the paritial identifier we're trying to complete.
413 // It is used when we query the index for more completion results.
414 struct SpecifiedScope {
415  // The scopes we should look in, determined by Sema.
416  //
417  // If the qualifier was fully resolved, we look for completions in these
418  // scopes; if there is an unresolved part of the qualifier, it should be
419  // resolved within these scopes.
420  //
421  // Examples of qualified completion:
422  //
423  // "::vec" => {""}
424  // "using namespace std; ::vec^" => {"", "std::"}
425  // "namespace ns {using namespace std;} ns::^" => {"ns::", "std::"}
426  // "std::vec^" => {""} // "std" unresolved
427  //
428  // Examples of unqualified completion:
429  //
430  // "vec^" => {""}
431  // "using namespace std; vec^" => {"", "std::"}
432  // "using namespace std; namespace ns { vec^ }" => {"ns::", "std::", ""}
433  //
434  // "" for global namespace, "ns::" for normal namespace.
435  std::vector<std::string> AccessibleScopes;
436  // The full scope qualifier as typed by the user (without the leading "::").
437  // Set if the qualifier is not fully resolved by Sema.
438  llvm::Optional<std::string> UnresolvedQualifier;
439 
440  // Construct scopes being queried in indexes.
441  // This method format the scopes to match the index request representation.
442  std::vector<std::string> scopesForIndexQuery() {
443  std::vector<std::string> Results;
444  for (llvm::StringRef AS : AccessibleScopes) {
445  Results.push_back(AS);
446  if (UnresolvedQualifier)
447  Results.back() += *UnresolvedQualifier;
448  }
449  return Results;
450  }
451 };
452 
453 // Get all scopes that will be queried in indexes.
454 std::vector<std::string> getQueryScopes(CodeCompletionContext &CCContext,
455  const SourceManager &SM) {
456  auto GetAllAccessibleScopes = [](CodeCompletionContext &CCContext) {
457  SpecifiedScope Info;
458  for (auto *Context : CCContext.getVisitedContexts()) {
459  if (isa<TranslationUnitDecl>(Context))
460  Info.AccessibleScopes.push_back(""); // global namespace
461  else if (const auto *NS = dyn_cast<NamespaceDecl>(Context))
462  Info.AccessibleScopes.push_back(NS->getQualifiedNameAsString() + "::");
463  }
464  return Info;
465  };
466 
467  auto SS = CCContext.getCXXScopeSpecifier();
468 
469  // Unqualified completion (e.g. "vec^").
470  if (!SS) {
471  // FIXME: Once we can insert namespace qualifiers and use the in-scope
472  // namespaces for scoring, search in all namespaces.
473  // FIXME: Capture scopes and use for scoring, for example,
474  // "using namespace std; namespace foo {v^}" =>
475  // foo::value > std::vector > boost::variant
476  return GetAllAccessibleScopes(CCContext).scopesForIndexQuery();
477  }
478 
479  // Qualified completion ("std::vec^"), we have two cases depending on whether
480  // the qualifier can be resolved by Sema.
481  if ((*SS)->isValid()) { // Resolved qualifier.
482  return GetAllAccessibleScopes(CCContext).scopesForIndexQuery();
483  }
484 
485  // Unresolved qualifier.
486  // FIXME: When Sema can resolve part of a scope chain (e.g.
487  // "known::unknown::id"), we should expand the known part ("known::") rather
488  // than treating the whole thing as unknown.
489  SpecifiedScope Info;
490  Info.AccessibleScopes.push_back(""); // global namespace
491 
492  Info.UnresolvedQualifier =
493  Lexer::getSourceText(CharSourceRange::getCharRange((*SS)->getRange()), SM,
494  clang::LangOptions())
495  .ltrim("::");
496  // Sema excludes the trailing "::".
497  if (!Info.UnresolvedQualifier->empty())
498  *Info.UnresolvedQualifier += "::";
499 
500  return Info.scopesForIndexQuery();
501 }
502 
503 // Should we perform index-based completion in a context of the specified kind?
504 // FIXME: consider allowing completion, but restricting the result types.
505 bool contextAllowsIndex(enum CodeCompletionContext::Kind K) {
506  switch (K) {
507  case CodeCompletionContext::CCC_TopLevel:
508  case CodeCompletionContext::CCC_ObjCInterface:
509  case CodeCompletionContext::CCC_ObjCImplementation:
510  case CodeCompletionContext::CCC_ObjCIvarList:
511  case CodeCompletionContext::CCC_ClassStructUnion:
512  case CodeCompletionContext::CCC_Statement:
513  case CodeCompletionContext::CCC_Expression:
514  case CodeCompletionContext::CCC_ObjCMessageReceiver:
515  case CodeCompletionContext::CCC_EnumTag:
516  case CodeCompletionContext::CCC_UnionTag:
517  case CodeCompletionContext::CCC_ClassOrStructTag:
518  case CodeCompletionContext::CCC_ObjCProtocolName:
519  case CodeCompletionContext::CCC_Namespace:
520  case CodeCompletionContext::CCC_Type:
521  case CodeCompletionContext::CCC_Name: // FIXME: why does ns::^ give this?
522  case CodeCompletionContext::CCC_PotentiallyQualifiedName:
523  case CodeCompletionContext::CCC_ParenthesizedExpression:
524  case CodeCompletionContext::CCC_ObjCInterfaceName:
525  case CodeCompletionContext::CCC_ObjCCategoryName:
526  return true;
527  case CodeCompletionContext::CCC_Other: // Be conservative.
528  case CodeCompletionContext::CCC_OtherWithMacros:
529  case CodeCompletionContext::CCC_DotMemberAccess:
530  case CodeCompletionContext::CCC_ArrowMemberAccess:
531  case CodeCompletionContext::CCC_ObjCPropertyAccess:
532  case CodeCompletionContext::CCC_MacroName:
533  case CodeCompletionContext::CCC_MacroNameUse:
534  case CodeCompletionContext::CCC_PreprocessorExpression:
535  case CodeCompletionContext::CCC_PreprocessorDirective:
536  case CodeCompletionContext::CCC_NaturalLanguage:
537  case CodeCompletionContext::CCC_SelectorName:
538  case CodeCompletionContext::CCC_TypeQualifiers:
539  case CodeCompletionContext::CCC_ObjCInstanceMessage:
540  case CodeCompletionContext::CCC_ObjCClassMessage:
541  case CodeCompletionContext::CCC_Recovery:
542  return false;
543  }
544  llvm_unreachable("unknown code completion context");
545 }
546 
547 // Some member calls are blacklisted because they're so rarely useful.
548 static bool isBlacklistedMember(const NamedDecl &D) {
549  // Destructor completion is rarely useful, and works inconsistently.
550  // (s.^ completes ~string, but s.~st^ is an error).
551  if (D.getKind() == Decl::CXXDestructor)
552  return true;
553  // Injected name may be useful for A::foo(), but who writes A::A::foo()?
554  if (auto *R = dyn_cast_or_null<RecordDecl>(&D))
555  if (R->isInjectedClassName())
556  return true;
557  // Explicit calls to operators are also rare.
558  auto NameKind = D.getDeclName().getNameKind();
559  if (NameKind == DeclarationName::CXXOperatorName ||
560  NameKind == DeclarationName::CXXLiteralOperatorName ||
561  NameKind == DeclarationName::CXXConversionFunctionName)
562  return true;
563  return false;
564 }
565 
566 // The CompletionRecorder captures Sema code-complete output, including context.
567 // It filters out ignored results (but doesn't apply fuzzy-filtering yet).
568 // It doesn't do scoring or conversion to CompletionItem yet, as we want to
569 // merge with index results first.
570 // Generally the fields and methods of this object should only be used from
571 // within the callback.
572 struct CompletionRecorder : public CodeCompleteConsumer {
573  CompletionRecorder(const CodeCompleteOptions &Opts,
574  llvm::unique_function<void()> ResultsCallback)
575  : CodeCompleteConsumer(Opts.getClangCompleteOpts(),
576  /*OutputIsBinary=*/false),
577  CCContext(CodeCompletionContext::CCC_Other), Opts(Opts),
578  CCAllocator(std::make_shared<GlobalCodeCompletionAllocator>()),
579  CCTUInfo(CCAllocator), ResultsCallback(std::move(ResultsCallback)) {
580  assert(this->ResultsCallback);
581  }
582 
583  std::vector<CodeCompletionResult> Results;
584  CodeCompletionContext CCContext;
585  Sema *CCSema = nullptr; // Sema that created the results.
586  // FIXME: Sema is scary. Can we store ASTContext and Preprocessor, instead?
587 
588  void ProcessCodeCompleteResults(class Sema &S, CodeCompletionContext Context,
589  CodeCompletionResult *InResults,
590  unsigned NumResults) override final {
591  // Results from recovery mode are generally useless, and the callback after
592  // recovery (if any) is usually more interesting. To make sure we handle the
593  // future callback from sema, we just ignore all callbacks in recovery mode,
594  // as taking only results from recovery mode results in poor completion
595  // results.
596  // FIXME: in case there is no future sema completion callback after the
597  // recovery mode, we might still want to provide some results (e.g. trivial
598  // identifier-based completion).
599  if (Context.getKind() == CodeCompletionContext::CCC_Recovery) {
600  log("Code complete: Ignoring sema code complete callback with Recovery "
601  "context.");
602  return;
603  }
604  // If a callback is called without any sema result and the context does not
605  // support index-based completion, we simply skip it to give way to
606  // potential future callbacks with results.
607  if (NumResults == 0 && !contextAllowsIndex(Context.getKind()))
608  return;
609  if (CCSema) {
610  log("Multiple code complete callbacks (parser backtracked?). "
611  "Dropping results from context {0}, keeping results from {1}.",
612  getCompletionKindString(Context.getKind()),
613  getCompletionKindString(this->CCContext.getKind()));
614  return;
615  }
616  // Record the completion context.
617  CCSema = &S;
618  CCContext = Context;
619 
620  // Retain the results we might want.
621  for (unsigned I = 0; I < NumResults; ++I) {
622  auto &Result = InResults[I];
623  // Drop hidden items which cannot be found by lookup after completion.
624  // Exception: some items can be named by using a qualifier.
625  if (Result.Hidden && (!Result.Qualifier || Result.QualifierIsInformative))
626  continue;
627  if (!Opts.IncludeIneligibleResults &&
628  (Result.Availability == CXAvailability_NotAvailable ||
629  Result.Availability == CXAvailability_NotAccessible))
630  continue;
631  if (Result.Declaration &&
632  !Context.getBaseType().isNull() // is this a member-access context?
633  && isBlacklistedMember(*Result.Declaration))
634  continue;
635  // We choose to never append '::' to completion results in clangd.
636  Result.StartsNestedNameSpecifier = false;
637  Results.push_back(Result);
638  }
639  ResultsCallback();
640  }
641 
642  CodeCompletionAllocator &getAllocator() override { return *CCAllocator; }
643  CodeCompletionTUInfo &getCodeCompletionTUInfo() override { return CCTUInfo; }
644 
645  // Returns the filtering/sorting name for Result, which must be from Results.
646  // Returned string is owned by this recorder (or the AST).
647  llvm::StringRef getName(const CodeCompletionResult &Result) {
648  switch (Result.Kind) {
649  case CodeCompletionResult::RK_Declaration:
650  if (auto *ID = Result.Declaration->getIdentifier())
651  return ID->getName();
652  break;
653  case CodeCompletionResult::RK_Keyword:
654  return Result.Keyword;
655  case CodeCompletionResult::RK_Macro:
656  return Result.Macro->getName();
657  case CodeCompletionResult::RK_Pattern:
658  return Result.Pattern->getTypedText();
659  }
660  auto *CCS = codeCompletionString(Result);
661  return CCS->getTypedText();
662  }
663 
664  // Build a CodeCompletion string for R, which must be from Results.
665  // The CCS will be owned by this recorder.
666  CodeCompletionString *codeCompletionString(const CodeCompletionResult &R) {
667  // CodeCompletionResult doesn't seem to be const-correct. We own it, anyway.
668  return const_cast<CodeCompletionResult &>(R).CreateCodeCompletionString(
669  *CCSema, CCContext, *CCAllocator, CCTUInfo,
670  /*IncludeBriefComments=*/false);
671  }
672 
673 private:
674  CodeCompleteOptions Opts;
675  std::shared_ptr<GlobalCodeCompletionAllocator> CCAllocator;
676  CodeCompletionTUInfo CCTUInfo;
677  llvm::unique_function<void()> ResultsCallback;
678 };
679 
680 class SignatureHelpCollector final : public CodeCompleteConsumer {
681 
682 public:
683  SignatureHelpCollector(const clang::CodeCompleteOptions &CodeCompleteOpts,
684  SignatureHelp &SigHelp)
685  : CodeCompleteConsumer(CodeCompleteOpts, /*OutputIsBinary=*/false),
686  SigHelp(SigHelp),
687  Allocator(std::make_shared<clang::GlobalCodeCompletionAllocator>()),
688  CCTUInfo(Allocator) {}
689 
690  void ProcessOverloadCandidates(Sema &S, unsigned CurrentArg,
691  OverloadCandidate *Candidates,
692  unsigned NumCandidates) override {
693  SigHelp.signatures.reserve(NumCandidates);
694  // FIXME(rwols): How can we determine the "active overload candidate"?
695  // Right now the overloaded candidates seem to be provided in a "best fit"
696  // order, so I'm not too worried about this.
697  SigHelp.activeSignature = 0;
698  assert(CurrentArg <= (unsigned)std::numeric_limits<int>::max() &&
699  "too many arguments");
700  SigHelp.activeParameter = static_cast<int>(CurrentArg);
701  for (unsigned I = 0; I < NumCandidates; ++I) {
702  const auto &Candidate = Candidates[I];
703  const auto *CCS = Candidate.CreateSignatureString(
704  CurrentArg, S, *Allocator, CCTUInfo, true);
705  assert(CCS && "Expected the CodeCompletionString to be non-null");
706  // FIXME: for headers, we need to get a comment from the index.
707  SigHelp.signatures.push_back(processOverloadCandidate(
708  Candidate, *CCS,
709  getParameterDocComment(S.getASTContext(), Candidate, CurrentArg,
710  /*CommentsFromHeaders=*/false)));
711  }
712  }
713 
714  GlobalCodeCompletionAllocator &getAllocator() override { return *Allocator; }
715 
716  CodeCompletionTUInfo &getCodeCompletionTUInfo() override { return CCTUInfo; }
717 
718 private:
719  // FIXME(ioeric): consider moving CodeCompletionString logic here to
720  // CompletionString.h.
721  SignatureInformation
722  processOverloadCandidate(const OverloadCandidate &Candidate,
723  const CodeCompletionString &CCS,
724  llvm::StringRef DocComment) const {
725  SignatureInformation Result;
726  const char *ReturnType = nullptr;
727 
728  Result.documentation = formatDocumentation(CCS, DocComment);
729 
730  for (const auto &Chunk : CCS) {
731  switch (Chunk.Kind) {
732  case CodeCompletionString::CK_ResultType:
733  // A piece of text that describes the type of an entity or,
734  // for functions and methods, the return type.
735  assert(!ReturnType && "Unexpected CK_ResultType");
736  ReturnType = Chunk.Text;
737  break;
738  case CodeCompletionString::CK_Placeholder:
739  // A string that acts as a placeholder for, e.g., a function call
740  // argument.
741  // Intentional fallthrough here.
742  case CodeCompletionString::CK_CurrentParameter: {
743  // A piece of text that describes the parameter that corresponds to
744  // the code-completion location within a function call, message send,
745  // macro invocation, etc.
746  Result.label += Chunk.Text;
747  ParameterInformation Info;
748  Info.label = Chunk.Text;
749  Result.parameters.push_back(std::move(Info));
750  break;
751  }
752  case CodeCompletionString::CK_Optional: {
753  // The rest of the parameters are defaulted/optional.
754  assert(Chunk.Optional &&
755  "Expected the optional code completion string to be non-null.");
756  Result.label +=
757  getOptionalParameters(*Chunk.Optional, Result.parameters);
758  break;
759  }
760  case CodeCompletionString::CK_VerticalSpace:
761  break;
762  default:
763  Result.label += Chunk.Text;
764  break;
765  }
766  }
767  if (ReturnType) {
768  Result.label += " -> ";
769  Result.label += ReturnType;
770  }
771  return Result;
772  }
773 
774  SignatureHelp &SigHelp;
775  std::shared_ptr<clang::GlobalCodeCompletionAllocator> Allocator;
776  CodeCompletionTUInfo CCTUInfo;
777 
778 }; // SignatureHelpCollector
779 
780 struct SemaCompleteInput {
782  const tooling::CompileCommand &Command;
783  PrecompiledPreamble const *Preamble;
784  StringRef Contents;
785  Position Pos;
786  IntrusiveRefCntPtr<vfs::FileSystem> VFS;
787  std::shared_ptr<PCHContainerOperations> PCHs;
788 };
789 
790 // Invokes Sema code completion on a file.
791 // If \p Includes is set, it will be updated based on the compiler invocation.
792 bool semaCodeComplete(std::unique_ptr<CodeCompleteConsumer> Consumer,
793  const clang::CodeCompleteOptions &Options,
794  const SemaCompleteInput &Input,
795  IncludeStructure *Includes = nullptr) {
796  trace::Span Tracer("Sema completion");
797  std::vector<const char *> ArgStrs;
798  for (const auto &S : Input.Command.CommandLine)
799  ArgStrs.push_back(S.c_str());
800 
801  if (Input.VFS->setCurrentWorkingDirectory(Input.Command.Directory)) {
802  log("Couldn't set working directory");
803  // We run parsing anyway, our lit-tests rely on results for non-existing
804  // working dirs.
805  }
806 
807  IgnoreDiagnostics DummyDiagsConsumer;
808  auto CI = createInvocationFromCommandLine(
809  ArgStrs,
810  CompilerInstance::createDiagnostics(new DiagnosticOptions,
811  &DummyDiagsConsumer, false),
812  Input.VFS);
813  if (!CI) {
814  elog("Couldn't create CompilerInvocation");
815  return false;
816  }
817  auto &FrontendOpts = CI->getFrontendOpts();
818  FrontendOpts.DisableFree = false;
819  FrontendOpts.SkipFunctionBodies = true;
820  CI->getLangOpts()->CommentOpts.ParseAllComments = true;
821  // Disable typo correction in Sema.
822  CI->getLangOpts()->SpellChecking = false;
823  // Setup code completion.
824  FrontendOpts.CodeCompleteOpts = Options;
825  FrontendOpts.CodeCompletionAt.FileName = Input.FileName;
826  auto Offset = positionToOffset(Input.Contents, Input.Pos);
827  if (!Offset) {
828  elog("Code completion position was invalid {0}", Offset.takeError());
829  return false;
830  }
831  std::tie(FrontendOpts.CodeCompletionAt.Line,
832  FrontendOpts.CodeCompletionAt.Column) =
833  offsetToClangLineColumn(Input.Contents, *Offset);
834 
835  std::unique_ptr<llvm::MemoryBuffer> ContentsBuffer =
836  llvm::MemoryBuffer::getMemBufferCopy(Input.Contents, Input.FileName);
837  // The diagnostic options must be set before creating a CompilerInstance.
838  CI->getDiagnosticOpts().IgnoreWarnings = true;
839  // We reuse the preamble whether it's valid or not. This is a
840  // correctness/performance tradeoff: building without a preamble is slow, and
841  // completion is latency-sensitive.
842  // NOTE: we must call BeginSourceFile after prepareCompilerInstance. Otherwise
843  // the remapped buffers do not get freed.
844  auto Clang = prepareCompilerInstance(
845  std::move(CI), Input.Preamble, std::move(ContentsBuffer),
846  std::move(Input.PCHs), std::move(Input.VFS), DummyDiagsConsumer);
847  Clang->setCodeCompletionConsumer(Consumer.release());
848 
850  if (!Action.BeginSourceFile(*Clang, Clang->getFrontendOpts().Inputs[0])) {
851  log("BeginSourceFile() failed when running codeComplete for {0}",
852  Input.FileName);
853  return false;
854  }
855  if (Includes)
856  Clang->getPreprocessor().addPPCallbacks(
857  collectIncludeStructureCallback(Clang->getSourceManager(), Includes));
858  if (!Action.Execute()) {
859  log("Execute() failed when running codeComplete for {0}", Input.FileName);
860  return false;
861  }
862  Action.EndSourceFile();
863 
864  return true;
865 }
866 
867 // Should we allow index completions in the specified context?
868 bool allowIndex(CodeCompletionContext &CC) {
869  if (!contextAllowsIndex(CC.getKind()))
870  return false;
871  // We also avoid ClassName::bar (but allow namespace::bar).
872  auto Scope = CC.getCXXScopeSpecifier();
873  if (!Scope)
874  return true;
875  NestedNameSpecifier *NameSpec = (*Scope)->getScopeRep();
876  if (!NameSpec)
877  return true;
878  // We only query the index when qualifier is a namespace.
879  // If it's a class, we rely solely on sema completions.
880  switch (NameSpec->getKind()) {
881  case NestedNameSpecifier::Global:
883  case NestedNameSpecifier::NamespaceAlias:
884  return true;
885  case NestedNameSpecifier::Super:
886  case NestedNameSpecifier::TypeSpec:
887  case NestedNameSpecifier::TypeSpecWithTemplate:
888  // Unresolved inside a template.
889  case NestedNameSpecifier::Identifier:
890  return false;
891  }
892  llvm_unreachable("invalid NestedNameSpecifier kind");
893 }
894 
895 } // namespace
896 
897 clang::CodeCompleteOptions CodeCompleteOptions::getClangCompleteOpts() const {
898  clang::CodeCompleteOptions Result;
899  Result.IncludeCodePatterns = EnableSnippets && IncludeCodePatterns;
900  Result.IncludeMacros = IncludeMacros;
901  Result.IncludeGlobals = true;
902  // We choose to include full comments and not do doxygen parsing in
903  // completion.
904  // FIXME: ideally, we should support doxygen in some form, e.g. do markdown
905  // formatting of the comments.
906  Result.IncludeBriefComments = false;
907 
908  // When an is used, Sema is responsible for completing the main file,
909  // the index can provide results from the preamble.
910  // Tell Sema not to deserialize the preamble to look for results.
911  Result.LoadExternal = !Index;
912 
913  return Result;
914 }
915 
916 // Runs Sema-based (AST) and Index-based completion, returns merged results.
917 //
918 // There are a few tricky considerations:
919 // - the AST provides information needed for the index query (e.g. which
920 // namespaces to search in). So Sema must start first.
921 // - we only want to return the top results (Opts.Limit).
922 // Building CompletionItems for everything else is wasteful, so we want to
923 // preserve the "native" format until we're done with scoring.
924 // - the data underlying Sema completion items is owned by the AST and various
925 // other arenas, which must stay alive for us to build CompletionItems.
926 // - we may get duplicate results from Sema and the Index, we need to merge.
927 //
928 // So we start Sema completion first, and do all our work in its callback.
929 // We use the Sema context information to query the index.
930 // Then we merge the two result sets, producing items that are Sema/Index/Both.
931 // These items are scored, and the top N are synthesized into the LSP response.
932 // Finally, we can clean up the data structures created by Sema completion.
933 //
934 // Main collaborators are:
935 // - semaCodeComplete sets up the compiler machinery to run code completion.
936 // - CompletionRecorder captures Sema completion results, including context.
937 // - SymbolIndex (Opts.Index) provides index completion results as Symbols
938 // - CompletionCandidates are the result of merging Sema and Index results.
939 // Each candidate points to an underlying CodeCompletionResult (Sema), a
940 // Symbol (Index), or both. It computes the result quality score.
941 // CompletionCandidate also does conversion to CompletionItem (at the end).
942 // - FuzzyMatcher scores how the candidate matches the partial identifier.
943 // This score is combined with the result quality score for the final score.
944 // - TopN determines the results with the best score.
947  IncludeStructure Includes; // Complete once the compiler runs.
948  const CodeCompleteOptions &Opts;
949  // Sema takes ownership of Recorder. Recorder is valid until Sema cleanup.
950  CompletionRecorder *Recorder = nullptr;
951  int NSema = 0, NIndex = 0, NBoth = 0; // Counters for logging.
952  bool Incomplete = false; // Would more be available with a higher limit?
953  llvm::Optional<FuzzyMatcher> Filter; // Initialized once Sema runs.
954  std::vector<std::string> QueryScopes; // Initialized once Sema runs.
955  // Include-insertion and proximity scoring rely on the include structure.
956  // This is available after Sema has run.
957  llvm::Optional<IncludeInserter> Inserter; // Available during runWithSema.
958  llvm::Optional<URIDistance> FileProximity; // Initialized once Sema runs.
959 
960 public:
961  // A CodeCompleteFlow object is only useful for calling run() exactly once.
962  CodeCompleteFlow(PathRef FileName, const IncludeStructure &Includes,
963  const CodeCompleteOptions &Opts)
964  : FileName(FileName), Includes(Includes), Opts(Opts) {}
965 
966  CodeCompleteResult run(const SemaCompleteInput &SemaCCInput) && {
967  trace::Span Tracer("CodeCompleteFlow");
968 
969  // We run Sema code completion first. It builds an AST and calculates:
970  // - completion results based on the AST.
971  // - partial identifier and context. We need these for the index query.
972  CodeCompleteResult Output;
973  auto RecorderOwner = llvm::make_unique<CompletionRecorder>(Opts, [&]() {
974  assert(Recorder && "Recorder is not set");
975  auto Style =
976  format::getStyle(format::DefaultFormatStyle, SemaCCInput.FileName,
977  format::DefaultFallbackStyle, SemaCCInput.Contents,
978  SemaCCInput.VFS.get());
979  if (!Style) {
980  log("getStyle() failed for file {0}: {1}. Fallback is LLVM style.",
981  SemaCCInput.FileName, Style.takeError());
982  Style = format::getLLVMStyle();
983  }
984  // If preprocessor was run, inclusions from preprocessor callback should
985  // already be added to Includes.
986  Inserter.emplace(
987  SemaCCInput.FileName, SemaCCInput.Contents, *Style,
988  SemaCCInput.Command.Directory,
989  Recorder->CCSema->getPreprocessor().getHeaderSearchInfo());
990  for (const auto &Inc : Includes.MainFileIncludes)
991  Inserter->addExisting(Inc);
992 
993  // Most of the cost of file proximity is in initializing the FileDistance
994  // structures based on the observed includes, once per query. Conceptually
995  // that happens here (though the per-URI-scheme initialization is lazy).
996  // The per-result proximity scoring is (amortized) very cheap.
997  FileDistanceOptions ProxOpts{}; // Use defaults.
998  const auto &SM = Recorder->CCSema->getSourceManager();
999  llvm::StringMap<SourceParams> ProxSources;
1000  for (auto &Entry : Includes.includeDepth(
1001  SM.getFileEntryForID(SM.getMainFileID())->getName())) {
1002  auto &Source = ProxSources[Entry.getKey()];
1003  Source.Cost = Entry.getValue() * ProxOpts.IncludeCost;
1004  // Symbols near our transitive includes are good, but only consider
1005  // things in the same directory or below it. Otherwise there can be
1006  // many false positives.
1007  if (Entry.getValue() > 0)
1008  Source.MaxUpTraversals = 1;
1009  }
1010  FileProximity.emplace(ProxSources, ProxOpts);
1011 
1012  Output = runWithSema();
1013  Inserter.reset(); // Make sure this doesn't out-live Clang.
1014  SPAN_ATTACH(Tracer, "sema_completion_kind",
1015  getCompletionKindString(Recorder->CCContext.getKind()));
1016  log("Code complete: sema context {0}, query scopes [{1}]",
1017  getCompletionKindString(Recorder->CCContext.getKind()),
1018  llvm::join(QueryScopes.begin(), QueryScopes.end(), ","));
1019  });
1020 
1021  Recorder = RecorderOwner.get();
1022  semaCodeComplete(std::move(RecorderOwner), Opts.getClangCompleteOpts(),
1023  SemaCCInput, &Includes);
1024 
1025  SPAN_ATTACH(Tracer, "sema_results", NSema);
1026  SPAN_ATTACH(Tracer, "index_results", NIndex);
1027  SPAN_ATTACH(Tracer, "merged_results", NBoth);
1028  SPAN_ATTACH(Tracer, "returned_results", int64_t(Output.Completions.size()));
1029  SPAN_ATTACH(Tracer, "incomplete", Output.HasMore);
1030  log("Code complete: {0} results from Sema, {1} from Index, "
1031  "{2} matched, {3} returned{4}.",
1032  NSema, NIndex, NBoth, Output.Completions.size(),
1033  Output.HasMore ? " (incomplete)" : "");
1034  assert(!Opts.Limit || Output.Completions.size() <= Opts.Limit);
1035  // We don't assert that isIncomplete means we hit a limit.
1036  // Indexes may choose to impose their own limits even if we don't have one.
1037  return Output;
1038  }
1039 
1040 private:
1041  // This is called by run() once Sema code completion is done, but before the
1042  // Sema data structures are torn down. It does all the real work.
1043  CodeCompleteResult runWithSema() {
1044  Filter = FuzzyMatcher(
1045  Recorder->CCSema->getPreprocessor().getCodeCompletionFilter());
1046  QueryScopes = getQueryScopes(Recorder->CCContext,
1047  Recorder->CCSema->getSourceManager());
1048  // Sema provides the needed context to query the index.
1049  // FIXME: in addition to querying for extra/overlapping symbols, we should
1050  // explicitly request symbols corresponding to Sema results.
1051  // We can use their signals even if the index can't suggest them.
1052  // We must copy index results to preserve them, but there are at most Limit.
1053  auto IndexResults = (Opts.Index && allowIndex(Recorder->CCContext))
1054  ? queryIndex()
1055  : SymbolSlab();
1056  // Merge Sema and Index results, score them, and pick the winners.
1057  auto Top = mergeResults(Recorder->Results, IndexResults);
1058  // Convert the results to final form, assembling the expensive strings.
1059  CodeCompleteResult Output;
1060  for (auto &C : Top) {
1061  Output.Completions.push_back(toCodeCompletion(C.first));
1062  Output.Completions.back().Score = C.second;
1063  }
1064  Output.HasMore = Incomplete;
1065  Output.Context = Recorder->CCContext.getKind();
1066  return Output;
1067  }
1068 
1069  SymbolSlab queryIndex() {
1070  trace::Span Tracer("Query index");
1071  SPAN_ATTACH(Tracer, "limit", int64_t(Opts.Limit));
1072 
1073  SymbolSlab::Builder ResultsBuilder;
1074  // Build the query.
1075  FuzzyFindRequest Req;
1076  if (Opts.Limit)
1077  Req.MaxCandidateCount = Opts.Limit;
1078  Req.Query = Filter->pattern();
1079  Req.RestrictForCodeCompletion = true;
1080  Req.Scopes = QueryScopes;
1081  // FIXME: we should send multiple weighted paths here.
1082  Req.ProximityPaths.push_back(FileName);
1083  vlog("Code complete: fuzzyFind(\"{0}\", scopes=[{1}])", Req.Query,
1084  llvm::join(Req.Scopes.begin(), Req.Scopes.end(), ","));
1085  // Run the query against the index.
1086  if (Opts.Index->fuzzyFind(
1087  Req, [&](const Symbol &Sym) { ResultsBuilder.insert(Sym); }))
1088  Incomplete = true;
1089  return std::move(ResultsBuilder).build();
1090  }
1091 
1092  // Merges Sema and Index results where possible, to form CompletionCandidates.
1093  // Groups overloads if desired, to form CompletionCandidate::Bundles.
1094  // The bundles are scored and top results are returned, best to worst.
1095  std::vector<ScoredBundle>
1096  mergeResults(const std::vector<CodeCompletionResult> &SemaResults,
1097  const SymbolSlab &IndexResults) {
1098  trace::Span Tracer("Merge and score results");
1099  std::vector<CompletionCandidate::Bundle> Bundles;
1100  llvm::DenseMap<size_t, size_t> BundleLookup;
1101  auto AddToBundles = [&](const CodeCompletionResult *SemaResult,
1102  const Symbol *IndexResult) {
1103  CompletionCandidate C;
1104  C.SemaResult = SemaResult;
1105  C.IndexResult = IndexResult;
1106  C.Name = IndexResult ? IndexResult->Name : Recorder->getName(*SemaResult);
1107  if (auto OverloadSet = Opts.BundleOverloads ? C.overloadSet() : 0) {
1108  auto Ret = BundleLookup.try_emplace(OverloadSet, Bundles.size());
1109  if (Ret.second)
1110  Bundles.emplace_back();
1111  Bundles[Ret.first->second].push_back(std::move(C));
1112  } else {
1113  Bundles.emplace_back();
1114  Bundles.back().push_back(std::move(C));
1115  }
1116  };
1117  llvm::DenseSet<const Symbol *> UsedIndexResults;
1118  auto CorrespondingIndexResult =
1119  [&](const CodeCompletionResult &SemaResult) -> const Symbol * {
1120  if (auto SymID = getSymbolID(SemaResult)) {
1121  auto I = IndexResults.find(*SymID);
1122  if (I != IndexResults.end()) {
1123  UsedIndexResults.insert(&*I);
1124  return &*I;
1125  }
1126  }
1127  return nullptr;
1128  };
1129  // Emit all Sema results, merging them with Index results if possible.
1130  for (auto &SemaResult : Recorder->Results)
1131  AddToBundles(&SemaResult, CorrespondingIndexResult(SemaResult));
1132  // Now emit any Index-only results.
1133  for (const auto &IndexResult : IndexResults) {
1134  if (UsedIndexResults.count(&IndexResult))
1135  continue;
1136  AddToBundles(/*SemaResult=*/nullptr, &IndexResult);
1137  }
1138  // We only keep the best N results at any time, in "native" format.
1140  Opts.Limit == 0 ? std::numeric_limits<size_t>::max() : Opts.Limit);
1141  for (auto &Bundle : Bundles)
1142  addCandidate(Top, std::move(Bundle));
1143  return std::move(Top).items();
1144  }
1145 
1146  Optional<float> fuzzyScore(const CompletionCandidate &C) {
1147  // Macros can be very spammy, so we only support prefix completion.
1148  // We won't end up with underfull index results, as macros are sema-only.
1149  if (C.SemaResult && C.SemaResult->Kind == CodeCompletionResult::RK_Macro &&
1150  !C.Name.startswith_lower(Filter->pattern()))
1151  return None;
1152  return Filter->match(C.Name);
1153  }
1154 
1155  // Scores a candidate and adds it to the TopN structure.
1156  void addCandidate(TopN<ScoredBundle, ScoredBundleGreater> &Candidates,
1157  CompletionCandidate::Bundle Bundle) {
1158  SymbolQualitySignals Quality;
1159  SymbolRelevanceSignals Relevance;
1160  Relevance.Context = Recorder->CCContext.getKind();
1162  Relevance.FileProximityMatch = FileProximity.getPointer();
1163  auto &First = Bundle.front();
1164  if (auto FuzzyScore = fuzzyScore(First))
1165  Relevance.NameMatch = *FuzzyScore;
1166  else
1167  return;
1169  bool FromIndex = false;
1170  for (const auto &Candidate : Bundle) {
1171  if (Candidate.IndexResult) {
1172  Quality.merge(*Candidate.IndexResult);
1173  Relevance.merge(*Candidate.IndexResult);
1174  Origin |= Candidate.IndexResult->Origin;
1175  FromIndex = true;
1176  }
1177  if (Candidate.SemaResult) {
1178  Quality.merge(*Candidate.SemaResult);
1179  Relevance.merge(*Candidate.SemaResult);
1180  Origin |= SymbolOrigin::AST;
1181  }
1182  }
1183 
1184  CodeCompletion::Scores Scores;
1185  Scores.Quality = Quality.evaluate();
1186  Scores.Relevance = Relevance.evaluate();
1187  Scores.Total = evaluateSymbolAndRelevance(Scores.Quality, Scores.Relevance);
1188  // NameMatch is in fact a multiplier on total score, so rescoring is sound.
1189  Scores.ExcludingName = Relevance.NameMatch
1190  ? Scores.Total / Relevance.NameMatch
1191  : Scores.Quality;
1192 
1193  dlog("CodeComplete: {0} ({1}) = {2}\n{3}{4}\n", First.Name,
1194  llvm::to_string(Origin), Scores.Total, llvm::to_string(Quality),
1195  llvm::to_string(Relevance));
1196 
1197  NSema += bool(Origin & SymbolOrigin::AST);
1198  NIndex += FromIndex;
1199  NBoth += bool(Origin & SymbolOrigin::AST) && FromIndex;
1200  if (Candidates.push({std::move(Bundle), Scores}))
1201  Incomplete = true;
1202  }
1203 
1204  CodeCompletion toCodeCompletion(const CompletionCandidate::Bundle &Bundle) {
1205  llvm::Optional<CodeCompletionBuilder> Builder;
1206  for (const auto &Item : Bundle) {
1207  CodeCompletionString *SemaCCS =
1208  Item.SemaResult ? Recorder->codeCompletionString(*Item.SemaResult)
1209  : nullptr;
1210  if (!Builder)
1211  Builder.emplace(Recorder->CCSema->getASTContext(), Item, SemaCCS,
1212  *Inserter, FileName, Opts);
1213  else
1214  Builder->add(Item, SemaCCS);
1215  }
1216  return Builder->build();
1217  }
1218 };
1219 
1221  const tooling::CompileCommand &Command,
1222  PrecompiledPreamble const *Preamble,
1223  const IncludeStructure &PreambleInclusions,
1224  StringRef Contents, Position Pos,
1225  IntrusiveRefCntPtr<vfs::FileSystem> VFS,
1226  std::shared_ptr<PCHContainerOperations> PCHs,
1227  CodeCompleteOptions Opts) {
1228  return CodeCompleteFlow(FileName, PreambleInclusions, Opts)
1229  .run({FileName, Command, Preamble, Contents, Pos, VFS, PCHs});
1230 }
1231 
1233  const tooling::CompileCommand &Command,
1234  PrecompiledPreamble const *Preamble,
1235  StringRef Contents, Position Pos,
1236  IntrusiveRefCntPtr<vfs::FileSystem> VFS,
1237  std::shared_ptr<PCHContainerOperations> PCHs) {
1238  SignatureHelp Result;
1239  clang::CodeCompleteOptions Options;
1240  Options.IncludeGlobals = false;
1241  Options.IncludeMacros = false;
1242  Options.IncludeCodePatterns = false;
1243  Options.IncludeBriefComments = false;
1244  IncludeStructure PreambleInclusions; // Unused for signatureHelp
1245  semaCodeComplete(llvm::make_unique<SignatureHelpCollector>(Options, Result),
1246  Options,
1247  {FileName, Command, Preamble, Contents, Pos, std::move(VFS),
1248  std::move(PCHs)});
1249  return Result;
1250 }
1251 
1252 bool isIndexedForCodeCompletion(const NamedDecl &ND, ASTContext &ASTCtx) {
1253  using namespace clang::ast_matchers;
1254  auto InTopLevelScope = hasDeclContext(
1255  anyOf(namespaceDecl(), translationUnitDecl(), linkageSpecDecl()));
1256  return !match(decl(anyOf(InTopLevelScope,
1257  hasDeclContext(
1258  enumDecl(InTopLevelScope, unless(isScoped()))))),
1259  ND, ASTCtx)
1260  .empty();
1261 }
1262 
1264  CompletionItem LSP;
1265  LSP.label = (HeaderInsertion ? Opts.IncludeIndicator.Insert
1266  : Opts.IncludeIndicator.NoInsert) +
1267  (Opts.ShowOrigins ? "[" + llvm::to_string(Origin) + "]" : "") +
1268  RequiredQualifier + Name + Signature;
1269 
1270  LSP.kind = Kind;
1271  LSP.detail = BundleSize > 1 ? llvm::formatv("[{0} overloads]", BundleSize)
1272  : ReturnType;
1273  if (!Header.empty())
1274  LSP.detail += "\n" + Header;
1275  LSP.documentation = Documentation;
1276  LSP.sortText = sortText(Score.Total, Name);
1277  LSP.filterText = Name;
1278  LSP.insertText = RequiredQualifier + Name;
1279  if (Opts.EnableSnippets)
1280  LSP.insertText += SnippetSuffix;
1283  if (HeaderInsertion)
1284  LSP.additionalTextEdits = {*HeaderInsertion};
1285  return LSP;
1286 }
1287 
1288 raw_ostream &operator<<(raw_ostream &OS, const CodeCompletion &C) {
1289  // For now just lean on CompletionItem.
1290  return OS << C.render(CodeCompleteOptions());
1291 }
1292 
1293 raw_ostream &operator<<(raw_ostream &OS, const CodeCompleteResult &R) {
1294  OS << "CodeCompleteResult: " << R.Completions.size() << (R.HasMore ? "+" : "")
1295  << " (" << getCompletionKindString(R.Context) << ")"
1296  << " items:\n";
1297  for (const auto &C : R.Completions)
1298  OS << C << "\n";
1299  return OS;
1300 }
1301 
1302 } // namespace clangd
1303 } // namespace clang
std::string insertText
A string that should be inserted to a document when selecting this completion.
Definition: Protocol.h:710
std::vector< std::string > AccessibleScopes
Sema * CCSema
llvm::StringRef Name
bool ShowOrigins
Expose origins of completion items in the label (for debugging).
Definition: CodeComplete.h:72
const tooling::CompileCommand & Command
bool BundleOverloads
Combine overloads into a single completion item where possible.
Definition: CodeComplete.h:58
std::pair< size_t, size_t > offsetToClangLineColumn(StringRef Code, size_t Offset)
Definition: SourceCode.cpp:155
CodeCompleteFlow(PathRef FileName, const IncludeStructure &Includes, const CodeCompleteOptions &Opts)
size_t Limit
Limit the number of results returned (0 means no limit).
Definition: CodeComplete.h:62
CodeCompletionContext::Kind Context
Definition: CodeComplete.h:148
void merge(const CodeCompletionResult &SemaCCResult)
Definition: Quality.cpp:169
std::string printQualifiedName(const NamedDecl &ND)
Returns the qualified name of ND.
Definition: AST.cpp:41
size_t MaxCandidateCount
The number of top candidates to return.
Definition: Index.h:303
CodeCompleteResult codeComplete(PathRef FileName, const tooling::CompileCommand &Command, PrecompiledPreamble const *Preamble, const IncludeStructure &PreambleInclusions, StringRef Contents, Position Pos, IntrusiveRefCntPtr< vfs::FileSystem > VFS, std::shared_ptr< PCHContainerOperations > PCHs, CodeCompleteOptions Opts)
Get code completions at a specified Pos in FileName.
bool RestrictForCodeCompletion
If set to true, only symbols for completion support will be considered.
Definition: Index.h:305
SignatureHelp signatureHelp(PathRef FileName, const tooling::CompileCommand &Command, PrecompiledPreamble const *Preamble, StringRef Contents, Position Pos, IntrusiveRefCntPtr< vfs::FileSystem > VFS, std::shared_ptr< PCHContainerOperations > PCHs)
Get signature help at a specified Pos in FileName.
CompletionItemKind kind
The kind of this completion item.
Definition: Protocol.h:691
bool EnableSnippets
When true, completion items will contain expandable code snippets in completion (e.g.
Definition: CodeComplete.h:40
clang::find_all_symbols::SymbolInfo::SymbolKind SymbolKind
Definition: SymbolInfo.cpp:22
std::vector< CodeCompletion > Completions
Definition: CodeComplete.h:146
const_iterator find(const SymbolID &SymID) const
Definition: Index.cpp:69
void merge(const CodeCompletionResult &SemaResult)
Definition: Quality.cpp:275
std::string sortText(float Score, llvm::StringRef Name)
Returns a string that sorts in the same order as (-Score, Tiebreak), for LSP.
Definition: Quality.cpp:385
virtual bool fuzzyFind(const FuzzyFindRequest &Req, llvm::function_ref< void(const Symbol &)> Callback) const =0
Matches symbols in the index fuzzily and applies Callback on each matched symbol before returning...
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:24
llvm::Optional< std::string > UnresolvedQualifier
std::vector< CodeCompletionResult > Results
CodeCompletionContext::Kind Context
Definition: Quality.h:103
std::string sortText
A string that should be used when comparing this item with other items.
Definition: Protocol.h:702
std::string getDocComment(const ASTContext &Ctx, const CodeCompletionResult &Result, bool CommentsFromHeaders)
Gets a minimally formatted documentation comment of Result, with comment markers stripped.
Documents should not be synced at all.
Attributes of a symbol that affect how much we like it.
Definition: Quality.h:49
std::string documentation
A human-readable string that represents a doc-comment.
Definition: Protocol.h:698
void vlog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:67
std::vector< TextEdit > additionalTextEdits
An optional array of additional text edits that are applied when selecting this completion.
Definition: Protocol.h:726
void elog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:56
StringRef Contents
std::vector< std::string > Scopes
If this is non-empty, symbols must be in at least one of the scopes (e.g.
Definition: Index.h:300
CompletionItemKind
The kind of a completion entry.
Definition: Protocol.h:644
std::string getReturnType(const CodeCompletionString &CCS)
Gets detail to be used as the detail field in an LSP completion item.
BindArgumentKind Kind
llvm::Expected< size_t > positionToOffset(StringRef Code, Position P, bool AllowColumnsBeyondLineLength)
Definition: SourceCode.cpp:83
std::string detail
A human-readable string with additional information about this item, like type or symbol information...
Definition: Protocol.h:695
enum clang::clangd::SymbolRelevanceSignals::QueryType Query
bool isIndexedForCodeCompletion(const NamedDecl &ND, ASTContext &ASTCtx)
std::string filterText
A string that should be used when filtering a set of completion items.
Definition: Protocol.h:706
llvm::unique_function< void()> Action
void log(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:62
std::string Query
A query string for the fuzzy find.
Definition: Index.h:293
#define dlog(...)
Definition: Logger.h:72
bool push(value_type &&V)
Definition: Quality.h:129
const Symbol * IndexResult
PathRef FileName
std::string formatDocumentation(const CodeCompletionString &CCS, llvm::StringRef DocComment)
Assembles formatted documentation for a completion result.
std::string ReturnType
PrecompiledPreamble const * Preamble
clang::CodeCompleteOptions getClangCompleteOpts() const
Returns options that can be passed to clang&#39;s completion engine.
std::string SnippetSuffix
Position Pos
IntrusiveRefCntPtr< vfs::FileSystem > VFS
float evaluateSymbolAndRelevance(float SymbolQuality, float SymbolRelevance)
Combine symbol quality and relevance into a single score.
Definition: Quality.cpp:367
FunctionInfo Info
const SymbolIndex * Index
If Index is set, it is used to augment the code completion results.
Definition: CodeComplete.h:79
Represents the signature of a callable.
Definition: Protocol.h:778
void getSignature(const CodeCompletionString &CCS, std::string *Signature, std::string *Snippet, std::string *RequiredQualifiers)
Formats the signature for an item, as a display string and snippet.
std::vector< Inclusion > MainFileIncludes
Definition: Headers.h:59
std::unique_ptr< PPCallbacks > collectIncludeStructureCallback(const SourceManager &SM, IncludeStructure *Out)
Returns a PPCallback that visits all inclusions in the main file.
Definition: Headers.cpp:74
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
std::vector< std::string > ProximityPaths
Contextually relevant files (e.g.
Definition: Index.h:308
std::string label
The label of this completion item.
Definition: Protocol.h:687
std::unique_ptr< CompilerInstance > prepareCompilerInstance(std::unique_ptr< clang::CompilerInvocation > CI, const PrecompiledPreamble *Preamble, std::unique_ptr< llvm::MemoryBuffer > Buffer, std::shared_ptr< PCHContainerOperations > PCHs, IntrusiveRefCntPtr< vfs::FileSystem > VFS, DiagnosticConsumer &DiagsClient)
Creates a compiler instance, configured so that:
Definition: Compiler.cpp:43
CompletionItem render(const CodeCompleteOptions &) const
llvm::StringMap< unsigned > includeDepth(llvm::StringRef Root) const
Definition: Headers.cpp:97
std::shared_ptr< PCHContainerOperations > PCHs
struct clang::clangd::CodeCompleteOptions::IncludeInsertionIndicator IncludeIndicator
std::pair< llvm::StringRef, llvm::StringRef > splitQualifiedName(llvm::StringRef QName)
From "a::b::c", return {"a::b::", "c"}.
Definition: SourceCode.cpp:166
const CodeCompletionResult * SemaResult
static llvm::Expected< std::string > resolve(const URI &U, llvm::StringRef HintPath="")
Resolves the absolute path of U.
Definition: URI.cpp:191
float NameMatch
0-1+ fuzzy-match score for unqualified name. Must be explicitly assigned.
Definition: Quality.h:78
const_iterator end() const
Definition: Index.h:246
static llvm::Expected< std::string > includeSpelling(const URI &U)
Gets the preferred spelling of this file for #include, if there is one, e.g.
Definition: URI.cpp:199
static llvm::Expected< URI > parse(llvm::StringRef Uri)
Parse a URI string "<scheme>:[//<authority>/]<path>".
Definition: URI.cpp:156
static std::string join(ArrayRef< SpecialMemberFunctionsCheck::SpecialMemberFunctionKind > SMFS, llvm::StringRef AndOr)
std::string getParameterDocComment(const ASTContext &Ctx, const CodeCompleteConsumer::OverloadCandidate &Result, unsigned ArgIndex, bool CommentsFromHeaders)
Gets a minimally formatted documentation for parameter of Result, corresponding to argument number Ar...
std::string Signature
Records an event whose duration is the lifetime of the Span object.
Definition: Trace.h:83
InsertTextFormat insertTextFormat
The format of the insert text.
Definition: Protocol.h:714
std::array< uint8_t, 20 > SymbolID
bool isLiteralInclude(llvm::StringRef Include)
Returns true if Include is literal include like "path" or <path>.
Definition: Headers.cpp:64
Attributes of a symbol-query pair that affect how much we like it.
Definition: Quality.h:76
CodeCompletionContext CCContext
#define SPAN_ATTACH(S, Name, Expr)
Attach a key-value pair to a Span event.
Definition: Trace.h:98
CodeCompleteResult run(const SemaCompleteInput &SemaCCInput) &&
raw_ostream & operator<<(raw_ostream &OS, const CodeCompletion &C)
TopN<T> is a lossy container that preserves only the "best" N elements.
Definition: Quality.h:121