clang-tools 8.0.0

FileIndex.cpp
Go to the documentation of this file.
1 //===--- FileIndex.cpp - Indexes for files. ------------------------ C++-*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9
10 #include "FileIndex.h"
11 #include "ClangdUnit.h"
12 #include "Logger.h"
13 #include "SymbolCollector.h"
14 #include "index/Index.h"
15 #include "index/MemIndex.h"
16 #include "index/Merge.h"
17 #include "index/dex/Dex.h"
18 #include "clang/Index/IndexingAction.h"
19 #include "clang/Lex/MacroInfo.h"
20 #include "clang/Lex/Preprocessor.h"
21 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/ADT/DenseSet.h"
23 #include "llvm/ADT/STLExtras.h"
24 #include "llvm/ADT/StringRef.h"
25 #include <memory>
26
27 namespace clang {
28 namespace clangd {
29
30 static std::pair<SymbolSlab, RefSlab>
31 indexSymbols(ASTContext &AST, std::shared_ptr<Preprocessor> PP,
32 llvm::ArrayRef<Decl *> DeclsToIndex, bool IsIndexMainAST) {
33 SymbolCollector::Options CollectorOpts;
34 // FIXME(ioeric): we might also want to collect include headers. We would need
35 // to make sure all includes are canonicalized (with CanonicalIncludes), which
36 // is not trivial given the current way of collecting symbols: we only have
37 // AST at this point, but we also need preprocessor callbacks (e.g.
38 // CommentHandler for IWYU pragma) to canonicalize includes.
39 CollectorOpts.CollectIncludePath = false;
40 CollectorOpts.CountReferences = false;
41 CollectorOpts.Origin = SymbolOrigin::Dynamic;
42
43 index::IndexingOptions IndexOpts;
44 // We only need declarations, because we don't count references.
45 IndexOpts.SystemSymbolFilter =
46 index::IndexingOptions::SystemSymbolFilterKind::DeclarationsOnly;
47 IndexOpts.IndexFunctionLocals = false;
48 if (IsIndexMainAST) {
49 // We only collect refs when indexing main AST.
50 CollectorOpts.RefFilter = RefKind::All;
51 }else {
52 IndexOpts.IndexMacrosInPreprocessor = true;
53 CollectorOpts.CollectMacro = true;
54 }
55
56 SymbolCollector Collector(std::move(CollectorOpts));
57 Collector.setPreprocessor(PP);
58 index::indexTopLevelDecls(AST, *PP, DeclsToIndex, Collector, IndexOpts);
59
60 const auto &SM = AST.getSourceManager();
61 const auto *MainFileEntry = SM.getFileEntryForID(SM.getMainFileID());
62 std::string FileName = MainFileEntry ? MainFileEntry->getName() : "";
63
64 auto Syms = Collector.takeSymbols();
65 auto Refs = Collector.takeRefs();
66 vlog("index AST for {0} (main={1}): \n"
67 " symbol slab: {2} symbols, {3} bytes\n"
68 " ref slab: {4} symbols, {5} refs, {6} bytes",
69 FileName, IsIndexMainAST, Syms.size(), Syms.bytes(), Refs.size(),
70 Refs.numRefs(), Refs.bytes());
71 return {std::move(Syms), std::move(Refs)};
72 }
73
74 std::pair<SymbolSlab, RefSlab> indexMainDecls(ParsedAST &AST) {
75 return indexSymbols(AST.getASTContext(), AST.getPreprocessorPtr(),
76 AST.getLocalTopLevelDecls(),
77 /*IsIndexMainAST=*/true);
78 }
79
80 SymbolSlab indexHeaderSymbols(ASTContext &AST,
81 std::shared_ptr<Preprocessor> PP) {
82 std::vector<Decl *> DeclsToIndex(
83 AST.getTranslationUnitDecl()->decls().begin(),
84 AST.getTranslationUnitDecl()->decls().end());
85 return indexSymbols(AST, std::move(PP), DeclsToIndex,
86 /*IsIndexMainAST=*/false)
87 .first;
88 }
89
90 void FileSymbols::update(PathRef Path, std::unique_ptr<SymbolSlab> Symbols,
91 std::unique_ptr<RefSlab> Refs) {
92 std::lock_guard<std::mutex> Lock(Mutex);
93 if (!Symbols)
94 FileToSymbols.erase(Path);
95 else
96 FileToSymbols[Path] = std::move(Symbols);
97 if (!Refs)
98 FileToRefs.erase(Path);
99 else
100 FileToRefs[Path] = std::move(Refs);
101 }
102
103 std::unique_ptr<SymbolIndex>
104 FileSymbols::buildIndex(IndexType Type, DuplicateHandling DuplicateHandle) {
105 std::vector<std::shared_ptr<SymbolSlab>> SymbolSlabs;
106 std::vector<std::shared_ptr<RefSlab>> RefSlabs;
107 {
108 std::lock_guard<std::mutex> Lock(Mutex);
109 for (const auto &FileAndSymbols : FileToSymbols)
110 SymbolSlabs.push_back(FileAndSymbols.second);
111 for (const auto &FileAndRefs : FileToRefs)
112 RefSlabs.push_back(FileAndRefs.second);
113 }
114 std::vector<const Symbol *> AllSymbols;
115 std::vector<Symbol> SymsStorage;
116 switch (DuplicateHandle) {
117 case DuplicateHandling::Merge: {
118 llvm::DenseMap<SymbolID, Symbol> Merged;
119 for (const auto &Slab : SymbolSlabs) {
120 for (const auto &Sym : *Slab) {
121 auto I = Merged.try_emplace(Sym.ID, Sym);
122 if (!I.second)
123 I.first->second = mergeSymbol(I.first->second, Sym);
124 }
125 }
126 SymsStorage.reserve(Merged.size());
127 for (auto &Sym : Merged) {
128 SymsStorage.push_back(std::move(Sym.second));
129 AllSymbols.push_back(&SymsStorage.back());
130 }
131 // FIXME: aggregate symbol reference count based on references.
132 break;
133 }
134 case DuplicateHandling::PickOne: {
135 llvm::DenseSet<SymbolID> AddedSymbols;
136 for (const auto &Slab : SymbolSlabs)
137 for (const auto &Sym : *Slab)
138 if (AddedSymbols.insert(Sym.ID).second)
139 AllSymbols.push_back(&Sym);
140 break;
141 }
142 }
143
144 std::vector<Ref> RefsStorage; // Contiguous ranges for each SymbolID.
145 llvm::DenseMap<SymbolID, llvm::ArrayRef<Ref>> AllRefs;
146 {
147 llvm::DenseMap<SymbolID, llvm::SmallVector<Ref, 4>> MergedRefs;
148 size_t Count = 0;
149 for (const auto &RefSlab : RefSlabs)
150 for (const auto &Sym : *RefSlab) {
151 MergedRefs[Sym.first].append(Sym.second.begin(), Sym.second.end());
152 Count += Sym.second.size();
153 }
154 RefsStorage.reserve(Count);
155 AllRefs.reserve(MergedRefs.size());
156 for (auto &Sym : MergedRefs) {
157 auto &SymRefs = Sym.second;
158 // Sorting isn't required, but yields more stable results over rebuilds.
159 llvm::sort(SymRefs);
160 llvm::copy(SymRefs, back_inserter(RefsStorage));
161 AllRefs.try_emplace(
162 Sym.first,
163 llvm::ArrayRef<Ref>(&RefsStorage[RefsStorage.size() - SymRefs.size()],
164 SymRefs.size()));
165 }
166 }
167
168 size_t StorageSize =
169 RefsStorage.size() * sizeof(Ref) + SymsStorage.size() * sizeof(Symbol);
170 for (const auto &Slab : SymbolSlabs)
171 StorageSize += Slab->bytes();
172 for (const auto &RefSlab : RefSlabs)
173 StorageSize += RefSlab->bytes();
174
175 // Index must keep the slabs and contiguous ranges alive.
176 switch (Type) {
177 case IndexType::Light:
178 return llvm::make_unique<MemIndex>(
179 llvm::make_pointee_range(AllSymbols), std::move(AllRefs),
180 std::make_tuple(std::move(SymbolSlabs), std::move(RefSlabs),
181 std::move(RefsStorage), std::move(SymsStorage)),
182 StorageSize);
183 case IndexType::Heavy:
184 return llvm::make_unique<dex::Dex>(
185 llvm::make_pointee_range(AllSymbols), std::move(AllRefs),
186 std::make_tuple(std::move(SymbolSlabs), std::move(RefSlabs),
187 std::move(RefsStorage), std::move(SymsStorage)),
188 StorageSize);
189 }
190 llvm_unreachable("Unknown clangd::IndexType");
191 }
192
193 FileIndex::FileIndex(bool UseDex)
194 : MergedIndex(&MainFileIndex, &PreambleIndex), UseDex(UseDex),
195 PreambleIndex(llvm::make_unique<MemIndex>()),
196 MainFileIndex(llvm::make_unique<MemIndex>()) {}
197
198 void FileIndex::updatePreamble(PathRef Path, ASTContext &AST,
199 std::shared_ptr<Preprocessor> PP) {
200 auto Symbols = indexHeaderSymbols(AST, std::move(PP));
201 PreambleSymbols.update(Path,
202 llvm::make_unique<SymbolSlab>(std::move(Symbols)),
203 llvm::make_unique<RefSlab>());
204 PreambleIndex.reset(
205 PreambleSymbols.buildIndex(UseDex ? IndexType::Heavy : IndexType::Light,
206 DuplicateHandling::PickOne));
207 }
208
209 void FileIndex::updateMain(PathRef Path, ParsedAST &AST) {
210 auto Contents = indexMainDecls(AST);
211 MainFileSymbols.update(
212 Path, llvm::make_unique<SymbolSlab>(std::move(Contents.first)),
213 llvm::make_unique<RefSlab>(std::move(Contents.second)));
214 MainFileIndex.reset(
215 MainFileSymbols.buildIndex(IndexType::Light, DuplicateHandling::PickOne));
216 }
217
218 } // namespace clangd
219 } // namespace clang
clang::clangd::SymbolCollector::Options::CollectMacro
bool CollectMacro
Collect macros.
Definition: SymbolCollector.h:75
clang::clangd::SymbolSlab
Definition: Index.h:292
clang::clangd::indexHeaderSymbols
SymbolSlab indexHeaderSymbols(ASTContext &AST, std::shared_ptr< Preprocessor > PP)
Idex declarations from AST and macros from PP that are declared in included headers.
Definition: FileIndex.cpp:80
Contents
llvm::StringRef Contents
Definition: CodeComplete.cpp:1007
llvm
Some operations such as code completion produce a set of candidates.
Definition: YAMLGenerator.cpp:28
clang::clangd::RefSlab
Definition: Index.h:388
Dex.h
This defines Dex - a symbol index implementation based on query iterators over symbol tokens...
Index.h
clang::clangd::IndexType
IndexType
Select between in-memory index implementations, which have tradeoffs.
Definition: FileIndex.h:30
clang::clangd::IndexType::Light
clang::clangd::SymbolCollector::takeSymbols
SymbolSlab takeSymbols()
Definition: SymbolCollector.h:106
clang::clangd::Ref
Definition: Index.h:373
clang::clangd::DuplicateHandling::Merge
clang::clangd::SymbolCollector
Collect declarations (symbols) from an AST.
Definition: SymbolCollector.h:44
MemIndex.h
clang::clangd::PathRef
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:24
clang::clangd::RefKind::All
clang::clangd::ParsedAST::getLocalTopLevelDecls
ArrayRef< Decl * > getLocalTopLevelDecls()
This function returns top-level decls present in the main file of the AST.
Definition: ClangdUnit.cpp:364
clang::clangd::SymbolCollector::Options::CountReferences
bool CountReferences
Definition: SymbolCollector.h:57
ClangdUnit.h
clang::clangd::vlog
void vlog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:68
clang::clangd::FileIndex::updateMain
void updateMain(PathRef Path, ParsedAST &AST)
Update symbols and references from main file Path with indexMainDecls.
Definition: FileIndex.cpp:209
clang::clangd::ParsedAST::getASTContext
ASTContext & getASTContext()
Note that the returned ast will not contain decls from the preamble that were not deserialized during...
Definition: ClangdUnit.cpp:348
clang::clangd::UseDex
static llvm::cl::opt< bool > UseDex("use-dex-index", llvm::cl::desc("Use experimental Dex dynamic index."), llvm::cl::init(false), llvm::cl::Hidden)
clang::clangd::DuplicateHandling
DuplicateHandling
How to handle duplicated symbols across multiple files.
Definition: FileIndex.h:38
clang::clangd::FileIndex::updatePreamble
void updatePreamble(PathRef Path, ASTContext &AST, std::shared_ptr< Preprocessor > PP)
Update preamble symbols of file Path with all declarations in AST and macros in PP.
Definition: FileIndex.cpp:198
clang::clangd::SymbolCollector::Options::CollectIncludePath
bool CollectIncludePath
Definition: SymbolCollector.h:52
clang::clangd::SymbolOrigin::AST
clang::clangd::MemIndex
MemIndex is a naive in-memory index suitable for a small set of symbols.
Definition: MemIndex.h:20
clang::clangd::ParsedAST::getPreprocessorPtr
std::shared_ptr< Preprocessor > getPreprocessorPtr()
Definition: ClangdUnit.cpp:356
clang::clangd::indexMainDecls
std::pair< SymbolSlab, RefSlab > indexMainDecls(ParsedAST &AST)
Retrieves symbols and refs of local top level decls in AST (i.e.
Definition: FileIndex.cpp:74
clang::clangd::Path
std::string Path
A typedef to represent a file path.
Definition: Path.h:21
clang::clangd::SymbolCollector::Options
Definition: SymbolCollector.h:46
clang::clangd::FileSymbols::buildIndex
std::unique_ptr< SymbolIndex > buildIndex(IndexType, DuplicateHandling DuplicateHandle=DuplicateHandling::PickOne)
Definition: FileIndex.cpp:104
clang::clangd::mergeSymbol
Symbol mergeSymbol(const Symbol &L, const Symbol &R)
Definition: Merge.cpp:119
FileName
PathRef FileName
Definition: CodeComplete.cpp:1004
clang::clangd::SymbolCollector::Options::Origin
SymbolOrigin Origin
Definition: SymbolCollector.h:70
clang::clangd::ParsedAST
Stores and provides access to parsed AST.
Definition: ClangdUnit.h:71
clang::clangd::FileIndex::FileIndex
FileIndex(bool UseDex=true)
Definition: FileIndex.cpp:193
clang::clangd::DuplicateHandling::PickOne
Logger.h
Merge.h
clang::clangd::IndexType::Heavy
clang::clangd::Symbol
Definition: Index.h:160
clang::clangd::indexSymbols
static std::pair< SymbolSlab, RefSlab > indexSymbols(ASTContext &AST, std::shared_ptr< Preprocessor > PP, llvm::ArrayRef< Decl *> DeclsToIndex, bool IsIndexMainAST)
Definition: FileIndex.cpp:31
clang
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
Definition: ChangeNamespace.cpp:18
SymbolCollector.h
clang::clangd::FileSymbols::update
void update(PathRef Path, std::unique_ptr< SymbolSlab > Slab, std::unique_ptr< RefSlab > Refs)
Updates all symbols and refs in a file.
Definition: FileIndex.cpp:90
clang::clangd::SymbolCollector::setPreprocessor
void setPreprocessor(std::shared_ptr< Preprocessor > PP) override
Definition: SymbolCollector.h:92
clang::clangd::RefSlab::bytes
size_t bytes() const
Definition: Index.h:405
clang::clangd::SymbolOrigin::Dynamic
FileIndex.h
clang::clangd::SymbolCollector::takeRefs
RefSlab takeRefs()
Definition: SymbolCollector.h:107
clang::clangd::MergedIndex
Definition: Merge.h:31
clang::clangd::SwapIndex::reset
void reset(std::unique_ptr< SymbolIndex >)
Definition: Index.cpp:162
clang::clangd::SymbolCollector::Options::RefFilter
RefKind RefFilter
The symbol ref kinds that will be collected.
Definition: SymbolCollector.h:63