26 #include "clang/AST/ASTContext.h"
27 #include "clang/Index/IndexingAction.h"
28 #include "clang/Index/IndexingOptions.h"
29 #include "clang/Lex/MacroInfo.h"
30 #include "clang/Lex/Preprocessor.h"
31 #include "llvm/ADT/DenseMap.h"
32 #include "llvm/ADT/Optional.h"
33 #include "llvm/ADT/STLExtras.h"
34 #include "llvm/ADT/StringMap.h"
35 #include "llvm/ADT/StringRef.h"
36 #include "llvm/Support/Error.h"
46 SlabTuple indexSymbols(ASTContext &AST, std::shared_ptr<Preprocessor>
PP,
47 llvm::ArrayRef<Decl *> DeclsToIndex,
48 const MainFileMacros *MacroRefsToIndex,
49 const CanonicalIncludes &Includes,
bool IsIndexMainAST,
50 llvm::StringRef Version) {
57 index::IndexingOptions IndexOpts;
59 IndexOpts.SystemSymbolFilter =
60 index::IndexingOptions::SystemSymbolFilterKind::DeclarationsOnly;
61 IndexOpts.IndexFunctionLocals =
false;
69 IndexOpts.IndexMacrosInPreprocessor =
true;
77 Collector.handleMacros(*MacroRefsToIndex);
78 index::indexTopLevelDecls(AST, *
PP, DeclsToIndex,
Collector, IndexOpts);
81 const auto *MainFileEntry = SM.getFileEntryForID(SM.getMainFileID());
83 std::string(MainFileEntry ? MainFileEntry->getName() :
"");
87 auto Relations =
Collector.takeRelations();
89 vlog(
"indexed {0} AST for {1} version {2}:\n"
90 " symbol slab: {3} symbols, {4} bytes\n"
91 " ref slab: {5} symbols, {6} refs, {7} bytes\n"
92 " relations slab: {8} relations, {9} bytes",
93 IsIndexMainAST ?
"file" :
"preamble",
FileName, Version, Syms.size(),
95 Relations.size(), Relations.bytes());
96 return std::make_tuple(std::move(Syms), std::move(
Refs),
97 std::move(Relations));
105 auto Entry = IG.try_emplace(URI).first;
106 auto &Node =
Entry->getValue();
107 Node = FullGraph.lookup(
Entry->getKey());
108 Node.URI =
Entry->getKey();
111 for (
auto &Include : Node.DirectIncludes) {
112 auto I = IG.try_emplace(Include).first;
113 I->getValue().URI = I->getKey();
114 Include = I->getKey();
121 :
Index(std::move(Input)) {
123 llvm::DenseMap<SymbolID, FileShard *> SymbolIDToFile;
127 for (
const auto &S : *Index.
Symbols) {
128 auto It = Shards.try_emplace(S.CanonicalDeclaration.FileURI);
129 It.first->getValue().Symbols.insert(&S);
130 SymbolIDToFile[S.ID] = &It.first->getValue();
133 S.Definition.FileURI != S.CanonicalDeclaration.FileURI) {
134 auto It = Shards.try_emplace(S.Definition.FileURI);
135 It.first->getValue().Symbols.insert(&S);
141 for (
const auto &SymRefs : *Index.
Refs) {
142 for (
const auto &R : SymRefs.second) {
143 const auto It = Shards.try_emplace(R.Location.FileURI);
144 It.first->getValue().Refs.insert(&R);
145 RefToSymID[&R] = SymRefs.first;
154 if (
auto *
File = SymbolIDToFile.lookup(R.Subject))
155 File->Relations.insert(&R);
160 const auto &FullGraph = *Index.
Sources;
161 for (
const auto &It : FullGraph) {
162 auto ShardIt = Shards.try_emplace(It.first());
163 ShardIt.first->getValue().IG = getSubGraph(It.first(), FullGraph);
170 std::vector<PathRef> Result;
171 Result.reserve(Shards.size());
172 for (
auto Key : Shards.keys())
173 Result.push_back(
Key);
177 llvm::Optional<IndexFileIn>
179 auto It = Shards.find(Uri);
180 if (It == Shards.end())
184 IF.
Sources = It->getValue().IG;
188 for (
const auto *S : It->getValue().Symbols)
190 IF.
Symbols = std::move(SymB).build();
193 for (
const auto *
Ref : It->getValue().Refs) {
194 auto SID = RefToSymID.lookup(
Ref);
197 IF.
Refs = std::move(RefB).build();
200 for (
const auto *Rel : It->getValue().Relations) {
205 return std::move(IF);
216 std::shared_ptr<Preprocessor>
PP,
218 std::vector<Decl *> DeclsToIndex(
219 AST.getTranslationUnitDecl()->decls().begin(),
220 AST.getTranslationUnitDecl()->decls().end());
221 return indexSymbols(AST, std::move(
PP), DeclsToIndex,
227 std::unique_ptr<SymbolSlab>
Symbols,
228 std::unique_ptr<RefSlab>
Refs,
229 std::unique_ptr<RelationSlab> Relations,
230 bool CountReferences) {
231 std::lock_guard<std::mutex> Lock(Mutex);
234 SymbolsSnapshot.erase(
Key);
238 RefsSnapshot.erase(
Key);
240 RefSlabAndCountReferences Item;
241 Item.CountReferences = CountReferences;
242 Item.Slab = std::move(
Refs);
243 RefsSnapshot[
Key] = std::move(Item);
246 RelatiosSnapshot.erase(
Key);
248 RelatiosSnapshot[
Key] = std::move(Relations);
251 std::unique_ptr<SymbolIndex>
254 std::vector<std::shared_ptr<SymbolSlab>> SymbolSlabs;
255 std::vector<std::shared_ptr<RefSlab>> RefSlabs;
256 std::vector<std::shared_ptr<RelationSlab>> RelationSlabs;
257 std::vector<RefSlab *> MainFileRefs;
259 std::lock_guard<std::mutex> Lock(Mutex);
260 for (
const auto &FileAndSymbols : SymbolsSnapshot)
261 SymbolSlabs.push_back(FileAndSymbols.second);
262 for (
const auto &FileAndRefs : RefsSnapshot) {
263 RefSlabs.push_back(FileAndRefs.second.Slab);
264 if (FileAndRefs.second.CountReferences)
265 MainFileRefs.push_back(RefSlabs.back().get());
267 for (
const auto &FileAndRelations : RelatiosSnapshot)
268 RelationSlabs.push_back(FileAndRelations.second);
271 *Version = this->Version;
273 std::vector<const Symbol *> AllSymbols;
274 std::vector<Symbol> SymsStorage;
275 switch (DuplicateHandle) {
277 llvm::DenseMap<SymbolID, Symbol> Merged;
278 for (
const auto &Slab : SymbolSlabs) {
279 for (
const auto &Sym : *Slab) {
280 assert(Sym.References == 0 &&
281 "Symbol with non-zero references sent to FileSymbols");
282 auto I = Merged.try_emplace(Sym.ID, Sym);
284 I.first->second =
mergeSymbol(I.first->second, Sym);
288 for (
const auto &Sym : *
Refs) {
289 auto It = Merged.find(Sym.first);
291 if (It == Merged.end())
293 It->getSecond().References += Sym.second.size();
295 SymsStorage.reserve(Merged.size());
296 for (
auto &Sym : Merged) {
297 SymsStorage.push_back(std::move(Sym.second));
298 AllSymbols.push_back(&SymsStorage.back());
303 llvm::DenseSet<SymbolID> AddedSymbols;
304 for (
const auto &Slab : SymbolSlabs)
305 for (
const auto &Sym : *Slab) {
306 assert(Sym.References == 0 &&
307 "Symbol with non-zero references sent to FileSymbols");
308 if (AddedSymbols.insert(Sym.ID).second)
309 AllSymbols.push_back(&Sym);
315 std::vector<Ref> RefsStorage;
316 llvm::DenseMap<SymbolID, llvm::ArrayRef<Ref>> AllRefs;
318 llvm::DenseMap<SymbolID, llvm::SmallVector<Ref, 4>> MergedRefs;
320 for (
const auto &
RefSlab : RefSlabs)
321 for (
const auto &Sym : *
RefSlab) {
322 MergedRefs[Sym.first].append(Sym.second.begin(), Sym.second.end());
323 Count += Sym.second.size();
325 RefsStorage.reserve(Count);
326 AllRefs.reserve(MergedRefs.size());
327 for (
auto &Sym : MergedRefs) {
328 auto &SymRefs = Sym.second;
331 llvm::copy(SymRefs, back_inserter(RefsStorage));
334 llvm::ArrayRef<Ref>(&RefsStorage[RefsStorage.size() - SymRefs.size()],
339 std::vector<Relation> AllRelations;
342 AllRelations.push_back(R);
346 RefsStorage.size() *
sizeof(
Ref) + SymsStorage.size() *
sizeof(
Symbol);
347 for (
const auto &Slab : SymbolSlabs)
348 StorageSize += Slab->bytes();
349 for (
const auto &
RefSlab : RefSlabs)
357 return std::make_unique<MemIndex>(
358 llvm::make_pointee_range(AllSymbols), std::move(AllRefs),
359 std::move(AllRelations),
360 std::make_tuple(std::move(SymbolSlabs), std::move(RefSlabs),
361 std::move(RefsStorage), std::move(SymsStorage)),
364 return std::make_unique<dex::Dex>(
365 llvm::make_pointee_range(AllSymbols), std::move(AllRefs),
366 std::move(AllRelations),
367 std::make_tuple(std::move(SymbolSlabs), std::move(RefSlabs),
368 std::move(RefsStorage), std::move(SymsStorage)),
371 llvm_unreachable(
"Unknown clangd::IndexType");
375 :
MergedIndex(&MainFileIndex, &PreambleIndex), UseDex(UseDex),
376 PreambleIndex(std::make_unique<
MemIndex>()),
377 MainFileIndex(std::make_unique<
MemIndex>()) {}
381 std::shared_ptr<Preprocessor>
PP,
388 auto IF = ShardedIndex.
getShard(Uri);
393 Uri, std::make_unique<SymbolSlab>(std::move(*IF->
Symbols)),
394 std::make_unique<RefSlab>(),
395 std::make_unique<RelationSlab>(std::move(*IF->
Relations)),
398 size_t IndexVersion = 0;
403 std::lock_guard<std::mutex> Lock(UpdateIndexMu);
404 if (IndexVersion <= PreambleIndexVersion) {
408 PreambleIndexVersion = IndexVersion;
409 PreambleIndex.
reset(std::move(NewIndex));
411 "Build dynamic index for header symbols with estimated memory usage of "
420 Path, std::make_unique<SymbolSlab>(std::move(std::get<0>(
Contents))),
421 std::make_unique<RefSlab>(std::move(std::get<1>(
Contents))),
422 std::make_unique<RelationSlab>(std::move(std::get<2>(
Contents))),
424 size_t IndexVersion = 0;
428 std::lock_guard<std::mutex> Lock(UpdateIndexMu);
429 if (IndexVersion <= MainIndexVersion) {
433 MainIndexVersion = IndexVersion;
434 MainFileIndex.
reset(std::move(NewIndex));
436 "Build dynamic index for main-file symbols with estimated memory usage "