19 #include "clang/Index/IndexingAction.h" 20 #include "clang/Lex/MacroInfo.h" 21 #include "clang/Lex/Preprocessor.h" 22 #include "llvm/ADT/DenseMap.h" 23 #include "llvm/ADT/DenseSet.h" 24 #include "llvm/ADT/STLExtras.h" 25 #include "llvm/ADT/StringRef.h" 32 llvm::ArrayRef<Decl *> DeclsToIndex,
34 bool IsIndexMainAST) {
41 index::IndexingOptions IndexOpts;
43 IndexOpts.SystemSymbolFilter =
44 index::IndexingOptions::SystemSymbolFilterKind::DeclarationsOnly;
45 IndexOpts.IndexFunctionLocals =
false;
53 IndexOpts.IndexMacrosInPreprocessor =
true;
60 index::indexTopLevelDecls(AST, *PP, DeclsToIndex, Collector, IndexOpts);
62 const auto &SM = AST.getSourceManager();
63 const auto *MainFileEntry = SM.getFileEntryForID(SM.getMainFileID());
64 std::string
FileName = MainFileEntry ? MainFileEntry->getName() :
"";
69 vlog(
"index AST for {0} (main={1}): \n" 70 " symbol slab: {2} symbols, {3} bytes\n" 71 " ref slab: {4} symbols, {5} refs, {6} bytes\n" 72 " relations slab: {7} relations, {8} bytes",
73 FileName, IsIndexMainAST, Syms.size(), Syms.bytes(),
Refs.
size(),
75 return std::make_tuple(std::move(Syms), std::move(
Refs),
87 std::vector<Decl *> DeclsToIndex(
88 AST.getTranslationUnitDecl()->decls().begin(),
89 AST.getTranslationUnitDecl()->decls().end());
90 return indexSymbols(AST, std::move(PP), DeclsToIndex, Includes,
95 std::unique_ptr<RefSlab>
Refs,
97 bool CountReferences) {
98 std::lock_guard<std::mutex> Lock(Mutex);
100 FileToSymbols.erase(Path);
102 FileToSymbols[
Path] = std::move(Symbols);
104 FileToRefs.erase(Path);
106 RefSlabAndCountReferences Item;
107 Item.CountReferences = CountReferences;
108 Item.Slab = std::move(Refs);
109 FileToRefs[
Path] = std::move(Item);
112 FileToRelations.erase(Path);
114 FileToRelations[
Path] = std::move(Relations);
117 std::unique_ptr<SymbolIndex>
119 std::vector<std::shared_ptr<SymbolSlab>> SymbolSlabs;
120 std::vector<std::shared_ptr<RefSlab>> RefSlabs;
121 std::vector<std::shared_ptr<RelationSlab>> RelationSlabs;
122 std::vector<RefSlab *> MainFileRefs;
124 std::lock_guard<std::mutex> Lock(Mutex);
125 for (
const auto &FileAndSymbols : FileToSymbols)
126 SymbolSlabs.push_back(FileAndSymbols.second);
127 for (
const auto &FileAndRefs : FileToRefs) {
128 RefSlabs.push_back(FileAndRefs.second.Slab);
129 if (FileAndRefs.second.CountReferences)
130 MainFileRefs.push_back(RefSlabs.back().get());
132 for (
const auto &FileAndRelations : FileToRelations)
133 RelationSlabs.push_back(FileAndRelations.second);
135 std::vector<const Symbol *> AllSymbols;
136 std::vector<Symbol> SymsStorage;
137 switch (DuplicateHandle) {
139 llvm::DenseMap<SymbolID, Symbol> Merged;
140 for (
const auto &Slab : SymbolSlabs) {
141 for (
const auto &Sym : *Slab) {
142 assert(Sym.References == 0 &&
143 "Symbol with non-zero references sent to FileSymbols");
144 auto I = Merged.try_emplace(Sym.ID, Sym);
146 I.first->second =
mergeSymbol(I.first->second, Sym);
150 for (
const auto &Sym : *
Refs) {
151 auto It = Merged.find(Sym.first);
153 if (It == Merged.end())
155 It->getSecond().References += Sym.second.size();
157 SymsStorage.reserve(Merged.size());
158 for (
auto &Sym : Merged) {
159 SymsStorage.push_back(std::move(Sym.second));
160 AllSymbols.push_back(&SymsStorage.back());
165 llvm::DenseSet<SymbolID> AddedSymbols;
166 for (
const auto &Slab : SymbolSlabs)
167 for (
const auto &Sym : *Slab) {
168 assert(Sym.References == 0 &&
169 "Symbol with non-zero references sent to FileSymbols");
170 if (AddedSymbols.insert(Sym.ID).second)
171 AllSymbols.push_back(&Sym);
177 std::vector<Ref> RefsStorage;
178 llvm::DenseMap<SymbolID, llvm::ArrayRef<Ref>> AllRefs;
180 llvm::DenseMap<SymbolID, llvm::SmallVector<Ref, 4>> MergedRefs;
182 for (
const auto &
RefSlab : RefSlabs)
183 for (
const auto &Sym : *
RefSlab) {
184 MergedRefs[Sym.first].append(Sym.second.begin(), Sym.second.end());
185 Count += Sym.second.size();
187 RefsStorage.reserve(Count);
188 AllRefs.reserve(MergedRefs.size());
189 for (
auto &Sym : MergedRefs) {
190 auto &SymRefs = Sym.second;
193 llvm::copy(SymRefs, back_inserter(RefsStorage));
196 llvm::ArrayRef<Ref>(&RefsStorage[RefsStorage.size() - SymRefs.size()],
201 std::vector<Relation> AllRelations;
204 AllRelations.push_back(R);
208 RefsStorage.size() *
sizeof(
Ref) + SymsStorage.size() *
sizeof(
Symbol);
209 for (
const auto &Slab : SymbolSlabs)
210 StorageSize += Slab->bytes();
211 for (
const auto &
RefSlab : RefSlabs)
219 return llvm::make_unique<MemIndex>(
220 llvm::make_pointee_range(AllSymbols), std::move(AllRefs),
221 std::move(AllRelations),
222 std::make_tuple(std::move(SymbolSlabs), std::move(RefSlabs),
223 std::move(RefsStorage), std::move(SymsStorage)),
226 return llvm::make_unique<dex::Dex>(
227 llvm::make_pointee_range(AllSymbols), std::move(AllRefs),
228 std::move(AllRelations),
229 std::make_tuple(std::move(SymbolSlabs), std::move(RefSlabs),
230 std::move(RefsStorage), std::move(SymsStorage)),
233 llvm_unreachable(
"Unknown clangd::IndexType");
237 :
MergedIndex(&MainFileIndex, &PreambleIndex), UseDex(UseDex),
242 std::shared_ptr<Preprocessor> PP,
246 Path, llvm::make_unique<SymbolSlab>(std::move(std::get<0>(Slabs))),
247 llvm::make_unique<RefSlab>(),
248 llvm::make_unique<RelationSlab>(std::move(std::get<2>(Slabs))),
258 Path, llvm::make_unique<SymbolSlab>(std::move(std::get<0>(
Contents))),
259 llvm::make_unique<RefSlab>(std::move(std::get<1>(
Contents))),
260 llvm::make_unique<RelationSlab>(std::move(std::get<2>(
Contents))),
std::tuple< SymbolSlab, RefSlab, RelationSlab > SlabTuple
bool CollectMacro
Collect macros.
Some operations such as code completion produce a set of candidates.
SymbolCollector::Options CollectorOpts
An efficient structure of storing large set of symbol references in memory.
This defines Dex - a symbol index implementation based on query iterators over symbol tokens...
IndexType
Select between in-memory index implementations, which have tradeoffs.
Represents a symbol occurrence in the source file.
SlabTuple indexMainDecls(ParsedAST &AST)
Retrieves symbols and refs of local top level decls in AST (i.e.
Collect declarations (symbols) from an AST.
void updatePreamble(PathRef Path, ASTContext &AST, std::shared_ptr< Preprocessor > PP, const CanonicalIncludes &Includes)
Update preamble symbols of file Path with all declarations in AST and macros in PP.
llvm::StringRef PathRef
A typedef to represent a ref to file path.
ArrayRef< Decl * > getLocalTopLevelDecls()
This function returns top-level decls present in the main file of the AST.
void vlog(const char *Fmt, Ts &&... Vals)
void updateMain(PathRef Path, ParsedAST &AST)
Update symbols and references from main file Path with indexMainDecls.
ASTContext & getASTContext()
Note that the returned ast will not contain decls from the preamble that were not deserialized during...
DuplicateHandling
How to handle duplicated symbols across multiple files.
Maps a definition location onto an #include file, based on a set of filename rules.
const CanonicalIncludes & getCanonicalIncludes() const
bool StoreAllDocumentation
If set to true, SymbolCollector will collect doc for all symbols.
MemIndex is a naive in-memory index suitable for a small set of symbols.
std::shared_ptr< Preprocessor > getPreprocessorPtr()
std::string Path
A typedef to represent a file path.
size_t size() const
Gets the number of symbols.
std::unique_ptr< SymbolIndex > buildIndex(IndexType, DuplicateHandling DuplicateHandle=DuplicateHandling::PickOne)
The index keeps the symbols alive.
Symbol mergeSymbol(const Symbol &L, const Symbol &R)
std::shared_ptr< SymbolCollector > Collector
Stores and provides access to parsed AST.
FileIndex(bool UseDex=true)
The class presents a C++ symbol, e.g.
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
RelationSlab takeRelations()
void setPreprocessor(std::shared_ptr< Preprocessor > PP) override
void update(PathRef Path, std::unique_ptr< SymbolSlab > Slab, std::unique_ptr< RefSlab > Refs, std::unique_ptr< RelationSlab > Relations, bool CountReferences)
Updates all symbols and refs in a file.
static SlabTuple indexSymbols(ASTContext &AST, std::shared_ptr< Preprocessor > PP, llvm::ArrayRef< Decl *> DeclsToIndex, const CanonicalIncludes &Includes, bool IsIndexMainAST)
const CanonicalIncludes * Includes
If set, this is used to map symbol #include path to a potentially different #include path...
void reset(std::unique_ptr< SymbolIndex >)
SlabTuple indexHeaderSymbols(ASTContext &AST, std::shared_ptr< Preprocessor > PP, const CanonicalIncludes &Includes)
Idex declarations from AST and macros from PP that are declared in included headers.
RefKind RefFilter
The symbol ref kinds that will be collected.