clang-tools 8.0.0

Dex.cpp
Go to the documentation of this file.
1 //===--- Dex.cpp - Dex Symbol Index Implementation --------------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9
10 #include "Dex.h"
11 #include "FileDistance.h"
12 #include "FuzzyMatch.h"
13 #include "Logger.h"
14 #include "Quality.h"
15 #include "Trace.h"
16 #include "index/Index.h"
17 #include "index/dex/Iterator.h"
18 #include "llvm/ADT/StringSet.h"
19 #include "llvm/Support/ScopedPrinter.h"
20 #include <algorithm>
21 #include <queue>
22
23 namespace clang {
24 namespace clangd {
25 namespace dex {
26
27 std::unique_ptr<SymbolIndex> Dex::build(SymbolSlab Symbols, RefSlab Refs) {
28 auto Size = Symbols.bytes() + Refs.bytes();
29 auto Data = std::make_pair(std::move(Symbols), std::move(Refs));
30 return llvm::make_unique<Dex>(Data.first, Data.second, std::move(Data), Size);
31 }
32
33 namespace {
34
35 // Mark symbols which are can be used for code completion.
36 const Token RestrictedForCodeCompletion =
37 Token(Token::Kind::Sentinel, "Restricted For Code Completion");
38
39 // Returns the tokens which are given symbol's characteristics. Currently, the
40 // generated tokens only contain fuzzy matching trigrams and symbol's scope,
41 // but in the future this will also return path proximity tokens and other
42 // types of tokens such as symbol type (if applicable).
43 // Returns the tokens which are given symbols's characteristics. For example,
44 // trigrams and scopes.
45 // FIXME(kbobyrev): Support more token types:
46 // * Types
47 // * Namespace proximity
48 std::vector<Token> generateSearchTokens(const Symbol &Sym) {
49 std::vector<Token> Result = generateIdentifierTrigrams(Sym.Name);
50 Result.emplace_back(Token::Kind::Scope, Sym.Scope);
51 // Skip token generation for symbols with unknown declaration location.
52 if (!llvm::StringRef(Sym.CanonicalDeclaration.FileURI).empty())
53 for (const auto &ProximityURI :
54 generateProximityURIs(Sym.CanonicalDeclaration.FileURI))
55 Result.emplace_back(Token::Kind::ProximityURI, ProximityURI);
56 if (Sym.Flags & Symbol::IndexedForCodeCompletion)
57 Result.emplace_back(RestrictedForCodeCompletion);
58 return Result;
59 }
60
61 // Constructs BOOST iterators for Path Proximities.
62 std::unique_ptr<Iterator> createFileProximityIterator(
63 llvm::ArrayRef<std::string> ProximityPaths,
64 const llvm::DenseMap<Token, PostingList> &InvertedIndex,
65 const Corpus &Corpus) {
66 std::vector<std::unique_ptr<Iterator>> BoostingIterators;
67 // Deduplicate parent URIs extracted from the ProximityPaths.
68 llvm::StringSet<> ParentURIs;
69 llvm::StringMap<SourceParams> Sources;
70 for (const auto &Path : ProximityPaths) {
71 Sources[Path] = SourceParams();
72 auto PathURI = URI::create(Path);
73 const auto PathProximityURIs = generateProximityURIs(PathURI.toString());
74 for (const auto &ProximityURI : PathProximityURIs)
75 ParentURIs.insert(ProximityURI);
76 }
77 // Use SymbolRelevanceSignals for symbol relevance evaluation: use defaults
78 // for all parameters except for Proximity Path distance signal.
79 SymbolRelevanceSignals PathProximitySignals;
80 // DistanceCalculator will find the shortest distance from ProximityPaths to
81 // any URI extracted from the ProximityPaths.
82 URIDistance DistanceCalculator(Sources);
83 PathProximitySignals.FileProximityMatch = &DistanceCalculator;
84 // Try to build BOOST iterator for each Proximity Path provided by
85 // ProximityPaths. Boosting factor should depend on the distance to the
86 // Proximity Path: the closer processed path is, the higher boosting factor.
87 for (const auto &ParentURI : ParentURIs.keys()) {
88 Token Tok(Token::Kind::ProximityURI, ParentURI);
89 const auto It = InvertedIndex.find(Tok);
90 if (It != InvertedIndex.end()) {
91 // FIXME(kbobyrev): Append LIMIT on top of every BOOST iterator.
92 PathProximitySignals.SymbolURI = ParentURI;
93 BoostingIterators.push_back(Corpus.boost(
94 It->second.iterator(&It->first), PathProximitySignals.evaluate()));
95 }
96 }
97 BoostingIterators.push_back(Corpus.all());
98 return Corpus.unionOf(std::move(BoostingIterators));
99 }
100
101 } // namespace
102
103 void Dex::buildIndex() {
104 this->Corpus = dex::Corpus(Symbols.size());
105 std::vector<std::pair<float, const Symbol *>> ScoredSymbols(Symbols.size());
106
107 for (size_t I = 0; I < Symbols.size(); ++I) {
108 const Symbol *Sym = Symbols[I];
109 LookupTable[Sym->ID] = Sym;
110 ScoredSymbols[I] = {quality(*Sym), Sym};
111 }
112
113 // Symbols are sorted by symbol qualities so that items in the posting lists
114 // are stored in the descending order of symbol quality.
115 llvm::sort(ScoredSymbols, std::greater<std::pair<float, const Symbol *>>());
116
117 // SymbolQuality was empty up until now.
118 SymbolQuality.resize(Symbols.size());
119 // Populate internal storage using Symbol + Score pairs.
120 for (size_t I = 0; I < ScoredSymbols.size(); ++I) {
121 SymbolQuality[I] = ScoredSymbols[I].first;
122 Symbols[I] = ScoredSymbols[I].second;
123 }
124
125 // Populate TempInvertedIndex with lists for index symbols.
126 llvm::DenseMap<Token, std::vector<DocID>> TempInvertedIndex;
127 for (DocID SymbolRank = 0; SymbolRank < Symbols.size(); ++SymbolRank) {
128 const auto *Sym = Symbols[SymbolRank];
129 for (const auto &Token : generateSearchTokens(*Sym))
130 TempInvertedIndex[Token].push_back(SymbolRank);
131 }
132
133 // Convert lists of items to posting lists.
134 for (const auto &TokenToPostingList : TempInvertedIndex)
135 InvertedIndex.insert(
136 {TokenToPostingList.first, PostingList(TokenToPostingList.second)});
137 }
138
139 std::unique_ptr<Iterator> Dex::iterator(const Token &Tok) const {
140 auto It = InvertedIndex.find(Tok);
141 return It == InvertedIndex.end() ? Corpus.none()
142 : It->second.iterator(&It->first);
143 }
144
145 /// Constructs iterators over tokens extracted from the query and exhausts it
146 /// while applying Callback to each symbol in the order of decreasing quality
147 /// of the matched symbols.
148 bool Dex::fuzzyFind(const FuzzyFindRequest &Req,
149 llvm::function_ref<void(const Symbol &)> Callback) const {
150 assert(!StringRef(Req.Query).contains("::") &&
151 "There must be no :: in query.");
152 trace::Span Tracer("Dex fuzzyFind");
153 FuzzyMatcher Filter(Req.Query);
154 // For short queries we use specialized trigrams that don't yield all results.
155 // Prevent clients from postfiltering them for longer queries.
156 bool More = !Req.Query.empty() && Req.Query.size() < 3;
157
158 std::vector<std::unique_ptr<Iterator>> Criteria;
159 const auto TrigramTokens = generateQueryTrigrams(Req.Query);
160
161 // Generate query trigrams and construct AND iterator over all query
162 // trigrams.
163 std::vector<std::unique_ptr<Iterator>> TrigramIterators;
164 for (const auto &Trigram : TrigramTokens)
165 TrigramIterators.push_back(iterator(Trigram));
166 Criteria.push_back(Corpus.intersect(move(TrigramIterators)));
167
168 // Generate scope tokens for search query.
169 std::vector<std::unique_ptr<Iterator>> ScopeIterators;
170 for (const auto &Scope : Req.Scopes)
171 ScopeIterators.push_back(iterator(Token(Token::Kind::Scope, Scope)));
172 if (Req.AnyScope)
173 ScopeIterators.push_back(
174 Corpus.boost(Corpus.all(), ScopeIterators.empty() ? 1.0 : 0.2));
175 Criteria.push_back(Corpus.unionOf(move(ScopeIterators)));
176
177 // Add proximity paths boosting (all symbols, some boosted).
178 Criteria.push_back(
179 createFileProximityIterator(Req.ProximityPaths, InvertedIndex, Corpus));
180
181 if (Req.RestrictForCodeCompletion)
182 Criteria.push_back(iterator(RestrictedForCodeCompletion));
183
184 // Use TRUE iterator if both trigrams and scopes from the query are not
185 // present in the symbol index.
186 auto Root = Corpus.intersect(move(Criteria));
187 // Retrieve more items than it was requested: some of the items with high
188 // final score might not be retrieved otherwise.
189 // FIXME(kbobyrev): Tune this ratio.
190 if (Req.Limit)
191 Root = Corpus.limit(move(Root), *Req.Limit * 100);
192 SPAN_ATTACH(Tracer, "query", llvm::to_string(*Root));
193 vlog("Dex query tree: {0}", *Root);
194
195 using IDAndScore = std::pair<DocID, float>;
196 std::vector<IDAndScore> IDAndScores = consume(*Root);
197
198 auto Compare = [](const IDAndScore &LHS, const IDAndScore &RHS) {
199 return LHS.second > RHS.second;
200 };
201 TopN<IDAndScore, decltype(Compare)> Top(
202 Req.Limit ? *Req.Limit : std::numeric_limits<size_t>::max(), Compare);
203 for (const auto &IDAndScore : IDAndScores) {
204 const DocID SymbolDocID = IDAndScore.first;
205 const auto *Sym = Symbols[SymbolDocID];
206 const llvm::Optional<float> Score = Filter.match(Sym->Name);
207 if (!Score)
208 continue;
209 // Combine Fuzzy Matching score, precomputed symbol quality and boosting
210 // score for a cumulative final symbol score.
211 const float FinalScore =
212 (*Score) * SymbolQuality[SymbolDocID] * IDAndScore.second;
213 // If Top.push(...) returns true, it means that it had to pop an item. In
214 // this case, it is possible to retrieve more symbols.
215 if (Top.push({SymbolDocID, FinalScore}))
216 More = true;
217 }
218
219 // Apply callback to the top Req.Limit items in the descending
220 // order of cumulative score.
221 for (const auto &Item : std::move(Top).items())
222 Callback(*Symbols[Item.first]);
223 return More;
224 }
225
226 void Dex::lookup(const LookupRequest &Req,
227 llvm::function_ref<void(const Symbol &)> Callback) const {
228 trace::Span Tracer("Dex lookup");
229 for (const auto &ID : Req.IDs) {
230 auto I = LookupTable.find(ID);
231 if (I != LookupTable.end())
232 Callback(*I->second);
233 }
234 }
235
236 void Dex::refs(const RefsRequest &Req,
237 llvm::function_ref<void(const Ref &)> Callback) const {
238 trace::Span Tracer("Dex refs");
239 uint32_t Remaining =
240 Req.Limit.getValueOr(std::numeric_limits<uint32_t>::max());
241 for (const auto &ID : Req.IDs)
242 for (const auto &Ref : Refs.lookup(ID)) {
243 if (Remaining > 0 && static_cast<int>(Req.Filter & Ref.Kind)) {
244 --Remaining;
245 Callback(Ref);
246 }
247 }
248 }
249
250 size_t Dex::estimateMemoryUsage() const {
251 size_t Bytes = Symbols.size() * sizeof(const Symbol *);
252 Bytes += SymbolQuality.size() * sizeof(float);
253 Bytes += LookupTable.getMemorySize();
254 Bytes += InvertedIndex.getMemorySize();
255 for (const auto &TokenToPostingList : InvertedIndex)
256 Bytes += TokenToPostingList.second.bytes();
257 Bytes += Refs.getMemorySize();
258 return Bytes + BackingDataSize;
259 }
260
261 std::vector<std::string> generateProximityURIs(llvm::StringRef URIPath) {
262 std::vector<std::string> Result;
263 auto ParsedURI = URI::parse(URIPath);
264 assert(ParsedURI &&
265 "Non-empty argument of generateProximityURIs() should be a valid "
266 "URI.");
267 llvm::StringRef Body = ParsedURI->body();
268 // FIXME(kbobyrev): Currently, this is a heuristic which defines the maximum
269 // size of resulting vector. Some projects might want to have higher limit if
270 // the file hierarchy is deeper. For the generic case, it would be useful to
271 // calculate Limit in the index build stage by calculating the maximum depth
272 // of the project source tree at runtime.
273 size_t Limit = 5;
274 // Insert original URI before the loop: this would save a redundant iteration
275 // with a URI parse.
276 Result.emplace_back(ParsedURI->toString());
277 while (!Body.empty() && --Limit > 0) {
278 // FIXME(kbobyrev): Parsing and encoding path to URIs is not necessary and
279 // could be optimized.
280 Body = llvm::sys::path::parent_path(Body, llvm::sys::path::Style::posix);
281 URI TokenURI(ParsedURI->scheme(), ParsedURI->authority(), Body);
282 if (!Body.empty())
283 Result.emplace_back(TokenURI.toString());
284 }
285 return Result;
286 }
287
288 } // namespace dex
289 } // namespace clangd
290 } // namespace clang
clang::clangd::SymbolSlab::bytes
size_t bytes() const
Definition: Index.h:307
clang::clangd::SymbolSlab
Definition: Index.h:292
clang::clangd::RefsRequest::IDs
llvm::DenseSet< SymbolID > IDs
Definition: Index.h:477
clang::clangd::FuzzyFindRequest::AnyScope
bool AnyScope
If set to true, allow symbols from any scope.
Definition: Index.h:449
Trace.h
clang::clangd::dex::Corpus::intersect
std::unique_ptr< Iterator > intersect(std::vector< std::unique_ptr< Iterator >> Children) const
Returns AND Iterator which performs the intersection of the PostingLists of its children.
Definition: Iterator.cpp:359
clang::clangd::dex::Dex::refs
void refs(const RefsRequest &Req, llvm::function_ref< void(const Ref &)> Callback) const override
Finds all occurrences (e.g.
Definition: Dex.cpp:236
clang::clangd::FuzzyFindRequest::RestrictForCodeCompletion
bool RestrictForCodeCompletion
If set to true, only symbols for completion support will be considered.
Definition: Index.h:454
clang::clangd::RefSlab
Definition: Index.h:388
Dex.h
This defines Dex - a symbol index implementation based on query iterators over symbol tokens...
Index.h
clang::clangd::dex::PostingList
PostingList is the storage of DocIDs which can be inserted to the Query Tree as a leaf by constructin...
Definition: PostingList.h:60
clang::clangd::FuzzyMatcher
Definition: FuzzyMatch.h:72
clang::clangd::LookupRequest::IDs
llvm::DenseSet< SymbolID > IDs
Definition: Index.h:473
clang::clangd::Ref
Definition: Index.h:373
clang::clangd::RefsRequest
Definition: Index.h:476
clang::clangd::Callback
llvm::unique_function< void(llvm::Expected< T >)> Callback
A Callback<T> is a void function that accepts Expected<T>.
Definition: Function.h:29
clang::clangd::Symbol::Scope
llvm::StringRef Scope
Definition: Index.h:168
clang::clangd::FuzzyFindRequest
Definition: Index.h:436
clang::clangd::SourceParams
Definition: FileDistance.h:63
clang::clangd::dex::generateIdentifierTrigrams
std::vector< Token > generateIdentifierTrigrams(llvm::StringRef Identifier)
Returns list of unique fuzzy-search trigrams from unqualified symbol.
Definition: Trigram.cpp:24
clang::clangd::dex::Dex::fuzzyFind
bool fuzzyFind(const FuzzyFindRequest &Req, llvm::function_ref< void(const Symbol &)> Callback) const override
Constructs iterators over tokens extracted from the query and exhausts it while applying Callback to ...
Definition: Dex.cpp:148
clang::clangd::vlog
void vlog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:68
clang::clangd::FuzzyFindRequest::Scopes
std::vector< std::string > Scopes
If this is non-empty, symbols must be in at least one of the scopes (e.g.
Definition: Index.h:446
clang::clangd::Symbol::ID
SymbolID ID
Definition: Index.h:162
clang::clangd::SymbolRelevanceSignals::evaluate
float evaluate() const
Definition: Quality.cpp:339
clang::clangd::dex::Token
A Token represents an attribute of a symbol, such as a particular trigram present in the name (used f...
Definition: Token.h:41
clang::clangd::RefsRequest::Filter
RefKind Filter
Definition: Index.h:478
clang::clangd::dex::consume
std::vector< std::pair< DocID, float > > consume(Iterator &It)
Advances the iterator until it is exhausted.
Definition: Iterator.cpp:351
clang::clangd::dex::Corpus::unionOf
std::unique_ptr< Iterator > unionOf(std::vector< std::unique_ptr< Iterator >> Children) const
Returns OR Iterator which performs the union of the PostingLists of its children. ...
Definition: Iterator.cpp:389
clang::clangd::SymbolRelevanceSignals::FileProximityMatch
URIDistance * FileProximityMatch
Definition: Quality.h:95
clang::clangd::Symbol::IndexedForCodeCompletion
Whether or not this symbol is meant to be used for the code completion.
Definition: Index.h:239
clang::clangd::dex::generateProximityURIs
std::vector< std::string > generateProximityURIs(llvm::StringRef URIPath)
Returns Search Token for a number of parent directories of given Path.
Definition: Dex.cpp:261
clang::clangd::dex::generateQueryTrigrams
std::vector< Token > generateQueryTrigrams(llvm::StringRef Query)
Returns list of unique fuzzy-search trigrams given a query.
Definition: Trigram.cpp:87
FileDistance.h
clang::clangd::Symbol::Flags
SymbolFlag Flags
Definition: Index.h:248
clang::clangd::dex::Corpus::limit
std::unique_ptr< Iterator > limit(std::unique_ptr< Iterator > Child, size_t Limit) const
Returns LIMIT iterator, which yields up to N elements of its child iterator.
Definition: Iterator.cpp:435
clang::clangd::dex::Dex::build
static std::unique_ptr< SymbolIndex > build(SymbolSlab, RefSlab)
Builds an index from slabs. The index takes ownership of the slab.
Definition: Dex.cpp:27
clang::clangd::dex::Token::Kind::Scope
Scope primitives, e.g.
clang::clangd::dex::Corpus::none
std::unique_ptr< Iterator > none() const
Returns FALSE Iterator which iterates over no documents.
Definition: Iterator.cpp:422
clang::clangd::Path
std::string Path
A typedef to represent a file path.
Definition: Path.h:21
clang::clangd::FuzzyFindRequest::Query
std::string Query
A query string for the fuzzy find.
Definition: Index.h:439
clang::clangd::FuzzyMatcher::match
llvm::Optional< float > match(llvm::StringRef Word)
Definition: FuzzyMatch.cpp:93
clang::clangd::Symbol::CanonicalDeclaration
SymbolLocation CanonicalDeclaration
Definition: Index.h:180
clang::clangd::TopN::push
bool push(value_type &&V)
Definition: Quality.h:156
clang::clangd::dex::DocID
uint32_t DocID
Symbol position in the list of all index symbols sorted by a pre-computed symbol quality.
Definition: Iterator.h:47
Quality.h
Result
llvm::Optional< llvm::Expected< tooling::AtomicChanges > > Result
Definition: ClangdServer.cpp:60
clang::clangd::dex::Dex::lookup
void lookup(const LookupRequest &Req, llvm::function_ref< void(const Symbol &)> Callback) const override
Looks up symbols with any of the given symbol IDs and applies Callback on each matched symbol...
Definition: Dex.cpp:226
clang::clangd::SymbolRelevanceSignals::SymbolURI
llvm::StringRef SymbolURI
These are used to calculate proximity between the index symbol and the query.
Definition: Quality.h:98
clang::clangd::URIDistance
Definition: FileDistance.h:93
Logger.h
clang::clangd::URI::create
static llvm::Expected< URI > create(llvm::StringRef AbsolutePath, llvm::StringRef Scheme)
Creates a URI for a file in the given scheme.
Definition: URI.cpp:188
clang::clangd::Symbol
Definition: Index.h:160
clang::clangd::LookupRequest
Definition: Index.h:472
clang
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
Definition: ChangeNamespace.cpp:18
clang::clangd::FuzzyFindRequest::ProximityPaths
std::vector< std::string > ProximityPaths
Contextually relevant files (e.g.
Definition: Index.h:457
clang::clangd::dex::Token::Kind::ProximityURI
Path Proximity URI to symbol declaration.
clang::clangd::Symbol::Name
llvm::StringRef Name
Definition: Index.h:166
clang::clangd::RefsRequest::Limit
llvm::Optional< uint32_t > Limit
If set, limit the number of refers returned from the index.
Definition: Index.h:482
Iterator.h
Symbol index queries consist of specific requirements for the requested symbol, such as high fuzzy ma...
clang::clangd::URI
A URI describes the location of a source file.
Definition: URI.h:29
clang::clangd::FuzzyFindRequest::Limit
llvm::Optional< uint32_t > Limit
The number of top candidates to return.
Definition: Index.h:452
clang::clangd::RefSlab::bytes
size_t bytes() const
Definition: Index.h:405
clang::clangd::dex::Token::Kind::Sentinel
Internal Token type for invalid/special tokens, e.g.
clang::clangd::URI::parse
static llvm::Expected< URI > parse(llvm::StringRef Uri)
Parse a URI string "<scheme>:[//<authority>/]<path>".
Definition: URI.cpp:166
clang::clangd::dex::Corpus::boost
std::unique_ptr< Iterator > boost(std::unique_ptr< Iterator > Child, float Factor) const
Returns BOOST iterator which multiplies the score of each item by given factor.
Definition: Iterator.cpp:426
clang::clangd::dex::Corpus::all
std::unique_ptr< Iterator > all() const
Returns TRUE Iterator which iterates over "virtual" PostingList containing all items in range [0...
Definition: Iterator.cpp:418
clang::clangd::dex::Dex::estimateMemoryUsage
size_t estimateMemoryUsage() const override
Returns estimated size of index (in bytes).
Definition: Dex.cpp:250
clang::clangd::dex::Corpus
Definition: Iterator.h:135
clang::clangd::trace::Span
Records an event whose duration is the lifetime of the Span object.
Definition: Trace.h:83
clang::clangd::SymbolLocation::FileURI
const char * FileURI
Definition: Index.h:72
clang::clangd::SymbolRelevanceSignals
Attributes of a symbol-query pair that affect how much we like it.
Definition: Quality.h:87
SPAN_ATTACH
#define SPAN_ATTACH(S, Name, Expr)
Attach a key-value pair to a Span event.
Definition: Trace.h:99
FuzzyMatch.h
clang::clangd::Ref::Kind
RefKind Kind
Definition: Index.h:376
clang::clangd::quality
float quality(const Symbol &S)
Definition: Index.cpp:69
clang::clangd::TopN
TopN<T> is a lossy container that preserves only the "best" N elements.
Definition: Quality.h:148