30 #include "clang/Basic/SourceLocation.h" 31 #include "clang/Basic/SourceManager.h" 32 #include "clang/Driver/Types.h" 33 #include "llvm/ADT/ArrayRef.h" 34 #include "llvm/ADT/DenseSet.h" 35 #include "llvm/ADT/Hashing.h" 36 #include "llvm/ADT/STLExtras.h" 37 #include "llvm/ADT/ScopeExit.h" 38 #include "llvm/ADT/StringMap.h" 39 #include "llvm/ADT/StringRef.h" 40 #include "llvm/ADT/StringSet.h" 41 #include "llvm/Support/Error.h" 42 #include "llvm/Support/Path.h" 43 #include "llvm/Support/Threading.h" 48 #include <condition_variable> 65 class URIToFileCache {
67 URIToFileCache(llvm::StringRef HintPath) : HintPath(HintPath) {}
69 llvm::StringRef resolve(llvm::StringRef FileURI) {
70 auto I = URIToPathCache.try_emplace(FileURI);
74 elog(
"Failed to parse URI {0}: {1}", FileURI, U.takeError());
75 assert(
false &&
"Failed to parse URI");
80 elog(
"Failed to resolve URI {0}: {1}", FileURI,
Path.takeError());
81 assert(
false &&
"Failed to resolve URI");
84 I.first->second = *
Path;
86 return I.first->second;
91 llvm::StringMap<std::string> URIToPathCache;
99 std::string FileURI = U.toString();
100 auto Entry = IG.try_emplace(FileURI).first;
101 auto &Node =
Entry->getValue();
102 Node = FullGraph.lookup(
Entry->getKey());
103 Node.URI =
Entry->getKey();
106 for (
auto &Include : Node.DirectIncludes) {
107 auto I = IG.try_emplace(Include).first;
108 I->getValue().URI = I->getKey();
109 Include = I->getKey();
118 llvm::SmallString<128> getAbsolutePath(
const tooling::CompileCommand &Cmd) {
119 llvm::SmallString<128> AbsolutePath;
120 if (llvm::sys::path::is_absolute(Cmd.Filename)) {
121 AbsolutePath = Cmd.Filename;
123 AbsolutePath = Cmd.Directory;
124 llvm::sys::path::append(AbsolutePath, Cmd.Filename);
125 llvm::sys::path::remove_dots(AbsolutePath,
true);
130 bool shardIsStale(
const LoadedShard &LS, llvm::vfs::FileSystem *
FS) {
131 auto Buf = FS->getBufferForFile(LS.AbsolutePath);
133 elog(
"Background-index: Couldn't read {0} to validate stored index: {1}",
134 LS.AbsolutePath, Buf.getError().message());
138 return digest(Buf->get()->getBuffer()) != LS.Digest;
148 CDB(CDB), BackgroundContext(std::move(BackgroundContext)),
149 Rebuilder(this, &IndexedSymbols, ThreadPoolSize),
150 IndexStorageFactory(std::move(IndexStorageFactory)),
152 CDB.watch([&](const std::vector<std::string> &ChangedFiles) {
155 assert(ThreadPoolSize > 0 &&
"Thread pool size can't be zero.");
156 assert(this->IndexStorageFactory &&
"Storage factory can not be null!");
157 for (
unsigned I = 0; I < ThreadPoolSize; ++I) {
158 ThreadPool.
runAsync(
"background-worker-" + llvm::Twine(I + 1), [
this] {
160 Queue.
work([&] { Rebuilder.
idle(); });
171 const std::vector<std::string> &ChangedFiles) {
175 log(
"Enqueueing {0} commands for indexing", ChangedFiles.size());
176 SPAN_ATTACH(Tracer,
"files", int64_t(ChangedFiles.size()));
178 auto NeedsReIndexing = loadProject(std::move(ChangedFiles));
180 std::shuffle(NeedsReIndexing.begin(), NeedsReIndexing.end(),
181 std::mt19937(std::random_device{}()));
182 std::vector<BackgroundQueue::Task> Tasks;
183 Tasks.reserve(NeedsReIndexing.size());
184 for (
auto &Cmd : NeedsReIndexing)
185 Tasks.push_back(indexFileTask(std::move(Cmd)));
186 Queue.
append(std::move(Tasks));
190 T.ThreadPri = llvm::ThreadPriority::Default;
195 Path = llvm::sys::path::filename(Path);
196 return Path.drop_back(llvm::sys::path::extension(Path).size());
200 BackgroundIndex::indexFileTask(tooling::CompileCommand Cmd) {
204 const std::string
FileName = Cmd.Filename;
205 if (
auto Error = index(std::move(Cmd)))
206 elog(
"Indexing {0} failed: {1}", FileName, std::move(
Error));
214 namespace types = clang::driver::types;
216 types::lookupTypeForExtension(llvm::sys::path::extension(Path).substr(1));
218 if (
Type != types::TY_INVALID && types::onlyPrecompileType(
Type))
225 void BackgroundIndex::update(
227 const llvm::StringMap<ShardVersion> &ShardVersionsSnapshot,
231 llvm::DenseSet<const Symbol *>
Symbols;
232 llvm::DenseSet<const Ref *>
Refs;
233 llvm::DenseSet<const Relation *>
Relations;
236 llvm::StringMap<File>
Files;
237 URIToFileCache URICache(MainFile);
238 for (
const auto &IndexIt : *Index.
Sources) {
239 const auto &IGN = IndexIt.getValue();
242 const auto AbsPath = URICache.resolve(IGN.URI);
243 const auto DigestIt = ShardVersionsSnapshot.find(AbsPath);
245 if (DigestIt == ShardVersionsSnapshot.end() ||
246 DigestIt->getValue().Digest != IGN.Digest ||
247 (DigestIt->getValue().HadErrors && !HadErrors))
248 Files.try_emplace(AbsPath).first->getValue().Digest = IGN.Digest;
251 llvm::DenseMap<SymbolID, File *> SymbolIDToFile;
252 for (
const auto &Sym : *Index.
Symbols) {
253 if (Sym.CanonicalDeclaration) {
254 auto DeclPath = URICache.resolve(Sym.CanonicalDeclaration.FileURI);
255 const auto FileIt = Files.find(DeclPath);
256 if (FileIt != Files.end()) {
257 FileIt->second.Symbols.insert(&Sym);
258 SymbolIDToFile[Sym.ID] = &FileIt->second;
265 if (Sym.Definition &&
266 Sym.Definition.FileURI != Sym.CanonicalDeclaration.FileURI) {
267 auto DefPath = URICache.resolve(Sym.Definition.FileURI);
268 const auto FileIt = Files.find(DefPath);
269 if (FileIt != Files.end())
270 FileIt->second.Symbols.insert(&Sym);
273 llvm::DenseMap<const Ref *, SymbolID> RefToIDs;
274 for (
const auto &SymRefs : *Index.
Refs) {
275 for (
const auto &R : SymRefs.second) {
276 auto Path = URICache.resolve(R.Location.FileURI);
277 const auto FileIt = Files.find(
Path);
278 if (FileIt != Files.end()) {
279 auto &F = FileIt->getValue();
280 RefToIDs[&R] = SymRefs.first;
285 for (
const auto &Rel : *Index.
Relations) {
286 const auto FileIt = SymbolIDToFile.find(Rel.Subject);
287 if (FileIt != SymbolIDToFile.end())
288 FileIt->second->Relations.insert(&Rel);
292 for (
const auto &FileIt : Files) {
293 llvm::StringRef
Path = FileIt.getKey();
297 for (
const auto *S : FileIt.second.Symbols)
299 for (
const auto *R : FileIt.second.Refs)
300 Refs.
insert(RefToIDs[R], *R);
301 for (
const auto *Rel : FileIt.second.Relations)
303 auto SS = llvm::make_unique<SymbolSlab>(std::move(Syms).build());
304 auto RS = llvm::make_unique<RefSlab>(std::move(Refs).build());
305 auto RelS = llvm::make_unique<RelationSlab>(std::move(Relations).build());
306 auto IG = llvm::make_unique<IncludeGraph>(
315 Shard.
Refs = RS.get();
321 if (Path == MainFile)
322 Shard.
Cmd = Index.
Cmd.getPointer();
325 elog(
"Failed to write background-index shard for file {0}: {1}", Path,
329 std::lock_guard<std::mutex> Lock(ShardVersionsMu);
330 auto Hash = FileIt.second.Digest;
331 auto DigestIt = ShardVersions.try_emplace(Path);
332 ShardVersion &SV = DigestIt.first->second;
335 if (!DigestIt.second && SV.Digest == Hash && SV.HadErrors && !HadErrors)
338 SV.HadErrors = HadErrors;
343 IndexedSymbols.update(Path, std::move(SS), std::move(RS), std::move(RelS),
349 llvm::Error BackgroundIndex::index(tooling::CompileCommand Cmd) {
352 auto AbsolutePath = getAbsolutePath(Cmd);
355 auto Buf = FS->getBufferForFile(AbsolutePath);
357 return llvm::errorCodeToError(Buf.getError());
358 auto Hash =
digest(Buf->get()->getBuffer());
361 llvm::StringMap<ShardVersion> ShardVersionsSnapshot;
363 std::lock_guard<std::mutex> Lock(ShardVersionsMu);
364 ShardVersionsSnapshot = ShardVersions;
367 vlog(
"Indexing {0} (digest:={1})", Cmd.Filename, llvm::toHex(Hash));
369 Inputs.
FS = std::move(FS);
370 Inputs.
FS->setCurrentWorkingDirectory(Cmd.Directory);
374 return llvm::createStringError(llvm::inconvertibleErrorCode(),
375 "Couldn't build compiler invocation");
378 std::move(*Buf), Inputs.
FS, IgnoreDiags);
380 return llvm::createStringError(llvm::inconvertibleErrorCode(),
381 "Couldn't build compiler instance");
386 IndexOpts.
FileFilter = [&ShardVersionsSnapshot](
const SourceManager &SM,
388 const auto *F = SM.getFileEntryForID(FID);
397 auto D = ShardVersionsSnapshot.find(*AbsPath);
398 if (
D != ShardVersionsSnapshot.end() &&
D->second.Digest == Digest &&
399 !
D->second.HadErrors)
416 const FrontendInputFile &Input = Clang->getFrontendOpts().Inputs.front();
417 if (!
Action->BeginSourceFile(*Clang, Input))
418 return llvm::createStringError(llvm::inconvertibleErrorCode(),
419 "BeginSourceFile() failed");
427 "Symbols, Refs and Sources must be set.");
429 log(
"Indexed {0} ({1} symbols, {2} refs, {3} files)",
436 bool HadErrors = Clang->hasDiagnostics() &&
437 Clang->getDiagnostics().hasUncompilableErrorOccurred();
439 log(
"Failed to compile {0}, index may be incomplete", AbsolutePath);
440 for (
auto &It : *Index.
Sources)
443 update(AbsolutePath, std::move(Index), ShardVersionsSnapshot, HadErrors);
445 Rebuilder.indexedTU();
446 return llvm::Error::success();
452 std::vector<tooling::CompileCommand>
453 BackgroundIndex::loadProject(std::vector<std::string> MainFiles) {
454 std::vector<tooling::CompileCommand> NeedsReIndexing;
456 Rebuilder.startLoading();
458 const std::vector<LoadedShard>
Result =
460 size_t LoadedShards = 0;
463 std::lock_guard<std::mutex> Lock(ShardVersionsMu);
464 for (
auto &LS : Result) {
469 ? llvm::make_unique<SymbolSlab>(std::move(*LS.Shard->Symbols))
471 auto RS = LS.Shard->Refs
472 ? llvm::make_unique<RefSlab>(std::move(*LS.Shard->Refs))
476 ? llvm::make_unique<RelationSlab>(std::move(*LS.Shard->Relations))
478 ShardVersion &SV = ShardVersions[LS.AbsolutePath];
479 SV.Digest = LS.Digest;
480 SV.HadErrors = LS.HadErrors;
483 IndexedSymbols.update(LS.AbsolutePath, std::move(SS), std::move(RS),
484 std::move(RelS), LS.CountReferences);
487 Rebuilder.loadedShard(LoadedShards);
488 Rebuilder.doneLoading();
491 llvm::DenseSet<PathRef> TUsToIndex;
494 for (
auto &LS : Result) {
495 if (!shardIsStale(LS, FS.get()))
497 PathRef TUForFile = LS.DependentTU;
498 assert(!TUForFile.empty() &&
"File without a TU!");
505 TUsToIndex.insert(TUForFile);
508 for (
PathRef TU : TUsToIndex) {
509 auto Cmd = CDB.getCompileCommand(TU);
512 NeedsReIndexing.emplace_back(std::move(*Cmd));
515 return NeedsReIndexing;
const tooling::CompileCommand * Cmd
llvm::Optional< SymbolSlab > Symbols
An immutable symbol container that stores a set of symbols.
llvm::Optional< tooling::CompileCommand > Cmd
Some operations such as code completion produce a set of candidates.
llvm::unique_function< BackgroundIndexStorage *(PathRef)> Factory
An efficient structure of storing large set of symbol references in memory.
void boostRelated(llvm::StringRef Path)
Boosts priority of indexing related to Path.
std::array< uint8_t, 8 > FileDigest
static llvm::StringRef filenameWithoutExtension(llvm::StringRef Path)
llvm::StringRef PathRef
A typedef to represent a ref to file path.
void insert(const Symbol &S)
Adds a symbol, overwriting any existing one with the same ID.
std::unique_ptr< CompilerInstance > prepareCompilerInstance(std::unique_ptr< clang::CompilerInvocation > CI, const PrecompiledPreamble *Preamble, std::unique_ptr< llvm::MemoryBuffer > Buffer, llvm::IntrusiveRefCntPtr< llvm::vfs::FileSystem > VFS, DiagnosticConsumer &DiagsClient)
llvm::Optional< RelationSlab > Relations
void vlog(const char *Fmt, Ts &&... Vals)
void elog(const char *Fmt, Ts &&... Vals)
llvm::Optional< IncludeGraph > Sources
const SymbolSlab * Symbols
MockFSProvider FSProvider
SymbolSlab::Builder is a mutable container that can 'freeze' to SymbolSlab.
void boost(llvm::StringRef Tag, unsigned NewPriority)
Provides compilation arguments used for parsing C and C++ files.
llvm::StringMap< IncludeGraphNode > IncludeGraph
Context clone() const
Clone this context object.
void runAsync(const llvm::Twine &Name, llvm::unique_function< void()> Action)
llvm::unique_function< void()> Action
MemIndex is a naive in-memory index suitable for a small set of symbols.
void log(const char *Fmt, Ts &&... Vals)
std::string Path
A typedef to represent a file path.
std::vector< LoadedShard > loadIndexShards(llvm::ArrayRef< Path > MainFiles, BackgroundIndexStorage::Factory &IndexStorageFactory, const GlobalCompilationDatabase &CDB)
Loads all shards for the TU MainFile from Storage.
std::unique_ptr< CompilerInvocation > buildCompilerInvocation(const ParseInputs &Inputs)
Builds compiler invocation that could be used to build AST or preamble.
void enqueue(const std::vector< std::string > &ChangedFiles)
void insert(const Relation &R)
Adds a relation to the slab.
const IncludeGraph * Sources
llvm::Optional< FileDigest > digestFile(const SourceManager &SM, FileID FID)
A context is an immutable container for per-request data that must be propagated through layers that ...
RelationSlab::Builder is a mutable container that can 'freeze' to RelationSlab.
std::function< bool(const SourceManager &, FileID)> FileFilter
If this is set, only collect symbols/references from a file if FileFilter(SM, FID) is true...
RefSlab::Builder is a mutable container that can 'freeze' to RefSlab.
WithContext replaces Context::current() with a provided scope.
FileDigest digest(llvm::StringRef Content)
virtual llvm::Error storeShard(llvm::StringRef ShardIdentifier, IndexFileOut Shard) const =0
const RelationSlab * Relations
llvm::Optional< RefSlab > Refs
BackgroundIndex(Context BackgroundContext, const FileSystemProvider &, const GlobalCompilationDatabase &CDB, BackgroundIndexStorage::Factory IndexStorageFactory, size_t ThreadPoolSize=llvm::heavyweight_hardware_concurrency())
If BuildIndexPeriodMs is greater than 0, the symbol index will only be rebuilt periodically (one per ...
static llvm::Expected< URI > create(llvm::StringRef AbsolutePath, llvm::StringRef Scheme)
Creates a URI for a file in the given scheme.
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
std::unique_ptr< FrontendAction > createStaticIndexingAction(SymbolCollector::Options Opts, std::function< void(SymbolSlab)> SymbolsCallback, std::function< void(RefSlab)> RefsCallback, std::function< void(RelationSlab)> RelationsCallback, std::function< void(IncludeGraph)> IncludeGraphCallback)
llvm::Optional< std::string > getCanonicalPath(const FileEntry *F, const SourceManager &SourceMgr)
Get the canonical path of F.
llvm::Optional< llvm::Expected< tooling::AtomicChanges > > Result
void insert(const SymbolID &ID, const Ref &S)
Adds a ref to the slab. Deep copy: Strings will be owned by the slab.
static llvm::Expected< std::string > resolve(const URI &U, llvm::StringRef HintPath="")
Resolves the absolute path of U.
static llvm::Expected< URI > parse(llvm::StringRef Uri)
Parse a URI string "<scheme>:[//<authority>/]<path>".
void append(std::vector< Task >)
A work item on the thread pool's queue.
Records an event whose duration is the lifetime of the Span object.
#define SPAN_ATTACH(S, Name, Expr)
Attach a key-value pair to a Span event.
llvm::StringMap< std::string > Files
void work(std::function< void()> OnIdle=nullptr)
const SymbolIndex * Index