30 #include "clang/Basic/SourceLocation.h" 31 #include "clang/Basic/SourceManager.h" 32 #include "clang/Driver/Types.h" 33 #include "llvm/ADT/ArrayRef.h" 34 #include "llvm/ADT/DenseSet.h" 35 #include "llvm/ADT/Hashing.h" 36 #include "llvm/ADT/STLExtras.h" 37 #include "llvm/ADT/ScopeExit.h" 38 #include "llvm/ADT/StringMap.h" 39 #include "llvm/ADT/StringRef.h" 40 #include "llvm/ADT/StringSet.h" 41 #include "llvm/Support/Error.h" 42 #include "llvm/Support/Path.h" 43 #include "llvm/Support/Threading.h" 48 #include <condition_variable> 65 class URIToFileCache {
67 URIToFileCache(llvm::StringRef HintPath) : HintPath(HintPath) {}
69 llvm::StringRef resolve(llvm::StringRef FileURI) {
70 auto I = URIToPathCache.try_emplace(FileURI);
74 elog(
"Failed to resolve URI {0}: {1}", FileURI,
Path.takeError());
75 assert(
false &&
"Failed to resolve URI");
78 I.first->second = *
Path;
80 return I.first->second;
85 llvm::StringMap<std::string> URIToPathCache;
93 std::string FileURI = U.toString();
94 auto Entry = IG.try_emplace(FileURI).first;
95 auto &Node =
Entry->getValue();
96 Node = FullGraph.lookup(
Entry->getKey());
97 Node.URI =
Entry->getKey();
100 for (
auto &Include : Node.DirectIncludes) {
101 auto I = IG.try_emplace(Include).first;
102 I->getValue().URI = I->getKey();
103 Include = I->getKey();
112 llvm::SmallString<128> getAbsolutePath(
const tooling::CompileCommand &Cmd) {
113 llvm::SmallString<128> AbsolutePath;
114 if (llvm::sys::path::is_absolute(Cmd.Filename)) {
115 AbsolutePath = Cmd.Filename;
117 AbsolutePath = Cmd.Directory;
118 llvm::sys::path::append(AbsolutePath, Cmd.Filename);
119 llvm::sys::path::remove_dots(AbsolutePath,
true);
124 bool shardIsStale(
const LoadedShard &LS, llvm::vfs::FileSystem *
FS) {
125 auto Buf = FS->getBufferForFile(LS.AbsolutePath);
127 elog(
"Background-index: Couldn't read {0} to validate stored index: {1}",
128 LS.AbsolutePath, Buf.getError().message());
132 return digest(Buf->get()->getBuffer()) != LS.Digest;
142 CDB(CDB), BackgroundContext(std::move(BackgroundContext)),
143 Rebuilder(this, &IndexedSymbols, ThreadPoolSize),
144 IndexStorageFactory(std::move(IndexStorageFactory)),
146 CDB.watch([&](const std::vector<std::string> &ChangedFiles) {
149 assert(ThreadPoolSize > 0 &&
"Thread pool size can't be zero.");
150 assert(this->IndexStorageFactory &&
"Storage factory can not be null!");
151 for (
unsigned I = 0; I < ThreadPoolSize; ++I) {
152 ThreadPool.
runAsync(
"background-worker-" + llvm::Twine(I + 1), [
this] {
154 Queue.
work([&] { Rebuilder.
idle(); });
165 const std::vector<std::string> &ChangedFiles) {
169 log(
"Enqueueing {0} commands for indexing", ChangedFiles.size());
170 SPAN_ATTACH(Tracer,
"files", int64_t(ChangedFiles.size()));
172 auto NeedsReIndexing = loadProject(std::move(ChangedFiles));
174 std::shuffle(NeedsReIndexing.begin(), NeedsReIndexing.end(),
175 std::mt19937(std::random_device{}()));
176 std::vector<BackgroundQueue::Task> Tasks;
177 Tasks.reserve(NeedsReIndexing.size());
178 for (
auto &Cmd : NeedsReIndexing)
179 Tasks.push_back(indexFileTask(std::move(Cmd)));
180 Queue.
append(std::move(Tasks));
184 T.ThreadPri = llvm::ThreadPriority::Default;
189 Path = llvm::sys::path::filename(Path);
190 return Path.drop_back(llvm::sys::path::extension(Path).size());
194 BackgroundIndex::indexFileTask(tooling::CompileCommand Cmd) {
198 const std::string
FileName = Cmd.Filename;
199 if (
auto Error = index(std::move(Cmd)))
200 elog(
"Indexing {0} failed: {1}", FileName, std::move(
Error));
215 void BackgroundIndex::update(
217 const llvm::StringMap<ShardVersion> &ShardVersionsSnapshot,
221 llvm::DenseSet<const Symbol *>
Symbols;
222 llvm::DenseSet<const Ref *>
Refs;
223 llvm::DenseSet<const Relation *> Relations;
226 llvm::StringMap<File>
Files;
227 URIToFileCache URICache(MainFile);
228 for (
const auto &IndexIt : *Index.
Sources) {
229 const auto &IGN = IndexIt.getValue();
232 const auto AbsPath = URICache.resolve(IGN.URI);
233 const auto DigestIt = ShardVersionsSnapshot.find(AbsPath);
235 if (DigestIt == ShardVersionsSnapshot.end() ||
236 DigestIt->getValue().Digest != IGN.Digest ||
237 (DigestIt->getValue().HadErrors && !HadErrors))
238 Files.try_emplace(AbsPath).first->getValue().Digest = IGN.Digest;
241 llvm::DenseMap<SymbolID, File *> SymbolIDToFile;
242 for (
const auto &Sym : *Index.
Symbols) {
243 if (Sym.CanonicalDeclaration) {
244 auto DeclPath = URICache.resolve(Sym.CanonicalDeclaration.FileURI);
245 const auto FileIt = Files.find(DeclPath);
246 if (FileIt != Files.end()) {
247 FileIt->second.Symbols.insert(&Sym);
248 SymbolIDToFile[Sym.ID] = &FileIt->second;
255 if (Sym.Definition &&
256 Sym.Definition.FileURI != Sym.CanonicalDeclaration.FileURI) {
257 auto DefPath = URICache.resolve(Sym.Definition.FileURI);
258 const auto FileIt = Files.find(DefPath);
259 if (FileIt != Files.end())
260 FileIt->second.Symbols.insert(&Sym);
263 llvm::DenseMap<const Ref *, SymbolID> RefToIDs;
264 for (
const auto &SymRefs : *Index.
Refs) {
265 for (
const auto &R : SymRefs.second) {
266 auto Path = URICache.resolve(R.Location.FileURI);
267 const auto FileIt = Files.find(
Path);
268 if (FileIt != Files.end()) {
269 auto &F = FileIt->getValue();
270 RefToIDs[&R] = SymRefs.first;
275 for (
const auto &Rel : *Index.
Relations) {
276 const auto FileIt = SymbolIDToFile.find(Rel.Subject);
277 if (FileIt != SymbolIDToFile.end())
278 FileIt->second->Relations.insert(&Rel);
282 for (
const auto &FileIt : Files) {
283 llvm::StringRef
Path = FileIt.getKey();
287 for (
const auto *S : FileIt.second.Symbols)
289 for (
const auto *R : FileIt.second.Refs)
290 Refs.
insert(RefToIDs[R], *R);
291 for (
const auto *Rel : FileIt.second.Relations)
293 auto SS = std::make_unique<SymbolSlab>(std::move(Syms).build());
294 auto RS = std::make_unique<RefSlab>(std::move(Refs).build());
295 auto RelS = std::make_unique<RelationSlab>(std::move(Relations).build());
296 auto IG = std::make_unique<IncludeGraph>(
305 Shard.
Refs = RS.get();
311 if (Path == MainFile)
312 Shard.
Cmd = Index.
Cmd.getPointer();
315 elog(
"Failed to write background-index shard for file {0}: {1}", Path,
319 std::lock_guard<std::mutex> Lock(ShardVersionsMu);
320 auto Hash = FileIt.second.Digest;
321 auto DigestIt = ShardVersions.try_emplace(Path);
322 ShardVersion &SV = DigestIt.first->second;
325 if (!DigestIt.second && SV.Digest == Hash && SV.HadErrors && !HadErrors)
328 SV.HadErrors = HadErrors;
333 IndexedSymbols.update(Path, std::move(SS), std::move(RS), std::move(RelS),
339 llvm::Error BackgroundIndex::index(tooling::CompileCommand Cmd) {
342 auto AbsolutePath = getAbsolutePath(Cmd);
345 auto Buf = FS->getBufferForFile(AbsolutePath);
347 return llvm::errorCodeToError(Buf.getError());
348 auto Hash =
digest(Buf->get()->getBuffer());
351 llvm::StringMap<ShardVersion> ShardVersionsSnapshot;
353 std::lock_guard<std::mutex> Lock(ShardVersionsMu);
354 ShardVersionsSnapshot = ShardVersions;
357 vlog(
"Indexing {0} (digest:={1})", Cmd.Filename, llvm::toHex(Hash));
359 Inputs.
FS = std::move(FS);
360 Inputs.
FS->setCurrentWorkingDirectory(Cmd.Directory);
365 return llvm::createStringError(llvm::inconvertibleErrorCode(),
366 "Couldn't build compiler invocation");
368 std::move(*Buf), Inputs.
FS, IgnoreDiags);
370 return llvm::createStringError(llvm::inconvertibleErrorCode(),
371 "Couldn't build compiler instance");
376 IndexOpts.
FileFilter = [&ShardVersionsSnapshot](
const SourceManager &SM,
378 const auto *F = SM.getFileEntryForID(FID);
387 auto D = ShardVersionsSnapshot.find(*AbsPath);
388 if (D != ShardVersionsSnapshot.end() && D->second.Digest == Digest &&
389 !D->second.HadErrors)
406 const FrontendInputFile &Input = Clang->getFrontendOpts().Inputs.front();
407 if (!
Action->BeginSourceFile(*Clang, Input))
408 return llvm::createStringError(llvm::inconvertibleErrorCode(),
409 "BeginSourceFile() failed");
417 "Symbols, Refs and Sources must be set.");
419 log(
"Indexed {0} ({1} symbols, {2} refs, {3} files)",
426 bool HadErrors = Clang->hasDiagnostics() &&
427 Clang->getDiagnostics().hasUncompilableErrorOccurred();
429 log(
"Failed to compile {0}, index may be incomplete", AbsolutePath);
430 for (
auto &It : *Index.
Sources)
433 update(AbsolutePath, std::move(Index), ShardVersionsSnapshot, HadErrors);
435 Rebuilder.indexedTU();
436 return llvm::Error::success();
442 std::vector<tooling::CompileCommand>
443 BackgroundIndex::loadProject(std::vector<std::string> MainFiles) {
444 std::vector<tooling::CompileCommand> NeedsReIndexing;
446 Rebuilder.startLoading();
448 const std::vector<LoadedShard> Result =
450 size_t LoadedShards = 0;
453 std::lock_guard<std::mutex> Lock(ShardVersionsMu);
454 for (
auto &LS : Result) {
459 ? std::make_unique<SymbolSlab>(std::move(*LS.Shard->Symbols))
461 auto RS = LS.Shard->Refs
462 ? std::make_unique<RefSlab>(std::move(*LS.Shard->Refs))
466 ? std::make_unique<RelationSlab>(std::move(*LS.Shard->Relations))
468 ShardVersion &SV = ShardVersions[LS.AbsolutePath];
469 SV.Digest = LS.Digest;
470 SV.HadErrors = LS.HadErrors;
473 IndexedSymbols.update(LS.AbsolutePath, std::move(SS), std::move(RS),
474 std::move(RelS), LS.CountReferences);
477 Rebuilder.loadedShard(LoadedShards);
478 Rebuilder.doneLoading();
481 llvm::DenseSet<PathRef> TUsToIndex;
484 for (
auto &LS : Result) {
485 if (!shardIsStale(LS, FS.get()))
487 PathRef TUForFile = LS.DependentTU;
488 assert(!TUForFile.empty() &&
"File without a TU!");
495 TUsToIndex.insert(TUForFile);
498 for (
PathRef TU : TUsToIndex) {
499 auto Cmd = CDB.getCompileCommand(TU);
502 NeedsReIndexing.emplace_back(std::move(*Cmd));
505 return NeedsReIndexing;
const tooling::CompileCommand * Cmd
llvm::Optional< SymbolSlab > Symbols
An immutable symbol container that stores a set of symbols.
llvm::Optional< tooling::CompileCommand > Cmd
llvm::unique_function< BackgroundIndexStorage *(PathRef)> Factory
An efficient structure of storing large set of symbol references in memory.
void boostRelated(llvm::StringRef Path)
Boosts priority of indexing related to Path.
std::array< uint8_t, 8 > FileDigest
static llvm::StringRef filenameWithoutExtension(llvm::StringRef Path)
llvm::StringRef PathRef
A typedef to represent a ref to file path.
void insert(const Symbol &S)
Adds a symbol, overwriting any existing one with the same ID.
std::unique_ptr< CompilerInstance > prepareCompilerInstance(std::unique_ptr< clang::CompilerInvocation > CI, const PrecompiledPreamble *Preamble, std::unique_ptr< llvm::MemoryBuffer > Buffer, llvm::IntrusiveRefCntPtr< llvm::vfs::FileSystem > VFS, DiagnosticConsumer &DiagsClient)
llvm::Optional< RelationSlab > Relations
void vlog(const char *Fmt, Ts &&... Vals)
void elog(const char *Fmt, Ts &&... Vals)
llvm::Optional< IncludeGraph > Sources
const SymbolSlab * Symbols
MockFSProvider FSProvider
SymbolSlab::Builder is a mutable container that can 'freeze' to SymbolSlab.
void boost(llvm::StringRef Tag, unsigned NewPriority)
Provides compilation arguments used for parsing C and C++ files.
llvm::StringMap< IncludeGraphNode > IncludeGraph
Context clone() const
Clone this context object.
void runAsync(const llvm::Twine &Name, llvm::unique_function< void()> Action)
llvm::unique_function< void()> Action
MemIndex is a naive in-memory index suitable for a small set of symbols.
void log(const char *Fmt, Ts &&... Vals)
std::string Path
A typedef to represent a file path.
std::vector< LoadedShard > loadIndexShards(llvm::ArrayRef< Path > MainFiles, BackgroundIndexStorage::Factory &IndexStorageFactory, const GlobalCompilationDatabase &CDB)
Loads all shards for the TU MainFile from Storage.
void enqueue(const std::vector< std::string > &ChangedFiles)
void insert(const Relation &R)
Adds a relation to the slab.
const IncludeGraph * Sources
llvm::Optional< FileDigest > digestFile(const SourceManager &SM, FileID FID)
A context is an immutable container for per-request data that must be propagated through layers that ...
RelationSlab::Builder is a mutable container that can 'freeze' to RelationSlab.
std::function< bool(const SourceManager &, FileID)> FileFilter
If this is set, only collect symbols/references from a file if FileFilter(SM, FID) is true...
RefSlab::Builder is a mutable container that can 'freeze' to RefSlab.
WithContext replaces Context::current() with a provided scope.
FileDigest digest(llvm::StringRef Content)
virtual llvm::Error storeShard(llvm::StringRef ShardIdentifier, IndexFileOut Shard) const =0
const RelationSlab * Relations
llvm::Optional< RefSlab > Refs
BackgroundIndex(Context BackgroundContext, const FileSystemProvider &, const GlobalCompilationDatabase &CDB, BackgroundIndexStorage::Factory IndexStorageFactory, size_t ThreadPoolSize=llvm::heavyweight_hardware_concurrency())
If BuildIndexPeriodMs is greater than 0, the symbol index will only be rebuilt periodically (one per ...
static llvm::Expected< URI > create(llvm::StringRef AbsolutePath, llvm::StringRef Scheme)
Creates a URI for a file in the given scheme.
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
std::unique_ptr< FrontendAction > createStaticIndexingAction(SymbolCollector::Options Opts, std::function< void(SymbolSlab)> SymbolsCallback, std::function< void(RefSlab)> RefsCallback, std::function< void(RelationSlab)> RelationsCallback, std::function< void(IncludeGraph)> IncludeGraphCallback)
llvm::Optional< std::string > getCanonicalPath(const FileEntry *F, const SourceManager &SourceMgr)
Get the canonical path of F.
std::unique_ptr< CompilerInvocation > buildCompilerInvocation(const ParseInputs &Inputs, clang::DiagnosticConsumer &D, std::vector< std::string > *CC1Args)
Builds compiler invocation that could be used to build AST or preamble.
void insert(const SymbolID &ID, const Ref &S)
Adds a ref to the slab. Deep copy: Strings will be owned by the slab.
bool isHeaderFile(llvm::StringRef FileName, llvm::Optional< LangOptions > LangOpts)
Infers whether this is a header from the FileName and LangOpts (if presents).
static llvm::Expected< std::string > resolve(const URI &U, llvm::StringRef HintPath="")
Resolves the absolute path of U.
void append(std::vector< Task >)
A work item on the thread pool's queue.
Records an event whose duration is the lifetime of the Span object.
#define SPAN_ATTACH(S, Name, Expr)
Attach a key-value pair to a Span event.
llvm::StringMap< std::string > Files
void work(std::function< void()> OnIdle=nullptr)