31 #include "clang/Basic/SourceLocation.h"
32 #include "clang/Basic/SourceManager.h"
33 #include "clang/Driver/Types.h"
34 #include "llvm/ADT/ArrayRef.h"
35 #include "llvm/ADT/DenseSet.h"
36 #include "llvm/ADT/Hashing.h"
37 #include "llvm/ADT/STLExtras.h"
38 #include "llvm/ADT/ScopeExit.h"
39 #include "llvm/ADT/StringMap.h"
40 #include "llvm/ADT/StringRef.h"
41 #include "llvm/ADT/StringSet.h"
42 #include "llvm/Support/Error.h"
43 #include "llvm/Support/Path.h"
44 #include "llvm/Support/Threading.h"
49 #include <condition_variable>
68 llvm::SmallString<128> getAbsolutePath(
const tooling::CompileCommand &Cmd) {
69 llvm::SmallString<128> AbsolutePath;
70 if (llvm::sys::path::is_absolute(Cmd.Filename)) {
71 AbsolutePath = Cmd.Filename;
73 AbsolutePath = Cmd.Directory;
74 llvm::sys::path::append(AbsolutePath, Cmd.Filename);
75 llvm::sys::path::remove_dots(AbsolutePath,
true);
80 bool shardIsStale(
const LoadedShard &LS, llvm::vfs::FileSystem *
FS) {
81 auto Buf =
FS->getBufferForFile(LS.AbsolutePath);
83 elog(
"Background-index: Couldn't read {0} to validate stored index: {1}",
84 LS.AbsolutePath, Buf.getError().message());
88 return digest(Buf->get()->getBuffer()) != LS.Digest;
100 BackgroundContext(std::move(BackgroundContext)),
101 ContextProvider(std::move(ContextProvider)),
102 Rebuilder(this, &IndexedSymbols, ThreadPoolSize),
103 IndexStorageFactory(std::move(IndexStorageFactory)),
104 Queue(std::move(OnProgress)),
106 CDB.watch([&](const std::vector<std::string> &ChangedFiles) {
109 assert(ThreadPoolSize > 0 &&
"Thread pool size can't be zero.");
110 assert(this->IndexStorageFactory &&
"Storage factory can not be null!");
111 for (
unsigned I = 0; I < ThreadPoolSize; ++I) {
112 ThreadPool.
runAsync(
"background-worker-" + llvm::Twine(I + 1), [
this] {
114 Queue.
work([&] { Rebuilder.
idle(); });
125 const std::vector<std::string> &ChangedFiles) {
129 llvm::Optional<WithContext> WithProvidedContext;
131 WithProvidedContext.emplace(ContextProvider(
""));
134 log(
"Enqueueing {0} commands for indexing", ChangedFiles.size());
137 auto NeedsReIndexing = loadProject(std::move(ChangedFiles));
139 std::shuffle(NeedsReIndexing.begin(), NeedsReIndexing.end(),
140 std::mt19937(std::random_device{}()));
141 std::vector<BackgroundQueue::Task> Tasks;
142 Tasks.reserve(NeedsReIndexing.size());
143 for (
auto &Cmd : NeedsReIndexing)
144 Tasks.push_back(indexFileTask(std::move(Cmd)));
145 Queue.
append(std::move(Tasks));
149 T.ThreadPri = llvm::ThreadPriority::Default;
154 Path = llvm::sys::path::filename(
Path);
155 return Path.drop_back(llvm::sys::path::extension(
Path).size());
158 BackgroundQueue::Task BackgroundIndex::indexFileTask(std::string
Path) {
160 BackgroundQueue::Task T([
this,
Path(std::move(
Path))] {
161 llvm::Optional<WithContext> WithProvidedContext;
163 WithProvidedContext.emplace(ContextProvider(
Path));
164 auto Cmd = CDB.getCompileCommand(
Path);
167 if (
auto Error = index(std::move(*Cmd)))
170 T.QueuePri = IndexFile;
171 T.Tag = std::move(
Tag);
183 void BackgroundIndex::update(
185 const llvm::StringMap<ShardVersion> &ShardVersionsSnapshot,
188 llvm::StringMap<std::pair<Path, FileDigest>> FilesToUpdate;
191 for (
const auto &IndexIt : *Index.Sources) {
192 const auto &IGN = IndexIt.getValue();
195 elog(
"Failed to resolve URI: {0}", AbsPath.takeError());
198 const auto DigestIt = ShardVersionsSnapshot.find(*AbsPath);
200 if (DigestIt == ShardVersionsSnapshot.end() ||
201 DigestIt->getValue().Digest != IGN.Digest ||
202 (DigestIt->getValue().HadErrors && !HadErrors))
203 FilesToUpdate[IGN.URI] = {std::move(*AbsPath), IGN.Digest};
207 FileShardedIndex ShardedIndex(std::move(Index));
210 for (
const auto &FileIt : FilesToUpdate) {
211 auto Uri = FileIt.first();
212 auto IF = ShardedIndex.getShard(Uri);
213 assert(IF &&
"no shard for file in Index.Sources?");
224 if (
auto Error = IndexStorageFactory(
Path)->storeShard(
Path, *IF))
225 elog(
"Failed to write background-index shard for file {0}: {1}",
Path,
229 std::lock_guard<std::mutex> Lock(ShardVersionsMu);
230 const auto &Hash = FileIt.getValue().second;
231 auto DigestIt = ShardVersions.try_emplace(
Path);
232 ShardVersion &SV = DigestIt.first->second;
235 if (!DigestIt.second && SV.Digest == Hash && SV.HadErrors && !HadErrors)
238 SV.HadErrors = HadErrors;
244 Path, std::make_unique<SymbolSlab>(std::move(*IF->Symbols)),
245 std::make_unique<RefSlab>(std::move(*IF->Refs)),
246 std::make_unique<RelationSlab>(std::move(*IF->Relations)),
252 llvm::Error BackgroundIndex::index(tooling::CompileCommand Cmd) {
253 trace::Span
Tracer(
"BackgroundIndex");
255 auto AbsolutePath = getAbsolutePath(Cmd);
257 auto FS = TFS.
view(Cmd.Directory);
258 auto Buf =
FS->getBufferForFile(AbsolutePath);
260 return llvm::errorCodeToError(Buf.getError());
261 auto Hash =
digest(Buf->get()->getBuffer());
264 llvm::StringMap<ShardVersion> ShardVersionsSnapshot;
266 std::lock_guard<std::mutex> Lock(ShardVersionsMu);
267 ShardVersionsSnapshot = ShardVersions;
270 vlog(
"Indexing {0} (digest:={1})", Cmd.Filename, llvm::toHex(Hash));
273 Inputs.CompileCommand = std::move(Cmd);
277 return llvm::createStringError(llvm::inconvertibleErrorCode(),
278 "Couldn't build compiler invocation");
284 return llvm::createStringError(llvm::inconvertibleErrorCode(),
285 "Couldn't build compiler instance");
287 SymbolCollector::Options IndexOpts;
290 IndexOpts.FileFilter = [&ShardVersionsSnapshot](
const SourceManager &SM,
292 const auto *F = SM.getFileEntryForID(FID);
301 auto D = ShardVersionsSnapshot.find(*AbsPath);
302 if (D != ShardVersionsSnapshot.end() && D->second.Digest == Digest &&
303 !D->second.HadErrors)
310 IndexOpts, [&](SymbolSlab S) { Index.Symbols = std::move(S); },
311 [&](RefSlab R) { Index.Refs = std::move(R); },
312 [&](RelationSlab R) { Index.Relations = std::move(R); },
313 [&](
IncludeGraph IG) { Index.Sources = std::move(IG); });
320 const FrontendInputFile &Input = Clang->getFrontendOpts().Inputs.front();
321 if (!
Action->BeginSourceFile(*Clang, Input))
322 return llvm::createStringError(llvm::inconvertibleErrorCode(),
323 "BeginSourceFile() failed");
329 Index.Cmd =
Inputs.CompileCommand;
330 assert(Index.Symbols && Index.Refs && Index.Sources &&
331 "Symbols, Refs and Sources must be set.");
333 log(
"Indexed {0} ({1} symbols, {2} refs, {3} files)",
334 Inputs.CompileCommand.Filename, Index.Symbols->size(),
335 Index.Refs->numRefs(), Index.Sources->size());
340 bool HadErrors = Clang->hasDiagnostics() &&
341 Clang->getDiagnostics().hasUncompilableErrorOccurred();
343 log(
"Failed to compile {0}, index may be incomplete", AbsolutePath);
344 for (
auto &It : *Index.Sources)
347 update(AbsolutePath, std::move(Index), ShardVersionsSnapshot, HadErrors);
350 return llvm::Error::success();
356 std::vector<std::string>
357 BackgroundIndex::loadProject(std::vector<std::string> MainFiles) {
360 llvm::erase_if(MainFiles, [&](
const std::string &TU) {
362 WithContext WithProvidedContext(ContextProvider(TU));
368 const std::vector<LoadedShard> Result =
370 size_t LoadedShards = 0;
373 std::lock_guard<std::mutex> Lock(ShardVersionsMu);
374 for (
auto &LS : Result) {
379 ? std::make_unique<SymbolSlab>(std::move(*LS.Shard->Symbols))
381 auto RS = LS.Shard->Refs
382 ? std::make_unique<RefSlab>(std::move(*LS.Shard->Refs))
386 ? std::make_unique<RelationSlab>(std::move(*LS.Shard->Relations))
388 ShardVersion &SV = ShardVersions[LS.AbsolutePath];
389 SV.Digest = LS.Digest;
390 SV.HadErrors = LS.HadErrors;
393 IndexedSymbols.
update(LS.AbsolutePath, std::move(SS), std::move(RS),
394 std::move(RelS), LS.CountReferences);
401 llvm::DenseSet<PathRef> TUsToIndex;
404 for (
auto &LS : Result) {
405 if (!shardIsStale(LS,
FS.get()))
407 PathRef TUForFile = LS.DependentTU;
408 assert(!TUForFile.empty() &&
"File without a TU!");
415 TUsToIndex.insert(TUForFile);
418 return {TUsToIndex.begin(), TUsToIndex.end()};