clang-tools 10.0.0

Background.h
Go to the documentation of this file.
1 //===--- Background.h - Build an index in a background thread ----*- C++-*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_BACKGROUND_H
10 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_BACKGROUND_H
11
12 #include "Context.h"
13 #include "FSProvider.h"
14 #include "GlobalCompilationDatabase.h"
15 #include "Path.h"
16 #include "SourceCode.h"
17 #include "Threading.h"
18 #include "index/BackgroundRebuild.h"
19 #include "index/FileIndex.h"
20 #include "index/Index.h"
21 #include "index/Serialization.h"
22 #include "clang/Tooling/CompilationDatabase.h"
23 #include "llvm/ADT/StringMap.h"
24 #include "llvm/Support/Threading.h"
25 #include <atomic>
26 #include <condition_variable>
27 #include <deque>
28 #include <mutex>
29 #include <queue>
30 #include <string>
31 #include <thread>
32 #include <vector>
33
34 namespace clang {
35 namespace clangd {
36
37 // Handles storage and retrieval of index shards. Both store and load
38 // operations can be called from multiple-threads concurrently.
39 class BackgroundIndexStorage {
40 public:
41 virtual ~BackgroundIndexStorage() = default;
42
43 // Shards of the index are stored and retrieved independently, keyed by shard
44 // identifier - in practice this is a source file name
45 virtual llvm::Error storeShard(llvm::StringRef ShardIdentifier,
46 IndexFileOut Shard) const = 0;
47
48 // Tries to load shard with given identifier, returns nullptr if shard
49 // couldn't be loaded.
50 virtual std::unique_ptr<IndexFileIn>
51 loadShard(llvm::StringRef ShardIdentifier) const = 0;
52
53 // The factory provides storage for each File.
54 // It keeps ownership of the storage instances, and should manage caching
55 // itself. Factory must be threadsafe and never returns nullptr.
56 using Factory = llvm::unique_function<BackgroundIndexStorage *(PathRef)>;
57
58 // Creates an Index Storage that saves shards into disk. Index storage uses
59 // CDBDirectory + ".clangd/index/" as the folder to save shards. CDBDirectory
60 // is the first directory containing a CDB in parent directories of a file, or
61 // user's home directory if none was found, e.g. standard library headers.
62 static Factory createDiskBackedStorageFactory(
63 std::function<llvm::Optional<ProjectInfo>(PathRef)> GetProjectInfo);
64 };
65
66 // A priority queue of tasks which can be run on (external) worker threads.
67 class BackgroundQueue {
68 public:
69 /// A work item on the thread pool's queue.
70 struct Task {
71 explicit Task(std::function<void()> Run) : Run(std::move(Run)) {}
72
73 std::function<void()> Run;
74 llvm::ThreadPriority ThreadPri = llvm::ThreadPriority::Background;
75 unsigned QueuePri = 0; // Higher-priority tasks will run first.
76 std::string Tag; // Allows priority to be boosted later.
77
78 bool operator<(const Task &O) const { return QueuePri < O.QueuePri; }
79 };
80
81 // Add tasks to the queue.
82 void push(Task);
83 void append(std::vector<Task>);
84 // Boost priority of current and new tasks with matching Tag, if they are
85 // lower priority.
86 // Reducing the boost of a tag affects future tasks but not current ones.
87 void boost(llvm::StringRef Tag, unsigned NewPriority);
88
89 // Process items on the queue until the queue is stopped.
90 // If the queue becomes empty, OnIdle will be called (on one worker).
91 void work(std::function<void()> OnIdle = nullptr);
92
93 // Stop processing new tasks, allowing all work() calls to return soon.
94 void stop();
95
96 // Disables thread priority lowering to ensure progress on loaded systems.
97 // Only affects tasks that run after the call.
98 static void preventThreadStarvationInTests();
99 LLVM_NODISCARD bool
100 blockUntilIdleForTest(llvm::Optional<double> TimeoutSeconds);
101
102 private:
103 std::mutex Mu;
104 unsigned NumActiveTasks = 0; // Only idle when queue is empty *and* no tasks.
105 std::condition_variable CV;
106 bool ShouldStop = false;
107 std::vector<Task> Queue; // max-heap
108 llvm::StringMap<unsigned> Boosts;
109 };
110
111 // Builds an in-memory index by by running the static indexer action over
112 // all commands in a compilation database. Indexing happens in the background.
113 // FIXME: it should also persist its state on disk for fast start.
114 // FIXME: it should watch for changes to files on disk.
115 class BackgroundIndex : public SwapIndex {
116 public:
117 /// If BuildIndexPeriodMs is greater than 0, the symbol index will only be
118 /// rebuilt periodically (one per \p BuildIndexPeriodMs); otherwise, index is
119 /// rebuilt for each indexed file.
120 BackgroundIndex(
121 Context BackgroundContext, const FileSystemProvider &,
122 const GlobalCompilationDatabase &CDB,
123 BackgroundIndexStorage::Factory IndexStorageFactory,
124 size_t ThreadPoolSize = llvm::heavyweight_hardware_concurrency());
125 ~BackgroundIndex(); // Blocks while the current task finishes.
126
127 // Enqueue translation units for indexing.
128 // The indexing happens in a background thread, so the symbols will be
129 // available sometime later.
130 void enqueue(const std::vector<std::string> &ChangedFiles) {
131 Queue.push(changedFilesTask(ChangedFiles));
132 }
133
134 /// Boosts priority of indexing related to Path.
135 /// Typically used to index TUs when headers are opened.
136 void boostRelated(llvm::StringRef Path);
137
138 // Cause background threads to stop after ther current task, any remaining
139 // tasks will be discarded.
140 void stop() {
141 Rebuilder.shutdown();
142 Queue.stop();
143 }
144
145 // Wait until the queue is empty, to allow deterministic testing.
146 LLVM_NODISCARD bool
147 blockUntilIdleForTest(llvm::Optional<double> TimeoutSeconds = 10) {
148 return Queue.blockUntilIdleForTest(TimeoutSeconds);
149 }
150
151 private:
152 /// Represents the state of a single file when indexing was performed.
153 struct ShardVersion {
154 FileDigest Digest{{0}};
155 bool HadErrors = false;
156 };
157
158 /// Given index results from a TU, only update symbols coming from files with
159 /// different digests than \p ShardVersionsSnapshot. Also stores new index
160 /// information on IndexStorage.
161 void update(llvm::StringRef MainFile, IndexFileIn Index,
162 const llvm::StringMap<ShardVersion> &ShardVersionsSnapshot,
163 bool HadErrors);
164
165 // configuration
166 const FileSystemProvider &FSProvider;
167 const GlobalCompilationDatabase &CDB;
168 Context BackgroundContext;
169
170 llvm::Error index(tooling::CompileCommand);
171
172 FileSymbols IndexedSymbols;
173 BackgroundIndexRebuilder Rebuilder;
174 llvm::StringMap<ShardVersion> ShardVersions; // Key is absolute file path.
175 std::mutex ShardVersionsMu;
176
177 BackgroundIndexStorage::Factory IndexStorageFactory;
178 // Tries to load shards for the MainFiles and their dependencies.
179 std::vector<tooling::CompileCommand>
180 loadProject(std::vector<std::string> MainFiles);
181
182 BackgroundQueue::Task
183 changedFilesTask(const std::vector<std::string> &ChangedFiles);
184 BackgroundQueue::Task indexFileTask(tooling::CompileCommand Cmd);
185
186 // from lowest to highest priority
187 enum QueuePriority {
188 IndexFile,
189 IndexBoostedFile,
190 LoadShards,
191 };
192 BackgroundQueue Queue;
193 AsyncTaskRunner ThreadPool;
194 GlobalCompilationDatabase::CommandChanged::Subscription CommandsChanged;
195 };
196
197 } // namespace clangd
198 } // namespace clang
199
200 #endif
clang::clangd::BackgroundIndexStorage::Factory
llvm::unique_function< BackgroundIndexStorage *(PathRef)> Factory
Definition: Background.h:56
clang::clangd::MessageType::Error
An error message.
Index.h
BackgroundRebuild.h
clang::clangd::FileDigest
std::array< uint8_t, 8 > FileDigest
Definition: SourceCode.h:38
clang::clangd::FileSymbols
A container of Symbols from several source files.
Definition: FileIndex.h:61
Tag
HTMLTag Tag
Definition: HTMLGenerator.cpp:90
clang::clangd::BackgroundIndexStorage::createDiskBackedStorageFactory
static Factory createDiskBackedStorageFactory(std::function< llvm::Optional< ProjectInfo >(PathRef)> GetProjectInfo)
Definition: BackgroundIndexStorage.cpp:146
SourceCode.h
clang::clangd::PathRef
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:23
Context.h
clang::clangd::BackgroundQueue::Task::Run
std::function< void()> Run
Definition: Background.h:73
clang::clangd::FileSystemProvider
Definition: FSProvider.h:20
clang::clangd::BackgroundQueue::Task::operator<
bool operator<(const Task &O) const
Definition: Background.h:78
Serialization.h
GlobalCompilationDatabase.h
clang::clangd::BackgroundQueue::Task::QueuePri
unsigned QueuePri
Definition: Background.h:75
FSProvider
MockFSProvider FSProvider
Definition: FindSymbolsTests.cpp:67
MainFile
std::string MainFile
Definition: HeadersTests.cpp:121
clang::clangd::BackgroundQueue
Definition: Background.h:67
clang::clangd::GlobalCompilationDatabase
Provides compilation arguments used for parsing C and C++ files.
Definition: GlobalCompilationDatabase.h:35
clang::clangd::IndexFileIn
Definition: Serialization.h:42
clang::clangd::BackgroundIndex
Definition: Background.h:115
clang::clangd::Path
std::string Path
A typedef to represent a file path.
Definition: Path.h:20
clang::clangd::BackgroundIndex::enqueue
void enqueue(const std::vector< std::string > &ChangedFiles)
Definition: Background.h:130
clang::clangd::BackgroundQueue::Task::Tag
std::string Tag
Definition: Background.h:76
clang::clangd::BackgroundIndexRebuilder
Definition: BackgroundRebuild.h:48
clang::clangd::AsyncTaskRunner
Runs tasks on separate (detached) threads and wait for all tasks to finish.
Definition: Threading.h:105
clang::clangd::Context
A context is an immutable container for per-request data that must be propagated through layers that ...
Definition: Context.h:69
clang::clangd::BackgroundIndex::stop
void stop()
Definition: Background.h:140
clang::clangd::BackgroundIndexStorage::loadShard
virtual std::unique_ptr< IndexFileIn > loadShard(llvm::StringRef ShardIdentifier) const =0
clang::clangd::SwapIndex
Definition: Index.h:128
clang::clangd::BackgroundIndexStorage::storeShard
virtual llvm::Error storeShard(llvm::StringRef ShardIdentifier, IndexFileOut Shard) const =0
clang
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
Definition: ApplyReplacements.h:27
clang::clangd::BackgroundIndex::blockUntilIdleForTest
LLVM_NODISCARD bool blockUntilIdleForTest(llvm::Optional< double > TimeoutSeconds=10)
Definition: Background.h:147
Path.h
clang::clangd::BackgroundQueue::Task::Task
Task(std::function< void()> Run)
Definition: Background.h:71
FSProvider.h
clang::clangd::IndexFileOut
Definition: Serialization.h:55
clang::clangd::BackgroundIndexStorage::~BackgroundIndexStorage
virtual ~BackgroundIndexStorage()=default
FileIndex.h
clang::clangd::BackgroundQueue::Task
A work item on the thread pool&#39;s queue.
Definition: Background.h:70
clang::clangd::BackgroundIndexStorage
Definition: Background.h:39
Index
const SymbolIndex * Index
Definition: Dexp.cpp:84
Threading.h