clang-tools 10.0.0

TUScheduler.h
Go to the documentation of this file.
1 //===--- TUScheduler.h -------------------------------------------*-C++-*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_TUSCHEDULER_H
10 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_TUSCHEDULER_H
11
12 #include "Compiler.h"
13 #include "Diagnostics.h"
14 #include "Function.h"
15 #include "GlobalCompilationDatabase.h"
16 #include "Path.h"
17 #include "Threading.h"
18 #include "index/CanonicalIncludes.h"
19 #include "llvm/ADT/Optional.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/StringMap.h"
22 #include "llvm/ADT/StringRef.h"
23
24 namespace clang {
25 namespace clangd {
26 class ParsedAST;
27 struct PreambleData;
28
29 /// Returns a number of a default async threads to use for TUScheduler.
30 /// Returned value is always >= 1 (i.e. will not cause requests to be processed
31 /// synchronously).
32 unsigned getDefaultAsyncThreadsCount();
33
34 struct InputsAndAST {
35 const ParseInputs &Inputs;
36 ParsedAST &AST;
37 };
38
39 struct InputsAndPreamble {
40 llvm::StringRef Contents;
41 const tooling::CompileCommand &Command;
42 // This can be nullptr if no preamble is available.
43 const PreambleData *Preamble;
44 };
45
46 /// Determines whether diagnostics should be generated for a file snapshot.
47 enum class WantDiagnostics {
48 Yes, /// Diagnostics must be generated for this snapshot.
49 No, /// Diagnostics must not be generated for this snapshot.
50 Auto, /// Diagnostics must be generated for this snapshot or a subsequent one,
51 /// within a bounded amount of time.
52 };
53
54 /// Configuration of the AST retention policy. This only covers retention of
55 /// *idle* ASTs. If queue has operations requiring the AST, they might be
56 /// kept in memory.
57 struct ASTRetentionPolicy {
58 /// Maximum number of ASTs to be retained in memory when there are no pending
59 /// requests for them.
60 unsigned MaxRetainedASTs = 3;
61 };
62
63 struct TUAction {
64 enum State {
65 Queued, // The TU is pending in the thread task queue to be built.
66 RunningAction, // Starting running actions on the TU.
67 BuildingPreamble, // The preamble of the TU is being built.
68 BuildingFile, // The TU is being built. It is only emitted when building
69 // the AST for diagnostics in write action (update).
70 Idle, // Indicates the worker thread is idle, and ready to run any upcoming
71 // actions.
72 };
73 TUAction(State S, llvm::StringRef Name) : S(S), Name(Name) {}
74 State S;
75 /// The name of the action currently running, e.g. Update, GoToDef, Hover.
76 /// Empty if we are in the idle state.
77 std::string Name;
78 };
79
80 // Internal status of the TU in TUScheduler.
81 struct TUStatus {
82 struct BuildDetails {
83 /// Indicates whether clang failed to build the TU.
84 bool BuildFailed = false;
85 /// Indicates whether we reused the prebuilt AST.
86 bool ReuseAST = false;
87 };
88 /// Serialize this to an LSP file status item.
89 FileStatus render(PathRef File) const;
90
91 TUAction Action;
92 BuildDetails Details;
93 };
94
95 class ParsingCallbacks {
96 public:
97 virtual ~ParsingCallbacks() = default;
98
99 /// Called on the AST that was built for emitting the preamble. The built AST
100 /// contains only AST nodes from the #include directives at the start of the
101 /// file. AST node in the current file should be observed on onMainAST call.
102 virtual void onPreambleAST(PathRef Path, ASTContext &Ctx,
103 std::shared_ptr<clang::Preprocessor> PP,
104 const CanonicalIncludes &) {}
105
106 /// The argument function is run under the critical section guarding against
107 /// races when closing the files.
108 using PublishFn = llvm::function_ref<void(llvm::function_ref<void()>)>;
109 /// Called on the AST built for the file itself. Note that preamble AST nodes
110 /// are not deserialized and should be processed in the onPreambleAST call
111 /// instead.
112 /// The \p AST always contains all AST nodes for the main file itself, and
113 /// only a portion of the AST nodes deserialized from the preamble. Note that
114 /// some nodes from the preamble may have been deserialized and may also be
115 /// accessed from the main file AST, e.g. redecls of functions from preamble,
116 /// etc. Clients are expected to process only the AST nodes from the main file
117 /// in this callback (obtained via ParsedAST::getLocalTopLevelDecls) to obtain
118 /// optimal performance.
119 ///
120 /// When information about the file (diagnostics, syntax highlighting) is
121 /// published to clients, this should be wrapped in Publish, e.g.
122 /// void onMainAST(...) {
123 /// Highlights = computeHighlights();
124 /// Publish([&] { notifyHighlights(Path, Highlights); });
125 /// }
126 /// This guarantees that clients will see results in the correct sequence if
127 /// the file is concurrently closed and/or reopened. (The lambda passed to
128 /// Publish() may never run in this case).
129 virtual void onMainAST(PathRef Path, ParsedAST &AST, PublishFn Publish) {}
130
131 /// Called whenever the AST fails to build. \p Diags will have the diagnostics
132 /// that led to failure.
133 virtual void onFailedAST(PathRef Path, std::vector<Diag> Diags,
134 PublishFn Publish) {}
135
136 /// Called whenever the TU status is updated.
137 virtual void onFileUpdated(PathRef File, const TUStatus &Status) {}
138 };
139
140 /// Handles running tasks for ClangdServer and managing the resources (e.g.,
141 /// preambles and ASTs) for opened files.
142 /// TUScheduler is not thread-safe, only one thread should be providing updates
143 /// and scheduling tasks.
144 /// Callbacks are run on a threadpool and it's appropriate to do slow work in
145 /// them. Each task has a name, used for tracing (should be UpperCamelCase).
146 /// FIXME(sammccall): pull out a scheduler options struct.
147 class TUScheduler {
148 public:
149 TUScheduler(const GlobalCompilationDatabase &CDB, unsigned AsyncThreadsCount,
150 bool StorePreamblesInMemory,
151 std::unique_ptr<ParsingCallbacks> ASTCallbacks,
152 std::chrono::steady_clock::duration UpdateDebounce,
153 ASTRetentionPolicy RetentionPolicy);
154 ~TUScheduler();
155
156 /// Returns estimated memory usage for each of the currently open files.
157 /// The order of results is unspecified.
158 std::vector<std::pair<Path, std::size_t>> getUsedBytesPerFile() const;
159
160 /// Returns a list of files with ASTs currently stored in memory. This method
161 /// is not very reliable and is only used for test. E.g., the results will not
162 /// contain files that currently run something over their AST.
163 std::vector<Path> getFilesWithCachedAST() const;
164
165 /// Schedule an update for \p File.
166 /// The compile command in \p Inputs is ignored; worker queries CDB to get
167 /// the actual compile command.
168 /// If diagnostics are requested (Yes), and the context is cancelled
169 /// before they are prepared, they may be skipped if eventual-consistency
170 /// permits it (i.e. WantDiagnostics is downgraded to Auto).
171 /// Returns true if the file was not previously tracked.
172 bool update(PathRef File, ParseInputs Inputs, WantDiagnostics WD);
173
174 /// Remove \p File from the list of tracked files and schedule removal of its
175 /// resources. Pending diagnostics for closed files may not be delivered, even
176 /// if requested with WantDiags::Auto or WantDiags::Yes.
177 void remove(PathRef File);
178
179 /// Returns the current contents of the buffer for File, per last update().
180 /// The returned StringRef may be invalidated by any write to TUScheduler.
181 llvm::StringRef getContents(PathRef File) const;
182
183 /// Returns a snapshot of all file buffer contents, per last update().
184 llvm::StringMap<std::string> getAllFileContents() const;
185
186 /// Schedule an async task with no dependencies.
187 void run(llvm::StringRef Name, llvm::unique_function<void()> Action);
188
189 /// Schedule an async read of the AST. \p Action will be called when AST is
190 /// ready. The AST passed to \p Action refers to the version of \p File
191 /// tracked at the time of the call, even if new updates are received before
192 /// \p Action is executed.
193 /// If an error occurs during processing, it is forwarded to the \p Action
194 /// callback.
195 /// If the context is cancelled before the AST is ready, the callback will
196 /// receive a CancelledError.
197 void runWithAST(llvm::StringRef Name, PathRef File,
198 Callback<InputsAndAST> Action);
199
200 /// Controls whether preamble reads wait for the preamble to be up-to-date.
201 enum PreambleConsistency {
202 /// The preamble is generated from the current version of the file.
203 /// If the content was recently updated, we will wait until we have a
204 /// preamble that reflects that update.
205 /// This is the slowest option, and may be delayed by other tasks.
206 Consistent,
207 /// The preamble may be generated from an older version of the file.
208 /// Reading from locations in the preamble may cause files to be re-read.
209 /// This gives callers two options:
210 /// - validate that the preamble is still valid, and only use it if so
211 /// - accept that the preamble contents may be outdated, and try to avoid
212 /// reading source code from headers.
213 /// This is the fastest option, usually a preamble is available immediately.
214 Stale,
215 /// Besides accepting stale preamble, this also allow preamble to be absent
216 /// (not ready or failed to build).
217 StaleOrAbsent,
218 };
219
220 /// Schedule an async read of the preamble.
221 /// If there's no up-to-date preamble, we follow the PreambleConsistency
222 /// policy.
223 /// If an error occurs, it is forwarded to the \p Action callback.
224 /// Context cancellation is ignored and should be handled by the Action.
225 /// (In practice, the Action is almost always executed immediately).
226 void runWithPreamble(llvm::StringRef Name, PathRef File,
227 PreambleConsistency Consistency,
228 Callback<InputsAndPreamble> Action);
229
230 /// Wait until there are no scheduled or running tasks.
231 /// Mostly useful for synchronizing tests.
232 bool blockUntilIdle(Deadline D) const;
233
234 private:
235 /// This class stores per-file data in the Files map.
236 struct FileData;
237
238 public:
239 /// Responsible for retaining and rebuilding idle ASTs. An implementation is
240 /// an LRU cache.
241 class ASTCache;
242
243 // The file being built/processed in the current thread. This is a hack in
244 // order to get the file name into the index implementations. Do not depend on
245 // this inside clangd.
246 // FIXME: remove this when there is proper index support via build system
247 // integration.
248 static llvm::Optional<llvm::StringRef> getFileBeingProcessedInContext();
249
250 private:
251 const GlobalCompilationDatabase &CDB;
252 const bool StorePreamblesInMemory;
253 std::unique_ptr<ParsingCallbacks> Callbacks; // not nullptr
254 Semaphore Barrier;
255 llvm::StringMap<std::unique_ptr<FileData>> Files;
256 std::unique_ptr<ASTCache> IdleASTs;
257 // None when running tasks synchronously and non-None when running tasks
258 // asynchronously.
259 llvm::Optional<AsyncTaskRunner> PreambleTasks;
260 llvm::Optional<AsyncTaskRunner> WorkerThreads;
261 std::chrono::steady_clock::duration UpdateDebounce;
262 };
263
264 } // namespace clangd
265 } // namespace clang
266
267 #endif
clang::clangd::TUScheduler::PreambleConsistency
PreambleConsistency
Controls whether preamble reads wait for the preamble to be up-to-date.
Definition: TUScheduler.h:201
clang::clangd::WantDiagnostics
WantDiagnostics
Determines whether diagnostics should be generated for a file snapshot.
Definition: TUScheduler.h:47
clang::clangd::TUScheduler::Stale
The preamble may be generated from an older version of the file.
Definition: TUScheduler.h:214
clang::clangd::WantDiagnostics::No
Diagnostics must be generated for this snapshot.
clang::clangd::InputsAndPreamble::Command
const tooling::CompileCommand & Command
Definition: TUScheduler.h:41
clang::clangd::ParsingCallbacks::onMainAST
virtual void onMainAST(PathRef Path, ParsedAST &AST, PublishFn Publish)
Called on the AST built for the file itself.
Definition: TUScheduler.h:129
clang::clangd::TUAction::Idle
Definition: TUScheduler.h:70
clang::clangd::TUAction::Queued
Definition: TUScheduler.h:65
clang::clangd::TUScheduler::StaleOrAbsent
Besides accepting stale preamble, this also allow preamble to be absent (not ready or failed to build...
Definition: TUScheduler.h:217
clang::clangd::InputsAndAST::AST
ParsedAST & AST
Definition: TUScheduler.h:36
clang::clangd::PathRef
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:23
clang::clangd::TUAction::RunningAction
Definition: TUScheduler.h:66
clang::clangd::TUAction::S
State S
Definition: TUScheduler.h:74
clang::clangd::Callback
llvm::unique_function< void(llvm::Expected< T >)> Callback
A Callback<T> is a void function that accepts Expected<T>.
Definition: Function.h:28
clang::clangd::TUStatus::Action
TUAction Action
Definition: TUScheduler.h:91
clang::clangd::Semaphore
Limits the number of threads that can acquire the lock at the same time.
Definition: Threading.h:41
clang::clangd::TUAction::BuildingFile
Definition: TUScheduler.h:68
GlobalCompilationDatabase.h
clang::clangd::ParsingCallbacks::onFileUpdated
virtual void onFileUpdated(PathRef File, const TUStatus &Status)
Called whenever the TU status is updated.
Definition: TUScheduler.h:137
CanonicalIncludes.h
clang::clangd::TUStatus::Details
BuildDetails Details
Definition: TUScheduler.h:92
clang::clangd::ASTRetentionPolicy
Configuration of the AST retention policy.
Definition: TUScheduler.h:57
clang::clangd::WantDiagnostics::Yes
clang::clangd::GlobalCompilationDatabase
Provides compilation arguments used for parsing C and C++ files.
Definition: GlobalCompilationDatabase.h:35
clang::clangd::CanonicalIncludes
Maps a definition location onto an #include file, based on a set of filename rules.
Definition: CanonicalIncludes.h:36
clang::clangd::ParsingCallbacks::onFailedAST
virtual void onFailedAST(PathRef Path, std::vector< Diag > Diags, PublishFn Publish)
Called whenever the AST fails to build.
Definition: TUScheduler.h:133
Ctx
Context Ctx
Definition: TUScheduler.cpp:237
clang::clangd::TUAction::State
State
Definition: TUScheduler.h:64
Compiler.h
Action
llvm::unique_function< void()> Action
Definition: TUScheduler.cpp:234
Diagnostics.h
clang::clangd::Path
std::string Path
A typedef to represent a file path.
Definition: Path.h:20
clang::clangd::InputsAndAST::Inputs
const ParseInputs & Inputs
Definition: TUScheduler.h:35
Name
static constexpr llvm::StringLiteral Name
Definition: UppercaseLiteralSuffixCheck.cpp:27
clang::clangd::InputsAndAST
Definition: TUScheduler.h:34
clang::clangd::TUStatus
Definition: TUScheduler.h:81
clang::clangd::ParsingCallbacks::onPreambleAST
virtual void onPreambleAST(PathRef Path, ASTContext &Ctx, std::shared_ptr< clang::Preprocessor > PP, const CanonicalIncludes &)
Called on the AST that was built for emitting the preamble.
Definition: TUScheduler.h:102
clang::clangd::TUStatus::BuildDetails
Definition: TUScheduler.h:82
clang::clangd::ParsedAST
Stores and provides access to parsed AST.
Definition: ParsedAST.h:46
clang::clangd::getDefaultAsyncThreadsCount
unsigned getDefaultAsyncThreadsCount()
Returns a number of a default async threads to use for TUScheduler.
Definition: TUScheduler.cpp:826
clang::clangd::TUAction
Definition: TUScheduler.h:63
clang::clangd::ParsingCallbacks
Definition: TUScheduler.h:95
clang::clangd::TUAction::TUAction
TUAction(State S, llvm::StringRef Name)
Definition: TUScheduler.h:73
clang::clangd::ParseInputs
Information required to run clang, e.g. to parse AST or do code completion.
Definition: Compiler.h:44
clang::clangd::TUScheduler::FileData
Definition: TUScheduler.cpp:843
clang
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
Definition: ApplyReplacements.h:27
Function.h
clang::tidy::bugprone::PP
static Preprocessor * PP
Definition: BadSignalToKillThreadCheck.cpp:28
Path.h
clang::clangd::TUAction::Name
std::string Name
The name of the action currently running, e.g.
Definition: TUScheduler.h:77
clang::clangd::Deadline
A point in time we can wait for.
Definition: Threading.h:58
clang::clangd::FileStatus
Clangd extension: indicates the current state of the file in clangd, sent from server via the textDoc...
Definition: Protocol.h:1202
clang::clangd::TUAction::BuildingPreamble
Definition: TUScheduler.h:67
clang::clangd::InputsAndPreamble
Definition: TUScheduler.h:39
clang::clangd::TUScheduler
Handles running tasks for ClangdServer and managing the resources (e.g., preambles and ASTs) for open...
Definition: TUScheduler.h:147
clang::clangd::InputsAndPreamble::Preamble
const PreambleData * Preamble
Definition: TUScheduler.h:43
clang::clangd::TUScheduler::ASTCache
An LRU cache of idle ASTs.
Definition: TUScheduler.cpp:87
clang::clangd::PreambleData
The parsed preamble and associated data.
Definition: Preamble.h:45
clang::clangd::CompletionItemKind::File
clang::clangd::InputsAndPreamble::Contents
llvm::StringRef Contents
Definition: TUScheduler.h:40
clang::clangd::WantDiagnostics::Auto
Diagnostics must not be generated for this snapshot.
Files
llvm::StringMap< std::string > Files
Definition: TUSchedulerTests.cpp:119
clang::clangd::TUScheduler::Consistent
The preamble is generated from the current version of the file.
Definition: TUScheduler.h:206
clang::clangd::ParsingCallbacks::PublishFn
llvm::function_ref< void(llvm::function_ref< void()>)> PublishFn
The argument function is run under the critical section guarding against races when closing the files...
Definition: TUScheduler.h:108
Threading.h