clang-tools 8.0.0

TUScheduler.h
Go to the documentation of this file.
1 //===--- TUScheduler.h -------------------------------------------*-C++-*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9
10 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_TUSCHEDULER_H
11 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_TUSCHEDULER_H
12
13 #include "ClangdUnit.h"
14 #include "Function.h"
15 #include "Threading.h"
16 #include "llvm/ADT/StringMap.h"
17 #include <future>
18
19 namespace clang {
20 namespace clangd {
21
22 /// Returns a number of a default async threads to use for TUScheduler.
23 /// Returned value is always >= 1 (i.e. will not cause requests to be processed
24 /// synchronously).
25 unsigned getDefaultAsyncThreadsCount();
26
27 struct InputsAndAST {
28 const ParseInputs &Inputs;
29 ParsedAST &AST;
30 };
31
32 struct InputsAndPreamble {
33 llvm::StringRef Contents;
34 const tooling::CompileCommand &Command;
35 const PreambleData *Preamble;
36 };
37
38 /// Determines whether diagnostics should be generated for a file snapshot.
39 enum class WantDiagnostics {
40 Yes, /// Diagnostics must be generated for this snapshot.
41 No, /// Diagnostics must not be generated for this snapshot.
42 Auto, /// Diagnostics must be generated for this snapshot or a subsequent one,
43 /// within a bounded amount of time.
44 };
45
46 /// Configuration of the AST retention policy. This only covers retention of
47 /// *idle* ASTs. If queue has operations requiring the AST, they might be
48 /// kept in memory.
49 struct ASTRetentionPolicy {
50 /// Maximum number of ASTs to be retained in memory when there are no pending
51 /// requests for them.
52 unsigned MaxRetainedASTs = 3;
53 };
54
55 struct TUAction {
56 enum State {
57 Queued, // The TU is pending in the thread task queue to be built.
58 RunningAction, // Starting running actions on the TU.
59 BuildingPreamble, // The preamble of the TU is being built.
60 BuildingFile, // The TU is being built. It is only emitted when building
61 // the AST for diagnostics in write action (update).
62 Idle, // Indicates the worker thread is idle, and ready to run any upcoming
63 // actions.
64 };
65 TUAction(State S, llvm::StringRef Name) : S(S), Name(Name) {}
66 State S;
67 /// The name of the action currently running, e.g. Update, GoToDef, Hover.
68 /// Empty if we are in the idle state.
69 std::string Name;
70 };
71
72 // Internal status of the TU in TUScheduler.
73 struct TUStatus {
74 struct BuildDetails {
75 /// Indicates whether clang failed to build the TU.
76 bool BuildFailed = false;
77 /// Indicates whether we reused the prebuilt AST.
78 bool ReuseAST = false;
79 };
80 /// Serialize this to an LSP file status item.
81 FileStatus render(PathRef File) const;
82
83 TUAction Action;
84 BuildDetails Details;
85 };
86
87 class ParsingCallbacks {
88 public:
89 virtual ~ParsingCallbacks() = default;
90
91 /// Called on the AST that was built for emitting the preamble. The built AST
92 /// contains only AST nodes from the #include directives at the start of the
93 /// file. AST node in the current file should be observed on onMainAST call.
94 virtual void onPreambleAST(PathRef Path, ASTContext &Ctx,
95 std::shared_ptr<clang::Preprocessor> PP) {}
96 /// Called on the AST built for the file itself. Note that preamble AST nodes
97 /// are not deserialized and should be processed in the onPreambleAST call
98 /// instead.
99 /// The \p AST always contains all AST nodes for the main file itself, and
100 /// only a portion of the AST nodes deserialized from the preamble. Note that
101 /// some nodes from the preamble may have been deserialized and may also be
102 /// accessed from the main file AST, e.g. redecls of functions from preamble,
103 /// etc. Clients are expected to process only the AST nodes from the main file
104 /// in this callback (obtained via ParsedAST::getLocalTopLevelDecls) to obtain
105 /// optimal performance.
106 virtual void onMainAST(PathRef Path, ParsedAST &AST) {}
107
108 /// Called whenever the diagnostics for \p File are produced.
109 virtual void onDiagnostics(PathRef File, std::vector<Diag> Diags) {}
110
111 /// Called whenever the TU status is updated.
112 virtual void onFileUpdated(PathRef File, const TUStatus &Status) {}
113 };
114
115 /// Handles running tasks for ClangdServer and managing the resources (e.g.,
116 /// preambles and ASTs) for opened files.
117 /// TUScheduler is not thread-safe, only one thread should be providing updates
118 /// and scheduling tasks.
119 /// Callbacks are run on a threadpool and it's appropriate to do slow work in
120 /// them. Each task has a name, used for tracing (should be UpperCamelCase).
121 /// FIXME(sammccall): pull out a scheduler options struct.
122 class TUScheduler {
123 public:
124 TUScheduler(unsigned AsyncThreadsCount, bool StorePreamblesInMemory,
125 std::unique_ptr<ParsingCallbacks> ASTCallbacks,
126 std::chrono::steady_clock::duration UpdateDebounce,
127 ASTRetentionPolicy RetentionPolicy);
128 ~TUScheduler();
129
130 /// Returns estimated memory usage for each of the currently open files.
131 /// The order of results is unspecified.
132 std::vector<std::pair<Path, std::size_t>> getUsedBytesPerFile() const;
133
134 /// Returns a list of files with ASTs currently stored in memory. This method
135 /// is not very reliable and is only used for test. E.g., the results will not
136 /// contain files that currently run something over their AST.
137 std::vector<Path> getFilesWithCachedAST() const;
138
139 /// Schedule an update for \p File. Adds \p File to a list of tracked files if
140 /// \p File was not part of it before.
141 /// If diagnostics are requested (Yes), and the context is cancelled before
142 /// they are prepared, they may be skipped if eventual-consistency permits it
143 /// (i.e. WantDiagnostics is downgraded to Auto).
144 void update(PathRef File, ParseInputs Inputs, WantDiagnostics WD);
145
146 /// Remove \p File from the list of tracked files and schedule removal of its
147 /// resources. Pending diagnostics for closed files may not be delivered, even
148 /// if requested with WantDiags::Auto or WantDiags::Yes.
149 void remove(PathRef File);
150
151 /// Schedule an async task with no dependencies.
152 void run(llvm::StringRef Name, llvm::unique_function<void()> Action);
153
154 /// Schedule an async read of the AST. \p Action will be called when AST is
155 /// ready. The AST passed to \p Action refers to the version of \p File
156 /// tracked at the time of the call, even if new updates are received before
157 /// \p Action is executed.
158 /// If an error occurs during processing, it is forwarded to the \p Action
159 /// callback.
160 /// If the context is cancelled before the AST is ready, the callback will
161 /// receive a CancelledError.
162 void runWithAST(llvm::StringRef Name, PathRef File,
163 Callback<InputsAndAST> Action);
164
165 /// Controls whether preamble reads wait for the preamble to be up-to-date.
166 enum PreambleConsistency {
167 /// The preamble is generated from the current version of the file.
168 /// If the content was recently updated, we will wait until we have a
169 /// preamble that reflects that update.
170 /// This is the slowest option, and may be delayed by other tasks.
171 Consistent,
172 /// The preamble may be generated from an older version of the file.
173 /// Reading from locations in the preamble may cause files to be re-read.
174 /// This gives callers two options:
175 /// - validate that the preamble is still valid, and only use it if so
176 /// - accept that the preamble contents may be outdated, and try to avoid
177 /// reading source code from headers.
178 /// This is the fastest option, usually a preamble is available immediately.
179 Stale,
180 };
181 /// Schedule an async read of the preamble.
182 /// If there's no preamble yet (because the file was just opened), we'll wait
183 /// for it to build. The result may be null if it fails to build or is empty.
184 /// If an error occurs, it is forwarded to the \p Action callback.
185 /// Context cancellation is ignored and should be handled by the Action.
186 /// (In practice, the Action is almost always executed immediately).
187 void runWithPreamble(llvm::StringRef Name, PathRef File,
188 PreambleConsistency Consistency,
189 Callback<InputsAndPreamble> Action);
190
191 /// Wait until there are no scheduled or running tasks.
192 /// Mostly useful for synchronizing tests.
193 bool blockUntilIdle(Deadline D) const;
194
195 private:
196 /// This class stores per-file data in the Files map.
197 struct FileData;
198
199 public:
200 /// Responsible for retaining and rebuilding idle ASTs. An implementation is
201 /// an LRU cache.
202 class ASTCache;
203
204 // The file being built/processed in the current thread. This is a hack in
205 // order to get the file name into the index implementations. Do not depend on
206 // this inside clangd.
207 // FIXME: remove this when there is proper index support via build system
208 // integration.
209 static llvm::Optional<llvm::StringRef> getFileBeingProcessedInContext();
210
211 private:
212 const bool StorePreamblesInMemory;
213 const std::shared_ptr<PCHContainerOperations> PCHOps;
214 std::unique_ptr<ParsingCallbacks> Callbacks; // not nullptr
215 Semaphore Barrier;
216 llvm::StringMap<std::unique_ptr<FileData>> Files;
217 std::unique_ptr<ASTCache> IdleASTs;
218 // None when running tasks synchronously and non-None when running tasks
219 // asynchronously.
220 llvm::Optional<AsyncTaskRunner> PreambleTasks;
221 llvm::Optional<AsyncTaskRunner> WorkerThreads;
222 std::chrono::steady_clock::duration UpdateDebounce;
223 };
224
225 /// Runs \p Action asynchronously with a new std::thread. The context will be
226 /// propagated.
227 template <typename T>
228 std::future<T> runAsync(llvm::unique_function<T()> Action) {
229 return std::async(
230 std::launch::async,
231 [](llvm::unique_function<T()> &&Action, Context Ctx) {
232 WithContext WithCtx(std::move(Ctx));
233 return Action();
234 },
235 std::move(Action), Context::current().clone());
236 }
237
238 } // namespace clangd
239 } // namespace clang
240
241 #endif
clang::clangd::TUScheduler::PreambleConsistency
PreambleConsistency
Controls whether preamble reads wait for the preamble to be up-to-date.
Definition: TUScheduler.h:166
clang::clangd::WantDiagnostics
WantDiagnostics
Determines whether diagnostics should be generated for a file snapshot.
Definition: TUScheduler.h:39
clang::clangd::TUScheduler::Stale
The preamble may be generated from an older version of the file.
Definition: TUScheduler.h:179
clang::clangd::WantDiagnostics::No
Diagnostics must be generated for this snapshot.
clang::clangd::InputsAndPreamble::Command
const tooling::CompileCommand & Command
Definition: TUScheduler.h:34
clang::clangd::TUAction::Idle
Definition: TUScheduler.h:62
clang::clangd::TUAction::Queued
Definition: TUScheduler.h:57
clang::clangd::InputsAndAST::AST
ParsedAST & AST
Definition: TUScheduler.h:29
clang::clangd::PathRef
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:24
clang::clangd::TUAction::RunningAction
Definition: TUScheduler.h:58
clang::clangd::TUAction::S
State S
Definition: TUScheduler.h:66
clang::clangd::Callback
llvm::unique_function< void(llvm::Expected< T >)> Callback
A Callback<T> is a void function that accepts Expected<T>.
Definition: Function.h:29
clang::clangd::TUStatus::Action
TUAction Action
Definition: TUScheduler.h:83
clang::clangd::ParsingCallbacks::onMainAST
virtual void onMainAST(PathRef Path, ParsedAST &AST)
Called on the AST built for the file itself.
Definition: TUScheduler.h:106
clang::clangd::Semaphore
Limits the number of threads that can acquire the lock at the same time.
Definition: Threading.h:41
ClangdUnit.h
clang::clangd::TUAction::BuildingFile
Definition: TUScheduler.h:60
clang::clangd::ParsingCallbacks::onFileUpdated
virtual void onFileUpdated(PathRef File, const TUStatus &Status)
Called whenever the TU status is updated.
Definition: TUScheduler.h:112
clang::clangd::TUStatus::Details
BuildDetails Details
Definition: TUScheduler.h:84
clang::clangd::ASTRetentionPolicy
Configuration of the AST retention policy.
Definition: TUScheduler.h:49
clang::clangd::runAsync
std::future< T > runAsync(llvm::unique_function< T()> Action)
Runs Action asynchronously with a new std::thread.
Definition: TUScheduler.h:228
clang::clangd::WantDiagnostics::Yes
Ctx
Context Ctx
Definition: TUScheduler.cpp:224
clang::clangd::TUAction::State
State
Definition: TUScheduler.h:56
clang::clangd::Context::clone
Context clone() const
Clone this context object.
Definition: Context.cpp:21
Action
llvm::unique_function< void()> Action
Definition: TUScheduler.cpp:221
clang::clangd::ParsingCallbacks::onPreambleAST
virtual void onPreambleAST(PathRef Path, ASTContext &Ctx, std::shared_ptr< clang::Preprocessor > PP)
Called on the AST that was built for emitting the preamble.
Definition: TUScheduler.h:94
clang::clangd::Path
std::string Path
A typedef to represent a file path.
Definition: Path.h:21
clang::clangd::Context::current
static const Context & current()
Returns the context for the current thread, creating it if needed.
Definition: Context.cpp:28
clang::clangd::InputsAndAST::Inputs
const ParseInputs & Inputs
Definition: TUScheduler.h:28
Name
static constexpr llvm::StringLiteral Name
Definition: UppercaseLiteralSuffixCheck.cpp:28
clang::clangd::InputsAndAST
Definition: TUScheduler.h:27
D
const Decl * D
Definition: XRefs.cpp:79
clang::clangd::ParsingCallbacks::onDiagnostics
virtual void onDiagnostics(PathRef File, std::vector< Diag > Diags)
Called whenever the diagnostics for File are produced.
Definition: TUScheduler.h:109
clang::clangd::TUStatus
Definition: TUScheduler.h:73
clang::clangd::Context
A context is an immutable container for per-request data that must be propagated through layers that ...
Definition: Context.h:70
clang::clangd::TUStatus::BuildDetails
Definition: TUScheduler.h:74
clang::clangd::ParsedAST
Stores and provides access to parsed AST.
Definition: ClangdUnit.h:71
clang::clangd::getDefaultAsyncThreadsCount
unsigned getDefaultAsyncThreadsCount()
Returns a number of a default async threads to use for TUScheduler.
Definition: TUScheduler.cpp:762
clang::clangd::WithContext
WithContext replaces Context::current() with a provided scope.
Definition: Context.h:190
clang::clangd::TUAction
Definition: TUScheduler.h:55
clang::clangd::ParsingCallbacks
Definition: TUScheduler.h:87
clang::clangd::TUAction::TUAction
TUAction(State S, llvm::StringRef Name)
Definition: TUScheduler.h:65
clang::clangd::ParseInputs
Information required to run clang, e.g. to parse AST or do code completion.
Definition: ClangdUnit.h:64
clang::clangd::TUScheduler::FileData
Definition: TUScheduler.cpp:779
clang
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
Definition: ChangeNamespace.cpp:18
Function.h
clang::clangd::TUAction::Name
std::string Name
The name of the action currently running, e.g.
Definition: TUScheduler.h:69
clang::clangd::Deadline
A point in time we can wait for.
Definition: Threading.h:58
clang::clangd::FileStatus
Clangd extension: indicates the current state of the file in clangd, sent from server via the textDoc...
Definition: Protocol.h:1006
clang::clangd::TUAction::BuildingPreamble
Definition: TUScheduler.h:59
clang::clangd::InputsAndPreamble
Definition: TUScheduler.h:32
clang::clangd::TUScheduler
Handles running tasks for ClangdServer and managing the resources (e.g., preambles and ASTs) for open...
Definition: TUScheduler.h:122
clang::clangd::InputsAndPreamble::Preamble
const PreambleData * Preamble
Definition: TUScheduler.h:35
clang::clangd::TUScheduler::ASTCache
An LRU cache of idle ASTs.
Definition: TUScheduler.cpp:79
clang::clangd::PreambleData
Definition: ClangdUnit.h:47
clang::clangd::CompletionItemKind::File
clang::clangd::InputsAndPreamble::Contents
llvm::StringRef Contents
Definition: TUScheduler.h:33
clang::clangd::WantDiagnostics::Auto
Diagnostics must not be generated for this snapshot.
clang::clangd::TUScheduler::Consistent
The preamble is generated from the current version of the file.
Definition: TUScheduler.h:171
Threading.h