clang-tools  9.0.0
TUScheduler.h
Go to the documentation of this file.
1 //===--- TUScheduler.h -------------------------------------------*-C++-*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_TUSCHEDULER_H
10 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_TUSCHEDULER_H
11 
12 #include "ClangdUnit.h"
13 #include "Function.h"
15 #include "Threading.h"
17 #include "llvm/ADT/Optional.h"
18 #include "llvm/ADT/StringMap.h"
19 #include "llvm/ADT/StringRef.h"
20 #include <future>
21 
22 namespace clang {
23 namespace clangd {
24 
25 /// Returns a number of a default async threads to use for TUScheduler.
26 /// Returned value is always >= 1 (i.e. will not cause requests to be processed
27 /// synchronously).
29 
30 struct InputsAndAST {
33 };
34 
36  llvm::StringRef Contents;
37  const tooling::CompileCommand &Command;
38  // This can be nullptr if no preamble is availble.
40 };
41 
42 /// Determines whether diagnostics should be generated for a file snapshot.
43 enum class WantDiagnostics {
44  Yes, /// Diagnostics must be generated for this snapshot.
45  No, /// Diagnostics must not be generated for this snapshot.
46  Auto, /// Diagnostics must be generated for this snapshot or a subsequent one,
47  /// within a bounded amount of time.
48 };
49 
50 /// Configuration of the AST retention policy. This only covers retention of
51 /// *idle* ASTs. If queue has operations requiring the AST, they might be
52 /// kept in memory.
54  /// Maximum number of ASTs to be retained in memory when there are no pending
55  /// requests for them.
56  unsigned MaxRetainedASTs = 3;
57 };
58 
59 struct TUAction {
60  enum State {
61  Queued, // The TU is pending in the thread task queue to be built.
62  RunningAction, // Starting running actions on the TU.
63  BuildingPreamble, // The preamble of the TU is being built.
64  BuildingFile, // The TU is being built. It is only emitted when building
65  // the AST for diagnostics in write action (update).
66  Idle, // Indicates the worker thread is idle, and ready to run any upcoming
67  // actions.
68  };
69  TUAction(State S, llvm::StringRef Name) : S(S), Name(Name) {}
71  /// The name of the action currently running, e.g. Update, GoToDef, Hover.
72  /// Empty if we are in the idle state.
73  std::string Name;
74 };
75 
76 // Internal status of the TU in TUScheduler.
77 struct TUStatus {
78  struct BuildDetails {
79  /// Indicates whether clang failed to build the TU.
80  bool BuildFailed = false;
81  /// Indicates whether we reused the prebuilt AST.
82  bool ReuseAST = false;
83  };
84  /// Serialize this to an LSP file status item.
85  FileStatus render(PathRef File) const;
86 
89 };
90 
92 public:
93  virtual ~ParsingCallbacks() = default;
94 
95  /// Called on the AST that was built for emitting the preamble. The built AST
96  /// contains only AST nodes from the #include directives at the start of the
97  /// file. AST node in the current file should be observed on onMainAST call.
98  virtual void onPreambleAST(PathRef Path, ASTContext &Ctx,
99  std::shared_ptr<clang::Preprocessor> PP,
100  const CanonicalIncludes &) {}
101  /// Called on the AST built for the file itself. Note that preamble AST nodes
102  /// are not deserialized and should be processed in the onPreambleAST call
103  /// instead.
104  /// The \p AST always contains all AST nodes for the main file itself, and
105  /// only a portion of the AST nodes deserialized from the preamble. Note that
106  /// some nodes from the preamble may have been deserialized and may also be
107  /// accessed from the main file AST, e.g. redecls of functions from preamble,
108  /// etc. Clients are expected to process only the AST nodes from the main file
109  /// in this callback (obtained via ParsedAST::getLocalTopLevelDecls) to obtain
110  /// optimal performance.
111  virtual void onMainAST(PathRef Path, ParsedAST &AST) {}
112 
113  /// Called whenever the diagnostics for \p File are produced.
114  virtual void onDiagnostics(PathRef File, std::vector<Diag> Diags) {}
115 
116  /// Called whenever the TU status is updated.
117  virtual void onFileUpdated(PathRef File, const TUStatus &Status) {}
118 };
119 
120 /// Handles running tasks for ClangdServer and managing the resources (e.g.,
121 /// preambles and ASTs) for opened files.
122 /// TUScheduler is not thread-safe, only one thread should be providing updates
123 /// and scheduling tasks.
124 /// Callbacks are run on a threadpool and it's appropriate to do slow work in
125 /// them. Each task has a name, used for tracing (should be UpperCamelCase).
126 /// FIXME(sammccall): pull out a scheduler options struct.
127 class TUScheduler {
128 public:
129  TUScheduler(const GlobalCompilationDatabase &CDB, unsigned AsyncThreadsCount,
130  bool StorePreamblesInMemory,
131  std::unique_ptr<ParsingCallbacks> ASTCallbacks,
132  std::chrono::steady_clock::duration UpdateDebounce,
133  ASTRetentionPolicy RetentionPolicy);
134  ~TUScheduler();
135 
136  /// Returns estimated memory usage for each of the currently open files.
137  /// The order of results is unspecified.
138  std::vector<std::pair<Path, std::size_t>> getUsedBytesPerFile() const;
139 
140  /// Returns a list of files with ASTs currently stored in memory. This method
141  /// is not very reliable and is only used for test. E.g., the results will not
142  /// contain files that currently run something over their AST.
143  std::vector<Path> getFilesWithCachedAST() const;
144 
145  /// Schedule an update for \p File.
146  /// The compile command in \p Inputs is ignored; worker queries CDB to get
147  /// the actual compile command.
148  /// If diagnostics are requested (Yes), and the context is cancelled
149  /// before they are prepared, they may be skipped if eventual-consistency
150  /// permits it (i.e. WantDiagnostics is downgraded to Auto).
151  /// Returns true if the file was not previously tracked.
152  bool update(PathRef File, ParseInputs Inputs, WantDiagnostics WD);
153 
154  /// Remove \p File from the list of tracked files and schedule removal of its
155  /// resources. Pending diagnostics for closed files may not be delivered, even
156  /// if requested with WantDiags::Auto or WantDiags::Yes.
157  void remove(PathRef File);
158 
159  /// Returns the current contents of the buffer for File, per last update().
160  /// The returned StringRef may be invalidated by any write to TUScheduler.
161  llvm::StringRef getContents(PathRef File) const;
162 
163  /// Schedule an async task with no dependencies.
164  void run(llvm::StringRef Name, llvm::unique_function<void()> Action);
165 
166  /// Schedule an async read of the AST. \p Action will be called when AST is
167  /// ready. The AST passed to \p Action refers to the version of \p File
168  /// tracked at the time of the call, even if new updates are received before
169  /// \p Action is executed.
170  /// If an error occurs during processing, it is forwarded to the \p Action
171  /// callback.
172  /// If the context is cancelled before the AST is ready, the callback will
173  /// receive a CancelledError.
174  void runWithAST(llvm::StringRef Name, PathRef File,
176 
177  /// Controls whether preamble reads wait for the preamble to be up-to-date.
179  /// The preamble is generated from the current version of the file.
180  /// If the content was recently updated, we will wait until we have a
181  /// preamble that reflects that update.
182  /// This is the slowest option, and may be delayed by other tasks.
184  /// The preamble may be generated from an older version of the file.
185  /// Reading from locations in the preamble may cause files to be re-read.
186  /// This gives callers two options:
187  /// - validate that the preamble is still valid, and only use it if so
188  /// - accept that the preamble contents may be outdated, and try to avoid
189  /// reading source code from headers.
190  /// This is the fastest option, usually a preamble is available immediately.
192  /// Besides accepting stale preamble, this also allow preamble to be absent
193  /// (not ready or failed to build).
195  };
196 
197  /// Schedule an async read of the preamble.
198  /// If there's no up-to-date preamble, we follow the PreambleConsistency
199  /// policy.
200  /// If an error occurs, it is forwarded to the \p Action callback.
201  /// Context cancellation is ignored and should be handled by the Action.
202  /// (In practice, the Action is almost always executed immediately).
203  void runWithPreamble(llvm::StringRef Name, PathRef File,
204  PreambleConsistency Consistency,
206 
207  /// Wait until there are no scheduled or running tasks.
208  /// Mostly useful for synchronizing tests.
209  bool blockUntilIdle(Deadline D) const;
210 
211 private:
212  /// This class stores per-file data in the Files map.
213  struct FileData;
214 
215 public:
216  /// Responsible for retaining and rebuilding idle ASTs. An implementation is
217  /// an LRU cache.
218  class ASTCache;
219 
220  // The file being built/processed in the current thread. This is a hack in
221  // order to get the file name into the index implementations. Do not depend on
222  // this inside clangd.
223  // FIXME: remove this when there is proper index support via build system
224  // integration.
225  static llvm::Optional<llvm::StringRef> getFileBeingProcessedInContext();
226 
227 private:
228  const GlobalCompilationDatabase &CDB;
229  const bool StorePreamblesInMemory;
230  std::unique_ptr<ParsingCallbacks> Callbacks; // not nullptr
231  Semaphore Barrier;
232  llvm::StringMap<std::unique_ptr<FileData>> Files;
233  std::unique_ptr<ASTCache> IdleASTs;
234  // None when running tasks synchronously and non-None when running tasks
235  // asynchronously.
236  llvm::Optional<AsyncTaskRunner> PreambleTasks;
237  llvm::Optional<AsyncTaskRunner> WorkerThreads;
238  std::chrono::steady_clock::duration UpdateDebounce;
239 };
240 
241 /// Runs \p Action asynchronously with a new std::thread. The context will be
242 /// propagated.
243 template <typename T>
244 std::future<T> runAsync(llvm::unique_function<T()> Action) {
245  return std::async(
246  std::launch::async,
247  [](llvm::unique_function<T()> &&Action, Context Ctx) {
248  WithContext WithCtx(std::move(Ctx));
249  return Action();
250  },
251  std::move(Action), Context::current().clone());
252 }
253 
254 } // namespace clangd
255 } // namespace clang
256 
257 #endif
PreambleConsistency
Controls whether preamble reads wait for the preamble to be up-to-date.
Definition: TUScheduler.h:178
WantDiagnostics
Determines whether diagnostics should be generated for a file snapshot.
Definition: TUScheduler.h:43
The preamble may be generated from an older version of the file.
Definition: TUScheduler.h:191
Diagnostics must be generated for this snapshot.
const tooling::CompileCommand & Command
Definition: TUScheduler.h:37
Besides accepting stale preamble, this also allow preamble to be absent (not ready or failed to build...
Definition: TUScheduler.h:194
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:23
llvm::unique_function< void(llvm::Expected< T >)> Callback
A Callback<T> is a void function that accepts Expected<T>.
Definition: Function.h:28
virtual void onMainAST(PathRef Path, ParsedAST &AST)
Called on the AST built for the file itself.
Definition: TUScheduler.h:111
Limits the number of threads that can acquire the lock at the same time.
Definition: Threading.h:40
virtual void onFileUpdated(PathRef File, const TUStatus &Status)
Called whenever the TU status is updated.
Definition: TUScheduler.h:117
BuildDetails Details
Definition: TUScheduler.h:88
Configuration of the AST retention policy.
Definition: TUScheduler.h:53
std::future< T > runAsync(llvm::unique_function< T()> Action)
Runs Action asynchronously with a new std::thread.
Definition: TUScheduler.h:244
Provides compilation arguments used for parsing C and C++ files.
Maps a definition location onto an #include file, based on a set of filename rules.
Context Ctx
Context clone() const
Clone this context object.
Definition: Context.cpp:20
llvm::unique_function< void()> Action
std::string Path
A typedef to represent a file path.
Definition: Path.h:20
static const Context & current()
Returns the context for the current thread, creating it if needed.
Definition: Context.cpp:27
const ParseInputs & Inputs
Definition: TUScheduler.h:31
static constexpr llvm::StringLiteral Name
const Decl * D
Definition: XRefs.cpp:868
virtual void onDiagnostics(PathRef File, std::vector< Diag > Diags)
Called whenever the diagnostics for File are produced.
Definition: TUScheduler.h:114
A context is an immutable container for per-request data that must be propagated through layers that ...
Definition: Context.h:69
virtual void onPreambleAST(PathRef Path, ASTContext &Ctx, std::shared_ptr< clang::Preprocessor > PP, const CanonicalIncludes &)
Called on the AST that was built for emitting the preamble.
Definition: TUScheduler.h:98
Stores and provides access to parsed AST.
Definition: ClangdUnit.h:73
unsigned getDefaultAsyncThreadsCount()
Returns a number of a default async threads to use for TUScheduler.
WithContext replaces Context::current() with a provided scope.
Definition: Context.h:189
TUAction(State S, llvm::StringRef Name)
Definition: TUScheduler.h:69
Information required to run clang, e.g. to parse AST or do code completion.
Definition: Compiler.h:44
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
std::string Name
The name of the action currently running, e.g.
Definition: TUScheduler.h:73
A point in time we can wait for.
Definition: Threading.h:57
Clangd extension: indicates the current state of the file in clangd, sent from server via the textDoc...
Definition: Protocol.h:1184
Handles running tasks for ClangdServer and managing the resources (e.g., preambles and ASTs) for open...
Definition: TUScheduler.h:127
const PreambleData * Preamble
Definition: TUScheduler.h:39
An LRU cache of idle ASTs.
Definition: TUScheduler.cpp:83
Diagnostics must not be generated for this snapshot.
llvm::StringMap< std::string > Files
The preamble is generated from the current version of the file.
Definition: TUScheduler.h:183