clang-tools 8.0.0

Background.h
Go to the documentation of this file.
1 //===--- Background.h - Build an index in a background thread ----*- C++-*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9
10 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_BACKGROUND_H
11 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_BACKGROUND_H
12
13 #include "Context.h"
14 #include "FSProvider.h"
15 #include "GlobalCompilationDatabase.h"
16 #include "Threading.h"
17 #include "index/FileIndex.h"
18 #include "index/Index.h"
19 #include "index/Serialization.h"
20 #include "clang/Tooling/CompilationDatabase.h"
21 #include "llvm/ADT/StringMap.h"
22 #include "llvm/Support/SHA1.h"
23 #include "llvm/Support/Threading.h"
24 #include <atomic>
25 #include <condition_variable>
26 #include <deque>
27 #include <mutex>
28 #include <string>
29 #include <thread>
30 #include <vector>
31
32 namespace clang {
33 namespace clangd {
34
35 // Handles storage and retrieval of index shards. Both store and load
36 // operations can be called from multiple-threads concurrently.
37 class BackgroundIndexStorage {
38 public:
39 virtual ~BackgroundIndexStorage() = default;
40
41 // Shards of the index are stored and retrieved independently, keyed by shard
42 // identifier - in practice this is a source file name
43 virtual llvm::Error storeShard(llvm::StringRef ShardIdentifier,
44 IndexFileOut Shard) const = 0;
45
46 // Tries to load shard with given identifier, returns nullptr if shard
47 // couldn't be loaded.
48 virtual std::unique_ptr<IndexFileIn>
49 loadShard(llvm::StringRef ShardIdentifier) const = 0;
50
51 // The factory provides storage for each CDB.
52 // It keeps ownership of the storage instances, and should manage caching
53 // itself. Factory must be threadsafe and never returns nullptr.
54 using Factory =
55 llvm::unique_function<BackgroundIndexStorage *(llvm::StringRef)>;
56
57 // Creates an Index Storage that saves shards into disk. Index storage uses
58 // CDBDirectory + ".clangd/index/" as the folder to save shards.
59 static Factory createDiskBackedStorageFactory();
60 };
61
62 // Builds an in-memory index by by running the static indexer action over
63 // all commands in a compilation database. Indexing happens in the background.
64 // FIXME: it should also persist its state on disk for fast start.
65 // FIXME: it should watch for changes to files on disk.
66 class BackgroundIndex : public SwapIndex {
67 public:
68 /// If BuildIndexPeriodMs is greater than 0, the symbol index will only be
69 /// rebuilt periodically (one per \p BuildIndexPeriodMs); otherwise, index is
70 /// rebuilt for each indexed file.
71 BackgroundIndex(
72 Context BackgroundContext, const FileSystemProvider &,
73 const GlobalCompilationDatabase &CDB,
74 BackgroundIndexStorage::Factory IndexStorageFactory,
75 size_t BuildIndexPeriodMs = 0,
76 size_t ThreadPoolSize = llvm::heavyweight_hardware_concurrency());
77 ~BackgroundIndex(); // Blocks while the current task finishes.
78
79 // Enqueue translation units for indexing.
80 // The indexing happens in a background thread, so the symbols will be
81 // available sometime later.
82 void enqueue(const std::vector<std::string> &ChangedFiles);
83
84 // Cause background threads to stop after ther current task, any remaining
85 // tasks will be discarded.
86 void stop();
87
88 // Wait until the queue is empty, to allow deterministic testing.
89 LLVM_NODISCARD bool
90 blockUntilIdleForTest(llvm::Optional<double> TimeoutSeconds = 10);
91
92 private:
93 /// Given index results from a TU, only update symbols coming from files with
94 /// different digests than \p DigestsSnapshot. Also stores new index
95 /// information on IndexStorage.
96 void update(llvm::StringRef MainFile, IndexFileIn Index,
97 const llvm::StringMap<FileDigest> &DigestsSnapshot,
98 BackgroundIndexStorage *IndexStorage);
99
100 // configuration
101 const FileSystemProvider &FSProvider;
102 const GlobalCompilationDatabase &CDB;
103 Context BackgroundContext;
104
105 // index state
106 llvm::Error index(tooling::CompileCommand,
107 BackgroundIndexStorage *IndexStorage);
108 void buildIndex(); // Rebuild index periodically every BuildIndexPeriodMs.
109 const size_t BuildIndexPeriodMs;
110 std::atomic<bool> SymbolsUpdatedSinceLastIndex;
111 std::mutex IndexMu;
112 std::condition_variable IndexCV;
113
114 FileSymbols IndexedSymbols;
115 llvm::StringMap<FileDigest> IndexedFileDigests; // Key is absolute file path.
116 std::mutex DigestsMu;
117
118 BackgroundIndexStorage::Factory IndexStorageFactory;
119 struct Source {
120 std::string Path;
121 bool NeedsReIndexing;
122 Source(llvm::StringRef Path, bool NeedsReIndexing)
123 : Path(Path), NeedsReIndexing(NeedsReIndexing) {}
124 };
125 // Loads the shards for a single TU and all of its dependencies. Returns the
126 // list of sources and whether they need to be re-indexed.
127 std::vector<Source> loadShard(const tooling::CompileCommand &Cmd,
128 BackgroundIndexStorage *IndexStorage,
129 llvm::StringSet<> &LoadedShards);
130 // Tries to load shards for the ChangedFiles.
131 std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>
132 loadShards(std::vector<std::string> ChangedFiles);
133 void enqueue(tooling::CompileCommand Cmd, BackgroundIndexStorage *Storage);
134
135 // queue management
136 using Task = std::function<void()>;
137 void run(); // Main loop executed by Thread. Runs tasks from Queue.
138 void enqueueTask(Task T, ThreadPriority Prioirty);
139 void enqueueLocked(tooling::CompileCommand Cmd,
140 BackgroundIndexStorage *IndexStorage);
141 std::mutex QueueMu;
142 unsigned NumActiveTasks = 0; // Only idle when queue is empty *and* no tasks.
143 std::condition_variable QueueCV;
144 bool ShouldStop = false;
145 std::deque<std::pair<Task, ThreadPriority>> Queue;
146 std::vector<std::thread> ThreadPool; // FIXME: Abstract this away.
147 GlobalCompilationDatabase::CommandChanged::Subscription CommandsChanged;
148 };
149
150 } // namespace clangd
151 } // namespace clang
152
153 #endif
Index.h
clang::clangd::FileSymbols
A container of Symbols from several source files.
Definition: FileIndex.h:58
clang::clangd::BackgroundIndexStorage::createDiskBackedStorageFactory
static Factory createDiskBackedStorageFactory()
Definition: BackgroundIndexStorage.cpp:158
clang::clangd::ThreadPriority
ThreadPriority
Definition: Threading.h:121
Context.h
clang::clangd::FileSystemProvider
Definition: FSProvider.h:21
Serialization.h
GlobalCompilationDatabase.h
clang::clangd::BackgroundIndexStorage::Factory
llvm::unique_function< BackgroundIndexStorage *(llvm::StringRef)> Factory
Definition: Background.h:55
clang::clangd::GlobalCompilationDatabase
Provides compilation arguments used for parsing C and C++ files.
Definition: GlobalCompilationDatabase.h:39
clang::clangd::IndexFileIn
Definition: Serialization.h:40
clang::clangd::BackgroundIndex
Definition: Background.h:66
clang::clangd::Path
std::string Path
A typedef to represent a file path.
Definition: Path.h:21
clang::clangd::Context
A context is an immutable container for per-request data that must be propagated through layers that ...
Definition: Context.h:70
clang::clangd::BackgroundIndexStorage::loadShard
virtual std::unique_ptr< IndexFileIn > loadShard(llvm::StringRef ShardIdentifier) const =0
clang::clangd::SwapIndex
Definition: Index.h:523
clang::clangd::BackgroundIndexStorage::storeShard
virtual llvm::Error storeShard(llvm::StringRef ShardIdentifier, IndexFileOut Shard) const =0
clang
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
Definition: ChangeNamespace.cpp:18
FSProvider.h
clang::clangd::IndexFileOut
Definition: Serialization.h:50
clang::clangd::BackgroundIndexStorage::~BackgroundIndexStorage
virtual ~BackgroundIndexStorage()=default
FileIndex.h
clang::clangd::BackgroundIndexStorage
Definition: Background.h:37
Index
const SymbolIndex * Index
Definition: Dexp.cpp:85
Threading.h