clang-tools 10.0.0

BackgroundIndexTests.cpp
Go to the documentation of this file.
1 #include "Headers.h"
2 #include "SyncAPI.h"
3 #include "TestFS.h"
4 #include "TestIndex.h"
5 #include "TestTU.h"
6 #include "index/Background.h"
7 #include "index/BackgroundRebuild.h"
8 #include "clang/Tooling/CompilationDatabase.h"
9 #include "llvm/Support/ScopedPrinter.h"
10 #include "llvm/Support/Threading.h"
11 #include "gmock/gmock.h"
12 #include "gtest/gtest.h"
13 #include <deque>
14 #include <thread>
15
16 using ::testing::_;
17 using ::testing::AllOf;
18 using ::testing::Contains;
19 using ::testing::ElementsAre;
20 using ::testing::Not;
21 using ::testing::UnorderedElementsAre;
22
23 namespace clang {
24 namespace clangd {
25
26 MATCHER_P(Named, N, "") { return arg.Name == N; }
27 MATCHER(Declared, "") {
28 return !StringRef(arg.CanonicalDeclaration.FileURI).empty();
29 }
30 MATCHER(Defined, "") { return !StringRef(arg.Definition.FileURI).empty(); }
31 MATCHER_P(FileURI, F, "") { return StringRef(arg.Location.FileURI) == F; }
32 ::testing::Matcher<const RefSlab &>
33 RefsAre(std::vector<::testing::Matcher<Ref>> Matchers) {
34 return ElementsAre(::testing::Pair(_, UnorderedElementsAreArray(Matchers)));
35 }
36 // URI cannot be empty since it references keys in the IncludeGraph.
37 MATCHER(EmptyIncludeNode, "") {
38 return arg.Flags == IncludeGraphNode::SourceFlag::None && !arg.URI.empty() &&
39 arg.Digest == FileDigest{{0}} && arg.DirectIncludes.empty();
40 }
41
42 MATCHER(HadErrors, "") {
43 return arg.Flags & IncludeGraphNode::SourceFlag::HadErrors;
44 }
45
46 MATCHER_P(NumReferences, N, "") { return arg.References == N; }
47
48 class MemoryShardStorage : public BackgroundIndexStorage {
49 mutable std::mutex StorageMu;
50 llvm::StringMap<std::string> &Storage;
51 size_t &CacheHits;
52
53 public:
54 MemoryShardStorage(llvm::StringMap<std::string> &Storage, size_t &CacheHits)
55 : Storage(Storage), CacheHits(CacheHits) {}
56 llvm::Error storeShard(llvm::StringRef ShardIdentifier,
57 IndexFileOut Shard) const override {
58 std::lock_guard<std::mutex> Lock(StorageMu);
59 AccessedPaths.insert(ShardIdentifier);
60 Storage[ShardIdentifier] = llvm::to_string(Shard);
61 return llvm::Error::success();
62 }
63 std::unique_ptr<IndexFileIn>
64 loadShard(llvm::StringRef ShardIdentifier) const override {
65 std::lock_guard<std::mutex> Lock(StorageMu);
66 AccessedPaths.insert(ShardIdentifier);
67 if (Storage.find(ShardIdentifier) == Storage.end()) {
68 return nullptr;
69 }
70 auto IndexFile = readIndexFile(Storage[ShardIdentifier]);
71 if (!IndexFile) {
72 ADD_FAILURE() << "Error while reading " << ShardIdentifier << ':'
73 << IndexFile.takeError();
74 return nullptr;
75 }
76 CacheHits++;
77 return std::make_unique<IndexFileIn>(std::move(*IndexFile));
78 }
79
80 mutable llvm::StringSet<> AccessedPaths;
81 };
82
83 class BackgroundIndexTest : public ::testing::Test {
84 protected:
85 BackgroundIndexTest() { BackgroundQueue::preventThreadStarvationInTests(); }
86 };
87
88 TEST_F(BackgroundIndexTest, NoCrashOnErrorFile) {
89 MockFSProvider FS;
90 FS.Files[testPath("root/A.cc")] = "error file";
91 llvm::StringMap<std::string> Storage;
92 size_t CacheHits = 0;
93 MemoryShardStorage MSS(Storage, CacheHits);
94 OverlayCDB CDB(/*Base=*/nullptr);
95 BackgroundIndex Idx(Context::empty(), FS, CDB,
96 [&](llvm::StringRef) { return &MSS; });
97
98 tooling::CompileCommand Cmd;
99 Cmd.Filename = testPath("root/A.cc");
100 Cmd.Directory = testPath("root");
101 Cmd.CommandLine = {"clang++", "-DA=1", testPath("root/A.cc")};
102 CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
103
104 ASSERT_TRUE(Idx.blockUntilIdleForTest());
105 }
106
107 TEST_F(BackgroundIndexTest, IndexTwoFiles) {
108 MockFSProvider FS;
109 // a.h yields different symbols when included by A.cc vs B.cc.
110 FS.Files[testPath("root/A.h")] = R"cpp(
111 void common();
112 void f_b();
113 #if A
114 class A_CC {};
115 #else
116 class B_CC{};
117 #endif
118 )cpp";
119 FS.Files[testPath("root/A.cc")] =
120 "#include \"A.h\"\nvoid g() { (void)common; }";
121 FS.Files[testPath("root/B.cc")] =
122 R"cpp(
123 #define A 0
124 #include "A.h"
125 void f_b() {
126 (void)common;
127 (void)common;
128 (void)common;
129 (void)common;
130 })cpp";
131 llvm::StringMap<std::string> Storage;
132 size_t CacheHits = 0;
133 MemoryShardStorage MSS(Storage, CacheHits);
134 OverlayCDB CDB(/*Base=*/nullptr);
135 BackgroundIndex Idx(Context::empty(), FS, CDB,
136 [&](llvm::StringRef) { return &MSS; });
137
138 tooling::CompileCommand Cmd;
139 Cmd.Filename = testPath("root/A.cc");
140 Cmd.Directory = testPath("root");
141 Cmd.CommandLine = {"clang++", "-DA=1", testPath("root/A.cc")};
142 CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
143
144 ASSERT_TRUE(Idx.blockUntilIdleForTest());
145 EXPECT_THAT(runFuzzyFind(Idx, ""),
146 UnorderedElementsAre(AllOf(Named("common"), NumReferences(1U)),
147 AllOf(Named("A_CC"), NumReferences(0U)),
148 AllOf(Named("g"), NumReferences(0U)),
149 AllOf(Named("f_b"), Declared(),
150 Not(Defined()), NumReferences(0U))));
151
152 Cmd.Filename = testPath("root/B.cc");
153 Cmd.CommandLine = {"clang++", Cmd.Filename};
154 CDB.setCompileCommand(testPath("root/B.cc"), Cmd);
155
156 ASSERT_TRUE(Idx.blockUntilIdleForTest());
157 // B_CC is dropped as we don't collect symbols from A.h in this compilation.
158 EXPECT_THAT(runFuzzyFind(Idx, ""),
159 UnorderedElementsAre(AllOf(Named("common"), NumReferences(5U)),
160 AllOf(Named("A_CC"), NumReferences(0U)),
161 AllOf(Named("g"), NumReferences(0U)),
162 AllOf(Named("f_b"), Declared(), Defined(),
163 NumReferences(1U))));
164
165 auto Syms = runFuzzyFind(Idx, "common");
166 EXPECT_THAT(Syms, UnorderedElementsAre(Named("common")));
167 auto Common = *Syms.begin();
168 EXPECT_THAT(getRefs(Idx, Common.ID),
169 RefsAre({FileURI("unittest:///root/A.h"),
170 FileURI("unittest:///root/A.cc"),
171 FileURI("unittest:///root/B.cc"),
172 FileURI("unittest:///root/B.cc"),
173 FileURI("unittest:///root/B.cc"),
174 FileURI("unittest:///root/B.cc")}));
175 }
176
177 TEST_F(BackgroundIndexTest, ShardStorageTest) {
178 MockFSProvider FS;
179 FS.Files[testPath("root/A.h")] = R"cpp(
180 void common();
181 void f_b();
182 class A_CC {};
183 )cpp";
184 std::string A_CC = "";
185 FS.Files[testPath("root/A.cc")] = R"cpp(
186 #include "A.h"
187 void g() { (void)common; }
188 class B_CC : public A_CC {};
189 )cpp";
190
191 llvm::StringMap<std::string> Storage;
192 size_t CacheHits = 0;
193 MemoryShardStorage MSS(Storage, CacheHits);
194
195 tooling::CompileCommand Cmd;
196 Cmd.Filename = testPath("root/A.cc");
197 Cmd.Directory = testPath("root");
198 Cmd.CommandLine = {"clang++", testPath("root/A.cc")};
199 // Check nothing is loaded from Storage, but A.cc and A.h has been stored.
200 {
201 OverlayCDB CDB(/*Base=*/nullptr);
202 BackgroundIndex Idx(Context::empty(), FS, CDB,
203 [&](llvm::StringRef) { return &MSS; });
204 CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
205 ASSERT_TRUE(Idx.blockUntilIdleForTest());
206 }
207 EXPECT_EQ(CacheHits, 0U);
208 EXPECT_EQ(Storage.size(), 2U);
209
210 {
211 OverlayCDB CDB(/*Base=*/nullptr);
212 BackgroundIndex Idx(Context::empty(), FS, CDB,
213 [&](llvm::StringRef) { return &MSS; });
214 CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
215 ASSERT_TRUE(Idx.blockUntilIdleForTest());
216 }
217 EXPECT_EQ(CacheHits, 2U); // Check both A.cc and A.h loaded from cache.
218 EXPECT_EQ(Storage.size(), 2U);
219
220 auto ShardHeader = MSS.loadShard(testPath("root/A.h"));
221 EXPECT_NE(ShardHeader, nullptr);
222 EXPECT_THAT(
223 *ShardHeader->Symbols,
224 UnorderedElementsAre(Named("common"), Named("A_CC"),
225 AllOf(Named("f_b"), Declared(), Not(Defined()))));
226 for (const auto &Ref : *ShardHeader->Refs)
227 EXPECT_THAT(Ref.second,
228 UnorderedElementsAre(FileURI("unittest:///root/A.h")));
229
230 auto ShardSource = MSS.loadShard(testPath("root/A.cc"));
231 EXPECT_NE(ShardSource, nullptr);
232 EXPECT_THAT(*ShardSource->Symbols,
233 UnorderedElementsAre(Named("g"), Named("B_CC")));
234 for (const auto &Ref : *ShardSource->Refs)
235 EXPECT_THAT(Ref.second,
236 UnorderedElementsAre(FileURI("unittest:///root/A.cc")));
237
238 // The BaseOf relationship between A_CC and B_CC is stored in the file
239 // containing the definition of the subject (A_CC)
240 SymbolID A = findSymbol(*ShardHeader->Symbols, "A_CC").ID;
241 SymbolID B = findSymbol(*ShardSource->Symbols, "B_CC").ID;
242 EXPECT_THAT(*ShardHeader->Relations,
243 UnorderedElementsAre(Relation{A, RelationKind::BaseOf, B}));
244 // (and not in the file containing the definition of the object (B_CC)).
245 EXPECT_EQ(ShardSource->Relations->size(), 0u);
246 }
247
248 TEST_F(BackgroundIndexTest, DirectIncludesTest) {
249 MockFSProvider FS;
250 FS.Files[testPath("root/B.h")] = "";
251 FS.Files[testPath("root/A.h")] = R"cpp(
252 #include "B.h"
253 void common();
254 void f_b();
255 class A_CC {};
256 )cpp";
257 std::string A_CC = "#include \"A.h\"\nvoid g() { (void)common; }";
258 FS.Files[testPath("root/A.cc")] = A_CC;
259
260 llvm::StringMap<std::string> Storage;
261 size_t CacheHits = 0;
262 MemoryShardStorage MSS(Storage, CacheHits);
263
264 tooling::CompileCommand Cmd;
265 Cmd.Filename = testPath("root/A.cc");
266 Cmd.Directory = testPath("root");
267 Cmd.CommandLine = {"clang++", testPath("root/A.cc")};
268 {
269 OverlayCDB CDB(/*Base=*/nullptr);
270 BackgroundIndex Idx(Context::empty(), FS, CDB,
271 [&](llvm::StringRef) { return &MSS; });
272 CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
273 ASSERT_TRUE(Idx.blockUntilIdleForTest());
274 }
275
276 auto ShardSource = MSS.loadShard(testPath("root/A.cc"));
277 EXPECT_TRUE(ShardSource->Sources);
278 EXPECT_EQ(ShardSource->Sources->size(), 2U); // A.cc, A.h
279 EXPECT_THAT(
280 ShardSource->Sources->lookup("unittest:///root/A.cc").DirectIncludes,
281 UnorderedElementsAre("unittest:///root/A.h"));
282 EXPECT_NE(ShardSource->Sources->lookup("unittest:///root/A.cc").Digest,
283 FileDigest{{0}});
284 EXPECT_THAT(ShardSource->Sources->lookup("unittest:///root/A.h"),
285 EmptyIncludeNode());
286
287 auto ShardHeader = MSS.loadShard(testPath("root/A.h"));
288 EXPECT_TRUE(ShardHeader->Sources);
289 EXPECT_EQ(ShardHeader->Sources->size(), 2U); // A.h, B.h
290 EXPECT_THAT(
291 ShardHeader->Sources->lookup("unittest:///root/A.h").DirectIncludes,
292 UnorderedElementsAre("unittest:///root/B.h"));
293 EXPECT_NE(ShardHeader->Sources->lookup("unittest:///root/A.h").Digest,
294 FileDigest{{0}});
295 EXPECT_THAT(ShardHeader->Sources->lookup("unittest:///root/B.h"),
296 EmptyIncludeNode());
297 }
298
299 TEST_F(BackgroundIndexTest, ShardStorageLoad) {
300 MockFSProvider FS;
301 FS.Files[testPath("root/A.h")] = R"cpp(
302 void common();
303 void f_b();
304 class A_CC {};
305 )cpp";
306 FS.Files[testPath("root/A.cc")] =
307 "#include \"A.h\"\nvoid g() { (void)common; }";
308
309 llvm::StringMap<std::string> Storage;
310 size_t CacheHits = 0;
311 MemoryShardStorage MSS(Storage, CacheHits);
312
313 tooling::CompileCommand Cmd;
314 Cmd.Filename = testPath("root/A.cc");
315 Cmd.Directory = testPath("root");
316 Cmd.CommandLine = {"clang++", testPath("root/A.cc")};
317 // Check nothing is loaded from Storage, but A.cc and A.h has been stored.
318 {
319 OverlayCDB CDB(/*Base=*/nullptr);
320 BackgroundIndex Idx(Context::empty(), FS, CDB,
321 [&](llvm::StringRef) { return &MSS; });
322 CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
323 ASSERT_TRUE(Idx.blockUntilIdleForTest());
324 }
325
326 // Change header.
327 FS.Files[testPath("root/A.h")] = R"cpp(
328 void common();
329 void f_b();
330 class A_CC {};
331 class A_CCnew {};
332 )cpp";
333 {
334 OverlayCDB CDB(/*Base=*/nullptr);
335 BackgroundIndex Idx(Context::empty(), FS, CDB,
336 [&](llvm::StringRef) { return &MSS; });
337 CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
338 ASSERT_TRUE(Idx.blockUntilIdleForTest());
339 }
340 EXPECT_EQ(CacheHits, 2U); // Check both A.cc and A.h loaded from cache.
341
342 // Check if the new symbol has arrived.
343 auto ShardHeader = MSS.loadShard(testPath("root/A.h"));
344 EXPECT_NE(ShardHeader, nullptr);
345 EXPECT_THAT(*ShardHeader->Symbols, Contains(Named("A_CCnew")));
346
347 // Change source.
348 FS.Files[testPath("root/A.cc")] =
349 "#include \"A.h\"\nvoid g() { (void)common; }\nvoid f_b() {}";
350 {
351 CacheHits = 0;
352 OverlayCDB CDB(/*Base=*/nullptr);
353 BackgroundIndex Idx(Context::empty(), FS, CDB,
354 [&](llvm::StringRef) { return &MSS; });
355 CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
356 ASSERT_TRUE(Idx.blockUntilIdleForTest());
357 }
358 EXPECT_EQ(CacheHits, 2U); // Check both A.cc and A.h loaded from cache.
359
360 // Check if the new symbol has arrived.
361 ShardHeader = MSS.loadShard(testPath("root/A.h"));
362 EXPECT_NE(ShardHeader, nullptr);
363 EXPECT_THAT(*ShardHeader->Symbols, Contains(Named("A_CCnew")));
364 auto ShardSource = MSS.loadShard(testPath("root/A.cc"));
365 EXPECT_NE(ShardSource, nullptr);
366 EXPECT_THAT(*ShardSource->Symbols,
367 Contains(AllOf(Named("f_b"), Declared(), Defined())));
368 }
369
370 TEST_F(BackgroundIndexTest, ShardStorageEmptyFile) {
371 MockFSProvider FS;
372 FS.Files[testPath("root/A.h")] = R"cpp(
373 void common();
374 void f_b();
375 class A_CC {};
376 )cpp";
377 FS.Files[testPath("root/B.h")] = R"cpp(
378 #include "A.h"
379 )cpp";
380 FS.Files[testPath("root/A.cc")] =
381 "#include \"B.h\"\nvoid g() { (void)common; }";
382
383 llvm::StringMap<std::string> Storage;
384 size_t CacheHits = 0;
385 MemoryShardStorage MSS(Storage, CacheHits);
386
387 tooling::CompileCommand Cmd;
388 Cmd.Filename = testPath("root/A.cc");
389 Cmd.Directory = testPath("root");
390 Cmd.CommandLine = {"clang++", testPath("root/A.cc")};
391 // Check that A.cc, A.h and B.h has been stored.
392 {
393 OverlayCDB CDB(/*Base=*/nullptr);
394 BackgroundIndex Idx(Context::empty(), FS, CDB,
395 [&](llvm::StringRef) { return &MSS; });
396 CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
397 ASSERT_TRUE(Idx.blockUntilIdleForTest());
398 }
399 EXPECT_THAT(Storage.keys(),
400 UnorderedElementsAre(testPath("root/A.cc"), testPath("root/A.h"),
401 testPath("root/B.h")));
402 auto ShardHeader = MSS.loadShard(testPath("root/B.h"));
403 EXPECT_NE(ShardHeader, nullptr);
404 EXPECT_TRUE(ShardHeader->Symbols->empty());
405
406 // Check that A.cc, A.h and B.h has been loaded.
407 {
408 CacheHits = 0;
409 OverlayCDB CDB(/*Base=*/nullptr);
410 BackgroundIndex Idx(Context::empty(), FS, CDB,
411 [&](llvm::StringRef) { return &MSS; });
412 CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
413 ASSERT_TRUE(Idx.blockUntilIdleForTest());
414 }
415 EXPECT_EQ(CacheHits, 3U);
416
417 // Update B.h to contain some symbols.
418 FS.Files[testPath("root/B.h")] = R"cpp(
419 #include "A.h"
420 void new_func();
421 )cpp";
422 // Check that B.h has been stored with new contents.
423 {
424 CacheHits = 0;
425 OverlayCDB CDB(/*Base=*/nullptr);
426 BackgroundIndex Idx(Context::empty(), FS, CDB,
427 [&](llvm::StringRef) { return &MSS; });
428 CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
429 ASSERT_TRUE(Idx.blockUntilIdleForTest());
430 }
431 EXPECT_EQ(CacheHits, 3U);
432 ShardHeader = MSS.loadShard(testPath("root/B.h"));
433 EXPECT_NE(ShardHeader, nullptr);
434 EXPECT_THAT(*ShardHeader->Symbols,
435 Contains(AllOf(Named("new_func"), Declared(), Not(Defined()))));
436 }
437
438 TEST_F(BackgroundIndexTest, NoDotsInAbsPath) {
439 MockFSProvider FS;
440 llvm::StringMap<std::string> Storage;
441 size_t CacheHits = 0;
442 MemoryShardStorage MSS(Storage, CacheHits);
443 OverlayCDB CDB(/*Base=*/nullptr);
444 BackgroundIndex Idx(Context::empty(), FS, CDB,
445 [&](llvm::StringRef) { return &MSS; });
446
447 tooling::CompileCommand Cmd;
448 FS.Files[testPath("root/A.cc")] = "";
449 Cmd.Filename = "../A.cc";
450 Cmd.Directory = testPath("root/build");
451 Cmd.CommandLine = {"clang++", "../A.cc"};
452 CDB.setCompileCommand(testPath("root/build/../A.cc"), Cmd);
453
454 FS.Files[testPath("root/B.cc")] = "";
455 Cmd.Filename = "./B.cc";
456 Cmd.Directory = testPath("root");
457 Cmd.CommandLine = {"clang++", "./B.cc"};
458 CDB.setCompileCommand(testPath("root/./B.cc"), Cmd);
459
460 ASSERT_TRUE(Idx.blockUntilIdleForTest());
461 for (llvm::StringRef AbsPath : MSS.AccessedPaths.keys()) {
462 EXPECT_FALSE(AbsPath.contains("./")) << AbsPath;
463 EXPECT_FALSE(AbsPath.contains("../")) << AbsPath;
464 }
465 }
466
467 TEST_F(BackgroundIndexTest, UncompilableFiles) {
468 MockFSProvider FS;
469 llvm::StringMap<std::string> Storage;
470 size_t CacheHits = 0;
471 MemoryShardStorage MSS(Storage, CacheHits);
472 OverlayCDB CDB(/*Base=*/nullptr);
473 BackgroundIndex Idx(Context::empty(), FS, CDB,
474 [&](llvm::StringRef) { return &MSS; });
475
476 tooling::CompileCommand Cmd;
477 FS.Files[testPath("A.h")] = "void foo();";
478 FS.Files[testPath("B.h")] = "#include \"C.h\"\nasdf;";
479 FS.Files[testPath("C.h")] = "";
480 FS.Files[testPath("A.cc")] = R"cpp(
481 #include "A.h"
482 #include "B.h"
483 #include "not_found_header.h"
484
485 void foo() {}
486 )cpp";
487 Cmd.Filename = "../A.cc";
488 Cmd.Directory = testPath("build");
489 Cmd.CommandLine = {"clang++", "../A.cc"};
490 CDB.setCompileCommand(testPath("build/../A.cc"), Cmd);
491 ASSERT_TRUE(Idx.blockUntilIdleForTest());
492
493 EXPECT_THAT(Storage.keys(), ElementsAre(testPath("A.cc"), testPath("A.h"),
494 testPath("B.h"), testPath("C.h")));
495
496 {
497 auto Shard = MSS.loadShard(testPath("A.cc"));
498 EXPECT_THAT(*Shard->Symbols, UnorderedElementsAre(Named("foo")));
499 EXPECT_THAT(Shard->Sources->keys(),
500 UnorderedElementsAre("unittest:///A.cc", "unittest:///A.h",
501 "unittest:///B.h"));
502 EXPECT_THAT(Shard->Sources->lookup("unittest:///A.cc"), HadErrors());
503 }
504
505 {
506 auto Shard = MSS.loadShard(testPath("A.h"));
507 EXPECT_THAT(*Shard->Symbols, UnorderedElementsAre(Named("foo")));
508 EXPECT_THAT(Shard->Sources->keys(),
509 UnorderedElementsAre("unittest:///A.h"));
510 EXPECT_THAT(Shard->Sources->lookup("unittest:///A.h"), HadErrors());
511 }
512
513 {
514 auto Shard = MSS.loadShard(testPath("B.h"));
515 EXPECT_THAT(*Shard->Symbols, UnorderedElementsAre(Named("asdf")));
516 EXPECT_THAT(Shard->Sources->keys(),
517 UnorderedElementsAre("unittest:///B.h", "unittest:///C.h"));
518 EXPECT_THAT(Shard->Sources->lookup("unittest:///B.h"), HadErrors());
519 }
520
521 {
522 auto Shard = MSS.loadShard(testPath("C.h"));
523 EXPECT_THAT(*Shard->Symbols, UnorderedElementsAre());
524 EXPECT_THAT(Shard->Sources->keys(),
525 UnorderedElementsAre("unittest:///C.h"));
526 EXPECT_THAT(Shard->Sources->lookup("unittest:///C.h"), HadErrors());
527 }
528 }
529
530 TEST_F(BackgroundIndexTest, CmdLineHash) {
531 MockFSProvider FS;
532 llvm::StringMap<std::string> Storage;
533 size_t CacheHits = 0;
534 MemoryShardStorage MSS(Storage, CacheHits);
535 OverlayCDB CDB(/*Base=*/nullptr);
536 BackgroundIndex Idx(Context::empty(), FS, CDB,
537 [&](llvm::StringRef) { return &MSS; });
538
539 tooling::CompileCommand Cmd;
540 FS.Files[testPath("A.cc")] = "#include \"A.h\"";
541 FS.Files[testPath("A.h")] = "";
542 Cmd.Filename = "../A.cc";
543 Cmd.Directory = testPath("build");
544 Cmd.CommandLine = {"clang++", "../A.cc", "-fsyntax-only"};
545 CDB.setCompileCommand(testPath("build/../A.cc"), Cmd);
546 ASSERT_TRUE(Idx.blockUntilIdleForTest());
547
548 EXPECT_THAT(Storage.keys(), ElementsAre(testPath("A.cc"), testPath("A.h")));
549 // Make sure we only store the Cmd for main file.
550 EXPECT_FALSE(MSS.loadShard(testPath("A.h"))->Cmd);
551
552 {
553 tooling::CompileCommand CmdStored = *MSS.loadShard(testPath("A.cc"))->Cmd;
554 EXPECT_EQ(CmdStored.CommandLine, Cmd.CommandLine);
555 EXPECT_EQ(CmdStored.Directory, Cmd.Directory);
556 }
557
558 // FIXME: Changing compile commands should be enough to invalidate the cache.
559 FS.Files[testPath("A.cc")] = " ";
560 Cmd.CommandLine = {"clang++", "../A.cc", "-Dfoo", "-fsyntax-only"};
561 CDB.setCompileCommand(testPath("build/../A.cc"), Cmd);
562 ASSERT_TRUE(Idx.blockUntilIdleForTest());
563
564 EXPECT_FALSE(MSS.loadShard(testPath("A.h"))->Cmd);
565
566 {
567 tooling::CompileCommand CmdStored = *MSS.loadShard(testPath("A.cc"))->Cmd;
568 EXPECT_EQ(CmdStored.CommandLine, Cmd.CommandLine);
569 EXPECT_EQ(CmdStored.Directory, Cmd.Directory);
570 }
571 }
572
573 class BackgroundIndexRebuilderTest : public testing::Test {
574 protected:
575 BackgroundIndexRebuilderTest()
576 : Target(std::make_unique<MemIndex>()),
577 Rebuilder(&Target, &Source, /*Threads=*/10) {
578 // Prepare FileSymbols with TestSymbol in it, for checkRebuild.
579 TestSymbol.ID = SymbolID("foo");
580 }
581
582 // Perform Action and determine whether it rebuilt the index or not.
583 bool checkRebuild(std::function<void()> Action) {
584 // Update name so we can tell if the index updates.
585 VersionStorage.push_back("Sym" + std::to_string(++VersionCounter));
586 TestSymbol.Name = VersionStorage.back();
587 SymbolSlab::Builder SB;
588 SB.insert(TestSymbol);
589 Source.update("", std::make_unique<SymbolSlab>(std::move(SB).build()),
590 nullptr, nullptr, false);
591 // Now maybe update the index.
592 Action();
593 // Now query the index to get the name count.
594 std::string ReadName;
595 LookupRequest Req;
596 Req.IDs.insert(TestSymbol.ID);
597 Target.lookup(Req, [&](const Symbol &S) { ReadName = S.Name; });
598 // The index was rebuild if the name is up to date.
599 return ReadName == VersionStorage.back();
600 }
601
602 Symbol TestSymbol;
603 FileSymbols Source;
604 SwapIndex Target;
605 BackgroundIndexRebuilder Rebuilder;
606
607 unsigned VersionCounter = 0;
608 std::deque<std::string> VersionStorage;
609 };
610
611 TEST_F(BackgroundIndexRebuilderTest, IndexingTUs) {
612 for (unsigned I = 0; I < Rebuilder.TUsBeforeFirstBuild - 1; ++I)
613 EXPECT_FALSE(checkRebuild([&] { Rebuilder.indexedTU(); }));
614 EXPECT_TRUE(checkRebuild([&] { Rebuilder.indexedTU(); }));
615 for (unsigned I = 0; I < Rebuilder.TUsBeforeRebuild - 1; ++I)
616 EXPECT_FALSE(checkRebuild([&] { Rebuilder.indexedTU(); }));
617 EXPECT_TRUE(checkRebuild([&] { Rebuilder.indexedTU(); }));
618 }
619
620 TEST_F(BackgroundIndexRebuilderTest, LoadingShards) {
621 Rebuilder.startLoading();
622 Rebuilder.loadedShard(10);
623 Rebuilder.loadedShard(20);
624 EXPECT_TRUE(checkRebuild([&] { Rebuilder.doneLoading(); }));
625
626 // No rebuild for no shards.
627 Rebuilder.startLoading();
628 EXPECT_FALSE(checkRebuild([&] { Rebuilder.doneLoading(); }));
629
630 // Loads can overlap.
631 Rebuilder.startLoading();
632 Rebuilder.loadedShard(1);
633 Rebuilder.startLoading();
634 Rebuilder.loadedShard(1);
635 EXPECT_FALSE(checkRebuild([&] { Rebuilder.doneLoading(); }));
636 Rebuilder.loadedShard(1);
637 EXPECT_TRUE(checkRebuild([&] { Rebuilder.doneLoading(); }));
638
639 // No rebuilding for indexed files while loading.
640 Rebuilder.startLoading();
641 for (unsigned I = 0; I < 3 * Rebuilder.TUsBeforeRebuild; ++I)
642 EXPECT_FALSE(checkRebuild([&] { Rebuilder.indexedTU(); }));
643 // But they get indexed when we're done, even if no shards were loaded.
644 EXPECT_TRUE(checkRebuild([&] { Rebuilder.doneLoading(); }));
645 }
646
647 TEST(BackgroundQueueTest, Priority) {
648 // Create high and low priority tasks.
649 // Once a bunch of high priority tasks have run, the queue is stopped.
650 // So the low priority tasks should never run.
651 BackgroundQueue Q;
652 std::atomic<unsigned> HiRan(0), LoRan(0);
653 BackgroundQueue::Task Lo([&] { ++LoRan; });
654 BackgroundQueue::Task Hi([&] {
655 if (++HiRan >= 10)
656 Q.stop();
657 });
658 Hi.QueuePri = 100;
659
660 // Enqueuing the low-priority ones first shouldn't make them run first.
661 Q.append(std::vector<BackgroundQueue::Task>(30, Lo));
662 for (unsigned I = 0; I < 30; ++I)
663 Q.push(Hi);
664
665 AsyncTaskRunner ThreadPool;
666 for (unsigned I = 0; I < 5; ++I)
667 ThreadPool.runAsync("worker", [&] { Q.work(); });
668 // We should test enqueue with active workers, but it's hard to avoid races.
669 // Just make sure we don't crash.
670 Q.push(Lo);
671 Q.append(std::vector<BackgroundQueue::Task>(2, Hi));
672
673 // After finishing, check the tasks that ran.
674 ThreadPool.wait();
675 EXPECT_GE(HiRan, 10u);
676 EXPECT_EQ(LoRan, 0u);
677 }
678
679 TEST(BackgroundQueueTest, Boost) {
680 std::string Sequence;
681
682 BackgroundQueue::Task A([&] { Sequence.push_back('A'); });
683 A.Tag = "A";
684 A.QueuePri = 1;
685
686 BackgroundQueue::Task B([&] { Sequence.push_back('B'); });
687 B.QueuePri = 2;
688 B.Tag = "B";
689
690 {
691 BackgroundQueue Q;
692 Q.append({A, B});
693 Q.work([&] { Q.stop(); });
694 EXPECT_EQ("BA", Sequence) << "priority order";
695 }
696 Sequence.clear();
697 {
698 BackgroundQueue Q;
699 Q.boost("A", 3);
700 Q.append({A, B});
701 Q.work([&] { Q.stop(); });
702 EXPECT_EQ("AB", Sequence) << "A was boosted before enqueueing";
703 }
704 Sequence.clear();
705 {
706 BackgroundQueue Q;
707 Q.append({A, B});
708 Q.boost("A", 3);
709 Q.work([&] { Q.stop(); });
710 EXPECT_EQ("AB", Sequence) << "A was boosted after enqueueing";
711 }
712 }
713
714 } // namespace clangd
715 } // namespace clang
clang::clangd::RefsAre
::testing::Matcher< const RefSlab & > RefsAre(std::vector<::testing::Matcher< Ref >> Matchers)
Definition: BackgroundIndexTests.cpp:33
clang::clangd::MATCHER_P
MATCHER_P(Named, N, "")
Definition: BackgroundIndexTests.cpp:26
clang::clangd::Relation
Represents a relation between two symbols.
Definition: Relation.h:29
clang::clangd::MessageType::Error
An error message.
BackgroundRebuild.h
clang::clangd::MockFSProvider::Files
llvm::StringMap< std::string > Files
Definition: TestFS.h:38
clang::clangd::BackgroundIndexRebuilderTest::Target
SwapIndex Target
Definition: BackgroundIndexTests.cpp:604
clang::clangd::FileDigest
std::array< uint8_t, 8 > FileDigest
Definition: SourceCode.h:38
clang::clangd::FileSymbols
A container of Symbols from several source files.
Definition: FileIndex.h:61
clang::clangd::MemoryShardStorage::storeShard
llvm::Error storeShard(llvm::StringRef ShardIdentifier, IndexFileOut Shard) const override
Definition: BackgroundIndexTests.cpp:56
clang::clangd::LookupRequest::IDs
llvm::DenseSet< SymbolID > IDs
Definition: Index.h:64
clang::clangd::BackgroundIndexRebuilderTest::checkRebuild
bool checkRebuild(std::function< void()> Action)
Definition: BackgroundIndexTests.cpp:583
clang::clangd::Ref
Represents a symbol occurrence in the source file.
Definition: Ref.h:52
clang::clangd::SymbolSlab::Builder::insert
void insert(const Symbol &S)
Adds a symbol, overwriting any existing one with the same ID.
Definition: Symbol.cpp:50
clang::clangd::BackgroundQueue::stop
void stop()
Definition: BackgroundQueue.cpp:59
FS
MockFSProvider FS
Definition: HeadersTests.cpp:119
clang::clangd::RelationKind::BaseOf
clang::clangd::readIndexFile
llvm::Expected< IndexFileIn > readIndexFile(llvm::StringRef Data)
Definition: Serialization.cpp:652
TestFS.h
clang::clangd::MemoryShardStorage::AccessedPaths
llvm::StringSet AccessedPaths
Definition: BackgroundIndexTests.cpp:80
TestTU.h
Headers.h
clang::clangd::BackgroundQueue::Task::QueuePri
unsigned QueuePri
Definition: Background.h:75
clang::clangd::SymbolSlab::Builder
SymbolSlab::Builder is a mutable container that can &#39;freeze&#39; to SymbolSlab.
Definition: Symbol.h:199
clang::clangd::Symbol::ID
SymbolID ID
The ID of the symbol.
Definition: Symbol.h:38
clang::clangd::MockFSProvider
Definition: TestFS.h:31
clang::clangd::BackgroundQueue::push
void push(Task)
Definition: BackgroundQueue.cpp:67
clang::clangd::BackgroundQueue::boost
void boost(llvm::StringRef Tag, unsigned NewPriority)
Definition: BackgroundQueue.cpp:88
clang::clangd::AsyncTaskRunner::wait
void wait() const
Definition: Threading.h:110
clang::clangd::BackgroundQueue
Definition: Background.h:67
clang::clangd::TEST_F
TEST_F(BackgroundIndexTest, NoCrashOnErrorFile)
Definition: BackgroundIndexTests.cpp:88
clang::clangd::IncludeGraphNode::SourceFlag::None
clang::clangd::BackgroundIndex
Definition: Background.h:115
clang::clangd::BackgroundQueue::preventThreadStarvationInTests
static void preventThreadStarvationInTests()
Definition: BackgroundQueue.cpp:16
clang::clangd::AsyncTaskRunner::runAsync
void runAsync(const llvm::Twine &Name, llvm::unique_function< void()> Action)
Definition: Threading.cpp:70
clang::clangd::TEST
TEST(BackgroundQueueTest, Priority)
Definition: BackgroundIndexTests.cpp:647
clang::clangd::testPath
std::string testPath(PathRef File)
Definition: TestFS.cpp:82
clang::clangd::BackgroundIndexRebuilderTest::Source
FileSymbols Source
Definition: BackgroundIndexTests.cpp:603
Action
llvm::unique_function< void()> Action
Definition: TUScheduler.cpp:234
clang::clangd::MemIndex
MemIndex is a naive in-memory index suitable for a small set of symbols.
Definition: MemIndex.h:19
clang::clangd::runFuzzyFind
SymbolSlab runFuzzyFind(const SymbolIndex &Index, llvm::StringRef Query)
Definition: SyncAPI.cpp:126
clang::clangd::MATCHER
MATCHER(Declared, "")
Definition: BackgroundIndexTests.cpp:27
clang::clangd::BackgroundQueue::Task::Tag
std::string Tag
Definition: Background.h:76
clang::clangd::BackgroundIndexRebuilder
Definition: BackgroundRebuild.h:48
clang::clangd::AsyncTaskRunner
Runs tasks on separate (detached) threads and wait for all tasks to finish.
Definition: Threading.h:105
clang::clangd::OverlayCDB::setCompileCommand
void setCompileCommand(PathRef File, llvm::Optional< tooling::CompileCommand > CompilationCommand)
Sets or clears the compilation command for a particular file.
Definition: GlobalCompilationDatabase.cpp:272
clang::clangd::IncludeGraphNode::SourceFlag::HadErrors
clang::clangd::BackgroundIndexTest::BackgroundIndexTest
BackgroundIndexTest()
Definition: BackgroundIndexTests.cpp:85
clang::clangd::SwapIndex
Definition: Index.h:128
clang::clangd::BackgroundIndexRebuilderTest::BackgroundIndexRebuilderTest
BackgroundIndexRebuilderTest()
Definition: BackgroundIndexTests.cpp:575
Background.h
clang::clangd::Symbol
The class presents a C++ symbol, e.g.
Definition: Symbol.h:36
clang::clangd::MemoryShardStorage::MemoryShardStorage
MemoryShardStorage(llvm::StringMap< std::string > &Storage, size_t &CacheHits)
Definition: BackgroundIndexTests.cpp:54
clang::clangd::LookupRequest
Definition: Index.h:63
clang
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
Definition: ApplyReplacements.h:27
clang::clangd::Symbol::Name
llvm::StringRef Name
The unqualified name of the symbol, e.g. "bar" (for ns::bar).
Definition: Symbol.h:42
clang::clangd::findSymbol
const Symbol & findSymbol(const SymbolSlab &Slab, llvm::StringRef QName)
Definition: TestTU.cpp:97
clang::clangd::MemoryShardStorage::loadShard
std::unique_ptr< IndexFileIn > loadShard(llvm::StringRef ShardIdentifier) const override
Definition: BackgroundIndexTests.cpp:64
clang::clangd::BackgroundIndexRebuilderTest::Rebuilder
BackgroundIndexRebuilder Rebuilder
Definition: BackgroundIndexTests.cpp:605
SyncAPI.h
clang::clangd::IndexFileOut
Definition: Serialization.h:55
TestIndex.h
clang::clangd::BackgroundQueue::append
void append(std::vector< Task >)
Definition: BackgroundQueue.cpp:77
clang::clangd::BackgroundIndexRebuilderTest::VersionStorage
std::deque< std::string > VersionStorage
Definition: BackgroundIndexTests.cpp:608
clang::clangd::BackgroundQueue::Task
A work item on the thread pool&#39;s queue.
Definition: Background.h:70
clang::clangd::BackgroundIndexRebuilderTest::TestSymbol
Symbol TestSymbol
Definition: BackgroundIndexTests.cpp:602
clang::clangd::OverlayCDB
Wraps another compilation database, and supports overriding the commands using an in-memory mapping...
Definition: GlobalCompilationDatabase.h:125
clang::clangd::Context::empty
static Context empty()
Returns an empty root context that contains no data.
Definition: Context.cpp:15
clang::clangd::BackgroundIndexStorage
Definition: Background.h:39
clang::doc::SymbolID
std::array< uint8_t, 20 > SymbolID
Definition: Representation.h:30
clang::clangd::MemoryShardStorage
Definition: BackgroundIndexTests.cpp:48
clang::clangd::BackgroundIndexTest
Definition: BackgroundIndexTests.cpp:83
clang::clangd::getRefs
RefSlab getRefs(const SymbolIndex &Index, SymbolID ID)
Definition: SyncAPI.cpp:139
clang::clangd::BackgroundIndexRebuilderTest
Definition: BackgroundIndexTests.cpp:573
clang::clangd::BackgroundQueue::work
void work(std::function< void()> OnIdle=nullptr)
Definition: BackgroundQueue.cpp:20
clang::clangd::SymbolID
Definition: SymbolID.h:31