clang-tools  9.0.0
BackgroundIndexTests.cpp
Go to the documentation of this file.
1 #include "Headers.h"
2 #include "SyncAPI.h"
3 #include "TestFS.h"
4 #include "TestIndex.h"
5 #include "TestTU.h"
6 #include "index/Background.h"
8 #include "clang/Tooling/CompilationDatabase.h"
9 #include "llvm/Support/ScopedPrinter.h"
10 #include "llvm/Support/Threading.h"
11 #include "gmock/gmock.h"
12 #include "gtest/gtest.h"
13 #include <deque>
14 #include <thread>
15 
16 using ::testing::_;
17 using ::testing::AllOf;
18 using ::testing::Contains;
19 using ::testing::ElementsAre;
20 using ::testing::Not;
21 using ::testing::UnorderedElementsAre;
22 
23 namespace clang {
24 namespace clangd {
25 
26 MATCHER_P(Named, N, "") { return arg.Name == N; }
27 MATCHER(Declared, "") {
28  return !StringRef(arg.CanonicalDeclaration.FileURI).empty();
29 }
30 MATCHER(Defined, "") { return !StringRef(arg.Definition.FileURI).empty(); }
31 MATCHER_P(FileURI, F, "") { return StringRef(arg.Location.FileURI) == F; }
32 ::testing::Matcher<const RefSlab &>
33 RefsAre(std::vector<::testing::Matcher<Ref>> Matchers) {
34  return ElementsAre(::testing::Pair(_, UnorderedElementsAreArray(Matchers)));
35 }
36 // URI cannot be empty since it references keys in the IncludeGraph.
37 MATCHER(EmptyIncludeNode, "") {
38  return arg.Flags == IncludeGraphNode::SourceFlag::None && !arg.URI.empty() &&
39  arg.Digest == FileDigest{{0}} && arg.DirectIncludes.empty();
40 }
41 
42 MATCHER(HadErrors, "") {
43  return arg.Flags & IncludeGraphNode::SourceFlag::HadErrors;
44 }
45 
46 MATCHER_P(NumReferences, N, "") { return arg.References == N; }
47 
49  mutable std::mutex StorageMu;
50  llvm::StringMap<std::string> &Storage;
51  size_t &CacheHits;
52 
53 public:
54  MemoryShardStorage(llvm::StringMap<std::string> &Storage, size_t &CacheHits)
55  : Storage(Storage), CacheHits(CacheHits) {}
56  llvm::Error storeShard(llvm::StringRef ShardIdentifier,
57  IndexFileOut Shard) const override {
58  std::lock_guard<std::mutex> Lock(StorageMu);
59  AccessedPaths.insert(ShardIdentifier);
60  Storage[ShardIdentifier] = llvm::to_string(Shard);
61  return llvm::Error::success();
62  }
63  std::unique_ptr<IndexFileIn>
64  loadShard(llvm::StringRef ShardIdentifier) const override {
65  std::lock_guard<std::mutex> Lock(StorageMu);
66  AccessedPaths.insert(ShardIdentifier);
67  if (Storage.find(ShardIdentifier) == Storage.end()) {
68  return nullptr;
69  }
70  auto IndexFile = readIndexFile(Storage[ShardIdentifier]);
71  if (!IndexFile) {
72  ADD_FAILURE() << "Error while reading " << ShardIdentifier << ':'
73  << IndexFile.takeError();
74  return nullptr;
75  }
76  CacheHits++;
77  return llvm::make_unique<IndexFileIn>(std::move(*IndexFile));
78  }
79 
80  mutable llvm::StringSet<> AccessedPaths;
81 };
82 
83 class BackgroundIndexTest : public ::testing::Test {
84 protected:
86 };
87 
88 TEST_F(BackgroundIndexTest, NoCrashOnErrorFile) {
90  FS.Files[testPath("root/A.cc")] = "error file";
91  llvm::StringMap<std::string> Storage;
92  size_t CacheHits = 0;
93  MemoryShardStorage MSS(Storage, CacheHits);
94  OverlayCDB CDB(/*Base=*/nullptr);
95  BackgroundIndex Idx(Context::empty(), FS, CDB,
96  [&](llvm::StringRef) { return &MSS; });
97 
98  tooling::CompileCommand Cmd;
99  Cmd.Filename = testPath("root/A.cc");
100  Cmd.Directory = testPath("root");
101  Cmd.CommandLine = {"clang++", "-DA=1", testPath("root/A.cc")};
102  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
103 
104  ASSERT_TRUE(Idx.blockUntilIdleForTest());
105 }
106 
107 TEST_F(BackgroundIndexTest, IndexTwoFiles) {
109  // a.h yields different symbols when included by A.cc vs B.cc.
110  FS.Files[testPath("root/A.h")] = R"cpp(
111  void common();
112  void f_b();
113  #if A
114  class A_CC {};
115  #else
116  class B_CC{};
117  #endif
118  )cpp";
119  FS.Files[testPath("root/A.cc")] =
120  "#include \"A.h\"\nvoid g() { (void)common; }";
121  FS.Files[testPath("root/B.cc")] =
122  R"cpp(
123  #define A 0
124  #include "A.h"
125  void f_b() {
126  (void)common;
127  (void)common;
128  (void)common;
129  (void)common;
130  })cpp";
131  llvm::StringMap<std::string> Storage;
132  size_t CacheHits = 0;
133  MemoryShardStorage MSS(Storage, CacheHits);
134  OverlayCDB CDB(/*Base=*/nullptr);
135  BackgroundIndex Idx(Context::empty(), FS, CDB,
136  [&](llvm::StringRef) { return &MSS; });
137 
138  tooling::CompileCommand Cmd;
139  Cmd.Filename = testPath("root/A.cc");
140  Cmd.Directory = testPath("root");
141  Cmd.CommandLine = {"clang++", "-DA=1", testPath("root/A.cc")};
142  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
143 
144  ASSERT_TRUE(Idx.blockUntilIdleForTest());
145  EXPECT_THAT(runFuzzyFind(Idx, ""),
146  UnorderedElementsAre(AllOf(Named("common"), NumReferences(1U)),
147  AllOf(Named("A_CC"), NumReferences(0U)),
148  AllOf(Named("g"), NumReferences(0U)),
149  AllOf(Named("f_b"), Declared(),
150  Not(Defined()), NumReferences(0U))));
151 
152  Cmd.Filename = testPath("root/B.cc");
153  Cmd.CommandLine = {"clang++", Cmd.Filename};
154  CDB.setCompileCommand(testPath("root/B.cc"), Cmd);
155 
156  ASSERT_TRUE(Idx.blockUntilIdleForTest());
157  // B_CC is dropped as we don't collect symbols from A.h in this compilation.
158  EXPECT_THAT(runFuzzyFind(Idx, ""),
159  UnorderedElementsAre(AllOf(Named("common"), NumReferences(5U)),
160  AllOf(Named("A_CC"), NumReferences(0U)),
161  AllOf(Named("g"), NumReferences(0U)),
162  AllOf(Named("f_b"), Declared(), Defined(),
163  NumReferences(1U))));
164 
165  auto Syms = runFuzzyFind(Idx, "common");
166  EXPECT_THAT(Syms, UnorderedElementsAre(Named("common")));
167  auto Common = *Syms.begin();
168  EXPECT_THAT(getRefs(Idx, Common.ID),
169  RefsAre({FileURI("unittest:///root/A.h"),
170  FileURI("unittest:///root/A.cc"),
171  FileURI("unittest:///root/B.cc"),
172  FileURI("unittest:///root/B.cc"),
173  FileURI("unittest:///root/B.cc"),
174  FileURI("unittest:///root/B.cc")}));
175 }
176 
177 TEST_F(BackgroundIndexTest, ShardStorageTest) {
179  FS.Files[testPath("root/A.h")] = R"cpp(
180  void common();
181  void f_b();
182  class A_CC {};
183  )cpp";
184  std::string A_CC = "";
185  FS.Files[testPath("root/A.cc")] = R"cpp(
186  #include "A.h"
187  void g() { (void)common; }
188  class B_CC : public A_CC {};
189  )cpp";
190 
191  llvm::StringMap<std::string> Storage;
192  size_t CacheHits = 0;
193  MemoryShardStorage MSS(Storage, CacheHits);
194 
195  tooling::CompileCommand Cmd;
196  Cmd.Filename = testPath("root/A.cc");
197  Cmd.Directory = testPath("root");
198  Cmd.CommandLine = {"clang++", testPath("root/A.cc")};
199  // Check nothing is loaded from Storage, but A.cc and A.h has been stored.
200  {
201  OverlayCDB CDB(/*Base=*/nullptr);
202  BackgroundIndex Idx(Context::empty(), FS, CDB,
203  [&](llvm::StringRef) { return &MSS; });
204  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
205  ASSERT_TRUE(Idx.blockUntilIdleForTest());
206  }
207  EXPECT_EQ(CacheHits, 0U);
208  EXPECT_EQ(Storage.size(), 2U);
209 
210  {
211  OverlayCDB CDB(/*Base=*/nullptr);
212  BackgroundIndex Idx(Context::empty(), FS, CDB,
213  [&](llvm::StringRef) { return &MSS; });
214  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
215  ASSERT_TRUE(Idx.blockUntilIdleForTest());
216  }
217  EXPECT_EQ(CacheHits, 2U); // Check both A.cc and A.h loaded from cache.
218  EXPECT_EQ(Storage.size(), 2U);
219 
220  auto ShardHeader = MSS.loadShard(testPath("root/A.h"));
221  EXPECT_NE(ShardHeader, nullptr);
222  EXPECT_THAT(
223  *ShardHeader->Symbols,
224  UnorderedElementsAre(Named("common"), Named("A_CC"),
225  AllOf(Named("f_b"), Declared(), Not(Defined()))));
226  for (const auto &Ref : *ShardHeader->Refs)
227  EXPECT_THAT(Ref.second,
228  UnorderedElementsAre(FileURI("unittest:///root/A.h")));
229 
230  auto ShardSource = MSS.loadShard(testPath("root/A.cc"));
231  EXPECT_NE(ShardSource, nullptr);
232  EXPECT_THAT(*ShardSource->Symbols,
233  UnorderedElementsAre(Named("g"), Named("B_CC")));
234  for (const auto &Ref : *ShardSource->Refs)
235  EXPECT_THAT(Ref.second,
236  UnorderedElementsAre(FileURI("unittest:///root/A.cc")));
237 
238  // The BaseOf relationship between A_CC and B_CC is stored in the file
239  // containing the definition of the subject (A_CC)
240  SymbolID A = findSymbol(*ShardHeader->Symbols, "A_CC").ID;
241  SymbolID B = findSymbol(*ShardSource->Symbols, "B_CC").ID;
242  EXPECT_THAT(
243  *ShardHeader->Relations,
244  UnorderedElementsAre(Relation{A, index::SymbolRole::RelationBaseOf, B}));
245  // (and not in the file containing the definition of the object (B_CC)).
246  EXPECT_EQ(ShardSource->Relations->size(), 0u);
247 }
248 
249 TEST_F(BackgroundIndexTest, DirectIncludesTest) {
251  FS.Files[testPath("root/B.h")] = "";
252  FS.Files[testPath("root/A.h")] = R"cpp(
253  #include "B.h"
254  void common();
255  void f_b();
256  class A_CC {};
257  )cpp";
258  std::string A_CC = "#include \"A.h\"\nvoid g() { (void)common; }";
259  FS.Files[testPath("root/A.cc")] = A_CC;
260 
261  llvm::StringMap<std::string> Storage;
262  size_t CacheHits = 0;
263  MemoryShardStorage MSS(Storage, CacheHits);
264 
265  tooling::CompileCommand Cmd;
266  Cmd.Filename = testPath("root/A.cc");
267  Cmd.Directory = testPath("root");
268  Cmd.CommandLine = {"clang++", testPath("root/A.cc")};
269  {
270  OverlayCDB CDB(/*Base=*/nullptr);
271  BackgroundIndex Idx(Context::empty(), FS, CDB,
272  [&](llvm::StringRef) { return &MSS; });
273  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
274  ASSERT_TRUE(Idx.blockUntilIdleForTest());
275  }
276 
277  auto ShardSource = MSS.loadShard(testPath("root/A.cc"));
278  EXPECT_TRUE(ShardSource->Sources);
279  EXPECT_EQ(ShardSource->Sources->size(), 2U); // A.cc, A.h
280  EXPECT_THAT(
281  ShardSource->Sources->lookup("unittest:///root/A.cc").DirectIncludes,
282  UnorderedElementsAre("unittest:///root/A.h"));
283  EXPECT_NE(ShardSource->Sources->lookup("unittest:///root/A.cc").Digest,
284  FileDigest{{0}});
285  EXPECT_THAT(ShardSource->Sources->lookup("unittest:///root/A.h"),
286  EmptyIncludeNode());
287 
288  auto ShardHeader = MSS.loadShard(testPath("root/A.h"));
289  EXPECT_TRUE(ShardHeader->Sources);
290  EXPECT_EQ(ShardHeader->Sources->size(), 2U); // A.h, B.h
291  EXPECT_THAT(
292  ShardHeader->Sources->lookup("unittest:///root/A.h").DirectIncludes,
293  UnorderedElementsAre("unittest:///root/B.h"));
294  EXPECT_NE(ShardHeader->Sources->lookup("unittest:///root/A.h").Digest,
295  FileDigest{{0}});
296  EXPECT_THAT(ShardHeader->Sources->lookup("unittest:///root/B.h"),
297  EmptyIncludeNode());
298 }
299 
300 TEST_F(BackgroundIndexTest, ShardStorageLoad) {
302  FS.Files[testPath("root/A.h")] = R"cpp(
303  void common();
304  void f_b();
305  class A_CC {};
306  )cpp";
307  FS.Files[testPath("root/A.cc")] =
308  "#include \"A.h\"\nvoid g() { (void)common; }";
309 
310  llvm::StringMap<std::string> Storage;
311  size_t CacheHits = 0;
312  MemoryShardStorage MSS(Storage, CacheHits);
313 
314  tooling::CompileCommand Cmd;
315  Cmd.Filename = testPath("root/A.cc");
316  Cmd.Directory = testPath("root");
317  Cmd.CommandLine = {"clang++", testPath("root/A.cc")};
318  // Check nothing is loaded from Storage, but A.cc and A.h has been stored.
319  {
320  OverlayCDB CDB(/*Base=*/nullptr);
321  BackgroundIndex Idx(Context::empty(), FS, CDB,
322  [&](llvm::StringRef) { return &MSS; });
323  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
324  ASSERT_TRUE(Idx.blockUntilIdleForTest());
325  }
326 
327  // Change header.
328  FS.Files[testPath("root/A.h")] = R"cpp(
329  void common();
330  void f_b();
331  class A_CC {};
332  class A_CCnew {};
333  )cpp";
334  {
335  OverlayCDB CDB(/*Base=*/nullptr);
336  BackgroundIndex Idx(Context::empty(), FS, CDB,
337  [&](llvm::StringRef) { return &MSS; });
338  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
339  ASSERT_TRUE(Idx.blockUntilIdleForTest());
340  }
341  EXPECT_EQ(CacheHits, 2U); // Check both A.cc and A.h loaded from cache.
342 
343  // Check if the new symbol has arrived.
344  auto ShardHeader = MSS.loadShard(testPath("root/A.h"));
345  EXPECT_NE(ShardHeader, nullptr);
346  EXPECT_THAT(*ShardHeader->Symbols, Contains(Named("A_CCnew")));
347 
348  // Change source.
349  FS.Files[testPath("root/A.cc")] =
350  "#include \"A.h\"\nvoid g() { (void)common; }\nvoid f_b() {}";
351  {
352  CacheHits = 0;
353  OverlayCDB CDB(/*Base=*/nullptr);
354  BackgroundIndex Idx(Context::empty(), FS, CDB,
355  [&](llvm::StringRef) { return &MSS; });
356  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
357  ASSERT_TRUE(Idx.blockUntilIdleForTest());
358  }
359  EXPECT_EQ(CacheHits, 2U); // Check both A.cc and A.h loaded from cache.
360 
361  // Check if the new symbol has arrived.
362  ShardHeader = MSS.loadShard(testPath("root/A.h"));
363  EXPECT_NE(ShardHeader, nullptr);
364  EXPECT_THAT(*ShardHeader->Symbols, Contains(Named("A_CCnew")));
365  auto ShardSource = MSS.loadShard(testPath("root/A.cc"));
366  EXPECT_NE(ShardSource, nullptr);
367  EXPECT_THAT(*ShardSource->Symbols,
368  Contains(AllOf(Named("f_b"), Declared(), Defined())));
369 }
370 
371 TEST_F(BackgroundIndexTest, ShardStorageEmptyFile) {
373  FS.Files[testPath("root/A.h")] = R"cpp(
374  void common();
375  void f_b();
376  class A_CC {};
377  )cpp";
378  FS.Files[testPath("root/B.h")] = R"cpp(
379  #include "A.h"
380  )cpp";
381  FS.Files[testPath("root/A.cc")] =
382  "#include \"B.h\"\nvoid g() { (void)common; }";
383 
384  llvm::StringMap<std::string> Storage;
385  size_t CacheHits = 0;
386  MemoryShardStorage MSS(Storage, CacheHits);
387 
388  tooling::CompileCommand Cmd;
389  Cmd.Filename = testPath("root/A.cc");
390  Cmd.Directory = testPath("root");
391  Cmd.CommandLine = {"clang++", testPath("root/A.cc")};
392  // Check that A.cc, A.h and B.h has been stored.
393  {
394  OverlayCDB CDB(/*Base=*/nullptr);
395  BackgroundIndex Idx(Context::empty(), FS, CDB,
396  [&](llvm::StringRef) { return &MSS; });
397  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
398  ASSERT_TRUE(Idx.blockUntilIdleForTest());
399  }
400  EXPECT_THAT(Storage.keys(),
401  UnorderedElementsAre(testPath("root/A.cc"), testPath("root/A.h"),
402  testPath("root/B.h")));
403  auto ShardHeader = MSS.loadShard(testPath("root/B.h"));
404  EXPECT_NE(ShardHeader, nullptr);
405  EXPECT_TRUE(ShardHeader->Symbols->empty());
406 
407  // Check that A.cc, A.h and B.h has been loaded.
408  {
409  CacheHits = 0;
410  OverlayCDB CDB(/*Base=*/nullptr);
411  BackgroundIndex Idx(Context::empty(), FS, CDB,
412  [&](llvm::StringRef) { return &MSS; });
413  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
414  ASSERT_TRUE(Idx.blockUntilIdleForTest());
415  }
416  EXPECT_EQ(CacheHits, 3U);
417 
418  // Update B.h to contain some symbols.
419  FS.Files[testPath("root/B.h")] = R"cpp(
420  #include "A.h"
421  void new_func();
422  )cpp";
423  // Check that B.h has been stored with new contents.
424  {
425  CacheHits = 0;
426  OverlayCDB CDB(/*Base=*/nullptr);
427  BackgroundIndex Idx(Context::empty(), FS, CDB,
428  [&](llvm::StringRef) { return &MSS; });
429  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
430  ASSERT_TRUE(Idx.blockUntilIdleForTest());
431  }
432  EXPECT_EQ(CacheHits, 3U);
433  ShardHeader = MSS.loadShard(testPath("root/B.h"));
434  EXPECT_NE(ShardHeader, nullptr);
435  EXPECT_THAT(*ShardHeader->Symbols,
436  Contains(AllOf(Named("new_func"), Declared(), Not(Defined()))));
437 }
438 
439 TEST_F(BackgroundIndexTest, NoDotsInAbsPath) {
441  llvm::StringMap<std::string> Storage;
442  size_t CacheHits = 0;
443  MemoryShardStorage MSS(Storage, CacheHits);
444  OverlayCDB CDB(/*Base=*/nullptr);
445  BackgroundIndex Idx(Context::empty(), FS, CDB,
446  [&](llvm::StringRef) { return &MSS; });
447 
448  tooling::CompileCommand Cmd;
449  FS.Files[testPath("root/A.cc")] = "";
450  Cmd.Filename = "../A.cc";
451  Cmd.Directory = testPath("root/build");
452  Cmd.CommandLine = {"clang++", "../A.cc"};
453  CDB.setCompileCommand(testPath("root/build/../A.cc"), Cmd);
454 
455  FS.Files[testPath("root/B.cc")] = "";
456  Cmd.Filename = "./B.cc";
457  Cmd.Directory = testPath("root");
458  Cmd.CommandLine = {"clang++", "./B.cc"};
459  CDB.setCompileCommand(testPath("root/./B.cc"), Cmd);
460 
461  ASSERT_TRUE(Idx.blockUntilIdleForTest());
462  for (llvm::StringRef AbsPath : MSS.AccessedPaths.keys()) {
463  EXPECT_FALSE(AbsPath.contains("./")) << AbsPath;
464  EXPECT_FALSE(AbsPath.contains("../")) << AbsPath;
465  }
466 }
467 
468 TEST_F(BackgroundIndexTest, UncompilableFiles) {
470  llvm::StringMap<std::string> Storage;
471  size_t CacheHits = 0;
472  MemoryShardStorage MSS(Storage, CacheHits);
473  OverlayCDB CDB(/*Base=*/nullptr);
474  BackgroundIndex Idx(Context::empty(), FS, CDB,
475  [&](llvm::StringRef) { return &MSS; });
476 
477  tooling::CompileCommand Cmd;
478  FS.Files[testPath("A.h")] = "void foo();";
479  FS.Files[testPath("B.h")] = "#include \"C.h\"\nasdf;";
480  FS.Files[testPath("C.h")] = "";
481  FS.Files[testPath("A.cc")] = R"cpp(
482  #include "A.h"
483  #include "B.h"
484  #include "not_found_header.h"
485 
486  void foo() {}
487  )cpp";
488  Cmd.Filename = "../A.cc";
489  Cmd.Directory = testPath("build");
490  Cmd.CommandLine = {"clang++", "../A.cc"};
491  CDB.setCompileCommand(testPath("build/../A.cc"), Cmd);
492  ASSERT_TRUE(Idx.blockUntilIdleForTest());
493 
494  EXPECT_THAT(Storage.keys(), ElementsAre(testPath("A.cc"), testPath("A.h"),
495  testPath("B.h"), testPath("C.h")));
496 
497  {
498  auto Shard = MSS.loadShard(testPath("A.cc"));
499  EXPECT_THAT(*Shard->Symbols, UnorderedElementsAre(Named("foo")));
500  EXPECT_THAT(Shard->Sources->keys(),
501  UnorderedElementsAre("unittest:///A.cc", "unittest:///A.h",
502  "unittest:///B.h"));
503  EXPECT_THAT(Shard->Sources->lookup("unittest:///A.cc"), HadErrors());
504  }
505 
506  {
507  auto Shard = MSS.loadShard(testPath("A.h"));
508  EXPECT_THAT(*Shard->Symbols, UnorderedElementsAre(Named("foo")));
509  EXPECT_THAT(Shard->Sources->keys(),
510  UnorderedElementsAre("unittest:///A.h"));
511  EXPECT_THAT(Shard->Sources->lookup("unittest:///A.h"), HadErrors());
512  }
513 
514  {
515  auto Shard = MSS.loadShard(testPath("B.h"));
516  EXPECT_THAT(*Shard->Symbols, UnorderedElementsAre(Named("asdf")));
517  EXPECT_THAT(Shard->Sources->keys(),
518  UnorderedElementsAre("unittest:///B.h", "unittest:///C.h"));
519  EXPECT_THAT(Shard->Sources->lookup("unittest:///B.h"), HadErrors());
520  }
521 
522  {
523  auto Shard = MSS.loadShard(testPath("C.h"));
524  EXPECT_THAT(*Shard->Symbols, UnorderedElementsAre());
525  EXPECT_THAT(Shard->Sources->keys(),
526  UnorderedElementsAre("unittest:///C.h"));
527  EXPECT_THAT(Shard->Sources->lookup("unittest:///C.h"), HadErrors());
528  }
529 }
530 
533  llvm::StringMap<std::string> Storage;
534  size_t CacheHits = 0;
535  MemoryShardStorage MSS(Storage, CacheHits);
536  OverlayCDB CDB(/*Base=*/nullptr, /*FallbackFlags=*/{},
537  /*ResourceDir=*/std::string(""));
538  BackgroundIndex Idx(Context::empty(), FS, CDB,
539  [&](llvm::StringRef) { return &MSS; });
540 
541  tooling::CompileCommand Cmd;
542  FS.Files[testPath("A.cc")] = "#include \"A.h\"";
543  FS.Files[testPath("A.h")] = "";
544  Cmd.Filename = "../A.cc";
545  Cmd.Directory = testPath("build");
546  Cmd.CommandLine = {"clang++", "../A.cc", "-fsyntax-only"};
547  CDB.setCompileCommand(testPath("build/../A.cc"), Cmd);
548  ASSERT_TRUE(Idx.blockUntilIdleForTest());
549 
550  EXPECT_THAT(Storage.keys(), ElementsAre(testPath("A.cc"), testPath("A.h")));
551  // Make sure we only store the Cmd for main file.
552  EXPECT_FALSE(MSS.loadShard(testPath("A.h"))->Cmd);
553 
554  {
555  tooling::CompileCommand CmdStored = *MSS.loadShard(testPath("A.cc"))->Cmd;
556  EXPECT_EQ(CmdStored.CommandLine, Cmd.CommandLine);
557  EXPECT_EQ(CmdStored.Directory, Cmd.Directory);
558  }
559 
560  // FIXME: Changing compile commands should be enough to invalidate the cache.
561  FS.Files[testPath("A.cc")] = " ";
562  Cmd.CommandLine = {"clang++", "../A.cc", "-Dfoo", "-fsyntax-only"};
563  CDB.setCompileCommand(testPath("build/../A.cc"), Cmd);
564  ASSERT_TRUE(Idx.blockUntilIdleForTest());
565 
566  EXPECT_FALSE(MSS.loadShard(testPath("A.h"))->Cmd);
567 
568  {
569  tooling::CompileCommand CmdStored = *MSS.loadShard(testPath("A.cc"))->Cmd;
570  EXPECT_EQ(CmdStored.CommandLine, Cmd.CommandLine);
571  EXPECT_EQ(CmdStored.Directory, Cmd.Directory);
572  }
573 }
574 
575 class BackgroundIndexRebuilderTest : public testing::Test {
576 protected:
578  : Target(llvm::make_unique<MemIndex>()),
579  Rebuilder(&Target, &Source, /*Threads=*/10) {
580  // Prepare FileSymbols with TestSymbol in it, for checkRebuild.
581  TestSymbol.ID = SymbolID("foo");
582  }
583 
584  // Perform Action and determine whether it rebuilt the index or not.
585  bool checkRebuild(std::function<void()> Action) {
586  // Update name so we can tell if the index updates.
587  VersionStorage.push_back("Sym" + std::to_string(++VersionCounter));
588  TestSymbol.Name = VersionStorage.back();
590  SB.insert(TestSymbol);
591  Source.update("", llvm::make_unique<SymbolSlab>(std::move(SB).build()),
592  nullptr, nullptr, false);
593  // Now maybe update the index.
594  Action();
595  // Now query the index to get the name count.
596  std::string ReadName;
597  LookupRequest Req;
598  Req.IDs.insert(TestSymbol.ID);
599  Target.lookup(Req, [&](const Symbol &S) { ReadName = S.Name; });
600  // The index was rebuild if the name is up to date.
601  return ReadName == VersionStorage.back();
602  }
603 
608 
609  unsigned VersionCounter = 0;
610  std::deque<std::string> VersionStorage;
611 };
612 
614  for (unsigned I = 0; I < Rebuilder.TUsBeforeFirstBuild - 1; ++I)
615  EXPECT_FALSE(checkRebuild([&] { Rebuilder.indexedTU(); }));
616  EXPECT_TRUE(checkRebuild([&] { Rebuilder.indexedTU(); }));
617  for (unsigned I = 0; I < Rebuilder.TUsBeforeRebuild - 1; ++I)
618  EXPECT_FALSE(checkRebuild([&] { Rebuilder.indexedTU(); }));
619  EXPECT_TRUE(checkRebuild([&] { Rebuilder.indexedTU(); }));
620 }
621 
623  Rebuilder.startLoading();
624  Rebuilder.loadedShard(10);
625  Rebuilder.loadedShard(20);
626  EXPECT_TRUE(checkRebuild([&] { Rebuilder.doneLoading(); }));
627 
628  // No rebuild for no shards.
629  Rebuilder.startLoading();
630  EXPECT_FALSE(checkRebuild([&] { Rebuilder.doneLoading(); }));
631 
632  // Loads can overlap.
633  Rebuilder.startLoading();
634  Rebuilder.loadedShard(1);
635  Rebuilder.startLoading();
636  Rebuilder.loadedShard(1);
637  EXPECT_FALSE(checkRebuild([&] { Rebuilder.doneLoading(); }));
638  Rebuilder.loadedShard(1);
639  EXPECT_TRUE(checkRebuild([&] { Rebuilder.doneLoading(); }));
640 
641  // No rebuilding for indexed files while loading.
642  Rebuilder.startLoading();
643  for (unsigned I = 0; I < 3 * Rebuilder.TUsBeforeRebuild; ++I)
644  EXPECT_FALSE(checkRebuild([&] { Rebuilder.indexedTU(); }));
645  // But they get indexed when we're done, even if no shards were loaded.
646  EXPECT_TRUE(checkRebuild([&] { Rebuilder.doneLoading(); }));
647 }
648 
649 TEST(BackgroundQueueTest, Priority) {
650  // Create high and low priority tasks.
651  // Once a bunch of high priority tasks have run, the queue is stopped.
652  // So the low priority tasks should never run.
653  BackgroundQueue Q;
654  std::atomic<unsigned> HiRan(0), LoRan(0);
655  BackgroundQueue::Task Lo([&] { ++LoRan; });
656  BackgroundQueue::Task Hi([&] {
657  if (++HiRan >= 10)
658  Q.stop();
659  });
660  Hi.QueuePri = 100;
661 
662  // Enqueuing the low-priority ones first shouldn't make them run first.
663  Q.append(std::vector<BackgroundQueue::Task>(30, Lo));
664  for (unsigned I = 0; I < 30; ++I)
665  Q.push(Hi);
666 
667  AsyncTaskRunner ThreadPool;
668  for (unsigned I = 0; I < 5; ++I)
669  ThreadPool.runAsync("worker", [&] { Q.work(); });
670  // We should test enqueue with active workers, but it's hard to avoid races.
671  // Just make sure we don't crash.
672  Q.push(Lo);
673  Q.append(std::vector<BackgroundQueue::Task>(2, Hi));
674 
675  // After finishing, check the tasks that ran.
676  ThreadPool.wait();
677  EXPECT_GE(HiRan, 10u);
678  EXPECT_EQ(LoRan, 0u);
679 }
680 
681 TEST(BackgroundQueueTest, Boost) {
682  std::string Sequence;
683 
684  BackgroundQueue::Task A([&] { Sequence.push_back('A'); });
685  A.Tag = "A";
686  A.QueuePri = 1;
687 
688  BackgroundQueue::Task B([&] { Sequence.push_back('B'); });
689  B.QueuePri = 2;
690  B.Tag = "B";
691 
692  {
693  BackgroundQueue Q;
694  Q.append({A, B});
695  Q.work([&] { Q.stop(); });
696  EXPECT_EQ("BA", Sequence) << "priority order";
697  }
698  Sequence.clear();
699  {
700  BackgroundQueue Q;
701  Q.boost("A", 3);
702  Q.append({A, B});
703  Q.work([&] { Q.stop(); });
704  EXPECT_EQ("AB", Sequence) << "A was boosted before enqueueing";
705  }
706  Sequence.clear();
707  {
708  BackgroundQueue Q;
709  Q.append({A, B});
710  Q.boost("A", 3);
711  Q.work([&] { Q.stop(); });
712  EXPECT_EQ("AB", Sequence) << "A was boosted after enqueueing";
713  }
714 }
715 
716 } // namespace clangd
717 } // namespace clang
::testing::Matcher< const RefSlab & > RefsAre(std::vector<::testing::Matcher< Ref >> Matchers)
MATCHER_P(Named, N, "")
Some operations such as code completion produce a set of candidates.
Represents a relation between two symbols.
Definition: Relation.h:25
llvm::StringMap< std::string > Files
Definition: TestFS.h:38
std::array< uint8_t, 8 > FileDigest
Definition: SourceCode.h:35
A container of Symbols from several source files.
Definition: FileIndex.h:59
llvm::Error storeShard(llvm::StringRef ShardIdentifier, IndexFileOut Shard) const override
llvm::DenseSet< SymbolID > IDs
Definition: Index.h:64
bool checkRebuild(std::function< void()> Action)
Represents a symbol occurrence in the source file.
Definition: Ref.h:52
void insert(const Symbol &S)
Adds a symbol, overwriting any existing one with the same ID.
Definition: Symbol.cpp:50
MockFSProvider FS
llvm::Expected< IndexFileIn > readIndexFile(llvm::StringRef Data)
SymbolSlab::Builder is a mutable container that can &#39;freeze&#39; to SymbolSlab.
Definition: Symbol.h:199
SymbolID ID
The ID of the symbol.
Definition: Symbol.h:38
void boost(llvm::StringRef Tag, unsigned NewPriority)
TEST_F(BackgroundIndexTest, NoCrashOnErrorFile)
static void preventThreadStarvationInTests()
void runAsync(const llvm::Twine &Name, llvm::unique_function< void()> Action)
Definition: Threading.cpp:70
TEST(BackgroundQueueTest, Priority)
std::string testPath(PathRef File)
Definition: TestFS.cpp:82
llvm::unique_function< void()> Action
MemIndex is a naive in-memory index suitable for a small set of symbols.
Definition: MemIndex.h:19
SymbolSlab runFuzzyFind(const SymbolIndex &Index, llvm::StringRef Query)
Definition: SyncAPI.cpp:129
MATCHER(Declared, "")
Runs tasks on separate (detached) threads and wait for all tasks to finish.
Definition: Threading.h:104
void setCompileCommand(PathRef File, llvm::Optional< tooling::CompileCommand > CompilationCommand)
Sets or clears the compilation command for a particular file.
The class presents a C++ symbol, e.g.
Definition: Symbol.h:36
MemoryShardStorage(llvm::StringMap< std::string > &Storage, size_t &CacheHits)
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
llvm::StringRef Name
The unqualified name of the symbol, e.g. "bar" (for ns::bar).
Definition: Symbol.h:42
const Symbol & findSymbol(const SymbolSlab &Slab, llvm::StringRef QName)
Definition: TestTU.cpp:93
std::unique_ptr< IndexFileIn > loadShard(llvm::StringRef ShardIdentifier) const override
void append(std::vector< Task >)
A work item on the thread pool&#39;s queue.
Definition: Background.h:70
Wraps another compilation database, and supports overriding the commands using an in-memory mapping...
static Context empty()
Returns an empty root context that contains no data.
Definition: Context.cpp:15
std::array< uint8_t, 20 > SymbolID
RefSlab getRefs(const SymbolIndex &Index, SymbolID ID)
Definition: SyncAPI.cpp:142
void work(std::function< void()> OnIdle=nullptr)