clang-tools  11.0.0
BackgroundIndexTests.cpp
Go to the documentation of this file.
1 #include "CompileCommands.h"
2 #include "Config.h"
3 #include "Headers.h"
4 #include "SyncAPI.h"
5 #include "TestFS.h"
6 #include "TestIndex.h"
7 #include "TestTU.h"
8 #include "index/Background.h"
10 #include "clang/Tooling/ArgumentsAdjusters.h"
11 #include "clang/Tooling/CompilationDatabase.h"
12 #include "llvm/Support/ScopedPrinter.h"
13 #include "llvm/Support/Threading.h"
14 #include "gmock/gmock.h"
15 #include "gtest/gtest.h"
16 #include <deque>
17 #include <thread>
18 
19 using ::testing::_;
20 using ::testing::AllOf;
21 using ::testing::Contains;
22 using ::testing::ElementsAre;
23 using ::testing::Not;
24 using ::testing::UnorderedElementsAre;
25 
26 namespace clang {
27 namespace clangd {
28 
29 MATCHER_P(Named, N, "") { return arg.Name == N; }
30 MATCHER_P(QName, N, "") { return (arg.Scope + arg.Name).str() == N; }
31 MATCHER(Declared, "") {
32  return !StringRef(arg.CanonicalDeclaration.FileURI).empty();
33 }
34 MATCHER(Defined, "") { return !StringRef(arg.Definition.FileURI).empty(); }
35 MATCHER_P(FileURI, F, "") { return StringRef(arg.Location.FileURI) == F; }
36 ::testing::Matcher<const RefSlab &>
37 RefsAre(std::vector<::testing::Matcher<Ref>> Matchers) {
38  return ElementsAre(::testing::Pair(_, UnorderedElementsAreArray(Matchers)));
39 }
40 // URI cannot be empty since it references keys in the IncludeGraph.
41 MATCHER(EmptyIncludeNode, "") {
42  return arg.Flags == IncludeGraphNode::SourceFlag::None && !arg.URI.empty() &&
43  arg.Digest == FileDigest{{0}} && arg.DirectIncludes.empty();
44 }
45 
46 MATCHER(HadErrors, "") {
47  return arg.Flags & IncludeGraphNode::SourceFlag::HadErrors;
48 }
49 
50 MATCHER_P(NumReferences, N, "") { return arg.References == N; }
51 
53  mutable std::mutex StorageMu;
54  llvm::StringMap<std::string> &Storage;
55  size_t &CacheHits;
56 
57 public:
58  MemoryShardStorage(llvm::StringMap<std::string> &Storage, size_t &CacheHits)
59  : Storage(Storage), CacheHits(CacheHits) {}
60  llvm::Error storeShard(llvm::StringRef ShardIdentifier,
61  IndexFileOut Shard) const override {
62  std::lock_guard<std::mutex> Lock(StorageMu);
63  AccessedPaths.insert(ShardIdentifier);
64  Storage[ShardIdentifier] = llvm::to_string(Shard);
65  return llvm::Error::success();
66  }
67  std::unique_ptr<IndexFileIn>
68  loadShard(llvm::StringRef ShardIdentifier) const override {
69  std::lock_guard<std::mutex> Lock(StorageMu);
70  AccessedPaths.insert(ShardIdentifier);
71  if (Storage.find(ShardIdentifier) == Storage.end()) {
72  return nullptr;
73  }
74  auto IndexFile = readIndexFile(Storage[ShardIdentifier]);
75  if (!IndexFile) {
76  ADD_FAILURE() << "Error while reading " << ShardIdentifier << ':'
77  << IndexFile.takeError();
78  return nullptr;
79  }
80  CacheHits++;
81  return std::make_unique<IndexFileIn>(std::move(*IndexFile));
82  }
83 
84  mutable llvm::StringSet<> AccessedPaths;
85 };
86 
87 class BackgroundIndexTest : public ::testing::Test {
88 protected:
90 };
91 
92 TEST_F(BackgroundIndexTest, NoCrashOnErrorFile) {
93  MockFS FS;
94  FS.Files[testPath("root/A.cc")] = "error file";
95  llvm::StringMap<std::string> Storage;
96  size_t CacheHits = 0;
97  MemoryShardStorage MSS(Storage, CacheHits);
98  OverlayCDB CDB(/*Base=*/nullptr);
99  BackgroundIndex Idx(Context::empty(), FS, CDB,
100  [&](llvm::StringRef) { return &MSS; });
101 
102  tooling::CompileCommand Cmd;
103  Cmd.Filename = testPath("root/A.cc");
104  Cmd.Directory = testPath("root");
105  Cmd.CommandLine = {"clang++", "-DA=1", testPath("root/A.cc")};
106  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
107 
108  ASSERT_TRUE(Idx.blockUntilIdleForTest());
109 }
110 
112  MockFS FS;
113  // Set up two identical TUs, foo and bar.
114  // They define foo::one and bar::one.
115  std::vector<tooling::CompileCommand> Cmds;
116  for (std::string Name : {"foo", "bar", "baz"}) {
117  std::string Filename = Name + ".cpp";
118  std::string Header = Name + ".h";
119  FS.Files[Filename] = "#include \"" + Header + "\"";
120  FS.Files[Header] = "namespace " + Name + " { int one; }";
121  tooling::CompileCommand Cmd;
122  Cmd.Filename = Filename;
123  Cmd.Directory = testRoot();
124  Cmd.CommandLine = {"clang++", Filename};
125  Cmds.push_back(std::move(Cmd));
126  }
127  // Context provider that installs a configuration mutating foo's command.
128  // This causes it to define foo::two instead of foo::one.
129  // It also disables indexing of baz entirely.
130  auto ContextProvider = [](PathRef P) {
131  Config C;
132  if (P.endswith("foo.cpp"))
133  C.CompileFlags.Edits.push_back(
134  [](std::vector<std::string> &Argv) { Argv.push_back("-Done=two"); });
135  if (P.endswith("baz.cpp"))
137  return Context::current().derive(Config::Key, std::move(C));
138  };
139  // Create the background index.
140  llvm::StringMap<std::string> Storage;
141  size_t CacheHits = 0;
142  MemoryShardStorage MSS(Storage, CacheHits);
143  // We need the CommandMangler, because that applies the config we're testing.
144  OverlayCDB CDB(/*Base=*/nullptr, /*FallbackFlags=*/{},
145  tooling::ArgumentsAdjuster(CommandMangler::forTests()));
146  BackgroundIndex Idx(
147  Context::empty(), FS, CDB, [&](llvm::StringRef) { return &MSS; },
148  /*ThreadPoolSize=*/4, /*OnProgress=*/nullptr, std::move(ContextProvider));
149  // Index the two files.
150  for (auto &Cmd : Cmds) {
151  std::string FullPath = testPath(Cmd.Filename);
152  CDB.setCompileCommand(FullPath, std::move(Cmd));
153  }
154  // Wait for both files to be indexed.
155  ASSERT_TRUE(Idx.blockUntilIdleForTest());
156  EXPECT_THAT(runFuzzyFind(Idx, ""),
157  UnorderedElementsAre(QName("foo"), QName("foo::two"),
158  QName("bar"), QName("bar::one")));
159 }
160 
161 TEST_F(BackgroundIndexTest, IndexTwoFiles) {
162  MockFS FS;
163  // a.h yields different symbols when included by A.cc vs B.cc.
164  FS.Files[testPath("root/A.h")] = R"cpp(
165  void common();
166  void f_b();
167  #if A
168  class A_CC {};
169  #else
170  class B_CC{};
171  #endif
172  )cpp";
173  FS.Files[testPath("root/A.cc")] =
174  "#include \"A.h\"\nvoid g() { (void)common; }";
175  FS.Files[testPath("root/B.cc")] =
176  R"cpp(
177  #define A 0
178  #include "A.h"
179  void f_b() {
180  (void)common;
181  (void)common;
182  (void)common;
183  (void)common;
184  })cpp";
185  llvm::StringMap<std::string> Storage;
186  size_t CacheHits = 0;
187  MemoryShardStorage MSS(Storage, CacheHits);
188  OverlayCDB CDB(/*Base=*/nullptr);
189  BackgroundIndex Idx(Context::empty(), FS, CDB,
190  [&](llvm::StringRef) { return &MSS; });
191 
192  tooling::CompileCommand Cmd;
193  Cmd.Filename = testPath("root/A.cc");
194  Cmd.Directory = testPath("root");
195  Cmd.CommandLine = {"clang++", "-DA=1", testPath("root/A.cc")};
196  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
197 
198  ASSERT_TRUE(Idx.blockUntilIdleForTest());
199  EXPECT_THAT(runFuzzyFind(Idx, ""),
200  UnorderedElementsAre(AllOf(Named("common"), NumReferences(1U)),
201  AllOf(Named("A_CC"), NumReferences(0U)),
202  AllOf(Named("g"), NumReferences(0U)),
203  AllOf(Named("f_b"), Declared(),
204  Not(Defined()), NumReferences(0U))));
205 
206  Cmd.Filename = testPath("root/B.cc");
207  Cmd.CommandLine = {"clang++", Cmd.Filename};
208  CDB.setCompileCommand(testPath("root/B.cc"), Cmd);
209 
210  ASSERT_TRUE(Idx.blockUntilIdleForTest());
211  // B_CC is dropped as we don't collect symbols from A.h in this compilation.
212  EXPECT_THAT(runFuzzyFind(Idx, ""),
213  UnorderedElementsAre(AllOf(Named("common"), NumReferences(5U)),
214  AllOf(Named("A_CC"), NumReferences(0U)),
215  AllOf(Named("g"), NumReferences(0U)),
216  AllOf(Named("f_b"), Declared(), Defined(),
217  NumReferences(1U))));
218 
219  auto Syms = runFuzzyFind(Idx, "common");
220  EXPECT_THAT(Syms, UnorderedElementsAre(Named("common")));
221  auto Common = *Syms.begin();
222  EXPECT_THAT(getRefs(Idx, Common.ID),
223  RefsAre({FileURI("unittest:///root/A.h"),
224  FileURI("unittest:///root/A.cc"),
225  FileURI("unittest:///root/B.cc"),
226  FileURI("unittest:///root/B.cc"),
227  FileURI("unittest:///root/B.cc"),
228  FileURI("unittest:///root/B.cc")}));
229 }
230 
231 TEST_F(BackgroundIndexTest, ShardStorageTest) {
232  MockFS FS;
233  FS.Files[testPath("root/A.h")] = R"cpp(
234  void common();
235  void f_b();
236  class A_CC {};
237  )cpp";
238  std::string A_CC = "";
239  FS.Files[testPath("root/A.cc")] = R"cpp(
240  #include "A.h"
241  void g() { (void)common; }
242  class B_CC : public A_CC {};
243  )cpp";
244 
245  llvm::StringMap<std::string> Storage;
246  size_t CacheHits = 0;
247  MemoryShardStorage MSS(Storage, CacheHits);
248 
249  tooling::CompileCommand Cmd;
250  Cmd.Filename = testPath("root/A.cc");
251  Cmd.Directory = testPath("root");
252  Cmd.CommandLine = {"clang++", testPath("root/A.cc")};
253  // Check nothing is loaded from Storage, but A.cc and A.h has been stored.
254  {
255  OverlayCDB CDB(/*Base=*/nullptr);
256  BackgroundIndex Idx(Context::empty(), FS, CDB,
257  [&](llvm::StringRef) { return &MSS; });
258  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
259  ASSERT_TRUE(Idx.blockUntilIdleForTest());
260  }
261  EXPECT_EQ(CacheHits, 0U);
262  EXPECT_EQ(Storage.size(), 2U);
263 
264  {
265  OverlayCDB CDB(/*Base=*/nullptr);
266  BackgroundIndex Idx(Context::empty(), FS, CDB,
267  [&](llvm::StringRef) { return &MSS; });
268  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
269  ASSERT_TRUE(Idx.blockUntilIdleForTest());
270  }
271  EXPECT_EQ(CacheHits, 2U); // Check both A.cc and A.h loaded from cache.
272  EXPECT_EQ(Storage.size(), 2U);
273 
274  auto ShardHeader = MSS.loadShard(testPath("root/A.h"));
275  EXPECT_NE(ShardHeader, nullptr);
276  EXPECT_THAT(
277  *ShardHeader->Symbols,
278  UnorderedElementsAre(Named("common"), Named("A_CC"),
279  AllOf(Named("f_b"), Declared(), Not(Defined()))));
280  for (const auto &Ref : *ShardHeader->Refs)
281  EXPECT_THAT(Ref.second,
282  UnorderedElementsAre(FileURI("unittest:///root/A.h")));
283 
284  auto ShardSource = MSS.loadShard(testPath("root/A.cc"));
285  EXPECT_NE(ShardSource, nullptr);
286  EXPECT_THAT(*ShardSource->Symbols,
287  UnorderedElementsAre(Named("g"), Named("B_CC")));
288  for (const auto &Ref : *ShardSource->Refs)
289  EXPECT_THAT(Ref.second,
290  UnorderedElementsAre(FileURI("unittest:///root/A.cc")));
291 
292  // The BaseOf relationship between A_CC and B_CC is stored in the file
293  // containing the definition of the subject (A_CC)
294  SymbolID A = findSymbol(*ShardHeader->Symbols, "A_CC").ID;
295  SymbolID B = findSymbol(*ShardSource->Symbols, "B_CC").ID;
296  EXPECT_THAT(*ShardHeader->Relations,
297  UnorderedElementsAre(Relation{A, RelationKind::BaseOf, B}));
298  // (and not in the file containing the definition of the object (B_CC)).
299  EXPECT_EQ(ShardSource->Relations->size(), 0u);
300 }
301 
302 TEST_F(BackgroundIndexTest, DirectIncludesTest) {
303  MockFS FS;
304  FS.Files[testPath("root/B.h")] = "";
305  FS.Files[testPath("root/A.h")] = R"cpp(
306  #include "B.h"
307  void common();
308  void f_b();
309  class A_CC {};
310  )cpp";
311  std::string A_CC = "#include \"A.h\"\nvoid g() { (void)common; }";
312  FS.Files[testPath("root/A.cc")] = A_CC;
313 
314  llvm::StringMap<std::string> Storage;
315  size_t CacheHits = 0;
316  MemoryShardStorage MSS(Storage, CacheHits);
317 
318  tooling::CompileCommand Cmd;
319  Cmd.Filename = testPath("root/A.cc");
320  Cmd.Directory = testPath("root");
321  Cmd.CommandLine = {"clang++", testPath("root/A.cc")};
322  {
323  OverlayCDB CDB(/*Base=*/nullptr);
324  BackgroundIndex Idx(Context::empty(), FS, CDB,
325  [&](llvm::StringRef) { return &MSS; });
326  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
327  ASSERT_TRUE(Idx.blockUntilIdleForTest());
328  }
329 
330  auto ShardSource = MSS.loadShard(testPath("root/A.cc"));
331  EXPECT_TRUE(ShardSource->Sources);
332  EXPECT_EQ(ShardSource->Sources->size(), 2U); // A.cc, A.h
333  EXPECT_THAT(
334  ShardSource->Sources->lookup("unittest:///root/A.cc").DirectIncludes,
335  UnorderedElementsAre("unittest:///root/A.h"));
336  EXPECT_NE(ShardSource->Sources->lookup("unittest:///root/A.cc").Digest,
337  FileDigest{{0}});
338  EXPECT_THAT(ShardSource->Sources->lookup("unittest:///root/A.h"),
339  EmptyIncludeNode());
340 
341  auto ShardHeader = MSS.loadShard(testPath("root/A.h"));
342  EXPECT_TRUE(ShardHeader->Sources);
343  EXPECT_EQ(ShardHeader->Sources->size(), 2U); // A.h, B.h
344  EXPECT_THAT(
345  ShardHeader->Sources->lookup("unittest:///root/A.h").DirectIncludes,
346  UnorderedElementsAre("unittest:///root/B.h"));
347  EXPECT_NE(ShardHeader->Sources->lookup("unittest:///root/A.h").Digest,
348  FileDigest{{0}});
349  EXPECT_THAT(ShardHeader->Sources->lookup("unittest:///root/B.h"),
350  EmptyIncludeNode());
351 }
352 
353 TEST_F(BackgroundIndexTest, ShardStorageLoad) {
354  MockFS FS;
355  FS.Files[testPath("root/A.h")] = R"cpp(
356  void common();
357  void f_b();
358  class A_CC {};
359  )cpp";
360  FS.Files[testPath("root/A.cc")] =
361  "#include \"A.h\"\nvoid g() { (void)common; }";
362 
363  llvm::StringMap<std::string> Storage;
364  size_t CacheHits = 0;
365  MemoryShardStorage MSS(Storage, CacheHits);
366 
367  tooling::CompileCommand Cmd;
368  Cmd.Filename = testPath("root/A.cc");
369  Cmd.Directory = testPath("root");
370  Cmd.CommandLine = {"clang++", testPath("root/A.cc")};
371  // Check nothing is loaded from Storage, but A.cc and A.h has been stored.
372  {
373  OverlayCDB CDB(/*Base=*/nullptr);
374  BackgroundIndex Idx(Context::empty(), FS, CDB,
375  [&](llvm::StringRef) { return &MSS; });
376  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
377  ASSERT_TRUE(Idx.blockUntilIdleForTest());
378  }
379 
380  // Change header.
381  FS.Files[testPath("root/A.h")] = R"cpp(
382  void common();
383  void f_b();
384  class A_CC {};
385  class A_CCnew {};
386  )cpp";
387  {
388  OverlayCDB CDB(/*Base=*/nullptr);
389  BackgroundIndex Idx(Context::empty(), FS, CDB,
390  [&](llvm::StringRef) { return &MSS; });
391  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
392  ASSERT_TRUE(Idx.blockUntilIdleForTest());
393  }
394  EXPECT_EQ(CacheHits, 2U); // Check both A.cc and A.h loaded from cache.
395 
396  // Check if the new symbol has arrived.
397  auto ShardHeader = MSS.loadShard(testPath("root/A.h"));
398  EXPECT_NE(ShardHeader, nullptr);
399  EXPECT_THAT(*ShardHeader->Symbols, Contains(Named("A_CCnew")));
400 
401  // Change source.
402  FS.Files[testPath("root/A.cc")] =
403  "#include \"A.h\"\nvoid g() { (void)common; }\nvoid f_b() {}";
404  {
405  CacheHits = 0;
406  OverlayCDB CDB(/*Base=*/nullptr);
407  BackgroundIndex Idx(Context::empty(), FS, CDB,
408  [&](llvm::StringRef) { return &MSS; });
409  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
410  ASSERT_TRUE(Idx.blockUntilIdleForTest());
411  }
412  EXPECT_EQ(CacheHits, 2U); // Check both A.cc and A.h loaded from cache.
413 
414  // Check if the new symbol has arrived.
415  ShardHeader = MSS.loadShard(testPath("root/A.h"));
416  EXPECT_NE(ShardHeader, nullptr);
417  EXPECT_THAT(*ShardHeader->Symbols, Contains(Named("A_CCnew")));
418  auto ShardSource = MSS.loadShard(testPath("root/A.cc"));
419  EXPECT_NE(ShardSource, nullptr);
420  EXPECT_THAT(*ShardSource->Symbols,
421  Contains(AllOf(Named("f_b"), Declared(), Defined())));
422 }
423 
424 TEST_F(BackgroundIndexTest, ShardStorageEmptyFile) {
425  MockFS FS;
426  FS.Files[testPath("root/A.h")] = R"cpp(
427  void common();
428  void f_b();
429  class A_CC {};
430  )cpp";
431  FS.Files[testPath("root/B.h")] = R"cpp(
432  #include "A.h"
433  )cpp";
434  FS.Files[testPath("root/A.cc")] =
435  "#include \"B.h\"\nvoid g() { (void)common; }";
436 
437  llvm::StringMap<std::string> Storage;
438  size_t CacheHits = 0;
439  MemoryShardStorage MSS(Storage, CacheHits);
440 
441  tooling::CompileCommand Cmd;
442  Cmd.Filename = testPath("root/A.cc");
443  Cmd.Directory = testPath("root");
444  Cmd.CommandLine = {"clang++", testPath("root/A.cc")};
445  // Check that A.cc, A.h and B.h has been stored.
446  {
447  OverlayCDB CDB(/*Base=*/nullptr);
448  BackgroundIndex Idx(Context::empty(), FS, CDB,
449  [&](llvm::StringRef) { return &MSS; });
450  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
451  ASSERT_TRUE(Idx.blockUntilIdleForTest());
452  }
453  EXPECT_THAT(Storage.keys(),
454  UnorderedElementsAre(testPath("root/A.cc"), testPath("root/A.h"),
455  testPath("root/B.h")));
456  auto ShardHeader = MSS.loadShard(testPath("root/B.h"));
457  EXPECT_NE(ShardHeader, nullptr);
458  EXPECT_TRUE(ShardHeader->Symbols->empty());
459 
460  // Check that A.cc, A.h and B.h has been loaded.
461  {
462  CacheHits = 0;
463  OverlayCDB CDB(/*Base=*/nullptr);
464  BackgroundIndex Idx(Context::empty(), FS, CDB,
465  [&](llvm::StringRef) { return &MSS; });
466  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
467  ASSERT_TRUE(Idx.blockUntilIdleForTest());
468  }
469  EXPECT_EQ(CacheHits, 3U);
470 
471  // Update B.h to contain some symbols.
472  FS.Files[testPath("root/B.h")] = R"cpp(
473  #include "A.h"
474  void new_func();
475  )cpp";
476  // Check that B.h has been stored with new contents.
477  {
478  CacheHits = 0;
479  OverlayCDB CDB(/*Base=*/nullptr);
480  BackgroundIndex Idx(Context::empty(), FS, CDB,
481  [&](llvm::StringRef) { return &MSS; });
482  CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
483  ASSERT_TRUE(Idx.blockUntilIdleForTest());
484  }
485  EXPECT_EQ(CacheHits, 3U);
486  ShardHeader = MSS.loadShard(testPath("root/B.h"));
487  EXPECT_NE(ShardHeader, nullptr);
488  EXPECT_THAT(*ShardHeader->Symbols,
489  Contains(AllOf(Named("new_func"), Declared(), Not(Defined()))));
490 }
491 
492 TEST_F(BackgroundIndexTest, NoDotsInAbsPath) {
493  MockFS FS;
494  llvm::StringMap<std::string> Storage;
495  size_t CacheHits = 0;
496  MemoryShardStorage MSS(Storage, CacheHits);
497  OverlayCDB CDB(/*Base=*/nullptr);
498  BackgroundIndex Idx(Context::empty(), FS, CDB,
499  [&](llvm::StringRef) { return &MSS; });
500  ASSERT_TRUE(Idx.blockUntilIdleForTest());
501 
502  tooling::CompileCommand Cmd;
503  FS.Files[testPath("root/A.cc")] = "";
504  Cmd.Filename = "../A.cc";
505  Cmd.Directory = testPath("root/build");
506  Cmd.CommandLine = {"clang++", "../A.cc"};
507  CDB.setCompileCommand(testPath("root/build/../A.cc"), Cmd);
508  ASSERT_TRUE(Idx.blockUntilIdleForTest());
509 
510  FS.Files[testPath("root/B.cc")] = "";
511  Cmd.Filename = "./B.cc";
512  Cmd.Directory = testPath("root");
513  Cmd.CommandLine = {"clang++", "./B.cc"};
514  CDB.setCompileCommand(testPath("root/./B.cc"), Cmd);
515  ASSERT_TRUE(Idx.blockUntilIdleForTest());
516 
517  for (llvm::StringRef AbsPath : MSS.AccessedPaths.keys()) {
518  EXPECT_FALSE(AbsPath.contains("./")) << AbsPath;
519  EXPECT_FALSE(AbsPath.contains("../")) << AbsPath;
520  }
521 }
522 
523 TEST_F(BackgroundIndexTest, UncompilableFiles) {
524  MockFS FS;
525  llvm::StringMap<std::string> Storage;
526  size_t CacheHits = 0;
527  MemoryShardStorage MSS(Storage, CacheHits);
528  OverlayCDB CDB(/*Base=*/nullptr);
529  BackgroundIndex Idx(Context::empty(), FS, CDB,
530  [&](llvm::StringRef) { return &MSS; });
531 
532  tooling::CompileCommand Cmd;
533  FS.Files[testPath("A.h")] = "void foo();";
534  FS.Files[testPath("B.h")] = "#include \"C.h\"\nasdf;";
535  FS.Files[testPath("C.h")] = "";
536  FS.Files[testPath("A.cc")] = R"cpp(
537  #include "A.h"
538  #include "B.h"
539  #include "not_found_header.h"
540 
541  void foo() {}
542  )cpp";
543  Cmd.Filename = "../A.cc";
544  Cmd.Directory = testPath("build");
545  Cmd.CommandLine = {"clang++", "../A.cc"};
546  CDB.setCompileCommand(testPath("build/../A.cc"), Cmd);
547  ASSERT_TRUE(Idx.blockUntilIdleForTest());
548 
549  EXPECT_THAT(Storage.keys(), ElementsAre(testPath("A.cc"), testPath("A.h"),
550  testPath("B.h"), testPath("C.h")));
551 
552  {
553  auto Shard = MSS.loadShard(testPath("A.cc"));
554  EXPECT_THAT(*Shard->Symbols, UnorderedElementsAre(Named("foo")));
555  EXPECT_THAT(Shard->Sources->keys(),
556  UnorderedElementsAre("unittest:///A.cc", "unittest:///A.h",
557  "unittest:///B.h"));
558  EXPECT_THAT(Shard->Sources->lookup("unittest:///A.cc"), HadErrors());
559  }
560 
561  {
562  auto Shard = MSS.loadShard(testPath("A.h"));
563  EXPECT_THAT(*Shard->Symbols, UnorderedElementsAre(Named("foo")));
564  EXPECT_THAT(Shard->Sources->keys(),
565  UnorderedElementsAre("unittest:///A.h"));
566  EXPECT_THAT(Shard->Sources->lookup("unittest:///A.h"), HadErrors());
567  }
568 
569  {
570  auto Shard = MSS.loadShard(testPath("B.h"));
571  EXPECT_THAT(*Shard->Symbols, UnorderedElementsAre(Named("asdf")));
572  EXPECT_THAT(Shard->Sources->keys(),
573  UnorderedElementsAre("unittest:///B.h", "unittest:///C.h"));
574  EXPECT_THAT(Shard->Sources->lookup("unittest:///B.h"), HadErrors());
575  }
576 
577  {
578  auto Shard = MSS.loadShard(testPath("C.h"));
579  EXPECT_THAT(*Shard->Symbols, UnorderedElementsAre());
580  EXPECT_THAT(Shard->Sources->keys(),
581  UnorderedElementsAre("unittest:///C.h"));
582  EXPECT_THAT(Shard->Sources->lookup("unittest:///C.h"), HadErrors());
583  }
584 }
585 
587  MockFS FS;
588  llvm::StringMap<std::string> Storage;
589  size_t CacheHits = 0;
590  MemoryShardStorage MSS(Storage, CacheHits);
591  OverlayCDB CDB(/*Base=*/nullptr);
592  BackgroundIndex Idx(Context::empty(), FS, CDB,
593  [&](llvm::StringRef) { return &MSS; });
594 
595  tooling::CompileCommand Cmd;
596  FS.Files[testPath("A.cc")] = "#include \"A.h\"";
597  FS.Files[testPath("A.h")] = "";
598  Cmd.Filename = "../A.cc";
599  Cmd.Directory = testPath("build");
600  Cmd.CommandLine = {"clang++", "../A.cc", "-fsyntax-only"};
601  CDB.setCompileCommand(testPath("build/../A.cc"), Cmd);
602  ASSERT_TRUE(Idx.blockUntilIdleForTest());
603 
604  EXPECT_THAT(Storage.keys(), ElementsAre(testPath("A.cc"), testPath("A.h")));
605  // Make sure we only store the Cmd for main file.
606  EXPECT_FALSE(MSS.loadShard(testPath("A.h"))->Cmd);
607 
608  {
609  tooling::CompileCommand CmdStored = *MSS.loadShard(testPath("A.cc"))->Cmd;
610  EXPECT_EQ(CmdStored.CommandLine, Cmd.CommandLine);
611  EXPECT_EQ(CmdStored.Directory, Cmd.Directory);
612  }
613 
614  // FIXME: Changing compile commands should be enough to invalidate the cache.
615  FS.Files[testPath("A.cc")] = " ";
616  Cmd.CommandLine = {"clang++", "../A.cc", "-Dfoo", "-fsyntax-only"};
617  CDB.setCompileCommand(testPath("build/../A.cc"), Cmd);
618  ASSERT_TRUE(Idx.blockUntilIdleForTest());
619 
620  EXPECT_FALSE(MSS.loadShard(testPath("A.h"))->Cmd);
621 
622  {
623  tooling::CompileCommand CmdStored = *MSS.loadShard(testPath("A.cc"))->Cmd;
624  EXPECT_EQ(CmdStored.CommandLine, Cmd.CommandLine);
625  EXPECT_EQ(CmdStored.Directory, Cmd.Directory);
626  }
627 }
628 
629 class BackgroundIndexRebuilderTest : public testing::Test {
630 protected:
632  : Target(std::make_unique<MemIndex>()),
633  Rebuilder(&Target, &Source, /*Threads=*/10) {
634  // Prepare FileSymbols with TestSymbol in it, for checkRebuild.
635  TestSymbol.ID = SymbolID("foo");
636  }
637 
638  // Perform Action and determine whether it rebuilt the index or not.
639  bool checkRebuild(std::function<void()> Action) {
640  // Update name so we can tell if the index updates.
641  VersionStorage.push_back("Sym" + std::to_string(++VersionCounter));
642  TestSymbol.Name = VersionStorage.back();
644  SB.insert(TestSymbol);
645  Source.update("", std::make_unique<SymbolSlab>(std::move(SB).build()),
646  nullptr, nullptr, false);
647  // Now maybe update the index.
648  Action();
649  // Now query the index to get the name count.
650  std::string ReadName;
651  LookupRequest Req;
652  Req.IDs.insert(TestSymbol.ID);
653  Target.lookup(Req,
654  [&](const Symbol &S) { ReadName = std::string(S.Name); });
655  // The index was rebuild if the name is up to date.
656  return ReadName == VersionStorage.back();
657  }
658 
663 
664  unsigned VersionCounter = 0;
665  std::deque<std::string> VersionStorage;
666 };
667 
669  for (unsigned I = 0; I < Rebuilder.TUsBeforeFirstBuild - 1; ++I)
670  EXPECT_FALSE(checkRebuild([&] { Rebuilder.indexedTU(); }));
671  EXPECT_TRUE(checkRebuild([&] { Rebuilder.indexedTU(); }));
672  for (unsigned I = 0; I < Rebuilder.TUsBeforeRebuild - 1; ++I)
673  EXPECT_FALSE(checkRebuild([&] { Rebuilder.indexedTU(); }));
674  EXPECT_TRUE(checkRebuild([&] { Rebuilder.indexedTU(); }));
675 }
676 
678  Rebuilder.startLoading();
679  Rebuilder.loadedShard(10);
680  Rebuilder.loadedShard(20);
681  EXPECT_TRUE(checkRebuild([&] { Rebuilder.doneLoading(); }));
682 
683  // No rebuild for no shards.
684  Rebuilder.startLoading();
685  EXPECT_FALSE(checkRebuild([&] { Rebuilder.doneLoading(); }));
686 
687  // Loads can overlap.
688  Rebuilder.startLoading();
689  Rebuilder.loadedShard(1);
690  Rebuilder.startLoading();
691  Rebuilder.loadedShard(1);
692  EXPECT_FALSE(checkRebuild([&] { Rebuilder.doneLoading(); }));
693  Rebuilder.loadedShard(1);
694  EXPECT_TRUE(checkRebuild([&] { Rebuilder.doneLoading(); }));
695 
696  // No rebuilding for indexed files while loading.
697  Rebuilder.startLoading();
698  for (unsigned I = 0; I < 3 * Rebuilder.TUsBeforeRebuild; ++I)
699  EXPECT_FALSE(checkRebuild([&] { Rebuilder.indexedTU(); }));
700  // But they get indexed when we're done, even if no shards were loaded.
701  EXPECT_TRUE(checkRebuild([&] { Rebuilder.doneLoading(); }));
702 }
703 
704 TEST(BackgroundQueueTest, Priority) {
705  // Create high and low priority tasks.
706  // Once a bunch of high priority tasks have run, the queue is stopped.
707  // So the low priority tasks should never run.
708  BackgroundQueue Q;
709  std::atomic<unsigned> HiRan(0), LoRan(0);
710  BackgroundQueue::Task Lo([&] { ++LoRan; });
711  BackgroundQueue::Task Hi([&] {
712  if (++HiRan >= 10)
713  Q.stop();
714  });
715  Hi.QueuePri = 100;
716 
717  // Enqueuing the low-priority ones first shouldn't make them run first.
718  Q.append(std::vector<BackgroundQueue::Task>(30, Lo));
719  for (unsigned I = 0; I < 30; ++I)
720  Q.push(Hi);
721 
722  AsyncTaskRunner ThreadPool;
723  for (unsigned I = 0; I < 5; ++I)
724  ThreadPool.runAsync("worker", [&] { Q.work(); });
725  // We should test enqueue with active workers, but it's hard to avoid races.
726  // Just make sure we don't crash.
727  Q.push(Lo);
728  Q.append(std::vector<BackgroundQueue::Task>(2, Hi));
729 
730  // After finishing, check the tasks that ran.
731  ThreadPool.wait();
732  EXPECT_GE(HiRan, 10u);
733  EXPECT_EQ(LoRan, 0u);
734 }
735 
736 TEST(BackgroundQueueTest, Boost) {
737  std::string Sequence;
738 
739  BackgroundQueue::Task A([&] { Sequence.push_back('A'); });
740  A.Tag = "A";
741  A.QueuePri = 1;
742 
743  BackgroundQueue::Task B([&] { Sequence.push_back('B'); });
744  B.QueuePri = 2;
745  B.Tag = "B";
746 
747  {
748  BackgroundQueue Q;
749  Q.append({A, B});
750  Q.work([&] { Q.stop(); });
751  EXPECT_EQ("BA", Sequence) << "priority order";
752  }
753  Sequence.clear();
754  {
755  BackgroundQueue Q;
756  Q.boost("A", 3);
757  Q.append({A, B});
758  Q.work([&] { Q.stop(); });
759  EXPECT_EQ("AB", Sequence) << "A was boosted before enqueueing";
760  }
761  Sequence.clear();
762  {
763  BackgroundQueue Q;
764  Q.append({A, B});
765  Q.boost("A", 3);
766  Q.work([&] { Q.stop(); });
767  EXPECT_EQ("AB", Sequence) << "A was boosted after enqueueing";
768  }
769 }
770 
771 TEST(BackgroundQueueTest, Progress) {
772  using testing::AnyOf;
775  // Verify values are sane.
776  // Items are enqueued one at a time (at least in this test).
777  EXPECT_THAT(New.Enqueued, AnyOf(S.Enqueued, S.Enqueued + 1));
778  // Items are completed one at a time.
779  EXPECT_THAT(New.Completed, AnyOf(S.Completed, S.Completed + 1));
780  // Items are started or completed one at a time.
781  EXPECT_THAT(New.Active, AnyOf(S.Active - 1, S.Active, S.Active + 1));
782  // Idle point only advances in time.
783  EXPECT_GE(New.LastIdle, S.LastIdle);
784  // Idle point is a task that has been completed in the past.
785  EXPECT_LE(New.LastIdle, New.Completed);
786  // LastIdle is now only if we're really idle.
787  EXPECT_EQ(New.LastIdle == New.Enqueued,
788  New.Completed == New.Enqueued && New.Active == 0u);
789  S = New;
790  });
791 
792  // Two types of tasks: a ping task enqueues a pong task.
793  // This avoids all enqueues followed by all completions (boring!)
794  std::atomic<int> PingCount(0), PongCount(0);
795  BackgroundQueue::Task Pong([&] { ++PongCount; });
796  BackgroundQueue::Task Ping([&] {
797  ++PingCount;
798  Q.push(Pong);
799  });
800 
801  for (int I = 0; I < 1000; ++I)
802  Q.push(Ping);
803  // Spin up some workers and stop while idle.
804  AsyncTaskRunner ThreadPool;
805  for (unsigned I = 0; I < 5; ++I)
806  ThreadPool.runAsync("worker", [&] { Q.work([&] { Q.stop(); }); });
807  ThreadPool.wait();
808 
809  // Everything's done, check final stats.
810  // Assertions above ensure we got from 0 to 2000 in a reasonable way.
811  EXPECT_EQ(PingCount.load(), 1000);
812  EXPECT_EQ(PongCount.load(), 1000);
813  EXPECT_EQ(S.Active, 0u);
814  EXPECT_EQ(S.Enqueued, 2000u);
815  EXPECT_EQ(S.Completed, 2000u);
816  EXPECT_EQ(S.LastIdle, 2000u);
817 }
818 
819 } // namespace clangd
820 } // namespace clang
clang::clangd::BackgroundIndexRebuilderTest::VersionStorage
std::deque< std::string > VersionStorage
Definition: BackgroundIndexTests.cpp:665
clang::clangd::BackgroundQueue
Definition: Background.h:67
clang::clangd::BackgroundQueue::Task
A work item on the thread pool's queue.
Definition: Background.h:70
clang::clangd::BackgroundQueue::Stats
Definition: Background.h:82
clang::clangd::TEST
TEST(BackgroundQueueTest, Priority)
Definition: BackgroundIndexTests.cpp:704
Headers.h
Background.h
clang::clangd::runFuzzyFind
SymbolSlab runFuzzyFind(const SymbolIndex &Index, llvm::StringRef Query)
Definition: SyncAPI.cpp:121
clang::clangd::RelationKind::BaseOf
clang::clangd::Config::BackgroundPolicy::Skip
clang::clangd::AsyncTaskRunner::wait
void wait() const
Definition: Threading.h:110
clang::clangd::testPath
std::string testPath(PathRef File, llvm::sys::path::Style Style)
Definition: TestFS.cpp:82
clang::clangd::Symbol::ID
SymbolID ID
The ID of the symbol.
Definition: Symbol.h:38
clang::clangd::BackgroundQueue::Task::Tag
std::string Tag
Definition: Background.h:76
clang::clangd::Config::Edits
std::vector< llvm::unique_function< void(std::vector< std::string > &) const > > Edits
Definition: Config.h:56
clang::clangd::Context::current
static const Context & current()
Returns the context for the current thread, creating it if needed.
Definition: Context.cpp:27
clang::clangd::BackgroundQueue::append
void append(std::vector< Task >)
Definition: BackgroundQueue.cpp:87
clang::clangd::BackgroundQueue::push
void push(Task)
Definition: BackgroundQueue.cpp:75
clang::clangd::SwapIndex
Definition: Index.h:128
TestTU.h
clang::clangd::MemoryShardStorage::AccessedPaths
llvm::StringSet AccessedPaths
Definition: BackgroundIndexTests.cpp:84
Filename
std::string Filename
Filename as a string.
Definition: IncludeOrderCheck.cpp:39
clang::clangd::Config::CompileFlags
struct clang::clangd::Config::@0 CompileFlags
Controls how the compile command for the current file is determined.
clang::clangd::BackgroundQueue::work
void work(std::function< void()> OnIdle=nullptr)
Definition: BackgroundQueue.cpp:21
clang::clangd::Config
Settings that express user/project preferences and control clangd behavior.
Definition: Config.h:40
clang::clangd::Relation
Represents a relation between two symbols.
Definition: Relation.h:29
clang::clangd::MemoryShardStorage::loadShard
std::unique_ptr< IndexFileIn > loadShard(llvm::StringRef ShardIdentifier) const override
Definition: BackgroundIndexTests.cpp:68
clang::clangd::TEST_F
TEST_F(BackgroundIndexTest, NoCrashOnErrorFile)
Definition: BackgroundIndexTests.cpp:92
clang::clangd::readIndexFile
llvm::Expected< IndexFileIn > readIndexFile(llvm::StringRef Data)
Definition: Serialization.cpp:657
BackgroundRebuild.h
clang::clangd::FileSymbols
A container of slabs associated with a key.
Definition: FileIndex.h:71
clang::clangd::BackgroundIndexRebuilderTest::Rebuilder
BackgroundIndexRebuilder Rebuilder
Definition: BackgroundIndexTests.cpp:662
clang::clangd::BackgroundIndexRebuilderTest::Target
SwapIndex Target
Definition: BackgroundIndexTests.cpp:661
Action
llvm::unique_function< void()> Action
Definition: TUScheduler.cpp:447
clang::clangd::IndexFileOut
Definition: Serialization.h:55
clang::clangd::IncludeGraphNode::SourceFlag::None
clang::clangd::RefsAre
::testing::Matcher< const RefSlab & > RefsAre(std::vector<::testing::Matcher< Ref >> Matchers)
Definition: BackgroundIndexTests.cpp:37
clang::clangd::BackgroundQueue::Stats::LastIdle
unsigned LastIdle
Definition: Background.h:86
clang::clangd::BackgroundQueue::stop
void stop()
Definition: BackgroundQueue.cpp:67
clang::clangd::getRefs
RefSlab getRefs(const SymbolIndex &Index, SymbolID ID)
Definition: SyncAPI.cpp:134
clang::clangd::BackgroundIndexRebuilderTest
Definition: BackgroundIndexTests.cpp:629
clang::clangd::testRoot
const char * testRoot()
Definition: TestFS.cpp:74
clang::clangd::MemoryShardStorage
Definition: BackgroundIndexTests.cpp:52
clang::clangd::BackgroundQueue::Stats::Completed
unsigned Completed
Definition: Background.h:85
clang::clangd::BackgroundQueue::boost
void boost(llvm::StringRef Tag, unsigned NewPriority)
Definition: BackgroundQueue.cpp:100
clang::clangd::MockFS
Definition: TestFS.h:34
CompileCommands.h
FS
MockFS FS
Definition: ClangdLSPServerTests.cpp:66
TestFS.h
clang::clangd::Symbol
The class presents a C++ symbol, e.g.
Definition: Symbol.h:36
clang::clangd::BackgroundQueue::preventThreadStarvationInTests
static void preventThreadStarvationInTests()
Definition: BackgroundQueue.cpp:17
Name
static constexpr llvm::StringLiteral Name
Definition: UppercaseLiteralSuffixCheck.cpp:27
SyncAPI.h
clang::clangd::FileSymbols::update
void update(llvm::StringRef Key, std::unique_ptr< SymbolSlab > Symbols, std::unique_ptr< RefSlab > Refs, std::unique_ptr< RelationSlab > Relations, bool CountReferences)
Updates all slabs associated with the Key.
Definition: FileIndex.cpp:226
clang::clangd::Symbol::Name
llvm::StringRef Name
The unqualified name of the symbol, e.g. "bar" (for ns::bar).
Definition: Symbol.h:42
clang::doc::SymbolID
std::array< uint8_t, 20 > SymbolID
Definition: Representation.h:30
clang::clangd::MemIndex
MemIndex is a naive in-memory index suitable for a small set of symbols.
Definition: MemIndex.h:19
clang::clangd::BackgroundIndexRebuilderTest::VersionCounter
unsigned VersionCounter
Definition: BackgroundIndexTests.cpp:664
clang::clangd::BackgroundIndexTest::BackgroundIndexTest
BackgroundIndexTest()
Definition: BackgroundIndexTests.cpp:89
clang::clangd::BackgroundQueue::Stats::Enqueued
unsigned Enqueued
Definition: Background.h:83
clang::clangd::Config::Background
BackgroundPolicy Background
Whether this TU should be indexed.
Definition: Config.h:63
clang::clangd::LookupRequest::IDs
llvm::DenseSet< SymbolID > IDs
Definition: Index.h:64
clang::clangd::MemoryShardStorage::MemoryShardStorage
MemoryShardStorage(llvm::StringMap< std::string > &Storage, size_t &CacheHits)
Definition: BackgroundIndexTests.cpp:58
Config.h
clang::clangd::LookupRequest
Definition: Index.h:63
clang::clangd::BackgroundIndexRebuilder
Definition: BackgroundRebuild.h:48
clang::clangd::Context::derive
Context derive(const Key< Type > &Key, typename std::decay< Type >::type Value) const &
Derives a child context It is safe to move or destroy a parent context after calling derive().
Definition: Context.h:121
clang::clangd::IncludeGraphNode::SourceFlag::HadErrors
TestIndex.h
clang::clangd::Ref
Represents a symbol occurrence in the source file.
Definition: Ref.h:87
clang::clangd::PathRef
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:23
clang::clangd::BackgroundIndexRebuilderTest::BackgroundIndexRebuilderTest
BackgroundIndexRebuilderTest()
Definition: BackgroundIndexTests.cpp:631
clang::clangd::MATCHER_P
MATCHER_P(Named, N, "")
Definition: BackgroundIndexTests.cpp:29
clang::clangd::AsyncTaskRunner::runAsync
void runAsync(const llvm::Twine &Name, llvm::unique_function< void()> Action)
Definition: Threading.cpp:72
clang::clangd::BackgroundQueue::Stats::Active
unsigned Active
Definition: Background.h:84
clang::clangd::FileDigest
std::array< uint8_t, 8 > FileDigest
Definition: SourceCode.h:40
clang::clangd::Config::Index
struct clang::clangd::Config::@1 Index
Controls background-index behavior.
clang::clangd::BackgroundIndexStorage
Definition: Background.h:39
clang::clangd::Context::empty
static Context empty()
Returns an empty root context that contains no data.
Definition: Context.cpp:15
clang
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
Definition: ApplyReplacements.h:27
clang::clangd::AsyncTaskRunner
Runs tasks on separate (detached) threads and wait for all tasks to finish.
Definition: Threading.h:105
clang::clangd::BackgroundQueue::Task::QueuePri
unsigned QueuePri
Definition: Background.h:75
clang::clangd::MockFS::Files
llvm::StringMap< std::string > Files
Definition: TestFS.h:41
clang::clangd::Config::Key
static clangd::Key< Config > Key
Context key which can be used to set the current Config.
Definition: Config.h:44
clang::clangd::BackgroundIndexRebuilderTest::TestSymbol
Symbol TestSymbol
Definition: BackgroundIndexTests.cpp:659
clang::clangd::SymbolSlab::Builder
SymbolSlab::Builder is a mutable container that can 'freeze' to SymbolSlab.
Definition: Symbol.h:200
clang::clangd::CommandMangler::forTests
static CommandMangler forTests()
Definition: CompileCommands.cpp:185
clang::clangd::BackgroundIndex
Definition: Background.h:129
clang::clangd::BackgroundIndexRebuilderTest::checkRebuild
bool checkRebuild(std::function< void()> Action)
Definition: BackgroundIndexTests.cpp:639
clang::clangd::SymbolID
Definition: SymbolID.h:31
clang::clangd::MessageType::Error
An error message.
clang::clangd::SwapIndex::lookup
void lookup(const LookupRequest &, llvm::function_ref< void(const Symbol &)>) const override
Looks up symbols with any of the given symbol IDs and applies Callback on each matched symbol.
Definition: Index.cpp:64
clang::clangd::findSymbol
const Symbol & findSymbol(const SymbolSlab &Slab, llvm::StringRef QName)
Definition: TestTU.cpp:144
clang::clangd::SymbolSlab::Builder::insert
void insert(const Symbol &S)
Adds a symbol, overwriting any existing one with the same ID.
Definition: Symbol.cpp:50
clang::clangd::MATCHER
MATCHER(Declared, "")
Definition: BackgroundIndexTests.cpp:31
clang::clangd::MemoryShardStorage::storeShard
llvm::Error storeShard(llvm::StringRef ShardIdentifier, IndexFileOut Shard) const override
Definition: BackgroundIndexTests.cpp:60
clang::clangd::BackgroundIndexRebuilderTest::Source
FileSymbols Source
Definition: BackgroundIndexTests.cpp:660
clang::clangd::BackgroundIndexTest
Definition: BackgroundIndexTests.cpp:87
clang::clangd::OverlayCDB
Wraps another compilation database, and supports overriding the commands using an in-memory mapping.
Definition: GlobalCompilationDatabase.h:124