clang-tools  10.0.0
SemanticHighlightingTests.cpp
Go to the documentation of this file.
1 //==- SemanticHighlightingTests.cpp - SemanticHighlighting tests-*- C++ -* -==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "Annotations.h"
10 #include "ClangdServer.h"
11 #include "Protocol.h"
12 #include "SemanticHighlighting.h"
13 #include "SourceCode.h"
14 #include "TestFS.h"
15 #include "TestTU.h"
16 #include "llvm/ADT/ArrayRef.h"
17 #include "llvm/ADT/StringRef.h"
18 #include "llvm/Support/Error.h"
19 #include "gmock/gmock.h"
20 #include <algorithm>
21 
22 namespace clang {
23 namespace clangd {
24 namespace {
25 
26 MATCHER_P(LineNumber, L, "") { return arg.Line == L; }
27 MATCHER(EmptyHighlightings, "") { return arg.Tokens.empty(); }
28 
29 std::vector<HighlightingToken>
30 makeHighlightingTokens(llvm::ArrayRef<Range> Ranges, HighlightingKind Kind) {
31  std::vector<HighlightingToken> Tokens(Ranges.size());
32  for (int I = 0, End = Ranges.size(); I < End; ++I) {
33  Tokens[I].R = Ranges[I];
34  Tokens[I].Kind = Kind;
35  }
36 
37  return Tokens;
38 }
39 
40 std::vector<HighlightingToken> getExpectedTokens(Annotations &Test) {
41  static const std::map<HighlightingKind, std::string> KindToString{
42  {HighlightingKind::Variable, "Variable"},
43  {HighlightingKind::LocalVariable, "LocalVariable"},
44  {HighlightingKind::Parameter, "Parameter"},
45  {HighlightingKind::Function, "Function"},
46  {HighlightingKind::Class, "Class"},
47  {HighlightingKind::Enum, "Enum"},
48  {HighlightingKind::Namespace, "Namespace"},
49  {HighlightingKind::EnumConstant, "EnumConstant"},
50  {HighlightingKind::Field, "Field"},
51  {HighlightingKind::StaticField, "StaticField"},
52  {HighlightingKind::Method, "Method"},
53  {HighlightingKind::StaticMethod, "StaticMethod"},
54  {HighlightingKind::Typedef, "Typedef"},
55  {HighlightingKind::DependentType, "DependentType"},
56  {HighlightingKind::DependentName, "DependentName"},
57  {HighlightingKind::TemplateParameter, "TemplateParameter"},
58  {HighlightingKind::Primitive, "Primitive"},
59  {HighlightingKind::Macro, "Macro"}};
60  std::vector<HighlightingToken> ExpectedTokens;
61  for (const auto &KindString : KindToString) {
62  std::vector<HighlightingToken> Toks = makeHighlightingTokens(
63  Test.ranges(KindString.second), KindString.first);
64  ExpectedTokens.insert(ExpectedTokens.end(), Toks.begin(), Toks.end());
65  }
66  llvm::sort(ExpectedTokens);
67  return ExpectedTokens;
68 }
69 
70 /// Annotates the input code with provided semantic highlightings. Results look
71 /// something like:
72 /// class $Class[[X]] {
73 /// $Primitive[[int]] $Field[[a]] = 0;
74 /// };
75 std::string annotate(llvm::StringRef Input,
76  llvm::ArrayRef<HighlightingToken> Tokens) {
77  assert(std::is_sorted(
78  Tokens.begin(), Tokens.end(),
79  [](const HighlightingToken &L, const HighlightingToken &R) {
80  return L.R.start < R.R.start;
81  }));
82 
83  std::string Result;
84  unsigned NextChar = 0;
85  for (auto &T : Tokens) {
86  unsigned StartOffset = llvm::cantFail(positionToOffset(Input, T.R.start));
87  unsigned EndOffset = llvm::cantFail(positionToOffset(Input, T.R.end));
88  assert(StartOffset <= EndOffset);
89  assert(NextChar <= StartOffset);
90 
91  Result += Input.substr(NextChar, StartOffset - NextChar);
92  Result += llvm::formatv("${0}[[{1}]]", T.Kind,
93  Input.substr(StartOffset, EndOffset - StartOffset));
94  NextChar = EndOffset;
95  }
96  Result += Input.substr(NextChar);
97  return Result;
98 }
99 
100 void checkHighlightings(llvm::StringRef Code,
101  std::vector<std::pair</*FileName*/ llvm::StringRef,
102  /*FileContent*/ llvm::StringRef>>
103  AdditionalFiles = {}) {
104  Annotations Test(Code);
105  TestTU TU;
106  TU.Code = Test.code();
107 
108  // FIXME: Auto-completion in a template requires disabling delayed template
109  // parsing.
110  TU.ExtraArgs.push_back("-fno-delayed-template-parsing");
111 
112  for (auto File : AdditionalFiles)
113  TU.AdditionalFiles.insert({File.first, File.second});
114  auto AST = TU.build();
115 
116  EXPECT_EQ(Code, annotate(Test.code(), getSemanticHighlightings(AST)));
117 }
118 
119 // Any annotations in OldCode and NewCode are converted into their corresponding
120 // HighlightingToken. The tokens are diffed against each other. Any lines where
121 // the tokens should diff must be marked with a ^ somewhere on that line in
122 // NewCode. If there are diffs that aren't marked with ^ the test fails. The
123 // test also fails if there are lines marked with ^ that don't differ.
124 void checkDiffedHighlights(llvm::StringRef OldCode, llvm::StringRef NewCode) {
125  Annotations OldTest(OldCode);
126  Annotations NewTest(NewCode);
127  std::vector<HighlightingToken> OldTokens = getExpectedTokens(OldTest);
128  std::vector<HighlightingToken> NewTokens = getExpectedTokens(NewTest);
129 
130  llvm::DenseMap<int, std::vector<HighlightingToken>> ExpectedLines;
131  for (const Position &Point : NewTest.points()) {
132  ExpectedLines[Point.line]; // Default initialize to an empty line. Tokens
133  // are inserted on these lines later.
134  }
135  std::vector<LineHighlightings> ExpectedLinePairHighlighting;
136  for (const HighlightingToken &Token : NewTokens) {
137  auto It = ExpectedLines.find(Token.R.start.line);
138  if (It != ExpectedLines.end())
139  It->second.push_back(Token);
140  }
141  for (auto &LineTokens : ExpectedLines)
142  ExpectedLinePairHighlighting.push_back(
143  {LineTokens.first, LineTokens.second, /*IsInactive = */ false});
144 
145  std::vector<LineHighlightings> ActualDiffed =
146  diffHighlightings(NewTokens, OldTokens);
147  EXPECT_THAT(ActualDiffed,
148  testing::UnorderedElementsAreArray(ExpectedLinePairHighlighting))
149  << OldCode;
150 }
151 
152 TEST(SemanticHighlighting, GetsCorrectTokens) {
153  const char *TestCases[] = {
154  R"cpp(
155  struct $Class[[AS]] {
156  double $Field[[SomeMember]];
157  };
158  struct {
159  } $Variable[[S]];
160  void $Function[[foo]](int $Parameter[[A]], $Class[[AS]] $Parameter[[As]]) {
161  $Primitive[[auto]] $LocalVariable[[VeryLongVariableName]] = 12312;
162  $Class[[AS]] $LocalVariable[[AA]];
163  $Primitive[[auto]] $LocalVariable[[L]] = $LocalVariable[[AA]].$Field[[SomeMember]] + $Parameter[[A]];
164  auto $LocalVariable[[FN]] = [ $LocalVariable[[AA]]](int $Parameter[[A]]) -> void {};
165  $LocalVariable[[FN]](12312);
166  }
167  )cpp",
168  R"cpp(
169  void $Function[[foo]](int);
170  void $Function[[Gah]]();
171  void $Function[[foo]]() {
172  auto $LocalVariable[[Bou]] = $Function[[Gah]];
173  }
174  struct $Class[[A]] {
175  void $Method[[abc]]();
176  };
177  )cpp",
178  R"cpp(
179  namespace $Namespace[[abc]] {
180  template<typename $TemplateParameter[[T]]>
181  struct $Class[[A]] {
182  $TemplateParameter[[T]] $Field[[t]];
183  };
184  }
185  template<typename $TemplateParameter[[T]]>
186  struct $Class[[C]] : $Namespace[[abc]]::$Class[[A]]<$TemplateParameter[[T]]> {
187  typename $TemplateParameter[[T]]::$DependentType[[A]]* $Field[[D]];
188  };
189  $Namespace[[abc]]::$Class[[A]]<int> $Variable[[AA]];
190  typedef $Namespace[[abc]]::$Class[[A]]<int> $Class[[AAA]];
191  struct $Class[[B]] {
192  $Class[[B]]();
193  ~$Class[[B]]();
194  void operator<<($Class[[B]]);
195  $Class[[AAA]] $Field[[AA]];
196  };
197  $Class[[B]]::$Class[[B]]() {}
198  $Class[[B]]::~$Class[[B]]() {}
199  void $Function[[f]] () {
200  $Class[[B]] $LocalVariable[[BB]] = $Class[[B]]();
201  $LocalVariable[[BB]].~$Class[[B]]();
202  $Class[[B]]();
203  }
204  )cpp",
205  R"cpp(
206  enum class $Enum[[E]] {
207  $EnumConstant[[A]],
208  $EnumConstant[[B]],
209  };
210  enum $Enum[[EE]] {
211  $EnumConstant[[Hi]],
212  };
213  struct $Class[[A]] {
214  $Enum[[E]] $Field[[EEE]];
215  $Enum[[EE]] $Field[[EEEE]];
216  };
217  int $Variable[[I]] = $EnumConstant[[Hi]];
218  $Enum[[E]] $Variable[[L]] = $Enum[[E]]::$EnumConstant[[B]];
219  )cpp",
220  R"cpp(
221  namespace $Namespace[[abc]] {
222  namespace {}
223  namespace $Namespace[[bcd]] {
224  struct $Class[[A]] {};
225  namespace $Namespace[[cde]] {
226  struct $Class[[A]] {
227  enum class $Enum[[B]] {
228  $EnumConstant[[Hi]],
229  };
230  };
231  }
232  }
233  }
234  using namespace $Namespace[[abc]]::$Namespace[[bcd]];
235  namespace $Namespace[[vwz]] =
236  $Namespace[[abc]]::$Namespace[[bcd]]::$Namespace[[cde]];
237  $Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[AA]];
238  $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]] $Variable[[AAA]] =
239  $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]]::$EnumConstant[[Hi]];
240  ::$Namespace[[vwz]]::$Class[[A]] $Variable[[B]];
241  ::$Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[BB]];
242  )cpp",
243  R"cpp(
244  struct $Class[[D]] {
245  double $Field[[C]];
246  };
247  struct $Class[[A]] {
248  double $Field[[B]];
249  $Class[[D]] $Field[[E]];
250  static double $StaticField[[S]];
251  static void $StaticMethod[[bar]]() {}
252  void $Method[[foo]]() {
253  $Field[[B]] = 123;
254  this->$Field[[B]] = 156;
255  this->$Method[[foo]]();
256  $Method[[foo]]();
257  $StaticMethod[[bar]]();
258  $StaticField[[S]] = 90.1;
259  }
260  };
261  void $Function[[foo]]() {
262  $Class[[A]] $LocalVariable[[AA]];
263  $LocalVariable[[AA]].$Field[[B]] += 2;
264  $LocalVariable[[AA]].$Method[[foo]]();
265  $LocalVariable[[AA]].$Field[[E]].$Field[[C]];
266  $Class[[A]]::$StaticField[[S]] = 90;
267  }
268  )cpp",
269  R"cpp(
270  struct $Class[[AA]] {
271  int $Field[[A]];
272  }
273  int $Variable[[B]];
274  $Class[[AA]] $Variable[[A]]{$Variable[[B]]};
275  )cpp",
276  R"cpp(
277  namespace $Namespace[[a]] {
278  struct $Class[[A]] {};
279  typedef char $Primitive[[C]];
280  }
281  typedef $Namespace[[a]]::$Class[[A]] $Class[[B]];
282  using $Class[[BB]] = $Namespace[[a]]::$Class[[A]];
283  enum class $Enum[[E]] {};
284  typedef $Enum[[E]] $Enum[[C]];
285  typedef $Enum[[C]] $Enum[[CC]];
286  using $Enum[[CD]] = $Enum[[CC]];
287  $Enum[[CC]] $Function[[f]]($Class[[B]]);
288  $Enum[[CD]] $Function[[f]]($Class[[BB]]);
289  typedef $Namespace[[a]]::$Primitive[[C]] $Primitive[[PC]];
290  typedef float $Primitive[[F]];
291  )cpp",
292  R"cpp(
293  template<typename $TemplateParameter[[T]], typename = void>
294  class $Class[[A]] {
295  $TemplateParameter[[T]] $Field[[AA]];
296  $TemplateParameter[[T]] $Method[[foo]]();
297  };
298  template<class $TemplateParameter[[TT]]>
299  class $Class[[B]] {
300  $Class[[A]]<$TemplateParameter[[TT]]> $Field[[AA]];
301  };
302  template<class $TemplateParameter[[TT]], class $TemplateParameter[[GG]]>
303  class $Class[[BB]] {};
304  template<class $TemplateParameter[[T]]>
305  class $Class[[BB]]<$TemplateParameter[[T]], int> {};
306  template<class $TemplateParameter[[T]]>
307  class $Class[[BB]]<$TemplateParameter[[T]], $TemplateParameter[[T]]*> {};
308 
309  template<template<class> class $TemplateParameter[[T]], class $TemplateParameter[[C]]>
310  $TemplateParameter[[T]]<$TemplateParameter[[C]]> $Function[[f]]();
311 
312  template<typename>
313  class $Class[[Foo]] {};
314 
315  template<typename $TemplateParameter[[T]]>
316  void $Function[[foo]]($TemplateParameter[[T]] ...);
317  )cpp",
318  R"cpp(
319  template <class $TemplateParameter[[T]]>
320  struct $Class[[Tmpl]] {$TemplateParameter[[T]] $Field[[x]] = 0;};
321  extern template struct $Class[[Tmpl]]<float>;
322  template struct $Class[[Tmpl]]<double>;
323  )cpp",
324  // This test is to guard against highlightings disappearing when using
325  // conversion operators as their behaviour in the clang AST differ from
326  // other CXXMethodDecls.
327  R"cpp(
328  class $Class[[Foo]] {};
329  struct $Class[[Bar]] {
330  explicit operator $Class[[Foo]]*() const;
331  explicit operator int() const;
332  operator $Class[[Foo]]();
333  };
334  void $Function[[f]]() {
335  $Class[[Bar]] $LocalVariable[[B]];
336  $Class[[Foo]] $LocalVariable[[F]] = $LocalVariable[[B]];
337  $Class[[Foo]] *$LocalVariable[[FP]] = ($Class[[Foo]]*)$LocalVariable[[B]];
338  int $LocalVariable[[I]] = (int)$LocalVariable[[B]];
339  }
340  )cpp"
341  R"cpp(
342  struct $Class[[B]] {};
343  struct $Class[[A]] {
344  $Class[[B]] $Field[[BB]];
345  $Class[[A]] &operator=($Class[[A]] &&$Parameter[[O]]);
346  };
347 
348  $Class[[A]] &$Class[[A]]::operator=($Class[[A]] &&$Parameter[[O]]) = default;
349  )cpp",
350  R"cpp(
351  enum $Enum[[En]] {
352  $EnumConstant[[EC]],
353  };
354  class $Class[[Foo]] {};
355  class $Class[[Bar]] {
356  $Class[[Foo]] $Field[[Fo]];
357  $Enum[[En]] $Field[[E]];
358  int $Field[[I]];
359  $Class[[Bar]] ($Class[[Foo]] $Parameter[[F]],
360  $Enum[[En]] $Parameter[[E]])
361  : $Field[[Fo]] ($Parameter[[F]]), $Field[[E]] ($Parameter[[E]]),
362  $Field[[I]] (123) {}
363  };
364  class $Class[[Bar2]] : public $Class[[Bar]] {
365  $Class[[Bar2]]() : $Class[[Bar]]($Class[[Foo]](), $EnumConstant[[EC]]) {}
366  };
367  )cpp",
368  R"cpp(
369  enum $Enum[[E]] {
370  $EnumConstant[[E]],
371  };
372  class $Class[[Foo]] {};
373  $Enum[[auto]] $Variable[[AE]] = $Enum[[E]]::$EnumConstant[[E]];
374  $Class[[auto]] $Variable[[AF]] = $Class[[Foo]]();
375  $Class[[decltype]](auto) $Variable[[AF2]] = $Class[[Foo]]();
376  $Class[[auto]] *$Variable[[AFP]] = &$Variable[[AF]];
377  $Enum[[auto]] &$Variable[[AER]] = $Variable[[AE]];
378  $Primitive[[auto]] $Variable[[Form]] = 10.2 + 2 * 4;
379  $Primitive[[decltype]]($Variable[[Form]]) $Variable[[F]] = 10;
380  auto $Variable[[Fun]] = []()->void{};
381  )cpp",
382  R"cpp(
383  class $Class[[G]] {};
384  template<$Class[[G]] *$TemplateParameter[[U]]>
385  class $Class[[GP]] {};
386  template<$Class[[G]] &$TemplateParameter[[U]]>
387  class $Class[[GR]] {};
388  template<int *$TemplateParameter[[U]]>
389  class $Class[[IP]] {
390  void $Method[[f]]() {
391  *$TemplateParameter[[U]] += 5;
392  }
393  };
394  template<unsigned $TemplateParameter[[U]] = 2>
395  class $Class[[Foo]] {
396  void $Method[[f]]() {
397  for(int $LocalVariable[[I]] = 0;
398  $LocalVariable[[I]] < $TemplateParameter[[U]];) {}
399  }
400  };
401 
402  $Class[[G]] $Variable[[L]];
403  void $Function[[f]]() {
404  $Class[[Foo]]<123> $LocalVariable[[F]];
405  $Class[[GP]]<&$Variable[[L]]> $LocalVariable[[LL]];
406  $Class[[GR]]<$Variable[[L]]> $LocalVariable[[LLL]];
407  }
408  )cpp",
409  R"cpp(
410  template<typename $TemplateParameter[[T]],
411  void ($TemplateParameter[[T]]::*$TemplateParameter[[method]])(int)>
412  struct $Class[[G]] {
413  void $Method[[foo]](
414  $TemplateParameter[[T]] *$Parameter[[O]]) {
415  ($Parameter[[O]]->*$TemplateParameter[[method]])(10);
416  }
417  };
418  struct $Class[[F]] {
419  void $Method[[f]](int);
420  };
421  template<void (*$TemplateParameter[[Func]])()>
422  struct $Class[[A]] {
423  void $Method[[f]]() {
424  (*$TemplateParameter[[Func]])();
425  }
426  };
427 
428  void $Function[[foo]]() {
429  $Class[[F]] $LocalVariable[[FF]];
430  $Class[[G]]<$Class[[F]], &$Class[[F]]::$Method[[f]]> $LocalVariable[[GG]];
431  $LocalVariable[[GG]].$Method[[foo]](&$LocalVariable[[FF]]);
432  $Class[[A]]<$Function[[foo]]> $LocalVariable[[AA]];
433  )cpp",
434  // Tokens that share a source range but have conflicting Kinds are not
435  // highlighted.
436  R"cpp(
437  #define $Macro[[DEF_MULTIPLE]](X) namespace X { class X { int X; }; }
438  #define $Macro[[DEF_CLASS]](T) class T {};
439  // Preamble ends.
440  $Macro[[DEF_MULTIPLE]](XYZ);
441  $Macro[[DEF_MULTIPLE]](XYZW);
442  $Macro[[DEF_CLASS]]($Class[[A]])
443  #define $Macro[[MACRO_CONCAT]](X, V, T) T foo##X = V
444  #define $Macro[[DEF_VAR]](X, V) int X = V
445  #define $Macro[[DEF_VAR_T]](T, X, V) T X = V
446  #define $Macro[[DEF_VAR_REV]](V, X) DEF_VAR(X, V)
447  #define $Macro[[CPY]](X) X
448  #define $Macro[[DEF_VAR_TYPE]](X, Y) X Y
449  #define $Macro[[SOME_NAME]] variable
450  #define $Macro[[SOME_NAME_SET]] variable2 = 123
451  #define $Macro[[INC_VAR]](X) X += 2
452  void $Function[[foo]]() {
453  $Macro[[DEF_VAR]]($LocalVariable[[X]], 123);
454  $Macro[[DEF_VAR_REV]](908, $LocalVariable[[XY]]);
455  int $Macro[[CPY]]( $LocalVariable[[XX]] );
456  $Macro[[DEF_VAR_TYPE]]($Class[[A]], $LocalVariable[[AA]]);
457  double $Macro[[SOME_NAME]];
458  int $Macro[[SOME_NAME_SET]];
459  $LocalVariable[[variable]] = 20.1;
460  $Macro[[MACRO_CONCAT]](var, 2, float);
461  $Macro[[DEF_VAR_T]]($Class[[A]], $Macro[[CPY]](
462  $Macro[[CPY]]($LocalVariable[[Nested]])),
463  $Macro[[CPY]]($Class[[A]]()));
464  $Macro[[INC_VAR]]($LocalVariable[[variable]]);
465  }
466  void $Macro[[SOME_NAME]]();
467  $Macro[[DEF_VAR]]($Variable[[XYZ]], 567);
468  $Macro[[DEF_VAR_REV]](756, $Variable[[AB]]);
469 
470  #define $Macro[[CALL_FN]](F) F();
471  #define $Macro[[DEF_FN]](F) void F ()
472  $Macro[[DEF_FN]]($Function[[g]]) {
473  $Macro[[CALL_FN]]($Function[[foo]]);
474  }
475  )cpp",
476  R"cpp(
477  #define $Macro[[fail]](expr) expr
478  #define $Macro[[assert]](COND) if (!(COND)) { fail("assertion failed" #COND); }
479  // Preamble ends.
480  int $Variable[[x]];
481  int $Variable[[y]];
482  int $Function[[f]]();
483  void $Function[[foo]]() {
484  $Macro[[assert]]($Variable[[x]] != $Variable[[y]]);
485  $Macro[[assert]]($Variable[[x]] != $Function[[f]]());
486  }
487  )cpp",
488  // highlighting all macro references
489  R"cpp(
490  #ifndef $Macro[[name]]
491  #define $Macro[[name]]
492  #endif
493 
494  #define $Macro[[test]]
495  #undef $Macro[[test]]
496 $InactiveCode[[]] #ifdef $Macro[[test]]
497 $InactiveCode[[]] #endif
498 
499 $InactiveCode[[]] #if defined($Macro[[test]])
500 $InactiveCode[[]] #endif
501  )cpp",
502  R"cpp(
503  struct $Class[[S]] {
504  float $Field[[Value]];
505  $Class[[S]] *$Field[[Next]];
506  };
507  $Class[[S]] $Variable[[Global]][2] = {$Class[[S]](), $Class[[S]]()};
508  void $Function[[f]]($Class[[S]] $Parameter[[P]]) {
509  int $LocalVariable[[A]][2] = {1,2};
510  auto [$Variable[[B1]], $Variable[[B2]]] = $LocalVariable[[A]];
511  auto [$Variable[[G1]], $Variable[[G2]]] = $Variable[[Global]];
512  $Class[[auto]] [$Variable[[P1]], $Variable[[P2]]] = $Parameter[[P]];
513  // Highlights references to BindingDecls.
514  $Variable[[B1]]++;
515  }
516  )cpp",
517  R"cpp(
518  template<class $TemplateParameter[[T]]>
519  class $Class[[A]] {
520  using $TemplateParameter[[TemplateParam1]] = $TemplateParameter[[T]];
521  typedef $TemplateParameter[[T]] $TemplateParameter[[TemplateParam2]];
522  using $Primitive[[IntType]] = int;
523 
524  using $Typedef[[Pointer]] = $TemplateParameter[[T]] *;
525  using $Typedef[[LVReference]] = $TemplateParameter[[T]] &;
526  using $Typedef[[RVReference]] = $TemplateParameter[[T]]&&;
527  using $Typedef[[Array]] = $TemplateParameter[[T]]*[3];
528  using $Typedef[[MemberPointer]] = int (A::*)(int);
529 
530  // Use various previously defined typedefs in a function type.
531  void $Method[[func]](
532  $Typedef[[Pointer]], $Typedef[[LVReference]], $Typedef[[RVReference]],
533  $Typedef[[Array]], $Typedef[[MemberPointer]]);
534  };
535  )cpp",
536  R"cpp(
537  template <class $TemplateParameter[[T]]>
538  void $Function[[phase1]]($TemplateParameter[[T]]);
539  template <class $TemplateParameter[[T]]>
540  void $Function[[foo]]($TemplateParameter[[T]] $Parameter[[P]]) {
541  $Function[[phase1]]($Parameter[[P]]);
542  $DependentName[[phase2]]($Parameter[[P]]);
543  }
544  )cpp",
545  R"cpp(
546  class $Class[[A]] {
547  template <class $TemplateParameter[[T]]>
548  void $Method[[bar]]($TemplateParameter[[T]]);
549  };
550 
551  template <class $TemplateParameter[[U]]>
552  void $Function[[foo]]($TemplateParameter[[U]] $Parameter[[P]]) {
553  $Class[[A]]().$Method[[bar]]($Parameter[[P]]);
554  }
555  )cpp",
556  R"cpp(
557  struct $Class[[A]] {
558  template <class $TemplateParameter[[T]]>
559  static void $StaticMethod[[foo]]($TemplateParameter[[T]]);
560  };
561 
562  template <class $TemplateParameter[[T]]>
563  struct $Class[[B]] {
564  void $Method[[bar]]() {
565  $Class[[A]]::$StaticMethod[[foo]]($TemplateParameter[[T]]());
566  }
567  };
568  )cpp",
569  R"cpp(
570  template <class $TemplateParameter[[T]]>
571  void $Function[[foo]](typename $TemplateParameter[[T]]::$DependentType[[Type]]
572  = $TemplateParameter[[T]]::$DependentName[[val]]);
573  )cpp",
574  R"cpp(
575  template <class $TemplateParameter[[T]]>
576  void $Function[[foo]]($TemplateParameter[[T]] $Parameter[[P]]) {
577  $Parameter[[P]].$DependentName[[Field]];
578  }
579  )cpp",
580  R"cpp(
581  template <class $TemplateParameter[[T]]>
582  class $Class[[A]] {
583  int $Method[[foo]]() {
584  return $TemplateParameter[[T]]::$DependentName[[Field]];
585  }
586  };
587  )cpp",
588  // Highlighting the using decl as the underlying using shadow decl.
589  R"cpp(
590  void $Function[[foo]]();
591  using ::$Function[[foo]];
592  )cpp",
593  // Highlighting of template template arguments.
594  R"cpp(
595  template <template <class> class $TemplateParameter[[TT]],
596  template <class> class ...$TemplateParameter[[TTs]]>
597  struct $Class[[Foo]] {
598  $Class[[Foo]]<$TemplateParameter[[TT]], $TemplateParameter[[TTs]]...>
599  *$Field[[t]];
600  }
601  )cpp",
602  // Inactive code highlighting
603  R"cpp(
604  // Code in the preamble.
605  // Inactive lines get an empty InactiveCode token at the beginning.
606 $InactiveCode[[]] #ifdef $Macro[[test]]
607 $InactiveCode[[]] #endif
608 
609  // A declaration to cause the preamble to end.
610  int $Variable[[EndPreamble]];
611 
612  // Code after the preamble.
613  // Code inside inactive blocks does not get regular highlightings
614  // because it's not part of the AST.
615 $InactiveCode[[]] #ifdef $Macro[[test]]
616 $InactiveCode[[]] int Inactive2;
617 $InactiveCode[[]] #endif
618 
619  #ifndef $Macro[[test]]
620  int $Variable[[Active1]];
621  #endif
622 
623 $InactiveCode[[]] #ifdef $Macro[[test]]
624 $InactiveCode[[]] int Inactive3;
625 $InactiveCode[[]] #else
626  int $Variable[[Active2]];
627  #endif
628  )cpp",
629  // Argument to 'sizeof...'
630  R"cpp(
631  template <typename... $TemplateParameter[[Elements]]>
632  struct $Class[[TupleSize]] {
633  static const int $StaticField[[size]] =
634 sizeof...($TemplateParameter[[Elements]]);
635  };
636  )cpp",
637  // More dependent types
638  R"cpp(
639  template <typename $TemplateParameter[[T]]>
640  struct $Class[[Waldo]] {
641  using $Typedef[[Location1]] = typename $TemplateParameter[[T]]
642  ::$DependentType[[Resolver]]::$DependentType[[Location]];
643  using $Typedef[[Location2]] = typename $TemplateParameter[[T]]
644  ::template $DependentType[[Resolver]]<$TemplateParameter[[T]]>
645  ::$DependentType[[Location]];
646  using $Typedef[[Location3]] = typename $TemplateParameter[[T]]
647  ::$DependentType[[Resolver]]
648  ::template $DependentType[[Location]]<$TemplateParameter[[T]]>;
649  static const int $StaticField[[Value]] = $TemplateParameter[[T]]
650  ::$DependentType[[Resolver]]::$DependentName[[Value]];
651  };
652  )cpp"};
653  for (const auto &TestCase : TestCases) {
654  checkHighlightings(TestCase);
655  }
656 
657  checkHighlightings(R"cpp(
658  class $Class[[A]] {
659  #include "imp.h"
660  };
661  #endif
662  )cpp",
663  {{"imp.h", R"cpp(
664  int someMethod();
665  void otherMethod();
666  )cpp"}});
667 
668  // A separate test for macros in headers.
669  checkHighlightings(R"cpp(
670  #include "imp.h"
671  $Macro[[DEFINE_Y]]
672  $Macro[[DXYZ_Y]](A);
673  )cpp",
674  {{"imp.h", R"cpp(
675  #define DXYZ(X) class X {};
676  #define DXYZ_Y(Y) DXYZ(x##Y)
677  #define DEFINE(X) int X;
678  #define DEFINE_Y DEFINE(Y)
679  )cpp"}});
680 }
681 
682 TEST(SemanticHighlighting, GeneratesHighlightsWhenFileChange) {
683  class HighlightingsCounterDiagConsumer : public DiagnosticsConsumer {
684  public:
685  std::atomic<int> Count = {0};
686 
687  void onDiagnosticsReady(PathRef, std::vector<Diag>) override {}
688  void onHighlightingsReady(
689  PathRef File, std::vector<HighlightingToken> Highlightings) override {
690  ++Count;
691  }
692  };
693 
694  auto FooCpp = testPath("foo.cpp");
695  MockFSProvider FS;
696  FS.Files[FooCpp] = "";
697 
698  MockCompilationDatabase MCD;
699  HighlightingsCounterDiagConsumer DiagConsumer;
700  ClangdServer Server(MCD, FS, DiagConsumer, ClangdServer::optsForTest());
701  Server.addDocument(FooCpp, "int a;");
702  ASSERT_TRUE(Server.blockUntilIdleForTest()) << "Waiting for server";
703  ASSERT_EQ(DiagConsumer.Count, 1);
704 }
705 
706 TEST(SemanticHighlighting, toSemanticHighlightingInformation) {
707  auto CreatePosition = [](int Line, int Character) -> Position {
708  Position Pos;
709  Pos.line = Line;
710  Pos.character = Character;
711  return Pos;
712  };
713 
714  std::vector<LineHighlightings> Tokens{
715  {3,
717  Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
719  Range{CreatePosition(3, 4), CreatePosition(3, 7)}}},
720  /* IsInactive = */ false},
721  {1,
723  Range{CreatePosition(1, 1), CreatePosition(1, 5)}}},
724  /* IsInactive = */ true}};
725  std::vector<SemanticHighlightingInformation> ActualResults =
727  std::vector<SemanticHighlightingInformation> ExpectedResults = {
728  {3, "AAAACAAEAAAAAAAEAAMAAw=="}, {1, "AAAAAQAEAAA="}};
729  EXPECT_EQ(ActualResults, ExpectedResults);
730 }
731 
732 TEST(SemanticHighlighting, HighlightingDiffer) {
733  struct {
734  llvm::StringRef OldCode;
735  llvm::StringRef NewCode;
736  } TestCases[]{{
737  R"(
738  $Variable[[A]]
739  $Class[[B]]
740  $Function[[C]]
741  )",
742  R"(
743  $Variable[[A]]
744  $Class[[D]]
745  $Function[[C]]
746  )"},
747  {
748  R"(
749  $Class[[C]]
750  $Field[[F]]
751  $Variable[[V]]
752  $Class[[C]] $Variable[[V]] $Field[[F]]
753  )",
754  R"(
755  $Class[[C]]
756  $Field[[F]]
757  ^$Function[[F]]
758  $Class[[C]] $Variable[[V]] $Field[[F]]
759  )"},
760  {
761  R"(
762 
763  $Class[[A]]
764  $Variable[[A]]
765  )",
766  R"(
767 
768  ^
769  ^$Class[[A]]
770  ^$Variable[[A]]
771  )"},
772  {
773  R"(
774  $Class[[C]]
775  $Field[[F]]
776  $Variable[[V]]
777  $Class[[C]] $Variable[[V]] $Field[[F]]
778  )",
779  R"(
780  $Class[[C]]
781  ^
782  ^
783  $Class[[C]] $Variable[[V]] $Field[[F]]
784  )"},
785  {
786  R"(
787  $Class[[A]]
788  $Variable[[A]]
789  $Variable[[A]]
790  )",
791  R"(
792  $Class[[A]]
793  ^$Variable[[AA]]
794  $Variable[[A]]
795  )"},
796  {
797  R"(
798  $Class[[A]]
799  $Variable[[A]]
800  )",
801  R"(
802  $Class[[A]]
803  $Variable[[A]]
804  ^$Class[[A]]
805  ^$Variable[[A]]
806  )"},
807  {
808  R"(
809  $Variable[[A]]
810  $Variable[[A]]
811  $Variable[[A]]
812  )",
813  R"(
814  ^$Class[[A]]
815  ^$Class[[A]]
816  ^$Class[[A]]
817  )"}};
818 
819  for (const auto &Test : TestCases)
820  checkDiffedHighlights(Test.OldCode, Test.NewCode);
821 }
822 
823 TEST(SemanticHighlighting, DiffBeyondTheEndOfFile) {
824  llvm::StringRef OldCode =
825  R"(
826  $Class[[A]]
827  $Variable[[A]]
828  $Class[[A]]
829  $Variable[[A]]
830  )";
831  llvm::StringRef NewCode =
832  R"(
833  $Class[[A]] // line 1
834  $Variable[[A]] // line 2
835  )";
836 
837  Annotations OldTest(OldCode);
838  Annotations NewTest(NewCode);
839  std::vector<HighlightingToken> OldTokens = getExpectedTokens(OldTest);
840  std::vector<HighlightingToken> NewTokens = getExpectedTokens(NewTest);
841 
842  auto ActualDiff = diffHighlightings(NewTokens, OldTokens);
843  EXPECT_THAT(ActualDiff,
844  testing::UnorderedElementsAre(
845  testing::AllOf(LineNumber(3), EmptyHighlightings()),
846  testing::AllOf(LineNumber(4), EmptyHighlightings())));
847 }
848 
849 } // namespace
850 } // namespace clangd
851 } // namespace clang
std::string Code
MATCHER_P(Named, N, "")
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:23
MockFSProvider FS
static Options optsForTest()
BindArgumentKind Kind
std::vector< SemanticHighlightingInformation > toSemanticHighlightingInformation(llvm::ArrayRef< LineHighlightings > Tokens)
Convert to LSP&#39;s semantic highlighting information.
TEST(BackgroundQueueTest, Priority)
llvm::Expected< size_t > positionToOffset(llvm::StringRef Code, Position P, bool AllowColumnsBeyondLineLength)
Turn a [line, column] pair into an offset in Code.
Definition: SourceCode.cpp:155
std::string testPath(PathRef File)
Definition: TestFS.cpp:82
MATCHER(Declared, "")
std::vector< LineHighlightings > diffHighlightings(ArrayRef< HighlightingToken > New, ArrayRef< HighlightingToken > Old)
Return a line-by-line diff between two highlightings.
Position Pos
Definition: SourceCode.cpp:772
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
ClangdServer Server
CharSourceRange Range
SourceRange for the file name.
std::vector< HighlightingToken > getSemanticHighlightings(ParsedAST &AST)
IgnoreDiagnostics DiagConsumer
static llvm::Optional< ParsedAST > build(std::unique_ptr< clang::CompilerInvocation > CI, llvm::ArrayRef< Diag > CompilerInvocationDiags, std::shared_ptr< const PreambleData > Preamble, std::unique_ptr< llvm::MemoryBuffer > Buffer, llvm::IntrusiveRefCntPtr< llvm::vfs::FileSystem > VFS, const SymbolIndex *Index, const ParseOptions &Opts)
Attempts to run Clang and store parsed AST.
Definition: ParsedAST.cpp:218