clang-tools  11.0.0
SemanticHighlightingTests.cpp
Go to the documentation of this file.
1 //==- SemanticHighlightingTests.cpp - SemanticHighlighting tests-*- C++ -* -==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "Annotations.h"
10 #include "ClangdServer.h"
11 #include "Protocol.h"
12 #include "SemanticHighlighting.h"
13 #include "SourceCode.h"
14 #include "TestFS.h"
15 #include "TestTU.h"
16 #include "llvm/ADT/ArrayRef.h"
17 #include "llvm/ADT/STLExtras.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/Support/Error.h"
20 #include "llvm/Support/ScopedPrinter.h"
21 #include "gmock/gmock.h"
22 #include <algorithm>
23 
24 namespace clang {
25 namespace clangd {
26 namespace {
27 
28 using testing::IsEmpty;
29 using testing::SizeIs;
30 
31 MATCHER_P(LineNumber, L, "") { return arg.Line == L; }
32 MATCHER(EmptyHighlightings, "") { return arg.Tokens.empty(); }
33 
34 std::vector<HighlightingToken>
35 makeHighlightingTokens(llvm::ArrayRef<Range> Ranges, HighlightingKind Kind) {
36  std::vector<HighlightingToken> Tokens(Ranges.size());
37  for (int I = 0, End = Ranges.size(); I < End; ++I) {
38  Tokens[I].R = Ranges[I];
39  Tokens[I].Kind = Kind;
40  }
41 
42  return Tokens;
43 }
44 
45 std::vector<HighlightingToken> getExpectedTokens(Annotations &Test) {
46  static const std::map<HighlightingKind, std::string> KindToString{
47  {HighlightingKind::Variable, "Variable"},
48  {HighlightingKind::LocalVariable, "LocalVariable"},
49  {HighlightingKind::Parameter, "Parameter"},
50  {HighlightingKind::Function, "Function"},
51  {HighlightingKind::Class, "Class"},
52  {HighlightingKind::Enum, "Enum"},
53  {HighlightingKind::Namespace, "Namespace"},
54  {HighlightingKind::EnumConstant, "EnumConstant"},
55  {HighlightingKind::Field, "Field"},
56  {HighlightingKind::StaticField, "StaticField"},
57  {HighlightingKind::Method, "Method"},
58  {HighlightingKind::StaticMethod, "StaticMethod"},
59  {HighlightingKind::Typedef, "Typedef"},
60  {HighlightingKind::DependentType, "DependentType"},
61  {HighlightingKind::DependentName, "DependentName"},
62  {HighlightingKind::TemplateParameter, "TemplateParameter"},
63  {HighlightingKind::Concept, "Concept"},
64  {HighlightingKind::Primitive, "Primitive"},
65  {HighlightingKind::Macro, "Macro"}};
66  std::vector<HighlightingToken> ExpectedTokens;
67  for (const auto &KindString : KindToString) {
68  std::vector<HighlightingToken> Toks = makeHighlightingTokens(
69  Test.ranges(KindString.second), KindString.first);
70  ExpectedTokens.insert(ExpectedTokens.end(), Toks.begin(), Toks.end());
71  }
72  llvm::sort(ExpectedTokens);
73  return ExpectedTokens;
74 }
75 
76 /// Annotates the input code with provided semantic highlightings. Results look
77 /// something like:
78 /// class $Class[[X]] {
79 /// $Primitive[[int]] $Field[[a]] = 0;
80 /// };
81 std::string annotate(llvm::StringRef Input,
82  llvm::ArrayRef<HighlightingToken> Tokens) {
83  assert(std::is_sorted(
84  Tokens.begin(), Tokens.end(),
85  [](const HighlightingToken &L, const HighlightingToken &R) {
86  return L.R.start < R.R.start;
87  }));
88 
89  std::string Result;
90  unsigned NextChar = 0;
91  for (auto &T : Tokens) {
92  unsigned StartOffset = llvm::cantFail(positionToOffset(Input, T.R.start));
93  unsigned EndOffset = llvm::cantFail(positionToOffset(Input, T.R.end));
94  assert(StartOffset <= EndOffset);
95  assert(NextChar <= StartOffset);
96 
97  Result += Input.substr(NextChar, StartOffset - NextChar);
98  Result += std::string(
99  llvm::formatv("${0}[[{1}]]", T.Kind,
100  Input.substr(StartOffset, EndOffset - StartOffset)));
101  NextChar = EndOffset;
102  }
103  Result += Input.substr(NextChar);
104  return Result;
105 }
106 
107 void checkHighlightings(llvm::StringRef Code,
108  std::vector<std::pair</*FileName*/ llvm::StringRef,
109  /*FileContent*/ llvm::StringRef>>
110  AdditionalFiles = {}) {
111  Annotations Test(Code);
112  TestTU TU;
113  TU.Code = std::string(Test.code());
114 
115  // FIXME: Auto-completion in a template requires disabling delayed template
116  // parsing.
117  TU.ExtraArgs.push_back("-fno-delayed-template-parsing");
118  TU.ExtraArgs.push_back("-std=c++20");
119 
120  for (auto File : AdditionalFiles)
121  TU.AdditionalFiles.insert({File.first, std::string(File.second)});
122  auto AST = TU.build();
123 
124  EXPECT_EQ(Code, annotate(Test.code(), getSemanticHighlightings(AST)));
125 }
126 
127 // Any annotations in OldCode and NewCode are converted into their corresponding
128 // HighlightingToken. The tokens are diffed against each other. Any lines where
129 // the tokens should diff must be marked with a ^ somewhere on that line in
130 // NewCode. If there are diffs that aren't marked with ^ the test fails. The
131 // test also fails if there are lines marked with ^ that don't differ.
132 void checkDiffedHighlights(llvm::StringRef OldCode, llvm::StringRef NewCode) {
133  Annotations OldTest(OldCode);
134  Annotations NewTest(NewCode);
135  std::vector<HighlightingToken> OldTokens = getExpectedTokens(OldTest);
136  std::vector<HighlightingToken> NewTokens = getExpectedTokens(NewTest);
137 
138  llvm::DenseMap<int, std::vector<HighlightingToken>> ExpectedLines;
139  for (const Position &Point : NewTest.points()) {
140  ExpectedLines[Point.line]; // Default initialize to an empty line. Tokens
141  // are inserted on these lines later.
142  }
143  std::vector<LineHighlightings> ExpectedLinePairHighlighting;
144  for (const HighlightingToken &Token : NewTokens) {
145  auto It = ExpectedLines.find(Token.R.start.line);
146  if (It != ExpectedLines.end())
147  It->second.push_back(Token);
148  }
149  for (auto &LineTokens : ExpectedLines)
150  ExpectedLinePairHighlighting.push_back(
151  {LineTokens.first, LineTokens.second, /*IsInactive = */ false});
152 
153  std::vector<LineHighlightings> ActualDiffed =
154  diffHighlightings(NewTokens, OldTokens);
155  EXPECT_THAT(ActualDiffed,
156  testing::UnorderedElementsAreArray(ExpectedLinePairHighlighting))
157  << OldCode;
158 }
159 
160 TEST(SemanticHighlighting, GetsCorrectTokens) {
161  const char *TestCases[] = {
162  R"cpp(
163  struct $Class[[AS]] {
164  double $Field[[SomeMember]];
165  };
166  struct {
167  } $Variable[[S]];
168  void $Function[[foo]](int $Parameter[[A]], $Class[[AS]] $Parameter[[As]]) {
169  $Primitive[[auto]] $LocalVariable[[VeryLongVariableName]] = 12312;
170  $Class[[AS]] $LocalVariable[[AA]];
171  $Primitive[[auto]] $LocalVariable[[L]] = $LocalVariable[[AA]].$Field[[SomeMember]] + $Parameter[[A]];
172  auto $LocalVariable[[FN]] = [ $LocalVariable[[AA]]](int $Parameter[[A]]) -> void {};
173  $LocalVariable[[FN]](12312);
174  }
175  )cpp",
176  R"cpp(
177  void $Function[[foo]](int);
178  void $Function[[Gah]]();
179  void $Function[[foo]]() {
180  auto $LocalVariable[[Bou]] = $Function[[Gah]];
181  }
182  struct $Class[[A]] {
183  void $Method[[abc]]();
184  };
185  )cpp",
186  R"cpp(
187  namespace $Namespace[[abc]] {
188  template<typename $TemplateParameter[[T]]>
189  struct $Class[[A]] {
190  $TemplateParameter[[T]] $Field[[t]];
191  };
192  }
193  template<typename $TemplateParameter[[T]]>
194  struct $Class[[C]] : $Namespace[[abc]]::$Class[[A]]<$TemplateParameter[[T]]> {
195  typename $TemplateParameter[[T]]::$DependentType[[A]]* $Field[[D]];
196  };
197  $Namespace[[abc]]::$Class[[A]]<int> $Variable[[AA]];
198  typedef $Namespace[[abc]]::$Class[[A]]<int> $Class[[AAA]];
199  struct $Class[[B]] {
200  $Class[[B]]();
201  ~$Class[[B]]();
202  void operator<<($Class[[B]]);
203  $Class[[AAA]] $Field[[AA]];
204  };
205  $Class[[B]]::$Class[[B]]() {}
206  $Class[[B]]::~$Class[[B]]() {}
207  void $Function[[f]] () {
208  $Class[[B]] $LocalVariable[[BB]] = $Class[[B]]();
209  $LocalVariable[[BB]].~$Class[[B]]();
210  $Class[[B]]();
211  }
212  )cpp",
213  R"cpp(
214  enum class $Enum[[E]] {
215  $EnumConstant[[A]],
216  $EnumConstant[[B]],
217  };
218  enum $Enum[[EE]] {
219  $EnumConstant[[Hi]],
220  };
221  struct $Class[[A]] {
222  $Enum[[E]] $Field[[EEE]];
223  $Enum[[EE]] $Field[[EEEE]];
224  };
225  int $Variable[[I]] = $EnumConstant[[Hi]];
226  $Enum[[E]] $Variable[[L]] = $Enum[[E]]::$EnumConstant[[B]];
227  )cpp",
228  R"cpp(
229  namespace $Namespace[[abc]] {
230  namespace {}
231  namespace $Namespace[[bcd]] {
232  struct $Class[[A]] {};
233  namespace $Namespace[[cde]] {
234  struct $Class[[A]] {
235  enum class $Enum[[B]] {
236  $EnumConstant[[Hi]],
237  };
238  };
239  }
240  }
241  }
242  using namespace $Namespace[[abc]]::$Namespace[[bcd]];
243  namespace $Namespace[[vwz]] =
244  $Namespace[[abc]]::$Namespace[[bcd]]::$Namespace[[cde]];
245  $Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[AA]];
246  $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]] $Variable[[AAA]] =
247  $Namespace[[vwz]]::$Class[[A]]::$Enum[[B]]::$EnumConstant[[Hi]];
248  ::$Namespace[[vwz]]::$Class[[A]] $Variable[[B]];
249  ::$Namespace[[abc]]::$Namespace[[bcd]]::$Class[[A]] $Variable[[BB]];
250  )cpp",
251  R"cpp(
252  struct $Class[[D]] {
253  double $Field[[C]];
254  };
255  struct $Class[[A]] {
256  double $Field[[B]];
257  $Class[[D]] $Field[[E]];
258  static double $StaticField[[S]];
259  static void $StaticMethod[[bar]]() {}
260  void $Method[[foo]]() {
261  $Field[[B]] = 123;
262  this->$Field[[B]] = 156;
263  this->$Method[[foo]]();
264  $Method[[foo]]();
265  $StaticMethod[[bar]]();
266  $StaticField[[S]] = 90.1;
267  }
268  };
269  void $Function[[foo]]() {
270  $Class[[A]] $LocalVariable[[AA]];
271  $LocalVariable[[AA]].$Field[[B]] += 2;
272  $LocalVariable[[AA]].$Method[[foo]]();
273  $LocalVariable[[AA]].$Field[[E]].$Field[[C]];
274  $Class[[A]]::$StaticField[[S]] = 90;
275  }
276  )cpp",
277  R"cpp(
278  struct $Class[[AA]] {
279  int $Field[[A]];
280  };
281  int $Variable[[B]];
282  $Class[[AA]] $Variable[[A]]{$Variable[[B]]};
283  )cpp",
284  R"cpp(
285  namespace $Namespace[[a]] {
286  struct $Class[[A]] {};
287  typedef char $Primitive[[C]];
288  }
289  typedef $Namespace[[a]]::$Class[[A]] $Class[[B]];
290  using $Class[[BB]] = $Namespace[[a]]::$Class[[A]];
291  enum class $Enum[[E]] {};
292  typedef $Enum[[E]] $Enum[[C]];
293  typedef $Enum[[C]] $Enum[[CC]];
294  using $Enum[[CD]] = $Enum[[CC]];
295  $Enum[[CC]] $Function[[f]]($Class[[B]]);
296  $Enum[[CD]] $Function[[f]]($Class[[BB]]);
297  typedef $Namespace[[a]]::$Primitive[[C]] $Primitive[[PC]];
298  typedef float $Primitive[[F]];
299  )cpp",
300  R"cpp(
301  template<typename $TemplateParameter[[T]], typename = void>
302  class $Class[[A]] {
303  $TemplateParameter[[T]] $Field[[AA]];
304  $TemplateParameter[[T]] $Method[[foo]]();
305  };
306  template<class $TemplateParameter[[TT]]>
307  class $Class[[B]] {
308  $Class[[A]]<$TemplateParameter[[TT]]> $Field[[AA]];
309  };
310  template<class $TemplateParameter[[TT]], class $TemplateParameter[[GG]]>
311  class $Class[[BB]] {};
312  template<class $TemplateParameter[[T]]>
313  class $Class[[BB]]<$TemplateParameter[[T]], int> {};
314  template<class $TemplateParameter[[T]]>
315  class $Class[[BB]]<$TemplateParameter[[T]], $TemplateParameter[[T]]*> {};
316 
317  template<template<class> class $TemplateParameter[[T]], class $TemplateParameter[[C]]>
318  $TemplateParameter[[T]]<$TemplateParameter[[C]]> $Function[[f]]();
319 
320  template<typename>
321  class $Class[[Foo]] {};
322 
323  template<typename $TemplateParameter[[T]]>
324  void $Function[[foo]]($TemplateParameter[[T]] ...);
325  )cpp",
326  R"cpp(
327  template <class $TemplateParameter[[T]]>
328  struct $Class[[Tmpl]] {$TemplateParameter[[T]] $Field[[x]] = 0;};
329  extern template struct $Class[[Tmpl]]<float>;
330  template struct $Class[[Tmpl]]<double>;
331  )cpp",
332  // This test is to guard against highlightings disappearing when using
333  // conversion operators as their behaviour in the clang AST differ from
334  // other CXXMethodDecls.
335  R"cpp(
336  class $Class[[Foo]] {};
337  struct $Class[[Bar]] {
338  explicit operator $Class[[Foo]]*() const;
339  explicit operator int() const;
340  operator $Class[[Foo]]();
341  };
342  void $Function[[f]]() {
343  $Class[[Bar]] $LocalVariable[[B]];
344  $Class[[Foo]] $LocalVariable[[F]] = $LocalVariable[[B]];
345  $Class[[Foo]] *$LocalVariable[[FP]] = ($Class[[Foo]]*)$LocalVariable[[B]];
346  int $LocalVariable[[I]] = (int)$LocalVariable[[B]];
347  }
348  )cpp"
349  R"cpp(
350  struct $Class[[B]] {};
351  struct $Class[[A]] {
352  $Class[[B]] $Field[[BB]];
353  $Class[[A]] &operator=($Class[[A]] &&$Parameter[[O]]);
354  };
355 
356  $Class[[A]] &$Class[[A]]::operator=($Class[[A]] &&$Parameter[[O]]) = default;
357  )cpp",
358  R"cpp(
359  enum $Enum[[En]] {
360  $EnumConstant[[EC]],
361  };
362  class $Class[[Foo]] {};
363  class $Class[[Bar]] {
364  public:
365  $Class[[Foo]] $Field[[Fo]];
366  $Enum[[En]] $Field[[E]];
367  int $Field[[I]];
368  $Class[[Bar]] ($Class[[Foo]] $Parameter[[F]],
369  $Enum[[En]] $Parameter[[E]])
370  : $Field[[Fo]] ($Parameter[[F]]), $Field[[E]] ($Parameter[[E]]),
371  $Field[[I]] (123) {}
372  };
373  class $Class[[Bar2]] : public $Class[[Bar]] {
374  $Class[[Bar2]]() : $Class[[Bar]]($Class[[Foo]](), $EnumConstant[[EC]]) {}
375  };
376  )cpp",
377  R"cpp(
378  enum $Enum[[E]] {
379  $EnumConstant[[E]],
380  };
381  class $Class[[Foo]] {};
382  $Enum[[auto]] $Variable[[AE]] = $Enum[[E]]::$EnumConstant[[E]];
383  $Class[[auto]] $Variable[[AF]] = $Class[[Foo]]();
384  $Class[[decltype]](auto) $Variable[[AF2]] = $Class[[Foo]]();
385  $Class[[auto]] *$Variable[[AFP]] = &$Variable[[AF]];
386  $Enum[[auto]] &$Variable[[AER]] = $Variable[[AE]];
387  $Primitive[[auto]] $Variable[[Form]] = 10.2 + 2 * 4;
388  $Primitive[[decltype]]($Variable[[Form]]) $Variable[[F]] = 10;
389  auto $Variable[[Fun]] = []()->void{};
390  )cpp",
391  R"cpp(
392  class $Class[[G]] {};
393  template<$Class[[G]] *$TemplateParameter[[U]]>
394  class $Class[[GP]] {};
395  template<$Class[[G]] &$TemplateParameter[[U]]>
396  class $Class[[GR]] {};
397  template<int *$TemplateParameter[[U]]>
398  class $Class[[IP]] {
399  void $Method[[f]]() {
400  *$TemplateParameter[[U]] += 5;
401  }
402  };
403  template<unsigned $TemplateParameter[[U]] = 2>
404  class $Class[[Foo]] {
405  void $Method[[f]]() {
406  for(int $LocalVariable[[I]] = 0;
407  $LocalVariable[[I]] < $TemplateParameter[[U]];) {}
408  }
409  };
410 
411  $Class[[G]] $Variable[[L]];
412  void $Function[[f]]() {
413  $Class[[Foo]]<123> $LocalVariable[[F]];
414  $Class[[GP]]<&$Variable[[L]]> $LocalVariable[[LL]];
415  $Class[[GR]]<$Variable[[L]]> $LocalVariable[[LLL]];
416  }
417  )cpp",
418  R"cpp(
419  template<typename $TemplateParameter[[T]],
420  void ($TemplateParameter[[T]]::*$TemplateParameter[[method]])(int)>
421  struct $Class[[G]] {
422  void $Method[[foo]](
423  $TemplateParameter[[T]] *$Parameter[[O]]) {
424  ($Parameter[[O]]->*$TemplateParameter[[method]])(10);
425  }
426  };
427  struct $Class[[F]] {
428  void $Method[[f]](int);
429  };
430  template<void (*$TemplateParameter[[Func]])()>
431  struct $Class[[A]] {
432  void $Method[[f]]() {
433  (*$TemplateParameter[[Func]])();
434  }
435  };
436 
437  void $Function[[foo]]() {
438  $Class[[F]] $LocalVariable[[FF]];
439  $Class[[G]]<$Class[[F]], &$Class[[F]]::$Method[[f]]> $LocalVariable[[GG]];
440  $LocalVariable[[GG]].$Method[[foo]](&$LocalVariable[[FF]]);
441  $Class[[A]]<$Function[[foo]]> $LocalVariable[[AA]];
442  }
443  )cpp",
444  // Tokens that share a source range but have conflicting Kinds are not
445  // highlighted.
446  R"cpp(
447  #define $Macro[[DEF_MULTIPLE]](X) namespace X { class X { int X; }; }
448  #define $Macro[[DEF_CLASS]](T) class T {};
449  // Preamble ends.
450  $Macro[[DEF_MULTIPLE]](XYZ);
451  $Macro[[DEF_MULTIPLE]](XYZW);
452  $Macro[[DEF_CLASS]]($Class[[A]])
453  #define $Macro[[MACRO_CONCAT]](X, V, T) T foo##X = V
454  #define $Macro[[DEF_VAR]](X, V) int X = V
455  #define $Macro[[DEF_VAR_T]](T, X, V) T X = V
456  #define $Macro[[DEF_VAR_REV]](V, X) DEF_VAR(X, V)
457  #define $Macro[[CPY]](X) X
458  #define $Macro[[DEF_VAR_TYPE]](X, Y) X Y
459  #define $Macro[[SOME_NAME]] variable
460  #define $Macro[[SOME_NAME_SET]] variable2 = 123
461  #define $Macro[[INC_VAR]](X) X += 2
462  void $Function[[foo]]() {
463  $Macro[[DEF_VAR]]($LocalVariable[[X]], 123);
464  $Macro[[DEF_VAR_REV]](908, $LocalVariable[[XY]]);
465  int $Macro[[CPY]]( $LocalVariable[[XX]] );
466  $Macro[[DEF_VAR_TYPE]]($Class[[A]], $LocalVariable[[AA]]);
467  double $Macro[[SOME_NAME]];
468  int $Macro[[SOME_NAME_SET]];
469  $LocalVariable[[variable]] = 20.1;
470  $Macro[[MACRO_CONCAT]](var, 2, float);
471  $Macro[[DEF_VAR_T]]($Class[[A]], $Macro[[CPY]](
472  $Macro[[CPY]]($LocalVariable[[Nested]])),
473  $Macro[[CPY]]($Class[[A]]()));
474  $Macro[[INC_VAR]]($LocalVariable[[variable]]);
475  }
476  void $Macro[[SOME_NAME]]();
477  $Macro[[DEF_VAR]]($Variable[[MMMMM]], 567);
478  $Macro[[DEF_VAR_REV]](756, $Variable[[AB]]);
479 
480  #define $Macro[[CALL_FN]](F) F();
481  #define $Macro[[DEF_FN]](F) void F ()
482  $Macro[[DEF_FN]]($Function[[g]]) {
483  $Macro[[CALL_FN]]($Function[[foo]]);
484  }
485  )cpp",
486  R"cpp(
487  #define $Macro[[fail]](expr) expr
488  #define $Macro[[assert]](COND) if (!(COND)) { fail("assertion failed" #COND); }
489  // Preamble ends.
490  int $Variable[[x]];
491  int $Variable[[y]];
492  int $Function[[f]]();
493  void $Function[[foo]]() {
494  $Macro[[assert]]($Variable[[x]] != $Variable[[y]]);
495  $Macro[[assert]]($Variable[[x]] != $Function[[f]]());
496  }
497  )cpp",
498  // highlighting all macro references
499  R"cpp(
500  #ifndef $Macro[[name]]
501  #define $Macro[[name]]
502  #endif
503 
504  #define $Macro[[test]]
505  #undef $Macro[[test]]
506 $InactiveCode[[]] #ifdef $Macro[[test]]
507 $InactiveCode[[]] #endif
508 
509 $InactiveCode[[]] #if defined($Macro[[test]])
510 $InactiveCode[[]] #endif
511  )cpp",
512  R"cpp(
513  struct $Class[[S]] {
514  float $Field[[Value]];
515  $Class[[S]] *$Field[[Next]];
516  };
517  $Class[[S]] $Variable[[Global]][2] = {$Class[[S]](), $Class[[S]]()};
518  void $Function[[f]]($Class[[S]] $Parameter[[P]]) {
519  int $LocalVariable[[A]][2] = {1,2};
520  auto [$Variable[[B1]], $Variable[[B2]]] = $LocalVariable[[A]];
521  auto [$Variable[[G1]], $Variable[[G2]]] = $Variable[[Global]];
522  $Class[[auto]] [$Variable[[P1]], $Variable[[P2]]] = $Parameter[[P]];
523  // Highlights references to BindingDecls.
524  $Variable[[B1]]++;
525  }
526  )cpp",
527  R"cpp(
528  template<class $TemplateParameter[[T]]>
529  class $Class[[A]] {
530  using $TemplateParameter[[TemplateParam1]] = $TemplateParameter[[T]];
531  typedef $TemplateParameter[[T]] $TemplateParameter[[TemplateParam2]];
532  using $Primitive[[IntType]] = int;
533 
534  using $Typedef[[Pointer]] = $TemplateParameter[[T]] *;
535  using $Typedef[[LVReference]] = $TemplateParameter[[T]] &;
536  using $Typedef[[RVReference]] = $TemplateParameter[[T]]&&;
537  using $Typedef[[Array]] = $TemplateParameter[[T]]*[3];
538  using $Typedef[[MemberPointer]] = int ($Class[[A]]::*)(int);
539 
540  // Use various previously defined typedefs in a function type.
541  void $Method[[func]](
542  $Typedef[[Pointer]], $Typedef[[LVReference]], $Typedef[[RVReference]],
543  $Typedef[[Array]], $Typedef[[MemberPointer]]);
544  };
545  )cpp",
546  R"cpp(
547  template <class $TemplateParameter[[T]]>
548  void $Function[[phase1]]($TemplateParameter[[T]]);
549  template <class $TemplateParameter[[T]]>
550  void $Function[[foo]]($TemplateParameter[[T]] $Parameter[[P]]) {
551  $Function[[phase1]]($Parameter[[P]]);
552  $DependentName[[phase2]]($Parameter[[P]]);
553  }
554  )cpp",
555  R"cpp(
556  class $Class[[A]] {
557  template <class $TemplateParameter[[T]]>
558  void $Method[[bar]]($TemplateParameter[[T]]);
559  };
560 
561  template <class $TemplateParameter[[U]]>
562  void $Function[[foo]]($TemplateParameter[[U]] $Parameter[[P]]) {
563  $Class[[A]]().$Method[[bar]]($Parameter[[P]]);
564  }
565  )cpp",
566  R"cpp(
567  struct $Class[[A]] {
568  template <class $TemplateParameter[[T]]>
569  static void $StaticMethod[[foo]]($TemplateParameter[[T]]);
570  };
571 
572  template <class $TemplateParameter[[T]]>
573  struct $Class[[B]] {
574  void $Method[[bar]]() {
575  $Class[[A]]::$StaticMethod[[foo]]($TemplateParameter[[T]]());
576  }
577  };
578  )cpp",
579  R"cpp(
580  template <class $TemplateParameter[[T]]>
581  void $Function[[foo]](typename $TemplateParameter[[T]]::$DependentType[[Type]]
582  = $TemplateParameter[[T]]::$DependentName[[val]]);
583  )cpp",
584  R"cpp(
585  template <class $TemplateParameter[[T]]>
586  void $Function[[foo]]($TemplateParameter[[T]] $Parameter[[P]]) {
587  $Parameter[[P]].$DependentName[[Field]];
588  }
589  )cpp",
590  R"cpp(
591  template <class $TemplateParameter[[T]]>
592  class $Class[[A]] {
593  int $Method[[foo]]() {
594  return $TemplateParameter[[T]]::$DependentName[[Field]];
595  }
596  };
597  )cpp",
598  // Highlighting the using decl as the underlying using shadow decl.
599  R"cpp(
600  void $Function[[foo]]();
601  using ::$Function[[foo]];
602  )cpp",
603  // Highlighting of template template arguments.
604  R"cpp(
605  template <template <class> class $TemplateParameter[[TT]],
606  template <class> class ...$TemplateParameter[[TTs]]>
607  struct $Class[[Foo]] {
608  $Class[[Foo]]<$TemplateParameter[[TT]], $TemplateParameter[[TTs]]...>
609  *$Field[[t]];
610  };
611  )cpp",
612  // Inactive code highlighting
613  R"cpp(
614  // Code in the preamble.
615  // Inactive lines get an empty InactiveCode token at the beginning.
616 $InactiveCode[[]] #ifdef $Macro[[test]]
617 $InactiveCode[[]] #endif
618 
619  // A declaration to cause the preamble to end.
620  int $Variable[[EndPreamble]];
621 
622  // Code after the preamble.
623  // Code inside inactive blocks does not get regular highlightings
624  // because it's not part of the AST.
625 $InactiveCode[[]] #ifdef $Macro[[test]]
626 $InactiveCode[[]] int Inactive2;
627 $InactiveCode[[]] #endif
628 
629  #ifndef $Macro[[test]]
630  int $Variable[[Active1]];
631  #endif
632 
633 $InactiveCode[[]] #ifdef $Macro[[test]]
634 $InactiveCode[[]] int Inactive3;
635 $InactiveCode[[]] #else
636  int $Variable[[Active2]];
637  #endif
638  )cpp",
639  // Argument to 'sizeof...'
640  R"cpp(
641  template <typename... $TemplateParameter[[Elements]]>
642  struct $Class[[TupleSize]] {
643  static const int $StaticField[[size]] =
644 sizeof...($TemplateParameter[[Elements]]);
645  };
646  )cpp",
647  // More dependent types
648  R"cpp(
649  template <typename $TemplateParameter[[T]]>
650  struct $Class[[Waldo]] {
651  using $Typedef[[Location1]] = typename $TemplateParameter[[T]]
652  ::$DependentType[[Resolver]]::$DependentType[[Location]];
653  using $Typedef[[Location2]] = typename $TemplateParameter[[T]]
654  ::template $DependentType[[Resolver]]<$TemplateParameter[[T]]>
655  ::$DependentType[[Location]];
656  using $Typedef[[Location3]] = typename $TemplateParameter[[T]]
657  ::$DependentType[[Resolver]]
658  ::template $DependentType[[Location]]<$TemplateParameter[[T]]>;
659  static const int $StaticField[[Value]] = $TemplateParameter[[T]]
660  ::$DependentType[[Resolver]]::$DependentName[[Value]];
661  };
662  )cpp",
663  // Dependent name with heuristic target
664  R"cpp(
665  template <typename>
666  struct $Class[[Foo]] {
667  int $Field[[Waldo]];
668  void $Method[[bar]]() {
669  $Class[[Foo]]().$Field[[Waldo]];
670  }
671  template <typename $TemplateParameter[[U]]>
672  void $Method[[bar1]]() {
673  $Class[[Foo]]<$TemplateParameter[[U]]>().$Field[[Waldo]];
674  }
675  };
676  )cpp",
677  // Concepts
678  R"cpp(
679  template <typename $TemplateParameter[[T]]>
680  concept $Concept[[Fooable]] =
681  requires($TemplateParameter[[T]] $Parameter[[F]]) {
682  $Parameter[[F]].$DependentName[[foo]]();
683  };
684  template <typename $TemplateParameter[[T]]>
685  requires $Concept[[Fooable]]<$TemplateParameter[[T]]>
686  void $Function[[bar]]($TemplateParameter[[T]] $Parameter[[F]]) {
687  $Parameter[[F]].$DependentName[[foo]]();
688  }
689  )cpp"};
690  for (const auto &TestCase : TestCases) {
691  checkHighlightings(TestCase);
692  }
693 
694  checkHighlightings(R"cpp(
695  class $Class[[A]] {
696  #include "imp.h"
697  };
698  )cpp",
699  {{"imp.h", R"cpp(
700  int someMethod();
701  void otherMethod();
702  )cpp"}});
703 
704  // A separate test for macros in headers.
705  checkHighlightings(R"cpp(
706  #include "imp.h"
707  $Macro[[DEFINE_Y]]
708  $Macro[[DXYZ_Y]](A);
709  )cpp",
710  {{"imp.h", R"cpp(
711  #define DXYZ(X) class X {};
712  #define DXYZ_Y(Y) DXYZ(x##Y)
713  #define DEFINE(X) int X;
714  #define DEFINE_Y DEFINE(Y)
715  )cpp"}});
716 }
717 
718 TEST(SemanticHighlighting, GeneratesHighlightsWhenFileChange) {
719  class HighlightingsCounter : public ClangdServer::Callbacks {
720  public:
721  std::atomic<int> Count = {0};
722 
723  void onHighlightingsReady(
724  PathRef File, llvm::StringRef Version,
725  std::vector<HighlightingToken> Highlightings) override {
726  ++Count;
727  }
728  };
729 
730  auto FooCpp = testPath("foo.cpp");
731  MockFS FS;
732  FS.Files[FooCpp] = "";
733 
734  MockCompilationDatabase MCD;
735  HighlightingsCounter Counter;
736  ClangdServer Server(MCD, FS, ClangdServer::optsForTest(), &Counter);
737  Server.addDocument(FooCpp, "int a;");
738  ASSERT_TRUE(Server.blockUntilIdleForTest()) << "Waiting for server";
739  ASSERT_EQ(Counter.Count, 1);
740 }
741 
742 // Ranges are highlighted as variables, unless highlighted as $Function etc.
743 std::vector<HighlightingToken> tokens(llvm::StringRef MarkedText) {
744  Annotations A(MarkedText);
745  std::vector<HighlightingToken> Results;
746  for (const Range& R : A.ranges())
747  Results.push_back({HighlightingKind::Variable, R});
748  for (unsigned I = 0; I < static_cast<unsigned>(HighlightingKind::LastKind); ++I) {
749  HighlightingKind Kind = static_cast<HighlightingKind>(I);
750  for (const Range& R : A.ranges(llvm::to_string(Kind)))
751  Results.push_back({Kind, R});
752  }
753  llvm::sort(Results);
754  return Results;
755 }
756 
757 TEST(SemanticHighlighting, toSemanticTokens) {
758  auto Results = toSemanticTokens(tokens(R"(
759  [[blah]]
760 
761  $Function[[big]] [[bang]]
762  )"));
763 
764  ASSERT_THAT(Results, SizeIs(3));
765  EXPECT_EQ(Results[0].tokenType, unsigned(HighlightingKind::Variable));
766  EXPECT_EQ(Results[0].deltaLine, 1u);
767  EXPECT_EQ(Results[0].deltaStart, 1u);
768  EXPECT_EQ(Results[0].length, 4u);
769 
770  EXPECT_EQ(Results[1].tokenType, unsigned(HighlightingKind::Function));
771  EXPECT_EQ(Results[1].deltaLine, 2u);
772  EXPECT_EQ(Results[1].deltaStart, 4u);
773  EXPECT_EQ(Results[1].length, 3u);
774 
775  EXPECT_EQ(Results[2].tokenType, unsigned(HighlightingKind::Variable));
776  EXPECT_EQ(Results[2].deltaLine, 0u);
777  EXPECT_EQ(Results[2].deltaStart, 4u);
778  EXPECT_EQ(Results[2].length, 4u);
779 }
780 
781 TEST(SemanticHighlighting, diffSemanticTokens) {
782  auto Before = toSemanticTokens(tokens(R"(
783  [[foo]] [[bar]] [[baz]]
784  [[one]] [[two]] [[three]]
785  )"));
786  EXPECT_THAT(diffTokens(Before, Before), IsEmpty());
787 
788  auto After = toSemanticTokens(tokens(R"(
789  [[foo]] [[hello]] [[world]] [[baz]]
790  [[one]] [[two]] [[three]]
791  )"));
792 
793  // Replace [bar, baz] with [hello, world, baz]
794  auto Diff = diffTokens(Before, After);
795  ASSERT_THAT(Diff, SizeIs(1));
796  EXPECT_EQ(1u, Diff.front().startToken);
797  EXPECT_EQ(2u, Diff.front().deleteTokens);
798  ASSERT_THAT(Diff.front().tokens, SizeIs(3));
799  // hello
800  EXPECT_EQ(0u, Diff.front().tokens[0].deltaLine);
801  EXPECT_EQ(4u, Diff.front().tokens[0].deltaStart);
802  EXPECT_EQ(5u, Diff.front().tokens[0].length);
803  // world
804  EXPECT_EQ(0u, Diff.front().tokens[1].deltaLine);
805  EXPECT_EQ(6u, Diff.front().tokens[1].deltaStart);
806  EXPECT_EQ(5u, Diff.front().tokens[1].length);
807  // baz
808  EXPECT_EQ(0u, Diff.front().tokens[2].deltaLine);
809  EXPECT_EQ(6u, Diff.front().tokens[2].deltaStart);
810  EXPECT_EQ(3u, Diff.front().tokens[2].length);
811 }
812 
813 TEST(SemanticHighlighting, toTheiaSemanticHighlightingInformation) {
814  auto CreatePosition = [](int Line, int Character) -> Position {
815  Position Pos;
816  Pos.line = Line;
817  Pos.character = Character;
818  return Pos;
819  };
820 
821  std::vector<LineHighlightings> Tokens{
822  {3,
824  Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
826  Range{CreatePosition(3, 4), CreatePosition(3, 7)}}},
827  /* IsInactive = */ false},
828  {1,
830  Range{CreatePosition(1, 1), CreatePosition(1, 5)}}},
831  /* IsInactive = */ true}};
832  std::vector<TheiaSemanticHighlightingInformation> ActualResults =
834  std::vector<TheiaSemanticHighlightingInformation> ExpectedResults = {
835  {3, "AAAACAAEAAAAAAAEAAMAAw=="}, {1, "AAAAAQAEAAA="}};
836  EXPECT_EQ(ActualResults, ExpectedResults);
837 }
838 
839 TEST(SemanticHighlighting, HighlightingDiffer) {
840  struct {
841  llvm::StringRef OldCode;
842  llvm::StringRef NewCode;
843  } TestCases[]{{
844  R"(
845  $Variable[[A]]
846  $Class[[B]]
847  $Function[[C]]
848  )",
849  R"(
850  $Variable[[A]]
851  $Class[[D]]
852  $Function[[C]]
853  )"},
854  {
855  R"(
856  $Class[[C]]
857  $Field[[F]]
858  $Variable[[V]]
859  $Class[[C]] $Variable[[V]] $Field[[F]]
860  )",
861  R"(
862  $Class[[C]]
863  $Field[[F]]
864  ^$Function[[F]]
865  $Class[[C]] $Variable[[V]] $Field[[F]]
866  )"},
867  {
868  R"(
869 
870  $Class[[A]]
871  $Variable[[A]]
872  )",
873  R"(
874 
875  ^
876  ^$Class[[A]]
877  ^$Variable[[A]]
878  )"},
879  {
880  R"(
881  $Class[[C]]
882  $Field[[F]]
883  $Variable[[V]]
884  $Class[[C]] $Variable[[V]] $Field[[F]]
885  )",
886  R"(
887  $Class[[C]]
888  ^
889  ^
890  $Class[[C]] $Variable[[V]] $Field[[F]]
891  )"},
892  {
893  R"(
894  $Class[[A]]
895  $Variable[[A]]
896  $Variable[[A]]
897  )",
898  R"(
899  $Class[[A]]
900  ^$Variable[[AA]]
901  $Variable[[A]]
902  )"},
903  {
904  R"(
905  $Class[[A]]
906  $Variable[[A]]
907  )",
908  R"(
909  $Class[[A]]
910  $Variable[[A]]
911  ^$Class[[A]]
912  ^$Variable[[A]]
913  )"},
914  {
915  R"(
916  $Variable[[A]]
917  $Variable[[A]]
918  $Variable[[A]]
919  )",
920  R"(
921  ^$Class[[A]]
922  ^$Class[[A]]
923  ^$Class[[A]]
924  )"}};
925 
926  for (const auto &Test : TestCases)
927  checkDiffedHighlights(Test.OldCode, Test.NewCode);
928 }
929 
930 TEST(SemanticHighlighting, DiffBeyondTheEndOfFile) {
931  llvm::StringRef OldCode =
932  R"(
933  $Class[[A]]
934  $Variable[[A]]
935  $Class[[A]]
936  $Variable[[A]]
937  )";
938  llvm::StringRef NewCode =
939  R"(
940  $Class[[A]] // line 1
941  $Variable[[A]] // line 2
942  )";
943 
944  Annotations OldTest(OldCode);
945  Annotations NewTest(NewCode);
946  std::vector<HighlightingToken> OldTokens = getExpectedTokens(OldTest);
947  std::vector<HighlightingToken> NewTokens = getExpectedTokens(NewTest);
948 
949  auto ActualDiff = diffHighlightings(NewTokens, OldTokens);
950  EXPECT_THAT(ActualDiff,
951  testing::UnorderedElementsAre(
952  testing::AllOf(LineNumber(3), EmptyHighlightings()),
953  testing::AllOf(LineNumber(4), EmptyHighlightings())));
954 }
955 
956 } // namespace
957 } // namespace clangd
958 } // namespace clang
Range
CharSourceRange Range
SourceRange for the file name.
Definition: IncludeOrderCheck.cpp:38
clang::clangd::TEST
TEST(BackgroundQueueTest, Priority)
Definition: BackgroundIndexTests.cpp:704
clang::clangd::testPath
std::string testPath(PathRef File, llvm::sys::path::Style Style)
Definition: TestFS.cpp:82
clang::clangd::HighlightingKind::Primitive
clang::clangd::CompletionItemKind::Field
clang::clangd::toTheiaSemanticHighlightingInformation
std::vector< TheiaSemanticHighlightingInformation > toTheiaSemanticHighlightingInformation(llvm::ArrayRef< LineHighlightings > Tokens)
Convert to LSP's semantic highlighting information.
Definition: SemanticHighlighting.cpp:555
TestTU.h
Kind
BindArgumentKind Kind
Definition: AvoidBindCheck.cpp:59
clang::clangd::HighlightingKind
HighlightingKind
Definition: SemanticHighlighting.h:40
clang::clangd::ClangdServer::optsForTest
static Options optsForTest()
Definition: ClangdServer.cpp:114
clang::clangd::HighlightingKind::Typedef
clang::clangd::HighlightingKind::TemplateParameter
clang::clangd::HighlightingKind::LastKind
clang::clangd::HighlightingKind::DependentName
clang::clangd::HighlightingKind::Macro
clang::clangd::Position::line
int line
Line position in a document (zero-based).
Definition: Protocol.h:146
clang::clangd::HighlightingKind::Variable
clang::clangd::CompletionItemKind::Enum
Protocol.h
clang::clangd::ParsedAST::build
static llvm::Optional< ParsedAST > build(llvm::StringRef Filename, const ParseInputs &Inputs, std::unique_ptr< clang::CompilerInvocation > CI, llvm::ArrayRef< Diag > CompilerInvocationDiags, std::shared_ptr< const PreambleData > Preamble)
Attempts to run Clang and store the parsed AST.
Definition: ParsedAST.cpp:246
Code
std::string Code
Definition: FindTargetTests.cpp:67
clang::clangd::diffHighlightings
std::vector< LineHighlightings > diffHighlightings(ArrayRef< HighlightingToken > New, ArrayRef< HighlightingToken > Old)
Return a line-by-line diff between two highlightings.
Definition: SemanticHighlighting.cpp:409
clang::clangd::HighlightingKind::Function
Line
int Line
Definition: PreprocessorTracker.cpp:513
clang::clangd::diffTokens
std::vector< SemanticTokensEdit > diffTokens(llvm::ArrayRef< SemanticToken > Old, llvm::ArrayRef< SemanticToken > New)
Definition: SemanticHighlighting.cpp:632
FS
MockFS FS
Definition: ClangdLSPServerTests.cpp:66
TestFS.h
clang::clangd::positionToOffset
llvm::Expected< size_t > positionToOffset(llvm::StringRef Code, Position P, bool AllowColumnsBeyondLineLength)
Turn a [line, column] pair into an offset in Code.
Definition: SourceCode.cpp:175
clang::clangd::HighlightingKind::Namespace
clang::clangd::HighlightingKind::Concept
clang::clangd::HighlightingKind::LocalVariable
clang::clangd::Position::character
int character
Character offset on a line in a document (zero-based).
Definition: Protocol.h:151
Results
std::vector< CodeCompletionResult > Results
Definition: CodeComplete.cpp:712
Annotations.h
clang::clangd::getSemanticHighlightings
std::vector< HighlightingToken > getSemanticHighlightings(ParsedAST &AST)
Definition: SemanticHighlighting.cpp:340
clang::clangd::HighlightingKind::DependentType
clang::clangd::HighlightingKind::Method
SourceCode.h
clang::clangd::CompletionItemKind::File
Counter
trace::Metric Counter
Definition: TraceTests.cpp:151
clang::clangd::HighlightingKind::Parameter
clang::clangd::PathRef
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:23
clang::clangd::toSemanticTokens
std::vector< SemanticToken > toSemanticTokens(llvm::ArrayRef< HighlightingToken > Tokens)
Definition: SemanticHighlighting.cpp:477
clang::clangd::MATCHER_P
MATCHER_P(Named, N, "")
Definition: BackgroundIndexTests.cpp:29
clang::clangd::HighlightingKind::StaticMethod
SemanticHighlighting.h
clang
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
Definition: ApplyReplacements.h:27
ClangdServer.h
clang::clangd::HighlightingKind::StaticField
clang::clangd::MockFS::Files
llvm::StringMap< std::string > Files
Definition: TestFS.h:41
Pos
Position Pos
Definition: SourceCode.cpp:649
clang::clangd::HighlightingKind::EnumConstant
clang::clangd::HighlightingKind::Class
clang::clangd::MATCHER
MATCHER(Declared, "")
Definition: BackgroundIndexTests.cpp:31