15 #include "clang/AST/ASTContext.h"
16 #include "clang/AST/Decl.h"
17 #include "clang/AST/DeclCXX.h"
18 #include "clang/AST/DeclarationName.h"
19 #include "clang/AST/ExprCXX.h"
20 #include "clang/AST/RecursiveASTVisitor.h"
21 #include "clang/AST/Type.h"
22 #include "clang/AST/TypeLoc.h"
23 #include "clang/Basic/LangOptions.h"
24 #include "clang/Basic/SourceLocation.h"
25 #include "clang/Basic/SourceManager.h"
26 #include "clang/Tooling/Syntax/Tokens.h"
27 #include "llvm/ADT/None.h"
28 #include "llvm/ADT/Optional.h"
29 #include "llvm/ADT/STLExtras.h"
30 #include "llvm/Support/Base64.h"
31 #include "llvm/Support/Casting.h"
40 bool canHighlightName(DeclarationName
Name) {
41 if (
Name.getNameKind() == DeclarationName::CXXConstructorName ||
42 Name.getNameKind() == DeclarationName::CXXUsingDirective)
44 auto *II =
Name.getAsIdentifierInfo();
45 return II && !II->getName().empty();
48 llvm::Optional<HighlightingKind> kindForType(
const Type *TP);
49 llvm::Optional<HighlightingKind> kindForDecl(
const NamedDecl *D) {
50 if (
auto *USD = dyn_cast<UsingShadowDecl>(D)) {
51 if (
auto *Target = USD->getTargetDecl())
54 if (
auto *TD = dyn_cast<TemplateDecl>(D)) {
55 if (
auto *Templated = TD->getTemplatedDecl())
58 if (
auto *TD = dyn_cast<TypedefNameDecl>(D)) {
60 if (
auto K = kindForType(TD->getUnderlyingType().getTypePtrOrNull()))
68 if (
auto *RD = llvm::dyn_cast<RecordDecl>(D)) {
74 if (isa<ClassTemplateDecl>(D) || isa<RecordDecl>(D) ||
75 isa<CXXConstructorDecl>(D))
77 if (
auto *
MD = dyn_cast<CXXMethodDecl>(D))
80 if (isa<FieldDecl>(D))
84 if (isa<EnumConstantDecl>(D))
86 if (isa<ParmVarDecl>(D))
88 if (
auto *VD = dyn_cast<VarDecl>(D))
89 return VD->isStaticDataMember()
93 if (isa<BindingDecl>(D))
95 if (isa<FunctionDecl>(D))
97 if (isa<NamespaceDecl>(D) || isa<NamespaceAliasDecl>(D) ||
98 isa<UsingDirectiveDecl>(D))
100 if (isa<TemplateTemplateParmDecl>(D) || isa<TemplateTypeParmDecl>(D) ||
101 isa<NonTypeTemplateParmDecl>(D))
103 if (isa<ConceptDecl>(D))
107 llvm::Optional<HighlightingKind> kindForType(
const Type *TP) {
110 if (TP->isBuiltinType())
112 if (
auto *TD = dyn_cast<TemplateTypeParmType>(TP))
113 return kindForDecl(TD->getDecl());
114 if (
auto *TD = TP->getAsTagDecl())
115 return kindForDecl(TD);
119 llvm::Optional<HighlightingKind> kindForReference(
const ReferenceLoc &R) {
120 llvm::Optional<HighlightingKind> Result;
121 for (
const NamedDecl *
Decl : R.Targets) {
122 if (!canHighlightName(
Decl->getDeclName()))
125 if (!
Kind || (Result &&
Kind != Result))
135 SourceLocation getHighlightableSpellingToken(SourceLocation L,
136 const SourceManager &SM) {
138 return SM.isWrittenInMainFile(L) ? L : SourceLocation{};
140 if (!SM.isMacroArgExpansion(L))
143 return getHighlightableSpellingToken(SM.getImmediateSpellingLoc(L), SM);
147 enum HighlightPriority { Dependent = 0, Resolved = 1 };
161 llvm::Optional<HighlightingToken>
162 resolveConflict(ArrayRef<HighlightingToken> Tokens) {
163 if (Tokens.size() == 1)
166 if (Tokens.size() != 2)
169 unsigned Priority1 = evaluateHighlightPriority(Tokens[0].
Kind);
170 unsigned Priority2 = evaluateHighlightPriority(Tokens[1].
Kind);
171 if (Priority1 == Priority2)
173 return Priority1 > Priority2 ? Tokens[0] : Tokens[1];
177 class HighlightingsBuilder {
179 HighlightingsBuilder(
const ParsedAST &
AST)
181 LangOpts(
AST.getLangOpts()) {}
183 void addToken(HighlightingToken T) { Tokens.push_back(T); }
189 const auto *Tok = TB.spelledTokenAt(
Loc);
194 Tokens.push_back(HighlightingToken{
Kind, std::move(
Range)});
197 std::vector<HighlightingToken> collect(ParsedAST &
AST) && {
201 auto Last = std::unique(Tokens.begin(), Tokens.end());
202 Tokens.erase(Last, Tokens.end());
207 std::vector<HighlightingToken> NonConflicting;
208 NonConflicting.reserve(Tokens.size());
209 for (ArrayRef<HighlightingToken> TokRef = Tokens; !TokRef.empty();) {
210 ArrayRef<HighlightingToken> Conflicting =
211 TokRef.take_while([&](
const HighlightingToken &T) {
214 return T.R == TokRef.front().R;
216 if (
auto Resolved = resolveConflict(Conflicting))
217 NonConflicting.push_back(*Resolved);
220 TokRef = TokRef.drop_front(Conflicting.size());
223 for (
const Range &R :
AST.getMacros().SkippedRanges) {
226 assert(R.start.line <= R.end.line);
227 for (
int Line = R.start.line;
Line <= R.end.line; ++
Line) {
233 {Position{
Line, 0}, Position{
Line, 0}}});
237 llvm::sort(NonConflicting);
238 return NonConflicting;
242 const syntax::TokenBuffer &TB;
244 const LangOptions &LangOpts;
245 std::vector<HighlightingToken> Tokens;
250 class CollectExtraHighlightings
251 :
public RecursiveASTVisitor<CollectExtraHighlightings> {
253 CollectExtraHighlightings(HighlightingsBuilder &H) : H(H) {}
255 bool VisitDecltypeTypeLoc(DecltypeTypeLoc L) {
256 if (
auto K = kindForType(L.getTypePtr()))
257 H.addToken(L.getBeginLoc(), *K);
261 bool VisitDeclaratorDecl(DeclaratorDecl *D) {
262 auto *AT = D->getType()->getContainedAutoType();
265 if (
auto K = kindForType(AT->getDeducedType().getTypePtrOrNull()))
266 H.addToken(D->getTypeSpecStartLoc(), *K);
270 bool VisitOverloadExpr(OverloadExpr *
E) {
271 if (!
E->decls().empty())
277 bool VisitCXXDependentScopeMemberExpr(CXXDependentScopeMemberExpr *
E) {
278 H.addToken(
E->getMemberNameInfo().getLoc(),
283 bool VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *
E) {
288 bool VisitDependentNameTypeLoc(DependentNameTypeLoc L) {
293 bool VisitDependentTemplateSpecializationTypeLoc(
294 DependentTemplateSpecializationTypeLoc L) {
304 bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc Q) {
305 if (NestedNameSpecifier *NNS = Q.getNestedNameSpecifier()) {
309 return RecursiveASTVisitor::TraverseNestedNameSpecifierLoc(Q);
313 HighlightingsBuilder &H;
316 void write32be(uint32_t I, llvm::raw_ostream &
OS) {
317 std::array<char, 4> Buf;
318 llvm::support::endian::write32be(Buf.data(), I);
319 OS.write(Buf.data(), Buf.size());
322 void write16be(uint16_t I, llvm::raw_ostream &
OS) {
323 std::array<char, 2> Buf;
324 llvm::support::endian::write16be(Buf.data(), I);
325 OS.write(Buf.data(), Buf.size());
330 ArrayRef<HighlightingToken>
331 takeLine(ArrayRef<HighlightingToken> AllTokens,
332 ArrayRef<HighlightingToken>::iterator StartLineIt,
int Line) {
333 return ArrayRef<HighlightingToken>(StartLineIt, AllTokens.end())
334 .take_while([
Line](
const HighlightingToken &Token) {
335 return Token.R.start.line ==
Line;
343 HighlightingsBuilder
Builder(AST);
345 CollectExtraHighlightings(
Builder).TraverseAST(C);
348 if (
auto Kind = kindForReference(R))
353 for (
const auto &M : SIDToRefs.second)
357 Builder.addToken({HighlightingKind::Macro, M});
359 return std::move(
Builder).collect(AST);
365 return OS <<
"Variable";
367 return OS <<
"LocalVariable";
369 return OS <<
"Parameter";
371 return OS <<
"Function";
373 return OS <<
"Method";
375 return OS <<
"StaticMethod";
377 return OS <<
"Field";
379 return OS <<
"StaticField";
381 return OS <<
"Class";
385 return OS <<
"EnumConstant";
387 return OS <<
"Typedef";
389 return OS <<
"DependentType";
391 return OS <<
"DependentName";
393 return OS <<
"Namespace";
395 return OS <<
"TemplateParameter";
397 return OS <<
"Concept";
399 return OS <<
"Primitive";
401 return OS <<
"Macro";
403 return OS <<
"InactiveCode";
405 llvm_unreachable(
"invalid HighlightingKind");
408 std::vector<LineHighlightings>
410 ArrayRef<HighlightingToken> Old) {
411 assert(std::is_sorted(New.begin(), New.end()) &&
412 "New must be a sorted vector");
413 assert(std::is_sorted(Old.begin(), Old.end()) &&
414 "Old must be a sorted vector");
428 std::vector<LineHighlightings> DiffedLines;
430 ArrayRef<HighlightingToken> NewLine(New.begin(),
431 static_cast<size_t>(0));
432 ArrayRef<HighlightingToken> OldLine(Old.begin(),
433 static_cast<size_t>(0));
434 auto NewEnd = New.end();
435 auto OldEnd = Old.end();
436 auto NextLineNumber = [&]() {
437 int NextNew = NewLine.end() != NewEnd ? NewLine.end()->R.start.line
438 : std::numeric_limits<int>::max();
439 int NextOld = OldLine.end() != OldEnd ? OldLine.end()->R.start.line
440 : std::numeric_limits<int>::max();
441 return std::min(NextNew, NextOld);
444 for (
int LineNumber = 0; NewLine.end() < NewEnd || OldLine.end() < OldEnd;
445 LineNumber = NextLineNumber()) {
446 NewLine = takeLine(New, NewLine.end(), LineNumber);
447 OldLine = takeLine(Old, OldLine.end(), LineNumber);
448 if (NewLine != OldLine) {
449 DiffedLines.push_back({LineNumber, NewLine,
false});
452 auto &AddedLine = DiffedLines.back();
454 if (T.Kind == HighlightingKind::InactiveCode) {
455 AddedLine.IsInactive = true;
467 return std::tie(L.
R, L.
Kind) == std::tie(R.
R, R.
Kind);
470 return std::tie(L.
R, L.
Kind) < std::tie(R.
R, R.
Kind);
476 std::vector<SemanticToken>
478 assert(std::is_sorted(Tokens.begin(), Tokens.end()));
479 std::vector<SemanticToken> Result;
483 if (Tok.Kind == HighlightingKind::InactiveCode)
485 Result.emplace_back();
489 assert(Tok.R.start.line >= Last->
R.
start.
line);
491 if (
Out.deltaLine == 0) {
495 Out.deltaStart = Tok.R.start.character;
498 Out.deltaLine = Tok.R.start.line;
499 Out.deltaStart = Tok.R.start.character;
501 assert(Tok.R.end.line == Tok.R.start.line);
502 Out.length = Tok.R.end.character - Tok.R.start.character;
503 Out.tokenType = static_cast<unsigned>(Tok.Kind);
511 case HighlightingKind::Variable:
512 case HighlightingKind::LocalVariable:
513 case HighlightingKind::StaticField:
515 case HighlightingKind::Parameter:
517 case HighlightingKind::Function:
519 case HighlightingKind::Method:
521 case HighlightingKind::StaticMethod:
524 case HighlightingKind::Field:
526 case HighlightingKind::Class:
528 case HighlightingKind::Enum:
530 case HighlightingKind::EnumConstant:
531 return "enumConstant";
532 case HighlightingKind::Typedef:
534 case HighlightingKind::DependentType:
536 case HighlightingKind::DependentName:
538 case HighlightingKind::Namespace:
540 case HighlightingKind::TemplateParameter:
541 return "typeParameter";
542 case HighlightingKind::Concept:
544 case HighlightingKind::Primitive:
546 case HighlightingKind::Macro:
548 case HighlightingKind::InactiveCode:
551 llvm_unreachable(
"unhandled HighlightingKind");
554 std::vector<TheiaSemanticHighlightingInformation>
556 llvm::ArrayRef<LineHighlightings> Tokens) {
557 if (Tokens.size() == 0)
562 std::vector<TheiaSemanticHighlightingInformation>
Lines;
563 Lines.reserve(Tokens.size());
564 for (
const auto &
Line : Tokens) {
565 llvm::SmallVector<char, 128> LineByteTokens;
566 llvm::raw_svector_ostream
OS(LineByteTokens);
567 for (
const auto &Token :
Line.Tokens) {
573 write32be(Token.R.start.character,
OS);
574 write16be(Token.R.end.character - Token.R.start.character,
OS);
575 write16be(static_cast<int>(Token.Kind),
OS);
578 Lines.push_back({
Line.Line, encodeBase64(LineByteTokens),
Line.IsInactive});
587 case HighlightingKind::Function:
588 return "entity.name.function.cpp";
589 case HighlightingKind::Method:
590 return "entity.name.function.method.cpp";
591 case HighlightingKind::StaticMethod:
592 return "entity.name.function.method.static.cpp";
593 case HighlightingKind::Variable:
594 return "variable.other.cpp";
595 case HighlightingKind::LocalVariable:
596 return "variable.other.local.cpp";
597 case HighlightingKind::Parameter:
598 return "variable.parameter.cpp";
599 case HighlightingKind::Field:
600 return "variable.other.field.cpp";
601 case HighlightingKind::StaticField:
602 return "variable.other.field.static.cpp";
603 case HighlightingKind::Class:
604 return "entity.name.type.class.cpp";
605 case HighlightingKind::Enum:
606 return "entity.name.type.enum.cpp";
607 case HighlightingKind::EnumConstant:
608 return "variable.other.enummember.cpp";
609 case HighlightingKind::Typedef:
610 return "entity.name.type.typedef.cpp";
611 case HighlightingKind::DependentType:
612 return "entity.name.type.dependent.cpp";
613 case HighlightingKind::DependentName:
614 return "entity.name.other.dependent.cpp";
615 case HighlightingKind::Namespace:
616 return "entity.name.namespace.cpp";
617 case HighlightingKind::TemplateParameter:
618 return "entity.name.type.template.cpp";
619 case HighlightingKind::Concept:
620 return "entity.name.type.concept.cpp";
621 case HighlightingKind::Primitive:
622 return "storage.type.primitive.cpp";
623 case HighlightingKind::Macro:
624 return "entity.name.function.preprocessor.cpp";
625 case HighlightingKind::InactiveCode:
626 return "meta.disabled";
628 llvm_unreachable(
"unhandled HighlightingKind");
631 std::vector<SemanticTokensEdit>
633 llvm::ArrayRef<SemanticToken> New) {
638 while (!Old.empty() && !New.empty() && Old.front() == New.front()) {
640 Old = Old.drop_front();
641 New = New.drop_front();
643 while (!Old.empty() && !New.empty() && Old.back() == New.back()) {
644 Old = Old.drop_back();
645 New = New.drop_back();
648 if (Old.empty() && New.empty())
652 Edit.deleteTokens = Old.size();
654 return {std::move(
Edit)};