13 #include "clang/AST/ASTContext.h" 14 #include "clang/AST/RecursiveASTVisitor.h" 21 class HighlightingTokenCollector
22 :
public RecursiveASTVisitor<HighlightingTokenCollector> {
23 std::vector<HighlightingToken>
Tokens;
25 const SourceManager &SM;
28 HighlightingTokenCollector(ParsedAST &
AST)
29 : Ctx(AST.getASTContext()), SM(AST.getSourceManager()) {}
31 std::vector<HighlightingToken> collectTokens() {
37 [](
const HighlightingToken &L,
const HighlightingToken &R) {
38 return std::tie(L.R, L.Kind) < std::tie(R.R, R.Kind);
40 auto Last = std::unique(Tokens.begin(), Tokens.end());
41 Tokens.erase(Last, Tokens.end());
45 bool VisitNamespaceAliasDecl(NamespaceAliasDecl *NAD) {
51 bool VisitMemberExpr(MemberExpr *ME) {
52 const auto *
MD = ME->getMemberDecl();
53 if (isa<CXXDestructorDecl>(
MD))
57 addToken(ME->getMemberLoc(),
MD);
61 bool VisitNamedDecl(NamedDecl *ND) {
65 if (
const auto *UD = dyn_cast<UsingDirectiveDecl>(ND)) {
73 if (ND->getDeclName().getNameKind() ==
74 DeclarationName::CXXConstructorName) {
75 addToken(ND->getLocation(), ND);
82 addToken(ND->getLocation(), ND);
86 bool VisitDeclRefExpr(DeclRefExpr *Ref) {
87 if (Ref->getNameInfo().getName().getNameKind() !=
92 addToken(Ref->getLocation(), Ref->getDecl());
96 bool VisitTypedefNameDecl(TypedefNameDecl *TD) {
97 if(
const auto *TSI = TD->getTypeSourceInfo())
98 addTypeLoc(TD->getLocation(), TSI->getTypeLoc());
102 bool VisitTemplateTypeParmTypeLoc(TemplateTypeParmTypeLoc &TL) {
104 addToken(TL.getBeginLoc(), TL.getDecl());
108 bool VisitTemplateSpecializationTypeLoc(TemplateSpecializationTypeLoc &TL) {
109 if (
const TemplateDecl *TD =
110 TL.getTypePtr()->getTemplateName().getAsTemplateDecl())
111 addToken(TL.getBeginLoc(), TD);
115 bool VisitTypeLoc(TypeLoc &TL) {
120 if (TL.getTypeLocClass() == TypeLoc::TypeLocClass::Elaborated)
123 addTypeLoc(TL.getBeginLoc(), TL);
127 bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc NNSLoc) {
128 if (NestedNameSpecifier *NNS = NNSLoc.getNestedNameSpecifier())
130 NNS->getKind() == NestedNameSpecifier::NamespaceAlias)
133 return RecursiveASTVisitor<
134 HighlightingTokenCollector>::TraverseNestedNameSpecifierLoc(NNSLoc);
138 void addTypeLoc(SourceLocation
Loc,
const TypeLoc &TL) {
139 if (
const Type *TP = TL.getTypePtr())
140 if (
const TagDecl *TD = TP->getAsTagDecl())
144 void addToken(SourceLocation Loc,
const NamedDecl *
D) {
145 if (D->getDeclName().isIdentifier() && D->getName().empty())
151 if (isa<ClassTemplateDecl>(D)) {
155 if (isa<RecordDecl>(D)) {
159 if (isa<CXXConstructorDecl>(D)) {
163 if (isa<CXXMethodDecl>(D)) {
167 if (isa<FieldDecl>(D)) {
171 if (isa<EnumDecl>(D)) {
175 if (isa<EnumConstantDecl>(D)) {
179 if (isa<VarDecl>(D)) {
183 if (isa<FunctionDecl>(D)) {
187 if (isa<NamespaceDecl>(D)) {
191 if (isa<NamespaceAliasDecl>(D)) {
195 if (isa<TemplateTemplateParmDecl>(D)) {
199 if (isa<TemplateTypeParmDecl>(D)) {
213 elog(
"Tried to add semantic token with an invalid range");
217 Tokens.push_back({
Kind, R.getValue()});
224 std::string encodeBase64(
const llvm::SmallVectorImpl<char> &Bytes) {
225 static const char Table[] =
"ABCDEFGHIJKLMNOPQRSTUVWXYZ" 226 "abcdefghijklmnopqrstuvwxyz" 230 for (I = 0; I + 2 < Bytes.size(); I += 3) {
231 uint32_t
X = (Bytes[I] << 16) + (Bytes[I + 1] << 8) + Bytes[I + 2];
232 Res += Table[(X >> 18) & 63];
233 Res += Table[(X >> 12) & 63];
234 Res += Table[(X >> 6) & 63];
235 Res += Table[X & 63];
237 if (I + 1 == Bytes.size()) {
238 uint32_t X = (Bytes[I] << 16);
239 Res += Table[(X >> 18) & 63];
240 Res += Table[(X >> 12) & 63];
242 }
else if (I + 2 == Bytes.size()) {
243 uint32_t X = (Bytes[I] << 16) + (Bytes[I + 1] << 8);
244 Res += Table[(X >> 18) & 63];
245 Res += Table[(X >> 12) & 63];
246 Res += Table[(X >> 6) & 63];
252 void write32be(uint32_t I, llvm::raw_ostream &OS) {
253 std::array<char, 4> Buf;
254 llvm::support::endian::write32be(Buf.data(), I);
255 OS.write(Buf.data(), Buf.size());
258 void write16be(uint16_t I, llvm::raw_ostream &OS) {
259 std::array<char, 2> Buf;
260 llvm::support::endian::write16be(Buf.data(), I);
261 OS.write(Buf.data(), Buf.size());
266 return Lhs.
Kind == Rhs.
Kind && Lhs.
R == Rhs.
R;
270 return HighlightingTokenCollector(AST).collectTokens();
273 std::vector<SemanticHighlightingInformation>
275 if (Tokens.size() == 0)
280 std::map<int, std::vector<HighlightingToken>> TokenLines;
282 TokenLines[Token.R.start.line].push_back(Token);
284 std::vector<SemanticHighlightingInformation>
Lines;
285 Lines.reserve(TokenLines.size());
286 for (
const auto &
Line : TokenLines) {
287 llvm::SmallVector<char, 128> LineByteTokens;
288 llvm::raw_svector_ostream OS(LineByteTokens);
289 for (
const auto &Token :
Line.second) {
295 write32be(Token.R.start.character, OS);
296 write16be(Token.R.end.character - Token.R.start.character, OS);
297 write16be(static_cast<int>(Token.Kind), OS);
300 Lines.push_back({
Line.first, encodeBase64(LineByteTokens)});
310 return "entity.name.function.cpp";
312 return "entity.name.function.method.cpp";
314 return "variable.other.cpp";
316 return "variable.other.field.cpp";
318 return "entity.name.type.class.cpp";
320 return "entity.name.type.enum.cpp";
322 return "variable.other.enummember.cpp";
324 return "entity.name.namespace.cpp";
326 return "entity.name.type.template.cpp";
328 llvm_unreachable(
"must not pass NumKinds to the function");
330 llvm_unreachable(
"unhandled HighlightingKind");
SourceLocation Loc
'#' location in the include directive
llvm::StringRef toTextMateScope(HighlightingKind Kind)
Converts a HighlightingKind to a corresponding TextMate scope (https://manual.macromates.com/en/language_grammars).
void elog(const char *Fmt, Ts &&... Vals)
bool operator==(const Ref &L, const Ref &R)
llvm::Optional< Range > getTokenRange(const SourceManager &SM, const LangOptions &LangOpts, SourceLocation TokLoc)
Returns the taken range at TokLoc.
std::vector< SemanticHighlightingInformation > toSemanticHighlightingInformation(llvm::ArrayRef< HighlightingToken > Tokens)
Stores and provides access to parsed AST.
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
static URISchemeRegistry::Add< TestScheme > X(TestScheme::Scheme, "Test schema")
static GeneratorRegistry::Add< MDGenerator > MD(MDGenerator::Format, "Generator for MD output.")
std::vector< HighlightingToken > getSemanticHighlightings(ParsedAST &AST)