clang-tools 8.0.0

Token.h
Go to the documentation of this file.
1 //===--- Token.h - Symbol Search primitive ----------------------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// Token objects represent a characteristic of a symbol, which can be used to
12 /// perform efficient search. Tokens are keys for inverted index which are
13 /// mapped to the corresponding posting lists.
14 ///
15 /// The symbol std::cout might have the tokens:
16 /// * Scope "std::"
17 /// * Trigram "cou"
18 /// * Trigram "out"
19 /// * Type "std::ostream"
20 ///
21 //===----------------------------------------------------------------------===//
22
23 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_DEX_TOKEN_H
24 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_DEX_TOKEN_H
25
26 #include "index/Index.h"
27 #include "llvm/ADT/DenseMap.h"
28 #include "llvm/Support/raw_ostream.h"
29 #include <string>
30 #include <vector>
31
32 namespace clang {
33 namespace clangd {
34 namespace dex {
35
36 /// A Token represents an attribute of a symbol, such as a particular trigram
37 /// present in the name (used for fuzzy search).
38 ///
39 /// Tokens can be used to perform more sophisticated search queries by
40 /// constructing complex iterator trees.
41 struct Token {
42 /// Kind specifies Token type which defines semantics for the internal
43 /// representation. Each Kind has different representation stored in Data
44 /// field.
45 // FIXME(kbobyrev): Storing Data hash would be more efficient than storing raw
46 // strings. For example, PathURI store URIs of each directory and its parents,
47 // which induces a lot of overhead because these paths tend to be long and
48 // each parent directory is a prefix.
49 enum class Kind {
50 /// Represents trigram used for fuzzy search of unqualified symbol names.
51 ///
52 /// Data contains 3 bytes with trigram contents.
53 Trigram,
54 /// Scope primitives, e.g. "symbol belongs to namespace foo::bar".
55 ///
56 /// Data stroes full scope name, e.g. "foo::bar::baz::" or "" (for global
57 /// scope).
58 Scope,
59 /// Path Proximity URI to symbol declaration.
60 ///
61 /// Data stores path URI of symbol declaration file or its parent.
62 ///
63 /// Example: "file:///path/to/clang-tools-extra/clangd/index/SymbolIndex.h"
64 /// and some amount of its parents.
65 ProximityURI,
66 /// Internal Token type for invalid/special tokens, e.g. empty tokens for
67 /// llvm::DenseMap.
68 Sentinel,
69 /// FIXME(kbobyrev): Add other Token Kinds
70 /// * Type with qualified type name or its USR
71 };
72
73 Token(Kind TokenKind, llvm::StringRef Data)
74 : Data(Data), TokenKind(TokenKind) {}
75
76 bool operator==(const Token &Other) const {
77 return TokenKind == Other.TokenKind && Data == Other.Data;
78 }
79
80 /// Representation which is unique among Token with the same Kind.
81 std::string Data;
82 Kind TokenKind;
83
84 friend llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Token &T) {
85 switch (T.TokenKind) {
86 case Kind::Trigram:
87 OS << "T=";
88 break;
89 case Kind::Scope:
90 OS << "S=";
91 break;
92 case Kind::ProximityURI:
93 OS << "U=";
94 break;
95 case Kind::Sentinel:
96 OS << "?=";
97 break;
98 }
99 return OS << T.Data;
100 }
101
102 private:
103 friend llvm::hash_code hash_value(const Token &Token) {
104 return llvm::hash_combine(static_cast<int>(Token.TokenKind), Token.Data);
105 }
106 };
107
108 } // namespace dex
109 } // namespace clangd
110 } // namespace clang
111
112 namespace llvm {
113
114 // Support Tokens as DenseMap keys.
115 template <> struct DenseMapInfo<clang::clangd::dex::Token> {
116 static inline clang::clangd::dex::Token getEmptyKey() {
117 return {clang::clangd::dex::Token::Kind::Sentinel, "EmptyKey"};
118 }
119
120 static inline clang::clangd::dex::Token getTombstoneKey() {
121 return {clang::clangd::dex::Token::Kind::Sentinel, "TombstoneKey"};
122 }
123
124 static unsigned getHashValue(const clang::clangd::dex::Token &Tag) {
125 return hash_value(Tag);
126 }
127
128 static bool isEqual(const clang::clangd::dex::Token &LHS,
129 const clang::clangd::dex::Token &RHS) {
130 return LHS == RHS;
131 }
132 };
133
134 } // namespace llvm
135
136 #endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_DEX_TOKEN_H
clang::clangd::dex::Token::hash_value
friend llvm::hash_code hash_value(const Token &Token)
Definition: Token.h:103
clang::clangd::dex::Token::TokenKind
Kind TokenKind
Definition: Token.h:82
llvm
Some operations such as code completion produce a set of candidates.
Definition: YAMLGenerator.cpp:28
Index.h
llvm::DenseMapInfo< clang::clangd::dex::Token >::getHashValue
static unsigned getHashValue(const clang::clangd::dex::Token &Tag)
Definition: Token.h:124
clang::clangd::dex::Token::Token
Token(Kind TokenKind, llvm::StringRef Data)
Definition: Token.h:73
clang::clangd::dex::Token::operator<<
friend llvm::raw_ostream & operator<<(llvm::raw_ostream &OS, const Token &T)
Definition: Token.h:84
clang::clangd::dex::Token::Kind::Trigram
Represents trigram used for fuzzy search of unqualified symbol names.
clang::clangd::dex::Token
A Token represents an attribute of a symbol, such as a particular trigram present in the name (used f...
Definition: Token.h:41
clang::clangd::dex::Token::Kind::Scope
Scope primitives, e.g.
clang
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
Definition: ChangeNamespace.cpp:18
clang::clangd::dex::Token::Kind::ProximityURI
Path Proximity URI to symbol declaration.
clang::clangd::dex::Token::Kind
Kind
Kind specifies Token type which defines semantics for the internal representation.
Definition: Token.h:49
clang::clangd::dex::Token::operator==
bool operator==(const Token &Other) const
Definition: Token.h:76
clang::clangd::dex::Token::Data
std::string Data
Representation which is unique among Token with the same Kind.
Definition: Token.h:81
llvm::DenseMapInfo< clang::clangd::dex::Token >::isEqual
static bool isEqual(const clang::clangd::dex::Token &LHS, const clang::clangd::dex::Token &RHS)
Definition: Token.h:128
llvm::DenseMapInfo< clang::clangd::dex::Token >::getEmptyKey
static clang::clangd::dex::Token getEmptyKey()
Definition: Token.h:116
clang::clangd::dex::Token::Kind::Sentinel
Internal Token type for invalid/special tokens, e.g.
llvm::DenseMapInfo< clang::clangd::dex::Token >::getTombstoneKey
static clang::clangd::dex::Token getTombstoneKey()
Definition: Token.h:120