clang-tools  7.0.0
Token.h
Go to the documentation of this file.
1 //===--- Token.h - Symbol Search primitive ----------------------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // Token objects represent a characteristic of a symbol, which can be used to
11 // perform efficient search. Tokens are keys for inverted index which are mapped
12 // to the corresponding posting lists.
13 //
14 // The symbol std::cout might have the tokens:
15 // * Scope "std::"
16 // * Trigram "cou"
17 // * Trigram "out"
18 // * Type "std::ostream"
19 //
20 //===----------------------------------------------------------------------===//
21 
22 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_DEX_TOKEN_H
23 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_DEX_TOKEN_H
24 
25 #include "llvm/ADT/DenseMap.h"
26 #include "llvm/Support/raw_ostream.h"
27 
28 #include <string>
29 #include <vector>
30 
31 namespace clang {
32 namespace clangd {
33 namespace dex {
34 
35 /// A Token represents an attribute of a symbol, such as a particular trigram
36 /// present in the name (used for fuzzy search).
37 ///
38 /// Tokens can be used to perform more sophisticated search queries by
39 /// constructing complex iterator trees.
40 struct Token {
41  /// Kind specifies Token type which defines semantics for the internal
42  /// representation. Each Kind has different representation stored in Data
43  /// field.
44  enum class Kind {
45  /// Represents trigram used for fuzzy search of unqualified symbol names.
46  ///
47  /// Data contains 3 bytes with trigram contents.
48  Trigram,
49  /// Scope primitives, e.g. "symbol belongs to namespace foo::bar".
50  ///
51  /// Data stroes full scope name , e.g. "foo::bar::baz::" or "" (for global
52  /// scope).
53  Scope,
54  /// Internal Token type for invalid/special tokens, e.g. empty tokens for
55  /// llvm::DenseMap.
56  Sentinel,
57  /// FIXME(kbobyrev): Add other Token Kinds
58  /// * Path with full or relative path to the directory in which symbol is
59  /// defined
60  /// * Type with qualified type name or its USR
61  };
62 
63  Token(Kind TokenKind, llvm::StringRef Data)
64  : Data(Data), TokenKind(TokenKind) {}
65 
66  bool operator==(const Token &Other) const {
67  return TokenKind == Other.TokenKind && Data == Other.Data;
68  }
69 
70  /// Representation which is unique among Token with the same Kind.
71  std::string Data;
73 
74  friend llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Token &T) {
75  return OS << T.Data;
76  }
77 
78 private:
79  friend llvm::hash_code hash_value(const Token &Token) {
80  return llvm::hash_combine(static_cast<int>(Token.TokenKind), Token.Data);
81  }
82 };
83 
84 } // namespace dex
85 } // namespace clangd
86 } // namespace clang
87 
88 namespace llvm {
89 
90 // Support Tokens as DenseMap keys.
91 template <> struct DenseMapInfo<clang::clangd::dex::Token> {
93  return {clang::clangd::dex::Token::Kind::Sentinel, "EmptyKey"};
94  }
95 
97  return {clang::clangd::dex::Token::Kind::Sentinel, "TombstoneKey"};
98  }
99 
100  static unsigned getHashValue(const clang::clangd::dex::Token &Tag) {
101  return hash_value(Tag);
102  }
103 
104  static bool isEqual(const clang::clangd::dex::Token &LHS,
105  const clang::clangd::dex::Token &RHS) {
106  return LHS == RHS;
107  }
108 };
109 
110 } // namespace llvm
111 
112 #endif
friend llvm::hash_code hash_value(const Token &Token)
Definition: Token.h:79
Some operations such as code completion produce a set of candidates.
static unsigned getHashValue(const clang::clangd::dex::Token &Tag)
Definition: Token.h:100
Token(Kind TokenKind, llvm::StringRef Data)
Definition: Token.h:63
friend llvm::raw_ostream & operator<<(llvm::raw_ostream &OS, const Token &T)
Definition: Token.h:74
Represents trigram used for fuzzy search of unqualified symbol names.
A Token represents an attribute of a symbol, such as a particular trigram present in the name (used f...
Definition: Token.h:40
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
Kind
Kind specifies Token type which defines semantics for the internal representation.
Definition: Token.h:44
bool operator==(const Token &Other) const
Definition: Token.h:66
std::string Data
Representation which is unique among Token with the same Kind.
Definition: Token.h:71
static bool isEqual(const clang::clangd::dex::Token &LHS, const clang::clangd::dex::Token &RHS)
Definition: Token.h:104
static clang::clangd::dex::Token getEmptyKey()
Definition: Token.h:92
Internal Token type for invalid/special tokens, e.g.
static clang::clangd::dex::Token getTombstoneKey()
Definition: Token.h:96