clang-tools  9.0.0
YAMLSerialization.cpp
Go to the documentation of this file.
1 //===-- YAMLSerialization.cpp ------------------------------------*- C++-*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // A YAML index file is a sequence of tagged entries.
10 // Each entry either encodes a Symbol or the list of references to a symbol
11 // (a "ref bundle").
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "Index.h"
16 #include "Relation.h"
17 #include "Serialization.h"
18 #include "SymbolLocation.h"
19 #include "SymbolOrigin.h"
20 #include "Trace.h"
21 #include "dex/Dex.h"
22 #include "llvm/ADT/Optional.h"
23 #include "llvm/ADT/SmallVector.h"
24 #include "llvm/ADT/StringRef.h"
25 #include "llvm/Support/Allocator.h"
26 #include "llvm/Support/Errc.h"
27 #include "llvm/Support/MemoryBuffer.h"
28 #include "llvm/Support/StringSaver.h"
29 #include "llvm/Support/YAMLTraits.h"
30 #include "llvm/Support/raw_ostream.h"
31 #include <cstdint>
32 
33 LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Symbol::IncludeHeaderWithReferences)
34 LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Ref)
35 
36 namespace {
37 using RefBundle =
38  std::pair<clang::clangd::SymbolID, std::vector<clang::clangd::Ref>>;
39 // This is a pale imitation of std::variant<Symbol, RefBundle, Relation>
40 struct VariantEntry {
41  llvm::Optional<clang::clangd::Symbol> Symbol;
42  llvm::Optional<RefBundle> Refs;
43  llvm::Optional<clang::clangd::Relation> Relation;
44 };
45 // A class helps YAML to serialize the 32-bit encoded position (Line&Column),
46 // as YAMLIO can't directly map bitfields.
47 struct YPosition {
48  uint32_t Line;
49  uint32_t Column;
50 };
51 
52 } // namespace
53 namespace llvm {
54 namespace yaml {
55 
56 using clang::clangd::Ref;
66 using clang::index::SymbolLanguage;
67 using clang::index::SymbolRole;
68 
69 // Helper to (de)serialize the SymbolID. We serialize it as a hex string.
72  NormalizedSymbolID(IO &, const SymbolID &ID) {
73  llvm::raw_string_ostream OS(HexString);
74  OS << ID;
75  }
76 
78  auto ID = SymbolID::fromStr(HexString);
79  if (!ID) {
80  I.setError(llvm::toString(ID.takeError()));
81  return SymbolID();
82  }
83  return *ID;
84  }
85 
86  std::string HexString;
87 };
88 
92  Flag = static_cast<uint8_t>(F);
93  }
94 
96  return static_cast<Symbol::SymbolFlag>(Flag);
97  }
98 
99  uint8_t Flag = 0;
100 };
101 
105  Origin = static_cast<uint8_t>(O);
106  }
107 
108  SymbolOrigin denormalize(IO &) { return static_cast<SymbolOrigin>(Origin); }
109 
110  uint8_t Origin = 0;
111 };
112 
113 template <> struct MappingTraits<YPosition> {
114  static void mapping(IO &IO, YPosition &Value) {
115  IO.mapRequired("Line", Value.Line);
116  IO.mapRequired("Column", Value.Column);
117  }
118 };
119 
123  NormalizedPosition(IO &, const Position &Pos) {
124  P.Line = Pos.line();
125  P.Column = Pos.column();
126  }
127 
129  Position Pos;
130  Pos.setLine(P.Line);
131  Pos.setColumn(P.Column);
132  return Pos;
133  }
134  YPosition P;
135 };
136 
139  NormalizedFileURI(IO &, const char *FileURI) { URI = FileURI; }
140 
141  const char *denormalize(IO &IO) {
142  assert(IO.getContext() &&
143  "Expecting an UniqueStringSaver to allocate data");
144  return static_cast<llvm::UniqueStringSaver *>(IO.getContext())
145  ->save(URI)
146  .data();
147  }
148 
149  std::string URI;
150 };
151 
152 template <> struct MappingTraits<SymbolLocation> {
153  static void mapping(IO &IO, SymbolLocation &Value) {
154  MappingNormalization<NormalizedFileURI, const char *> NFile(IO,
155  Value.FileURI);
156  IO.mapRequired("FileURI", NFile->URI);
157  MappingNormalization<NormalizedPosition, SymbolLocation::Position> NStart(
158  IO, Value.Start);
159  IO.mapRequired("Start", NStart->P);
160  MappingNormalization<NormalizedPosition, SymbolLocation::Position> NEnd(
161  IO, Value.End);
162  IO.mapRequired("End", NEnd->P);
163  }
164 };
165 
166 template <> struct MappingTraits<SymbolInfo> {
167  static void mapping(IO &io, SymbolInfo &SymInfo) {
168  // FIXME: expose other fields?
169  io.mapRequired("Kind", SymInfo.Kind);
170  io.mapRequired("Lang", SymInfo.Lang);
171  }
172 };
173 
174 template <>
175 struct MappingTraits<clang::clangd::Symbol::IncludeHeaderWithReferences> {
176  static void mapping(IO &io,
178  io.mapRequired("Header", Inc.IncludeHeader);
179  io.mapRequired("References", Inc.References);
180  }
181 };
182 
183 template <> struct MappingTraits<Symbol> {
184  static void mapping(IO &IO, Symbol &Sym) {
185  MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO, Sym.ID);
186  MappingNormalization<NormalizedSymbolFlag, Symbol::SymbolFlag> NSymbolFlag(
187  IO, Sym.Flags);
188  MappingNormalization<NormalizedSymbolOrigin, SymbolOrigin> NSymbolOrigin(
189  IO, Sym.Origin);
190  IO.mapRequired("ID", NSymbolID->HexString);
191  IO.mapRequired("Name", Sym.Name);
192  IO.mapRequired("Scope", Sym.Scope);
193  IO.mapRequired("SymInfo", Sym.SymInfo);
194  IO.mapOptional("CanonicalDeclaration", Sym.CanonicalDeclaration,
195  SymbolLocation());
196  IO.mapOptional("Definition", Sym.Definition, SymbolLocation());
197  IO.mapOptional("References", Sym.References, 0u);
198  IO.mapOptional("Origin", NSymbolOrigin->Origin);
199  IO.mapOptional("Flags", NSymbolFlag->Flag);
200  IO.mapOptional("Signature", Sym.Signature);
201  IO.mapOptional("TemplateSpecializationArgs",
203  IO.mapOptional("CompletionSnippetSuffix", Sym.CompletionSnippetSuffix);
204  IO.mapOptional("Documentation", Sym.Documentation);
205  IO.mapOptional("ReturnType", Sym.ReturnType);
206  IO.mapOptional("Type", Sym.Type);
207  IO.mapOptional("IncludeHeaders", Sym.IncludeHeaders);
208  }
209 };
210 
211 template <> struct ScalarEnumerationTraits<SymbolLanguage> {
212  static void enumeration(IO &IO, SymbolLanguage &Value) {
213  IO.enumCase(Value, "C", SymbolLanguage::C);
214  IO.enumCase(Value, "Cpp", SymbolLanguage::CXX);
215  IO.enumCase(Value, "ObjC", SymbolLanguage::ObjC);
216  IO.enumCase(Value, "Swift", SymbolLanguage::Swift);
217  }
218 };
219 
220 template <> struct ScalarEnumerationTraits<SymbolKind> {
221  static void enumeration(IO &IO, SymbolKind &Value) {
222 #define DEFINE_ENUM(name) IO.enumCase(Value, #name, SymbolKind::name)
223 
225  DEFINE_ENUM(Function);
226  DEFINE_ENUM(Module);
227  DEFINE_ENUM(Namespace);
228  DEFINE_ENUM(NamespaceAlias);
229  DEFINE_ENUM(Macro);
230  DEFINE_ENUM(Enum);
231  DEFINE_ENUM(Struct);
232  DEFINE_ENUM(Class);
233  DEFINE_ENUM(Protocol);
234  DEFINE_ENUM(Extension);
235  DEFINE_ENUM(Union);
236  DEFINE_ENUM(TypeAlias);
237  DEFINE_ENUM(Function);
238  DEFINE_ENUM(Variable);
239  DEFINE_ENUM(Field);
240  DEFINE_ENUM(EnumConstant);
241  DEFINE_ENUM(InstanceMethod);
242  DEFINE_ENUM(ClassMethod);
243  DEFINE_ENUM(StaticMethod);
244  DEFINE_ENUM(InstanceProperty);
245  DEFINE_ENUM(ClassProperty);
246  DEFINE_ENUM(StaticProperty);
247  DEFINE_ENUM(Constructor);
248  DEFINE_ENUM(Destructor);
249  DEFINE_ENUM(ConversionFunction);
250  DEFINE_ENUM(Parameter);
251  DEFINE_ENUM(Using);
252 
253 #undef DEFINE_ENUM
254  }
255 };
256 
257 template <> struct MappingTraits<RefBundle> {
258  static void mapping(IO &IO, RefBundle &Refs) {
259  MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO,
260  Refs.first);
261  IO.mapRequired("ID", NSymbolID->HexString);
262  IO.mapRequired("References", Refs.second);
263  }
264 };
265 
268  NormalizedRefKind(IO &, RefKind O) { Kind = static_cast<uint8_t>(O); }
269 
270  RefKind denormalize(IO &) { return static_cast<RefKind>(Kind); }
271 
272  uint8_t Kind = 0;
273 };
274 
275 template <> struct MappingTraits<Ref> {
276  static void mapping(IO &IO, Ref &R) {
277  MappingNormalization<NormalizedRefKind, RefKind> NKind(IO, R.Kind);
278  IO.mapRequired("Kind", NKind->Kind);
279  IO.mapRequired("Location", R.Location);
280  }
281 };
282 
285  NormalizedSymbolRole(IO &IO, SymbolRole R) {
286  Kind = static_cast<uint8_t>(clang::clangd::symbolRoleToRelationKind(R));
287  }
288 
289  SymbolRole denormalize(IO &IO) {
291  static_cast<RelationKind>(Kind));
292  }
293 
294  uint8_t Kind = 0;
295 };
296 
297 template <> struct MappingTraits<SymbolID> {
298  static void mapping(IO &IO, SymbolID &ID) {
299  MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO, ID);
300  IO.mapRequired("ID", NSymbolID->HexString);
301  }
302 };
303 
304 template <> struct MappingTraits<Relation> {
305  static void mapping(IO &IO, Relation &Relation) {
306  MappingNormalization<NormalizedSymbolRole, SymbolRole> NRole(
307  IO, Relation.Predicate);
308  IO.mapRequired("Subject", Relation.Subject);
309  IO.mapRequired("Predicate", NRole->Kind);
310  IO.mapRequired("Object", Relation.Object);
311  }
312 };
313 
314 template <> struct MappingTraits<VariantEntry> {
315  static void mapping(IO &IO, VariantEntry &Variant) {
316  if (IO.mapTag("!Symbol", Variant.Symbol.hasValue())) {
317  if (!IO.outputting())
318  Variant.Symbol.emplace();
319  MappingTraits<Symbol>::mapping(IO, *Variant.Symbol);
320  } else if (IO.mapTag("!Refs", Variant.Refs.hasValue())) {
321  if (!IO.outputting())
322  Variant.Refs.emplace();
323  MappingTraits<RefBundle>::mapping(IO, *Variant.Refs);
324  } else if (IO.mapTag("!Relations", Variant.Relation.hasValue())) {
325  if (!IO.outputting())
326  Variant.Relation.emplace();
327  MappingTraits<Relation>::mapping(IO, *Variant.Relation);
328  }
329  }
330 };
331 
332 } // namespace yaml
333 } // namespace llvm
334 
335 namespace clang {
336 namespace clangd {
337 
338 void writeYAML(const IndexFileOut &O, llvm::raw_ostream &OS) {
339  llvm::yaml::Output Yout(OS);
340  for (const auto &Sym : *O.Symbols) {
341  VariantEntry Entry;
342  Entry.Symbol = Sym;
343  Yout << Entry;
344  }
345  if (O.Refs)
346  for (auto &Sym : *O.Refs) {
347  VariantEntry Entry;
348  Entry.Refs = Sym;
349  Yout << Entry;
350  }
351  if (O.Relations)
352  for (auto &R : *O.Relations) {
353  VariantEntry Entry;
354  Entry.Relation = R;
355  Yout << Entry;
356  }
357 }
358 
359 llvm::Expected<IndexFileIn> readYAML(llvm::StringRef Data) {
363  llvm::BumpPtrAllocator
364  Arena; // store the underlying data of Position::FileURI.
365  llvm::UniqueStringSaver Strings(Arena);
366  llvm::yaml::Input Yin(Data, &Strings);
367  while (Yin.setCurrentDocument()) {
368  llvm::yaml::EmptyContext Ctx;
369  VariantEntry Variant;
370  yamlize(Yin, Variant, true, Ctx);
371  if (Yin.error())
372  return llvm::errorCodeToError(Yin.error());
373 
374  if (Variant.Symbol)
375  Symbols.insert(*Variant.Symbol);
376  if (Variant.Refs)
377  for (const auto &Ref : Variant.Refs->second)
378  Refs.insert(Variant.Refs->first, Ref);
379  if (Variant.Relation)
380  Relations.insert(*Variant.Relation);
381  Yin.nextDocument();
382  }
383 
385  Result.Symbols.emplace(std::move(Symbols).build());
386  Result.Refs.emplace(std::move(Refs).build());
387  Result.Relations.emplace(std::move(Relations).build());
388  return std::move(Result);
389 }
390 
391 std::string toYAML(const Symbol &S) {
392  std::string Buf;
393  {
394  llvm::raw_string_ostream OS(Buf);
395  llvm::yaml::Output Yout(OS);
396  Symbol Sym = S; // copy: Yout<< requires mutability.
397  Yout << Sym;
398  }
399  return Buf;
400 }
401 
402 std::string toYAML(const std::pair<SymbolID, llvm::ArrayRef<Ref>> &Data) {
403  RefBundle Refs = {Data.first, Data.second};
404  std::string Buf;
405  {
406  llvm::raw_string_ostream OS(Buf);
407  llvm::yaml::Output Yout(OS);
408  Yout << Refs;
409  }
410  return Buf;
411 }
412 
413 std::string toYAML(const Relation &R) {
414  std::string Buf;
415  {
416  llvm::raw_string_ostream OS(Buf);
417  llvm::yaml::Output Yout(OS);
418  Relation Rel = R; // copy: Yout<< requires mutability.
419  Yout << Rel;
420  }
421  return Buf;
422 }
423 
424 } // namespace clangd
425 } // namespace clang
llvm::Expected< IndexFileIn > readYAML(llvm::StringRef)
llvm::Optional< SymbolSlab > Symbols
Definition: Serialization.h:43
Symbol::SymbolFlag denormalize(IO &)
Some operations such as code completion produce a set of candidates.
Represents a relation between two symbols.
Definition: Relation.h:25
static void mapping(IO &io, SymbolInfo &SymInfo)
This defines Dex - a symbol index implementation based on query iterators over symbol tokens...
static void mapping(IO &io, clang::clangd::Symbol::IncludeHeaderWithReferences &Inc)
unsigned References
The number of translation units that reference this symbol and include this header.
Definition: Symbol.h:104
void writeYAML(const IndexFileOut &, llvm::raw_ostream &)
RelationKind symbolRoleToRelationKind(index::SymbolRole Role)
clang::find_all_symbols::SymbolInfo::SymbolKind SymbolKind
Definition: SymbolInfo.cpp:21
Represents a symbol occurrence in the source file.
Definition: Ref.h:52
void insert(const Symbol &S)
Adds a symbol, overwriting any existing one with the same ID.
Definition: Symbol.cpp:50
static llvm::StringRef toString(SpecialMemberFunctionsCheck::SpecialMemberFunctionKind K)
int Column
static void enumeration(IO &IO, SymbolKind &Value)
llvm::StringRef Scope
The containing namespace. e.g. "" (global), "ns::" (top-level namespace).
Definition: Symbol.h:44
llvm::Optional< RelationSlab > Relations
Definition: Serialization.h:45
static void mapping(IO &IO, Ref &R)
const SymbolSlab * Symbols
Definition: Serialization.h:56
unsigned References
The number of translation units that reference this symbol from their main file.
Definition: Symbol.h:59
SymbolSlab::Builder is a mutable container that can &#39;freeze&#39; to SymbolSlab.
Definition: Symbol.h:199
SymbolID ID
The ID of the symbol.
Definition: Symbol.h:38
static void mapping(IO &IO, Symbol &Sym)
index::SymbolInfo SymInfo
The symbol information, like symbol kind.
Definition: Symbol.h:40
BindArgumentKind Kind
static void mapping(IO &IO, SymbolID &ID)
llvm::BumpPtrAllocator Arena
SymbolLocation Definition
The location of the symbol&#39;s definition, if one was found.
Definition: Symbol.h:47
Context Ctx
clang::find_all_symbols::SymbolInfo SymbolInfo
NormalizedSymbolFlag(IO &, Symbol::SymbolFlag F)
index::SymbolRole Predicate
Definition: Relation.h:27
llvm::SmallVector< IncludeHeaderWithReferences, 1 > IncludeHeaders
One Symbol can potentially be incuded via different headers.
Definition: Symbol.h:111
SymbolFlag Flags
Definition: Symbol.h:128
llvm::StringRef Signature
A brief description of the symbol that can be appended in the completion candidate list...
Definition: Symbol.h:65
SymbolLocation Location
The source location where the symbol is named.
Definition: Ref.h:54
static void enumeration(IO &IO, SymbolLanguage &Value)
llvm::StringRef Documentation
Documentation including comment for the symbol declaration.
Definition: Symbol.h:76
SymbolLocation CanonicalDeclaration
The location of the preferred declaration of the symbol.
Definition: Symbol.h:56
RelationSlab Relations
void insert(const Relation &R)
Adds a relation to the slab.
Definition: Relation.h:69
RefKind
Describes the kind of a cross-reference.
Definition: Ref.h:28
NormalizedSymbolRole(IO &IO, SymbolRole R)
SymbolSlab Symbols
std::string toYAML(const std::pair< SymbolID, llvm::ArrayRef< Ref >> &Data)
RelationSlab::Builder is a mutable container that can &#39;freeze&#39; to RelationSlab.
Definition: Relation.h:66
RefSlab::Builder is a mutable container that can &#39;freeze&#39; to RefSlab.
Definition: Ref.h:93
static void mapping(IO &IO, SymbolLocation &Value)
std::vector< llvm::StringRef > Strings
NormalizedPosition(IO &, const Position &Pos)
const RelationSlab * Relations
Definition: Serialization.h:58
llvm::Optional< RefSlab > Refs
Definition: Serialization.h:44
llvm::StringRef IncludeHeader
This can be either a URI of the header to be #include&#39;d for this symbol, or a literal header quoted w...
Definition: Symbol.h:101
The class presents a C++ symbol, e.g.
Definition: Symbol.h:36
Position Start
The symbol range, using half-open range [Start, End).
index::SymbolRole relationKindToSymbolRole(RelationKind Kind)
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
llvm::StringRef Name
The unqualified name of the symbol, e.g. "bar" (for ns::bar).
Definition: Symbol.h:42
static void mapping(IO &IO, RefBundle &Refs)
NormalizedSymbolID(IO &, const SymbolID &ID)
NormalizedSymbolOrigin(IO &, SymbolOrigin O)
llvm::Optional< llvm::Expected< tooling::AtomicChanges > > Result
Definition: Rename.cpp:36
static void mapping(IO &IO, VariantEntry &Variant)
void insert(const SymbolID &ID, const Ref &S)
Adds a ref to the slab. Deep copy: Strings will be owned by the slab.
Definition: Ref.cpp:34
llvm::StringRef CompletionSnippetSuffix
What to insert when completing this symbol, after the symbol name.
Definition: Symbol.h:74
#define DEFINE_ENUM(name)
RefSlab Refs
static void mapping(IO &IO, YPosition &Value)
llvm::StringRef Type
Raw representation of the OpaqueType of the symbol, used for scoring purposes.
Definition: Symbol.h:85
SymbolOrigin Origin
Where this symbol came from. Usually an index provides a constant value.
Definition: Symbol.h:61
NormalizedFileURI(IO &, const char *FileURI)
llvm::StringRef TemplateSpecializationArgs
Argument list in human-readable format, will be displayed to help disambiguate between different spec...
Definition: Symbol.h:69
llvm::StringRef ReturnType
Type when this symbol is used in an expression.
Definition: Symbol.h:80
std::array< uint8_t, 20 > SymbolID
static void mapping(IO &IO, Relation &Relation)
RefKind Kind
Definition: Ref.h:55