13 #include "clang/AST/ASTTypeTraits.h"
14 #include "clang/AST/Decl.h"
15 #include "clang/AST/DeclCXX.h"
16 #include "clang/AST/Expr.h"
17 #include "clang/AST/ExprCXX.h"
18 #include "clang/AST/PrettyPrinter.h"
19 #include "clang/AST/RecursiveASTVisitor.h"
20 #include "clang/AST/TypeLoc.h"
21 #include "clang/Basic/OperatorKinds.h"
22 #include "clang/Basic/SourceLocation.h"
23 #include "clang/Basic/SourceManager.h"
24 #include "clang/Basic/TokenKinds.h"
25 #include "clang/Lex/Lexer.h"
26 #include "clang/Tooling/Syntax/Tokens.h"
27 #include "llvm/ADT/STLExtras.h"
28 #include "llvm/ADT/StringExtras.h"
29 #include "llvm/Support/Casting.h"
30 #include "llvm/Support/raw_ostream.h"
37 using Node = SelectionTree::Node;
38 using ast_type_traits::DynTypedNode;
41 void recordMetrics(
const SelectionTree &S) {
42 static constexpr trace::Metric SelectionUsedRecovery(
44 static constexpr trace::Metric RecoveryType(
"selection_recovery_type",
46 const auto *Common = S.commonAncestor();
47 for (
const auto *N = Common; N; N = N->Parent) {
48 if (
const auto *RE = N->ASTNode.get<RecoveryExpr>()) {
49 SelectionUsedRecovery.record(1);
50 RecoveryType.record(RE->isTypeDependent() ? 0 : 1);
55 SelectionUsedRecovery.record(0);
76 template <
typename T>
class IntervalSet {
78 IntervalSet(llvm::ArrayRef<T>
Range) { UnclaimedRanges.insert(
Range); }
83 llvm::SmallVector<llvm::ArrayRef<T>, 4> erase(llvm::ArrayRef<T> Claim) {
84 llvm::SmallVector<llvm::ArrayRef<T>, 4>
Out;
93 auto Overlap = std::make_pair(
94 UnclaimedRanges.lower_bound({Claim.begin(), Claim.begin()}),
95 UnclaimedRanges.lower_bound({Claim.end(), Claim.end()}));
97 if (Overlap.first != UnclaimedRanges.begin()) {
100 if (Overlap.first->end() <= Claim.begin())
103 if (Overlap.first == Overlap.second)
107 auto OutFirst =
Out.insert(
Out.end(), Overlap.first, Overlap.second);
111 llvm::ArrayRef<T> RemainingHead, RemainingTail;
112 if (Claim.begin() > OutFirst->begin()) {
113 RemainingHead = {OutFirst->begin(), Claim.begin()};
114 *OutFirst = {Claim.begin(), OutFirst->end()};
116 if (Claim.end() <
Out.back().end()) {
117 RemainingTail = {Claim.end(),
Out.back().end()};
118 Out.back() = {
Out.back().begin(), Claim.end()};
122 UnclaimedRanges.erase(Overlap.first, Overlap.second);
124 if (!RemainingHead.empty())
125 UnclaimedRanges.insert(RemainingHead);
126 if (!RemainingTail.empty())
127 UnclaimedRanges.insert(RemainingTail);
133 using TokenRange = llvm::ArrayRef<T>;
135 bool operator()(llvm::ArrayRef<T> L, llvm::ArrayRef<T> R)
const {
136 return L.begin() < R.begin();
141 std::set<llvm::ArrayRef<T>, RangeLess> UnclaimedRanges;
149 static_cast<SelectionTree::Selection>(
157 if (Result == NoTokens)
159 else if (Result != New)
166 bool shouldIgnore(
const syntax::Token &Tok) {
167 return Tok.kind() == tok::comment || Tok.kind() == tok::semi;
172 bool isFirstExpansion(FileID Target, SourceLocation SpellingLoc,
173 const SourceManager &SM) {
174 SourceLocation Prev = SpellingLoc;
178 SourceLocation Next = SM.getMacroArgExpandedLocation(Prev);
181 if (SM.getFileID(Next) == Target)
186 if (SM.getFileID(Next) == SM.getFileID(Prev))
207 class SelectionTester {
210 SelectionTester(
const syntax::TokenBuffer &Buf, FileID SelFile,
211 unsigned SelBegin,
unsigned SelEnd,
const SourceManager &SM)
212 : SelFile(SelFile), SM(SM) {
214 auto AllSpelledTokens = Buf.spelledTokens(SelFile);
215 const syntax::Token *SelFirst =
216 llvm::partition_point(AllSpelledTokens, [&](
const syntax::Token &Tok) {
217 return SM.getFileOffset(Tok.endLocation()) <= SelBegin;
219 const syntax::Token *SelLimit = std::partition_point(
220 SelFirst, AllSpelledTokens.end(), [&](
const syntax::Token &Tok) {
221 return SM.getFileOffset(Tok.location()) < SelEnd;
224 for (
const syntax::Token *T = SelFirst; T < SelLimit; ++T) {
225 if (shouldIgnore(*T))
227 SpelledTokens.emplace_back();
228 Tok &S = SpelledTokens.back();
229 S.Offset = SM.getFileOffset(T->location());
230 if (S.Offset >= SelBegin && S.Offset + T->length() <= SelEnd)
240 test(llvm::ArrayRef<syntax::Token> ExpandedTokens)
const {
241 if (SpelledTokens.empty())
244 while (!ExpandedTokens.empty()) {
246 FileID FID = SM.getFileID(ExpandedTokens.front().location());
247 auto Batch = ExpandedTokens.take_while([&](
const syntax::Token &T) {
248 return SM.getFileID(T.location()) == FID;
250 assert(!Batch.empty());
251 ExpandedTokens = ExpandedTokens.drop_front(Batch.size());
253 update(Result, testChunk(FID, Batch));
261 bool mayHit(SourceRange R)
const {
262 if (SpelledTokens.empty())
264 auto B = SM.getDecomposedLoc(R.getBegin());
265 auto E = SM.getDecomposedLoc(R.getEnd());
266 if (B.first == SelFile &&
E.first == SelFile)
267 if (
E.second < SpelledTokens.front().Offset ||
268 B.second > SpelledTokens.back().Offset)
276 testChunk(FileID FID, llvm::ArrayRef<syntax::Token> Batch)
const {
277 assert(!Batch.empty());
278 SourceLocation StartLoc = Batch.front().location();
287 if (FID == SelFile) {
288 return testTokenRange(SM.getFileOffset(Batch.front().location()),
289 SM.getFileOffset(Batch.back().location()));
294 if (StartLoc.isFileID()) {
295 for (SourceLocation
Loc = Batch.front().location();
Loc.isValid();
296 Loc = SM.getIncludeLoc(SM.getFileID(
Loc))) {
297 if (SM.getFileID(
Loc) == SelFile)
299 return testToken(SM.getFileOffset(
Loc));
304 assert(StartLoc.isMacroID());
306 SourceLocation ArgStart = SM.getTopMacroCallerLoc(StartLoc);
307 if (SM.getFileID(ArgStart) == SelFile) {
308 if (isFirstExpansion(FID, ArgStart, SM)) {
309 SourceLocation ArgEnd =
310 SM.getTopMacroCallerLoc(Batch.back().location());
311 return testTokenRange(SM.getFileOffset(ArgStart),
312 SM.getFileOffset(ArgEnd));
320 auto Expansion = SM.getDecomposedExpansionLoc(StartLoc);
321 if (Expansion.first == SelFile)
323 return testToken(Expansion.second);
330 assert(Begin <= End);
332 if (End < SpelledTokens.front().Offset ||
333 Begin > SpelledTokens.back().Offset)
337 auto B = llvm::partition_point(
338 SpelledTokens, [&](
const Tok &T) {
return T.Offset < Begin; });
339 auto E = std::partition_point(
340 B, SpelledTokens.end(), [&](
const Tok &T) {
return T.Offset <= End; });
343 bool ExtendsOutsideSelection = Begin < SpelledTokens.front().Offset ||
344 End > SpelledTokens.back().Offset;
347 for (
auto It = B; It !=
E; ++It)
348 update(Result, It->Selected);
355 if (
Offset < SpelledTokens.front().Offset ||
356 Offset > SpelledTokens.back().Offset)
359 auto It = llvm::partition_point(
360 SpelledTokens, [&](
const Tok &T) {
return T.Offset <
Offset; });
361 if (It != SpelledTokens.end() && It->Offset ==
Offset)
370 std::vector<Tok> SpelledTokens;
372 const SourceManager &SM;
376 void printNodeKind(llvm::raw_ostream &
OS,
const DynTypedNode &N) {
377 if (
const TypeLoc *TL = N.get<TypeLoc>()) {
380 if (TL->getTypeLocClass() == TypeLoc::Qualified)
381 OS <<
"QualifiedTypeLoc";
383 OS << TL->getType()->getTypeClassName() <<
"TypeLoc";
385 OS << N.getNodeKind().asStringRef();
390 std::string printNodeToString(
const DynTypedNode &N,
const PrintingPolicy &
PP) {
392 llvm::raw_string_ostream
OS(S);
393 printNodeKind(
OS, N);
395 return std::move(
OS.str());
399 bool isImplicit(
const Stmt *S) {
403 if (
auto *ICE = llvm::dyn_cast<ImplicitCastExpr>(S))
404 S = ICE->getSubExprAsWritten();
407 if (
auto *CTI = llvm::dyn_cast<CXXThisExpr>(S))
408 if (CTI->isImplicit())
411 if (
auto *DRE = llvm::dyn_cast<DeclRefExpr>(S)) {
412 if (
auto *FD = llvm::dyn_cast<FunctionDecl>(DRE->getDecl())) {
413 switch (FD->getOverloadedOperator()) {
433 class SelectionVisitor :
public RecursiveASTVisitor<SelectionVisitor> {
437 static std::deque<Node> collect(ASTContext &
AST,
438 const syntax::TokenBuffer &Tokens,
439 const PrintingPolicy &
PP,
unsigned Begin,
440 unsigned End, FileID
File) {
441 SelectionVisitor V(
AST, Tokens,
PP, Begin, End,
File);
443 assert(V.Stack.size() == 1 &&
"Unpaired push/pop?");
444 assert(V.Stack.top() == &V.Nodes.front());
445 return std::move(V.Nodes);
459 bool TraverseDecl(
Decl *
X) {
460 if (
X && isa<TranslationUnitDecl>(
X))
461 return Base::TraverseDecl(
X);
463 if (
X &&
X->isImplicit())
465 return traverseNode(
X, [&] {
return Base::TraverseDecl(
X); });
467 bool TraverseTypeLoc(TypeLoc
X) {
468 return traverseNode(&
X, [&] {
return Base::TraverseTypeLoc(
X); });
470 bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc
X) {
472 &
X, [&] {
return Base::TraverseNestedNameSpecifierLoc(
X); });
474 bool TraverseConstructorInitializer(CXXCtorInitializer *
X) {
476 X, [&] {
return Base::TraverseConstructorInitializer(
X); });
479 bool dataTraverseStmtPre(Stmt *
X) {
480 if (!
X || isImplicit(
X))
482 auto N = DynTypedNode::create(*
X);
483 if (canSafelySkipNode(N))
486 if (shouldSkipChildren(
X)) {
492 bool dataTraverseStmtPost(Stmt *
X) {
501 bool TraverseQualifiedTypeLoc(QualifiedTypeLoc QX) {
502 return traverseNode<TypeLoc>(
503 &QX, [&] {
return TraverseTypeLoc(QX.getUnqualifiedLoc()); });
506 bool TraverseNestedNameSpecifier(NestedNameSpecifier *) {
return true; }
507 bool TraverseType(QualType) {
return true; }
512 bool TraverseCXXForRangeStmt(CXXForRangeStmt *S) {
513 return traverseNode(S, [&] {
514 return TraverseStmt(S->getInit()) && TraverseDecl(S->getLoopVariable()) &&
515 TraverseStmt(S->getRangeInit()) && TraverseStmt(S->getBody());
519 bool TraverseOpaqueValueExpr(OpaqueValueExpr *
E) {
520 return traverseNode(
E, [&] {
return TraverseStmt(
E->getSourceExpr()); });
523 bool TraversePseudoObjectExpr(PseudoObjectExpr *
E) {
524 return traverseNode(
E, [&] {
return TraverseStmt(
E->getSyntacticForm()); });
528 using Base = RecursiveASTVisitor<SelectionVisitor>;
530 SelectionVisitor(ASTContext &
AST,
const syntax::TokenBuffer &Tokens,
531 const PrintingPolicy &
PP,
unsigned SelBegin,
unsigned SelEnd,
533 : SM(
AST.getSourceManager()), LangOpts(
AST.getLangOpts()),
537 TokenBuf(Tokens), SelChecker(Tokens, SelFile, SelBegin, SelEnd, SM),
538 UnclaimedExpandedTokens(Tokens.expandedTokens()) {
540 Nodes.emplace_back();
541 Nodes.back().ASTNode = DynTypedNode::create(*
AST.getTranslationUnitDecl());
542 Nodes.back().Parent =
nullptr;
544 Stack.push(&Nodes.back());
549 template <
typename T,
typename Func>
550 bool traverseNode(T *Node,
const Func &Body) {
553 auto N = DynTypedNode::create(*Node);
554 if (canSafelySkipNode(N))
556 push(DynTypedNode::create(*Node));
589 bool canSafelySkipNode(
const DynTypedNode &N) {
590 SourceRange S = N.getSourceRange();
591 if (
auto *TL = N.get<TypeLoc>()) {
601 if (
auto DT = TL->getAs<DecltypeTypeLoc>())
602 S.setEnd(DT.getUnderlyingExpr()->getEndLoc());
604 if (!SelChecker.mayHit(S)) {
605 dlog(
"{1}skip: {0}", printNodeToString(N, PrintPolicy), indent());
606 dlog(
"{1}skipped range = {0}", S.printToString(SM), indent(1));
614 bool shouldSkipChildren(
const Stmt *
X)
const {
618 return llvm::isa<UserDefinedLiteral>(
X);
623 void push(DynTypedNode Node) {
624 SourceRange Early = earlySourceRange(Node);
625 dlog(
"{1}push: {0}", printNodeToString(Node, PrintPolicy), indent());
626 Nodes.emplace_back();
627 Nodes.back().ASTNode = std::move(Node);
628 Nodes.back().Parent = Stack.top();
629 Nodes.back().Selected = NoTokens;
630 Stack.push(&Nodes.back());
631 claimRange(Early, Nodes.back().Selected);
637 Node &N = *Stack.top();
638 dlog(
"{1}pop: {0}", printNodeToString(N.ASTNode, PrintPolicy), indent(-1));
639 claimRange(N.ASTNode.getSourceRange(), N.Selected);
640 if (N.Selected == NoTokens)
642 if (N.Selected || !N.Children.empty()) {
644 N.Parent->Children.push_back(&N);
647 assert(&N == &Nodes.back());
656 SourceRange earlySourceRange(
const DynTypedNode &N) {
657 if (
const Decl *D = N.get<
Decl>()) {
664 if (isa<CXXConstructorDecl>(D) || isa<CXXDeductionGuideDecl>(D))
665 return SourceRange();
672 if (
const auto *DD = llvm::dyn_cast<DeclaratorDecl>(D))
673 return DD->getLocation();
674 }
else if (
const auto *CCI = N.get<CXXCtorInitializer>()) {
676 return CCI->getMemberLocation();
678 return SourceRange();
686 for (
const auto &ClaimedRange :
687 UnclaimedExpandedTokens.erase(TokenBuf.expandedTokens(S)))
688 update(Result, SelChecker.test(ClaimedRange));
690 if (Result && Result != NoTokens)
691 dlog(
"{1}hit selection: {0}", S.printToString(SM), indent());
694 std::string indent(
int Offset = 0) {
696 int Amount = int(Stack.size()) +
Offset;
698 return std::string(Amount,
' ');
702 const LangOptions &LangOpts;
704 const PrintingPolicy &PrintPolicy;
706 const syntax::TokenBuffer &TokenBuf;
707 std::stack<Node *> Stack;
708 SelectionTester SelChecker;
709 IntervalSet<syntax::Token> UnclaimedExpandedTokens;
710 std::deque<Node> Nodes;
716 const PrintingPolicy &
PP) {
717 llvm::SmallString<256> Result;
719 llvm::raw_svector_ostream
OS(Result);
722 auto Pos = Result.find(
'\n');
723 if (
Pos != llvm::StringRef::npos) {
725 !llvm::all_of(llvm::StringRef(Result).drop_front(
Pos), llvm::isSpace);
733 void SelectionTree::print(llvm::raw_ostream &
OS,
const SelectionTree::Node &N,
740 printNodeKind(
OS, N.ASTNode);
742 for (
const Node *Child : N.Children)
743 print(
OS, *Child, Indent + 2);
748 llvm::raw_string_ostream
OS(S);
750 return std::move(
OS.str());
757 static llvm::SmallVector<std::pair<unsigned, unsigned>, 2>
759 const auto &SM = Tokens.sourceManager();
760 SourceLocation
Loc = SM.getComposedLoc(SM.getMainFileID(),
Offset);
761 llvm::SmallVector<std::pair<unsigned, unsigned>, 2> Result;
763 for (
const syntax::Token &Tok :
764 llvm::reverse(spelledTokensTouching(
Loc, Tokens))) {
765 if (shouldIgnore(Tok))
767 unsigned Offset = Tokens.sourceManager().getFileOffset(Tok.location());
776 const syntax::TokenBuffer &Tokens,
777 unsigned Begin,
unsigned End,
788 const syntax::TokenBuffer &Tokens,
789 unsigned int Begin,
unsigned int End) {
790 llvm::Optional<SelectionTree> Result;
792 Result = std::move(T);
795 return std::move(*Result);
799 unsigned Begin,
unsigned End)
800 : PrintPolicy(
AST.getLangOpts()) {
803 const SourceManager &SM =
AST.getSourceManager();
804 FileID FID = SM.getMainFileID();
805 PrintPolicy.TerseOutput =
true;
806 PrintPolicy.IncludeNewlines =
false;
808 dlog(
"Computing selection for {0}",
809 SourceRange(SM.getComposedLoc(FID, Begin), SM.getComposedLoc(FID, End))
811 Nodes = SelectionVisitor::collect(
AST, Tokens, PrintPolicy, Begin, End, FID);
812 Root = Nodes.empty() ? nullptr : &Nodes.front();
813 recordMetrics(*
this);
814 dlog(
"Built selection tree\n{0}", *
this);
818 const Node *Ancestor = Root;
820 Ancestor = Ancestor->
Children.front();
824 return Ancestor != Root ? Ancestor :
nullptr;
828 for (
const Node *CurrentNode =
this; CurrentNode !=
nullptr;
829 CurrentNode = CurrentNode->
Parent) {
830 if (
const Decl *Current = CurrentNode->ASTNode.get<
Decl>()) {
831 if (CurrentNode !=
this)
832 if (
auto *DC = dyn_cast<DeclContext>(Current))
834 return *Current->getDeclContext();
837 llvm_unreachable(
"A tree must always be rooted at TranslationUnitDecl.");
842 Children.front()->ASTNode.getSourceRange() == ASTNode.getSourceRange())
843 return Children.front()->ignoreImplicit();