clang-tools  9.0.0
BackgroundIndexStorage.cpp
Go to the documentation of this file.
1 //== BackgroundIndexStorage.cpp - Provide caching support to BackgroundIndex ==/
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
10 #include "Logger.h"
11 #include "Path.h"
12 #include "index/Background.h"
13 #include "llvm/ADT/Optional.h"
14 #include "llvm/ADT/STLExtras.h"
15 #include "llvm/ADT/ScopeExit.h"
16 #include "llvm/ADT/SmallString.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/Support/Error.h"
20 #include "llvm/Support/FileSystem.h"
21 #include "llvm/Support/MemoryBuffer.h"
22 #include "llvm/Support/Path.h"
23 #include <functional>
24 
25 namespace clang {
26 namespace clangd {
27 namespace {
28 
29 std::string getShardPathFromFilePath(llvm::StringRef ShardRoot,
30  llvm::StringRef FilePath) {
31  llvm::SmallString<128> ShardRootSS(ShardRoot);
32  llvm::sys::path::append(ShardRootSS, llvm::sys::path::filename(FilePath) +
33  "." + llvm::toHex(digest(FilePath)) +
34  ".idx");
35  return ShardRootSS.str();
36 }
37 
39 writeAtomically(llvm::StringRef OutPath,
40  llvm::function_ref<void(llvm::raw_ostream &)> Writer) {
41  // Write to a temporary file first.
42  llvm::SmallString<128> TempPath;
43  int FD;
44  auto EC =
45  llvm::sys::fs::createUniqueFile(OutPath + ".tmp.%%%%%%%%", FD, TempPath);
46  if (EC)
47  return llvm::errorCodeToError(EC);
48  // Make sure temp file is destroyed on failure.
49  auto RemoveOnFail =
50  llvm::make_scope_exit([TempPath] { llvm::sys::fs::remove(TempPath); });
51  llvm::raw_fd_ostream OS(FD, /*shouldClose=*/true);
52  Writer(OS);
53  OS.close();
54  if (OS.has_error())
55  return llvm::errorCodeToError(OS.error());
56  // Then move to real location.
57  EC = llvm::sys::fs::rename(TempPath, OutPath);
58  if (EC)
59  return llvm::errorCodeToError(EC);
60  // If everything went well, we already moved the file to another name. So
61  // don't delete the file, as the name might be taken by another file.
62  RemoveOnFail.release();
63  return llvm::ErrorSuccess();
64 }
65 
66 // Uses disk as a storage for index shards. Creates a directory called
67 // ".clangd/index/" under the path provided during construction.
68 class DiskBackedIndexStorage : public BackgroundIndexStorage {
69  std::string DiskShardRoot;
70 
71 public:
72  // Sets DiskShardRoot to (Directory + ".clangd/index/") which is the base
73  // directory for all shard files.
74  DiskBackedIndexStorage(llvm::StringRef Directory) {
75  llvm::SmallString<128> CDBDirectory(Directory);
76  llvm::sys::path::append(CDBDirectory, ".clangd", "index");
77  DiskShardRoot = CDBDirectory.str();
78  std::error_code OK;
79  std::error_code EC = llvm::sys::fs::create_directories(DiskShardRoot);
80  if (EC != OK) {
81  elog("Failed to create directory {0} for index storage: {1}",
82  DiskShardRoot, EC.message());
83  }
84  }
85 
86  std::unique_ptr<IndexFileIn>
87  loadShard(llvm::StringRef ShardIdentifier) const override {
88  const std::string ShardPath =
89  getShardPathFromFilePath(DiskShardRoot, ShardIdentifier);
90  auto Buffer = llvm::MemoryBuffer::getFile(ShardPath);
91  if (!Buffer)
92  return nullptr;
93  if (auto I = readIndexFile(Buffer->get()->getBuffer()))
94  return llvm::make_unique<IndexFileIn>(std::move(*I));
95  else
96  elog("Error while reading shard {0}: {1}", ShardIdentifier,
97  I.takeError());
98  return nullptr;
99  }
100 
101  llvm::Error storeShard(llvm::StringRef ShardIdentifier,
102  IndexFileOut Shard) const override {
103  return writeAtomically(
104  getShardPathFromFilePath(DiskShardRoot, ShardIdentifier),
105  [&Shard](llvm::raw_ostream &OS) { OS << Shard; });
106  }
107 };
108 
109 // Doesn't persist index shards anywhere (used when the CDB dir is unknown).
110 // We could consider indexing into ~/.clangd/ or so instead.
111 class NullStorage : public BackgroundIndexStorage {
112 public:
113  std::unique_ptr<IndexFileIn>
114  loadShard(llvm::StringRef ShardIdentifier) const override {
115  return nullptr;
116  }
117 
118  llvm::Error storeShard(llvm::StringRef ShardIdentifier,
119  IndexFileOut Shard) const override {
120  vlog("Couldn't find project for {0}, indexing in-memory only",
121  ShardIdentifier);
122  return llvm::Error::success();
123  }
124 };
125 
126 // Creates and owns IndexStorages for multiple CDBs.
127 class DiskBackedIndexStorageManager {
128 public:
129  DiskBackedIndexStorageManager(
130  std::function<llvm::Optional<ProjectInfo>(PathRef)> GetProjectInfo)
131  : IndexStorageMapMu(llvm::make_unique<std::mutex>()),
132  GetProjectInfo(std::move(GetProjectInfo)) {
133  llvm::SmallString<128> HomeDir;
134  llvm::sys::path::home_directory(HomeDir);
135  this->HomeDir = HomeDir.str().str();
136  }
137 
138  // Creates or fetches to storage from cache for the specified project.
139  BackgroundIndexStorage *operator()(PathRef File) {
140  std::lock_guard<std::mutex> Lock(*IndexStorageMapMu);
141  Path CDBDirectory = HomeDir;
142  if (auto PI = GetProjectInfo(File))
143  CDBDirectory = PI->SourceRoot;
144  auto &IndexStorage = IndexStorageMap[CDBDirectory];
145  if (!IndexStorage)
146  IndexStorage = create(CDBDirectory);
147  return IndexStorage.get();
148  }
149 
150 private:
151  std::unique_ptr<BackgroundIndexStorage> create(PathRef CDBDirectory) {
152  if (CDBDirectory.empty()) {
153  elog("Tried to create storage for empty directory!");
154  return llvm::make_unique<NullStorage>();
155  }
156  return llvm::make_unique<DiskBackedIndexStorage>(CDBDirectory);
157  }
158 
159  Path HomeDir;
160 
161  llvm::StringMap<std::unique_ptr<BackgroundIndexStorage>> IndexStorageMap;
162  std::unique_ptr<std::mutex> IndexStorageMapMu;
163 
164  std::function<llvm::Optional<ProjectInfo>(PathRef)> GetProjectInfo;
165 };
166 
167 } // namespace
168 
171  std::function<llvm::Optional<ProjectInfo>(PathRef)> GetProjectInfo) {
172  return DiskBackedIndexStorageManager(std::move(GetProjectInfo));
173 }
174 
175 } // namespace clangd
176 } // namespace clang
Some operations such as code completion produce a set of candidates.
llvm::unique_function< BackgroundIndexStorage *(PathRef)> Factory
Definition: Background.h:56
static Factory createDiskBackedStorageFactory(std::function< llvm::Optional< ProjectInfo >(PathRef)> GetProjectInfo)
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:23
llvm::Expected< IndexFileIn > readIndexFile(llvm::StringRef Data)
void vlog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:67
void elog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:56
std::string Path
A typedef to represent a file path.
Definition: Path.h:20
llvm::StringRef Directory
FileDigest digest(llvm::StringRef Content)
Definition: SourceCode.cpp:467
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//