TARAXA
Loading...
Searching...
No Matches
storage.hpp
Go to the documentation of this file.
1#pragma once
2
3#include <rocksdb/db.h>
4#include <rocksdb/options.h>
5#include <rocksdb/slice.h>
6#include <rocksdb/write_batch.h>
7
8#include <filesystem>
9#include <functional>
10#include <regex>
11
12#include "common/types.hpp"
13#include "dag/dag_block.hpp"
14#include "logger/logging.hpp"
15#include "pbft/pbft_block.hpp"
16#include "pbft/period_data.hpp"
21#include "vote/pillar_vote.hpp"
23
24namespace taraxa {
25namespace fs = std::filesystem;
26struct SortitionParamsChange;
27
28namespace pillar_chain {
29struct PillarBlockData;
30class PillarBlock;
31} // namespace pillar_chain
32
42
43enum class PbftMgrField : uint8_t { Round = 0, Step };
44
51
52enum class DBMetaKeys { LAST_NUMBER = 1 };
53
54class DbException : public std::exception {
55 public:
56 explicit DbException(const std::string& desc) : desc_(desc) {}
57 virtual ~DbException() = default;
58
59 DbException(const DbException&) = default;
63
64 virtual const char* what() const noexcept { return desc_.c_str(); }
65
66 private:
67 const std::string desc_;
68};
69
70using Batch = rocksdb::WriteBatch;
71using Slice = rocksdb::Slice;
72using OnEntry = std::function<void(Slice const&, Slice const&)>;
73
74class DbStorage : public std::enable_shared_from_this<DbStorage> {
75 public:
76 class Column {
77 std::string const name_;
78
79 public:
80 size_t const ordinal_;
81 const rocksdb::Comparator* comparator_;
82
83 Column(std::string name, size_t ordinal, const rocksdb::Comparator* comparator)
84 : name_(std::move(name)), ordinal_(ordinal), comparator_(comparator) {}
85
86 Column(std::string name, size_t ordinal) : name_(std::move(name)), ordinal_(ordinal), comparator_(nullptr) {}
87
88 auto const& name() const { return ordinal_ ? name_ : rocksdb::kDefaultColumnFamilyName; }
89 };
90
91 class Columns {
92 static inline std::vector<Column> all_;
93
94 public:
95 static inline auto const& all = all_;
96
97#define COLUMN(__name__) static inline auto const __name__ = all_.emplace_back(#__name__, all_.size())
98#define COLUMN_W_COMP(__name__, ...) \
99 static inline auto const __name__ = all_.emplace_back(#__name__, all_.size(), __VA_ARGS__)
100
101 // do not change/move
102 COLUMN(default_column);
103 // migrations
104 COLUMN(migrations);
105 // Contains full data for an executed PBFT block including PBFT block, cert votes, dag blocks and transactions
106 COLUMN_W_COMP(period_data, getIntComparator<PbftPeriod>());
107 COLUMN(genesis);
108 COLUMN(dag_blocks);
109 COLUMN_W_COMP(dag_blocks_level, getIntComparator<uint64_t>());
110 COLUMN(transactions);
111 COLUMN(trx_period);
112 COLUMN(status);
113 COLUMN(pbft_mgr_round_step);
114 COLUMN(pbft_mgr_status);
115 COLUMN(cert_voted_block_in_round); // Cert voted block + round -> node voted for this block
116 COLUMN(proposed_pbft_blocks); // Proposed pbft blocks
117 COLUMN(pbft_head);
118 COLUMN(latest_round_own_votes); // own votes of any type for the latest round
119 COLUMN(latest_round_two_t_plus_one_votes); // 2t+1 votes bundles of any type for the latest round
120 COLUMN(extra_reward_votes); // extra reward votes on top of 2t+1 cert votes bundle from
121 // latest_round_two_t_plus_one_votes
122 COLUMN(pbft_block_period);
123 COLUMN(dag_block_period);
124 COLUMN_W_COMP(proposal_period_levels_map, getIntComparator<uint64_t>());
125 COLUMN(final_chain_meta);
126 COLUMN(final_chain_blk_by_number);
127 COLUMN(final_chain_blk_hash_by_number);
128 COLUMN(final_chain_blk_number_by_hash);
129 COLUMN(final_chain_receipt_by_trx_hash);
130 COLUMN(final_chain_log_blooms_index);
131 COLUMN_W_COMP(sortition_params_change, getIntComparator<PbftPeriod>());
132
133 COLUMN_W_COMP(block_rewards_stats, getIntComparator<uint64_t>());
134
135 // Finalized pillar blocks
136 COLUMN_W_COMP(pillar_block, getIntComparator<PbftPeriod>());
137 // Current pillar block data - current pillar block + current vote counts
138 COLUMN(current_pillar_block_data);
139 // Current pillar block own pillar vote
140 COLUMN(current_pillar_block_own_vote);
141 // system transactions that is not a part of the block
142 COLUMN(system_transaction);
143 // system transactions hashes by period
144 COLUMN(period_system_transactions);
145 // final chain receipts by period
146 COLUMN_W_COMP(final_chain_receipt_by_period, getIntComparator<PbftPeriod>());
147
148#undef COLUMN
149#undef COLUMN_W_COMP
150 };
151
152 auto handle(Column const& col) const { return handles_[col.ordinal_]; }
153 rocksdb::ReadOptions read_options_;
154
155 rocksdb::WriteOptions async_write_;
156 rocksdb::WriteOptions sync_write_;
157
158 private:
159 fs::path path_;
160 fs::path db_path_;
162 const std::string kDbDir = "db";
163 const std::string kStateDbDir = "state_db";
164 std::unique_ptr<rocksdb::DB> db_;
165 std::vector<rocksdb::ColumnFamilyHandle*> handles_;
167 std::atomic<uint64_t> dag_blocks_count_;
168 std::atomic<uint64_t> dag_edge_count_;
169 const uint32_t kDbSnapshotsEachNblock = 0;
170 std::atomic<bool> snapshots_enabled_ = true;
171 const uint32_t kDbSnapshotsMaxCount = 0;
172 std::set<PbftPeriod> snapshots_;
174
178
180
181 public:
182 explicit DbStorage(fs::path const& base_path, uint32_t db_snapshot_each_n_pbft_block = 0, uint32_t max_open_files = 0,
183 uint32_t db_max_snapshots = 0, PbftPeriod db_revert_to_period = 0, bool rebuild = false);
184 ~DbStorage();
185
186 DbStorage(const DbStorage&) = delete;
187 DbStorage(DbStorage&&) = delete;
188 DbStorage& operator=(const DbStorage&) = delete;
190
191 auto const& path() const { return path_; }
192 auto dbStoragePath() const { return db_path_; }
193 auto stateDbStoragePath() const { return state_db_path_; }
194 static Batch createWriteBatch();
195 void commitWriteBatch(Batch& write_batch, const rocksdb::WriteOptions& opts);
196 void commitWriteBatch(Batch& write_batch) { commitWriteBatch(write_batch, async_write_); }
197
198 void rebuildColumns(const rocksdb::Options& options);
199 bool createSnapshot(PbftPeriod period);
200 void deleteSnapshot(PbftPeriod period);
201 void recoverToPeriod(PbftPeriod period);
202 void loadSnapshots();
203 void disableSnapshots();
204 void enableSnapshots();
205 void updateDbVersions();
206 void deleteColumnData(const Column& c);
207
208 void replaceColumn(const Column& to_be_replaced_col, std::unique_ptr<rocksdb::ColumnFamilyHandle>&& replacing_col);
209 std::unique_ptr<rocksdb::ColumnFamilyHandle> copyColumn(rocksdb::ColumnFamilyHandle* orig_column,
210 const std::string& new_col_name, bool move_data = false);
211
212 // For removal of LOG.old.* files in the database
213 void removeTempFiles() const;
214 void removeFilesWithPattern(const std::string& directory, const std::regex& pattern) const;
215 void deleteTmpDirectories(const std::string& path) const;
216
217 uint32_t getMajorVersion() const;
218 std::unique_ptr<rocksdb::Iterator> getColumnIterator(const Column& c);
219 std::unique_ptr<rocksdb::Iterator> getColumnIterator(rocksdb::ColumnFamilyHandle* c);
220
221 // Genesis
222 void setGenesisHash(const h256& genesis_hash);
223 std::optional<h256> getGenesisHash();
224
225 // Period data
226 void savePeriodData(const PeriodData& period_data, Batch& write_batch);
227 void clearPeriodDataHistory(PbftPeriod period, uint64_t dag_level_to_keep, PbftPeriod last_block_number);
229 std::optional<PeriodData> getPeriodData(PbftPeriod period) const;
230 std::optional<PbftBlock> getPbftBlock(PbftPeriod period) const;
231 std::vector<std::shared_ptr<PbftVote>> getPeriodCertVotes(PbftPeriod period) const;
233 SharedTransactions transactionsFromPeriodDataRlp(PbftPeriod period, const dev::RLP& period_data_rlp) const;
234 std::optional<SharedTransactions> getPeriodTransactions(PbftPeriod period) const;
235 std::vector<std::shared_ptr<PillarVote>> getPeriodPillarVotes(PbftPeriod period) const;
236 uint64_t getEarliestBlockNumber() const;
237
238 // Pillar chain
239 void savePillarBlock(const std::shared_ptr<pillar_chain::PillarBlock>& pillar_block);
240 std::shared_ptr<pillar_chain::PillarBlock> getPillarBlock(PbftPeriod period) const;
241 std::shared_ptr<pillar_chain::PillarBlock> getLatestPillarBlock() const;
242 void saveOwnPillarBlockVote(const std::shared_ptr<PillarVote>& vote);
243 std::shared_ptr<PillarVote> getOwnPillarBlockVote() const;
244 void saveCurrentPillarBlockData(const pillar_chain::CurrentPillarBlockDataDb& current_pillar_block_data);
245 std::optional<pillar_chain::CurrentPillarBlockDataDb> getCurrentPillarBlockData() const;
246
247 // DAG
248 void saveDagBlock(const std::shared_ptr<DagBlock>& blk, Batch* write_batch_p = nullptr);
249 std::shared_ptr<DagBlock> getDagBlock(blk_hash_t const& hash);
250 bool dagBlockInDb(blk_hash_t const& hash);
251 std::set<blk_hash_t> getBlocksByLevel(level_t level);
253 std::vector<std::shared_ptr<DagBlock>> getDagBlocksAtLevel(level_t level, int number_of_levels);
254 void updateDagBlockCounters(std::vector<std::shared_ptr<DagBlock>> blks);
255 std::map<level_t, std::vector<std::shared_ptr<DagBlock>>> getNonfinalizedDagBlocks();
256 void removeDagBlockBatch(Batch& write_batch, blk_hash_t const& hash);
257 void removeDagBlock(blk_hash_t const& hash);
258 // Sortition params
259 void saveSortitionParamsChange(PbftPeriod period, const SortitionParamsChange& params, Batch& batch);
260 std::deque<SortitionParamsChange> getLastSortitionParams(size_t count);
261 std::optional<SortitionParamsChange> getParamsChangeForPeriod(PbftPeriod period);
262
263 // Transaction
264 std::shared_ptr<Transaction> getTransaction(trx_hash_t const& hash) const;
265 std::shared_ptr<Transaction> getTransaction(PbftPeriod period, uint32_t position) const;
266
268 bool transactionInDb(trx_hash_t const& hash);
269 bool transactionFinalized(trx_hash_t const& hash);
270 std::vector<bool> transactionsInDb(std::vector<trx_hash_t> const& trx_hashes);
271 std::vector<bool> transactionsFinalized(std::vector<trx_hash_t> const& trx_hashes);
272 void addTransactionToBatch(Transaction const& trx, Batch& write_batch);
273 void removeTransactionToBatch(trx_hash_t const& trx, Batch& write_batch);
274
275 void addTransactionLocationToBatch(Batch& write_batch, trx_hash_t const& trx, PbftPeriod period, uint32_t position,
276 bool is_system = false);
277 std::optional<TransactionLocation> getTransactionLocation(trx_hash_t const& hash) const;
278 std::unordered_map<trx_hash_t, PbftPeriod> getAllTransactionPeriod();
279 uint64_t getTransactionCount(PbftPeriod period) const;
281 std::optional<TransactionReceipt> getTransactionReceipt(EthBlockNumber blk_n, uint64_t position) const;
282
290 SharedTransactions getFinalizedTransactions(std::vector<trx_hash_t> const& trx_hashes) const;
291
292 // System transaction
294 std::shared_ptr<Transaction> getSystemTransaction(const trx_hash_t& hash) const;
295 void addPeriodSystemTransactions(Batch& write_batch, SharedTransactions trxs, PbftPeriod period);
296 std::vector<trx_hash_t> getPeriodSystemTransactionsHashes(PbftPeriod period) const;
298
299 // PBFT manager
300 uint32_t getPbftMgrField(PbftMgrField field);
301 void savePbftMgrField(PbftMgrField field, uint32_t value);
302 void addPbftMgrFieldToBatch(PbftMgrField field, uint32_t value, Batch& write_batch);
303
305 void savePbftMgrStatus(PbftMgrStatus field, bool const& value);
306 void addPbftMgrStatusToBatch(PbftMgrStatus field, bool const& value, Batch& write_batch);
307
308 void saveCertVotedBlockInRound(PbftRound round, const std::shared_ptr<PbftBlock>& block);
309 std::optional<std::pair<PbftRound, std::shared_ptr<PbftBlock>>> getCertVotedBlockInRound() const;
310 void removeCertVotedBlockInRound(Batch& write_batch);
311
312 // pbft_blocks
313 std::optional<PbftBlock> getPbftBlock(blk_hash_t const& hash);
314 bool pbftBlockInDb(blk_hash_t const& hash);
315
316 // Proposed pbft blocks
317 void saveProposedPbftBlock(const std::shared_ptr<PbftBlock>& block);
318 void removeProposedPbftBlock(const blk_hash_t& block_hash, Batch& write_batch);
319 std::vector<std::shared_ptr<PbftBlock>> getProposedPbftBlocks();
320
321 // pbft_blocks (head)
322 std::string getPbftHead(blk_hash_t const& hash);
323 void savePbftHead(blk_hash_t const& hash, std::string const& pbft_chain_head_str);
324 void addPbftHeadToBatch(taraxa::blk_hash_t const& head_hash, std::string const& head_str, Batch& write_batch);
325
326 // status
327 uint64_t getStatusField(StatusDbField const& field);
328 void saveStatusField(StatusDbField const& field, uint64_t value);
329 void addStatusFieldToBatch(StatusDbField const& field, uint64_t value, Batch& write_batch);
330
331 // Own votes for the latest round
332 void saveOwnVerifiedVote(const std::shared_ptr<PbftVote>& vote);
333 std::vector<std::shared_ptr<PbftVote>> getOwnVerifiedVotes();
334 void clearOwnVerifiedVotes(Batch& write_batch, const std::vector<std::shared_ptr<PbftVote>>& own_verified_votes);
335
336 // 2t+1 votes bundles for the latest round
337 void replaceTwoTPlusOneVotes(TwoTPlusOneVotedBlockType type, const std::vector<std::shared_ptr<PbftVote>>& votes);
339 const std::vector<std::shared_ptr<PbftVote>>& votes, Batch& write_batch);
340 std::vector<std::shared_ptr<PbftVote>> getAllTwoTPlusOneVotes();
341
342 // Reward votes - cert votes for the latest finalized block
343 void removeExtraRewardVotes(const std::vector<vote_hash_t>& votes, Batch& write_batch);
344 void saveExtraRewardVote(const std::shared_ptr<PbftVote>& vote);
345 std::vector<std::shared_ptr<PbftVote>> getRewardVotes();
346
347 // period_pbft_block
348 void addPbftBlockPeriodToBatch(PbftPeriod period, taraxa::blk_hash_t const& pbft_block_hash, Batch& write_batch);
349 std::pair<bool, PbftPeriod> getPeriodFromPbftHash(taraxa::blk_hash_t const& pbft_block_hash);
350 // dag_block_period
351 std::shared_ptr<std::pair<PbftPeriod, uint32_t>> getDagBlockPeriod(blk_hash_t const& hash);
352 void addDagBlockPeriodToBatch(blk_hash_t const& hash, PbftPeriod period, uint32_t position, Batch& write_batch);
353
354 uint64_t getDagBlocksCount() const { return dag_blocks_count_.load(); }
355 uint64_t getDagEdgeCount() const { return dag_edge_count_.load(); }
356
360
361 std::vector<blk_hash_t> getFinalizedDagBlockHashesByPeriod(PbftPeriod period);
362 std::vector<std::shared_ptr<DagBlock>> getFinalizedDagBlockByPeriod(PbftPeriod period);
363 std::pair<blk_hash_t, std::vector<std::shared_ptr<DagBlock>>> getLastPbftBlockHashAndFinalizedDagBlockByPeriod(
364 PbftPeriod period);
365
366 // DPOS level to proposal period map
367 std::optional<uint64_t> getProposalPeriodForDagLevel(uint64_t level);
368 void saveProposalPeriodDagLevelsMap(uint64_t level, PbftPeriod period);
369 void addProposalPeriodDagLevelsMapToBatch(uint64_t level, PbftPeriod period, Batch& write_batch);
370
373
374 void compactColumn(Column const& column) { db_->CompactRange({}, handle(column), nullptr, nullptr); }
375
376 inline static bytes asBytes(std::string const& b) {
377 return bytes((byte const*)b.data(), (byte const*)(b.data() + b.size()));
378 }
379
380 template <typename T>
381 inline static Slice make_slice(T const* begin, size_t size) {
382 if (!size) {
383 return {};
384 }
385 return {reinterpret_cast<char const*>(begin), size};
386 }
387
388 inline static Slice toSlice(dev::bytesConstRef const& b) { return make_slice(b.data(), b.size()); }
389
390 template <unsigned N>
391 inline static Slice toSlice(dev::FixedHash<N> const& h) {
392 return make_slice(h.data(), N);
393 }
394
395 inline static Slice toSlice(dev::bytes const& b) { return make_slice(b.data(), b.size()); }
396
397 template <class N>
398 inline static auto toSlice(N const& n) -> std::enable_if_t<std::is_integral_v<N> || std::is_enum_v<N>, Slice> {
399 return make_slice(&n, sizeof(N));
400 }
401
402 inline static Slice toSlice(std::string const& str) { return make_slice(str.data(), str.size()); }
403
404 inline static auto const& toSlice(Slice const& s) { return s; }
405
406 template <typename T>
407 inline static auto toSlices(std::vector<T> const& keys) {
408 std::vector<Slice> ret;
409 ret.reserve(keys.size());
410 for (auto const& k : keys) {
411 ret.emplace_back(toSlice(k));
412 }
413 return ret;
414 }
415
416 inline static auto const& toSlices(std::vector<Slice> const& ss) { return ss; }
417
418 template <typename K>
419 std::string lookup(K const& key, Column const& column) const {
420 std::string value;
421 auto status = db_->Get(read_options_, handle(column), toSlice(key), &value);
422 if (status.IsNotFound()) {
423 return value;
424 }
425 checkStatus(status);
426 return value;
427 }
428
429 template <typename Int, typename K>
430 auto lookup_int(K const& key, Column const& column) -> std::enable_if_t<std::is_integral_v<Int>, std::optional<Int>> {
431 auto str = lookup(key, column);
432 if (str.empty()) {
433 return std::nullopt;
434 }
435 return *reinterpret_cast<Int*>(str.data());
436 }
437
438 template <typename K>
439 bool exist(K const& key, Column const& column) {
440 std::string value;
441 // KeyMayExist can lead to a few false positives, but not false negatives.
442 if (db_->KeyMayExist(read_options_, handle(column), toSlice(key), &value)) {
443 auto status = db_->Get(read_options_, handle(column), toSlice(key), &value);
444 if (status.IsNotFound()) {
445 return false;
446 }
447 checkStatus(status);
448 return !value.empty();
449 }
450 return false;
451 }
452
453 static void checkStatus(rocksdb::Status const& status);
454
455 template <typename K, typename V>
456 void insert(rocksdb::ColumnFamilyHandle* col, const K& k, const V& v) {
457 checkStatus(db_->Put(async_write_, col, toSlice(k), toSlice(v)));
458 }
459
460 template <typename K, typename V>
461 void insert(Column const& col, K const& k, V const& v) {
462 checkStatus(db_->Put(async_write_, handle(col), toSlice(k), toSlice(v)));
463 }
464
465 template <typename K, typename V>
466 void insert(Batch& batch, Column const& col, K const& k, V const& v) {
467 checkStatus(batch.Put(handle(col), toSlice(k), toSlice(v)));
468 }
469
470 template <typename K, typename V>
471 void insert(Batch& batch, rocksdb::ColumnFamilyHandle* col, K const& k, V const& v) {
472 checkStatus(batch.Put(col, toSlice(k), toSlice(v)));
473 }
474
475 template <typename K>
476 void remove(Column const& col, K const& k) {
477 checkStatus(db_->Delete(async_write_, handle(col), toSlice(k)));
478 }
479
480 template <typename K>
481 void remove(Batch& batch, Column const& col, K const& k) {
482 checkStatus(batch.Delete(handle(col), toSlice(k)));
483 }
484
485 void forEach(Column const& col, OnEntry const& f);
486};
487
488} // namespace taraxa
::byte * data()
Definition FixedHash.h:189
Definition RLP.h:56
Definition vector_ref.h:23
_T * data() const
Definition vector_ref.h:76
size_t size() const
Definition vector_ref.h:82
Definition storage.hpp:54
virtual const char * what() const noexcept
Definition storage.hpp:64
virtual ~DbException()=default
const std::string desc_
Definition storage.hpp:67
DbException(const std::string &desc)
Definition storage.hpp:56
DbException(const DbException &)=default
DbException(DbException &&)=default
DbException & operator=(DbException &&)=delete
DbException & operator=(const DbException &)=delete
Definition storage.hpp:76
Column(std::string name, size_t ordinal, const rocksdb::Comparator *comparator)
Definition storage.hpp:83
const rocksdb::Comparator * comparator_
Definition storage.hpp:81
Column(std::string name, size_t ordinal)
Definition storage.hpp:86
auto const & name() const
Definition storage.hpp:88
size_t const ordinal_
Definition storage.hpp:80
std::string const name_
Definition storage.hpp:77
Definition storage.hpp:91
static auto const & all
Definition storage.hpp:95
COLUMN(period_system_transactions)
COLUMN(cert_voted_block_in_round)
COLUMN_W_COMP(dag_blocks_level, getIntComparator< uint64_t >())
COLUMN(current_pillar_block_own_vote)
COLUMN(final_chain_meta)
COLUMN(dag_block_period)
COLUMN_W_COMP(pillar_block, getIntComparator< PbftPeriod >())
COLUMN(final_chain_receipt_by_trx_hash)
COLUMN_W_COMP(block_rewards_stats, getIntComparator< uint64_t >())
COLUMN(final_chain_blk_by_number)
COLUMN(extra_reward_votes)
COLUMN(final_chain_blk_hash_by_number)
COLUMN(pbft_block_period)
COLUMN(latest_round_two_t_plus_one_votes)
COLUMN(system_transaction)
COLUMN(final_chain_log_blooms_index)
COLUMN(latest_round_own_votes)
COLUMN_W_COMP(proposal_period_levels_map, getIntComparator< uint64_t >())
COLUMN(pbft_mgr_round_step)
static std::vector< Column > all_
Definition storage.hpp:92
COLUMN_W_COMP(period_data, getIntComparator< PbftPeriod >())
COLUMN(pbft_mgr_status)
COLUMN(proposed_pbft_blocks)
COLUMN_W_COMP(sortition_params_change, getIntComparator< PbftPeriod >())
COLUMN(current_pillar_block_data)
COLUMN(final_chain_blk_number_by_hash)
COLUMN_W_COMP(final_chain_receipt_by_period, getIntComparator< PbftPeriod >())
Definition storage.hpp:74
void addStatusFieldToBatch(StatusDbField const &field, uint64_t value, Batch &write_batch)
Definition storage.cpp:1064
void addDagBlockPeriodToBatch(blk_hash_t const &hash, PbftPeriod period, uint32_t position, Batch &write_batch)
Definition storage.cpp:1277
auto const & path() const
Definition storage.hpp:191
uint64_t getTransactionCount(PbftPeriod period) const
Definition storage.cpp:875
std::atomic< uint64_t > dag_blocks_count_
Definition storage.hpp:167
SharedTransactions getPeriodSystemTransactions(PbftPeriod period) const
Definition storage.cpp:940
static auto toSlices(std::vector< T > const &keys)
Definition storage.hpp:407
bool dagBlockInDb(blk_hash_t const &hash)
Definition storage.cpp:461
void remove(Batch &batch, Column const &col, K const &k)
Definition storage.hpp:481
std::shared_ptr< DagBlock > getDagBlock(blk_hash_t const &hash)
Definition storage.cpp:444
fs::path db_path_
Definition storage.hpp:160
std::shared_ptr< Transaction > getTransaction(trx_hash_t const &hash) const
Definition storage.cpp:850
void disableSnapshots()
Definition storage.cpp:393
std::vector< std::shared_ptr< PbftVote > > getAllTwoTPlusOneVotes()
Definition storage.cpp:1206
std::optional< uint64_t > getProposalPeriodForDagLevel(uint64_t level)
Definition storage.cpp:1324
std::mutex dag_blocks_mutex_
Definition storage.hpp:166
uint32_t getPbftMgrField(PbftMgrField field)
Definition storage.cpp:1070
std::optional< PeriodData > getPeriodData(PbftPeriod period) const
Definition storage.cpp:714
void saveProposedPbftBlock(const std::shared_ptr< PbftBlock > &block)
Definition storage.cpp:815
void removeProposedPbftBlock(const blk_hash_t &block_hash, Batch &write_batch)
Definition storage.cpp:819
void addProposalPeriodDagLevelsMapToBatch(uint64_t level, PbftPeriod period, Batch &write_batch)
Definition storage.cpp:1343
void saveCertVotedBlockInRound(PbftRound round, const std::shared_ptr< PbftBlock > &block)
Definition storage.cpp:1107
std::unordered_map< trx_hash_t, PbftPeriod > getAllTransactionPeriod()
Definition storage.cpp:802
void saveOwnPillarBlockVote(const std::shared_ptr< PillarVote > &vote)
Definition storage.cpp:746
rocksdb::WriteOptions async_write_
Definition storage.hpp:155
std::shared_ptr< pillar_chain::PillarBlock > getPillarBlock(PbftPeriod period) const
Definition storage.cpp:727
void addPbftMgrStatusToBatch(PbftMgrStatus field, bool const &value, Batch &write_batch)
Definition storage.cpp:1103
void removeExtraRewardVotes(const std::vector< vote_hash_t > &votes, Batch &write_batch)
Definition storage.cpp:1226
void rebuildColumns(const rocksdb::Options &options)
Definition storage.cpp:229
void addTransactionLocationToBatch(Batch &write_batch, trx_hash_t const &trx, PbftPeriod period, uint32_t position, bool is_system=false)
Definition storage.cpp:772
SharedTransactions transactionsFromPeriodDataRlp(PbftPeriod period, const dev::RLP &period_data_rlp) const
Definition storage.cpp:966
blk_hash_t getPeriodBlockHash(PbftPeriod period) const
Definition storage.cpp:842
std::vector< std::shared_ptr< PbftBlock > > getProposedPbftBlocks()
Definition storage.cpp:823
std::vector< std::shared_ptr< DagBlock > > getDagBlocksAtLevel(level_t level, int number_of_levels)
Definition storage.cpp:485
std::shared_ptr< PillarVote > getOwnPillarBlockVote() const
Definition storage.cpp:750
std::atomic< bool > snapshots_enabled_
Definition storage.hpp:170
DbStorage & operator=(DbStorage &&)=delete
uint32_t kMajorVersion_
Definition storage.hpp:175
const uint32_t kDbSnapshotsMaxCount
Definition storage.hpp:171
void addPeriodSystemTransactions(Batch &write_batch, SharedTransactions trxs, PbftPeriod period)
Definition storage.cpp:923
static auto const & toSlices(std::vector< Slice > const &ss)
Definition storage.hpp:416
std::optional< h256 > getGenesisHash()
Definition storage.cpp:403
void compactColumn(Column const &column)
Definition storage.hpp:374
SharedTransactions getAllNonfinalizedTransactions()
Definition storage.cpp:510
void insert(Batch &batch, Column const &col, K const &k, V const &v)
Definition storage.hpp:466
auto stateDbStoragePath() const
Definition storage.hpp:193
uint64_t getDagBlocksCount() const
Definition storage.hpp:354
std::vector< blk_hash_t > getFinalizedDagBlockHashesByPeriod(PbftPeriod period)
Definition storage.cpp:1286
bool transactionInDb(trx_hash_t const &hash)
Definition storage.cpp:1030
void saveOwnVerifiedVote(const std::shared_ptr< PbftVote > &vote)
Definition storage.cpp:1161
bool minor_version_changed_
Definition storage.hpp:177
bool createSnapshot(PbftPeriod period)
Definition storage.cpp:312
SharedTransactions getFinalizedTransactions(std::vector< trx_hash_t > const &trx_hashes) const
Gets finalized transactions from provided hashes.
Definition storage.cpp:884
std::map< level_t, std::vector< std::shared_ptr< DagBlock > > > getNonfinalizedDagBlocks()
Definition storage.cpp:500
void saveStatusField(StatusDbField const &field, uint64_t value)
Definition storage.cpp:1060
void saveCurrentPillarBlockData(const pillar_chain::CurrentPillarBlockDataDb &current_pillar_block_data)
Definition storage.cpp:759
rocksdb::ReadOptions read_options_
Definition storage.hpp:153
std::unique_ptr< rocksdb::DB > db_
Definition storage.hpp:164
void commitWriteBatch(Batch &write_batch, const rocksdb::WriteOptions &opts)
Definition storage.cpp:438
uint64_t getEarliestBlockNumber() const
Definition storage.cpp:603
bool exist(K const &key, Column const &column)
Definition storage.hpp:439
void saveExtraRewardVote(const std::shared_ptr< PbftVote > &vote)
Definition storage.cpp:1232
void removeDagBlock(blk_hash_t const &hash)
Definition storage.cpp:523
fs::path path_
Definition storage.hpp:159
void insert(Column const &col, K const &k, V const &v)
Definition storage.hpp:461
std::vector< bool > transactionsInDb(std::vector< trx_hash_t > const &trx_hashes)
Definition storage.cpp:1038
std::pair< blk_hash_t, std::vector< std::shared_ptr< DagBlock > > > getLastPbftBlockHashAndFinalizedDagBlockByPeriod(PbftPeriod period)
Definition storage.cpp:1310
void savePeriodData(const PeriodData &period_data, Batch &write_batch)
Definition storage.cpp:687
void deleteColumnData(const Column &c)
Definition storage.cpp:218
void recoverToPeriod(PbftPeriod period)
Definition storage.cpp:345
fs::path state_db_path_
Definition storage.hpp:161
std::shared_ptr< pillar_chain::PillarBlock > getLatestPillarBlock() const
Definition storage.cpp:736
std::optional< TransactionLocation > getTransactionLocation(trx_hash_t const &hash) const
Definition storage.cpp:784
bool getPbftMgrStatus(PbftMgrStatus field)
Definition storage.cpp:1089
bool major_version_changed_
Definition storage.hpp:176
void savePillarBlock(const std::shared_ptr< pillar_chain::PillarBlock > &pillar_block)
Definition storage.cpp:723
void savePbftMgrField(PbftMgrField field, uint32_t value)
Definition storage.cpp:1081
logger::Logger logger_
Definition storage.hpp:179
std::optional< std::pair< PbftRound, std::shared_ptr< PbftBlock > > > getCertVotedBlockInRound() const
Definition storage.cpp:1116
static Slice make_slice(T const *begin, size_t size)
Definition storage.hpp:381
std::vector< bool > transactionsFinalized(std::vector< trx_hash_t > const &trx_hashes)
Definition storage.cpp:792
bool transactionFinalized(trx_hash_t const &hash)
Definition storage.cpp:1034
std::deque< SortitionParamsChange > getLastSortitionParams(size_t count)
Definition storage.cpp:578
static auto toSlice(N const &n) -> std::enable_if_t< std::is_integral_v< N >||std::is_enum_v< N >, Slice >
Definition storage.hpp:398
void deleteSnapshot(PbftPeriod period)
Definition storage.cpp:377
std::shared_ptr< std::pair< PbftPeriod, uint32_t > > getDagBlockPeriod(blk_hash_t const &hash)
Definition storage.cpp:1264
void forEach(Column const &col, OnEntry const &f)
Definition storage.cpp:1347
void remove(Column const &col, K const &k)
Definition storage.hpp:476
DbStorage(DbStorage &&)=delete
std::set< PbftPeriod > snapshots_
Definition storage.hpp:172
void removeCertVotedBlockInRound(Batch &write_batch)
Definition storage.cpp:1132
void savePbftMgrStatus(PbftMgrStatus field, bool const &value)
Definition storage.cpp:1099
void replaceColumn(const Column &to_be_replaced_col, std::unique_ptr< rocksdb::ColumnFamilyHandle > &&replacing_col)
Definition storage.cpp:200
std::optional< PbftBlock > getPbftBlock(PbftPeriod period) const
Definition storage.cpp:832
void addPbftBlockPeriodToBatch(PbftPeriod period, taraxa::blk_hash_t const &pbft_block_hash, Batch &write_batch)
Definition storage.cpp:1247
void addSystemTransactionToBatch(Batch &write_batch, SharedTransaction trx)
Definition storage.cpp:910
auto getNumTransactionExecuted()
Definition storage.hpp:357
void commitWriteBatch(Batch &write_batch)
Definition storage.hpp:196
static void checkStatus(rocksdb::Status const &status)
Definition storage.cpp:430
std::optional< SharedTransactions > getPeriodTransactions(PbftPeriod period) const
Definition storage.cpp:977
static Batch createWriteBatch()
Definition storage.cpp:436
void clearOwnVerifiedVotes(Batch &write_batch, const std::vector< std::shared_ptr< PbftVote > > &own_verified_votes)
Definition storage.cpp:1176
void clearPeriodDataHistory(PbftPeriod period, uint64_t dag_level_to_keep, PbftPeriod last_block_number)
Definition storage.cpp:605
rocksdb::WriteOptions sync_write_
Definition storage.hpp:156
level_t getLastBlocksLevel() const
Definition storage.cpp:475
void addTransactionToBatch(Transaction const &trx, Batch &write_batch)
Definition storage.cpp:1022
std::set< blk_hash_t > getBlocksByLevel(level_t level)
Definition storage.cpp:469
void setGenesisHash(const h256 &genesis_hash)
Definition storage.cpp:397
auto dbStoragePath() const
Definition storage.hpp:192
std::string lookup(K const &key, Column const &column) const
Definition storage.hpp:419
uint32_t getMajorVersion() const
Definition storage.cpp:420
std::vector< std::shared_ptr< DagBlock > > getFinalizedDagBlockByPeriod(PbftPeriod period)
Definition storage.cpp:1299
const std::string kStateDbDir
Definition storage.hpp:163
void saveDagBlock(const std::shared_ptr< DagBlock > &blk, Batch *write_batch_p=nullptr)
Definition storage.cpp:546
~DbStorage()
Definition storage.cpp:411
std::vector< std::shared_ptr< PbftVote > > getOwnVerifiedVotes()
Definition storage.cpp:1165
const std::string kDbDir
Definition storage.hpp:162
void deleteTmpDirectories(const std::string &path) const
Definition storage.cpp:134
std::vector< std::shared_ptr< PillarVote > > getPeriodPillarVotes(PbftPeriod period) const
Definition storage.cpp:1007
dev::bytes getPeriodDataRaw(PbftPeriod period) const
Definition storage.cpp:710
std::unique_ptr< rocksdb::ColumnFamilyHandle > copyColumn(rocksdb::ColumnFamilyHandle *orig_column, const std::string &new_col_name, bool move_data=false)
Definition storage.cpp:156
std::pair< bool, PbftPeriod > getPeriodFromPbftHash(taraxa::blk_hash_t const &pbft_block_hash)
Definition storage.cpp:1252
void insert(rocksdb::ColumnFamilyHandle *col, const K &k, const V &v)
Definition storage.hpp:456
auto getNumTransactionInDag()
Definition storage.hpp:358
void enableSnapshots()
Definition storage.cpp:395
void saveProposalPeriodDagLevelsMap(uint64_t level, PbftPeriod period)
Definition storage.cpp:1339
std::string getPbftHead(blk_hash_t const &hash)
Definition storage.cpp:1148
void updateDagBlockCounters(std::vector< std::shared_ptr< DagBlock > > blks)
Definition storage.cpp:525
void replaceTwoTPlusOneVotes(TwoTPlusOneVotedBlockType type, const std::vector< std::shared_ptr< PbftVote > > &votes)
Definition storage.cpp:1183
void removeTempFiles() const
Definition storage.cpp:109
bool hasMinorVersionChanged()
Definition storage.hpp:371
uint64_t getStatusField(StatusDbField const &field)
Definition storage.cpp:1049
auto handle(Column const &col) const
Definition storage.hpp:152
std::optional< pillar_chain::CurrentPillarBlockDataDb > getCurrentPillarBlockData() const
Definition storage.cpp:763
std::atomic< uint64_t > dag_edge_count_
Definition storage.hpp:168
std::vector< rocksdb::ColumnFamilyHandle * > handles_
Definition storage.hpp:165
SharedTransactionReceipts getBlockReceipts(PbftPeriod period) const
Definition storage.cpp:998
static Slice toSlice(dev::bytesConstRef const &b)
Definition storage.hpp:388
static bytes asBytes(std::string const &b)
Definition storage.hpp:376
std::vector< std::shared_ptr< PbftVote > > getRewardVotes()
Definition storage.cpp:1236
uint64_t getDagEdgeCount() const
Definition storage.hpp:355
std::optional< TransactionReceipt > getTransactionReceipt(EthBlockNumber blk_n, uint64_t position) const
Definition storage.cpp:986
void updateDbVersions()
Definition storage.cpp:150
void addPbftHeadToBatch(taraxa::blk_hash_t const &head_hash, std::string const &head_str, Batch &write_batch)
Definition storage.cpp:1156
void savePbftHead(blk_hash_t const &hash, std::string const &pbft_chain_head_str)
Definition storage.cpp:1152
auto lookup_int(K const &key, Column const &column) -> std::enable_if_t< std::is_integral_v< Int >, std::optional< Int > >
Definition storage.hpp:430
void saveSortitionParamsChange(PbftPeriod period, const SortitionParamsChange &params, Batch &batch)
Definition storage.cpp:574
void addPbftMgrFieldToBatch(PbftMgrField field, uint32_t value, Batch &write_batch)
Definition storage.cpp:1085
std::vector< std::shared_ptr< PbftVote > > getPeriodCertVotes(PbftPeriod period) const
Definition storage.cpp:952
DbStorage & operator=(const DbStorage &)=delete
auto getNumBlockExecuted()
Definition storage.hpp:359
bool hasMajorVersionChanged()
Definition storage.hpp:372
std::vector< trx_hash_t > getPeriodSystemTransactionsHashes(PbftPeriod period) const
Definition storage.cpp:932
bool pbftBlockInDb(blk_hash_t const &hash)
Definition storage.cpp:1144
void removeDagBlockBatch(Batch &write_batch, blk_hash_t const &hash)
Definition storage.cpp:519
std::shared_ptr< Transaction > getSystemTransaction(const trx_hash_t &hash) const
Definition storage.cpp:914
void removeTransactionToBatch(trx_hash_t const &trx, Batch &write_batch)
Definition storage.cpp:1026
static Slice toSlice(dev::bytes const &b)
Definition storage.hpp:395
std::optional< SortitionParamsChange > getParamsChangeForPeriod(PbftPeriod period)
Definition storage.cpp:590
DbStorage(const DbStorage &)=delete
const uint32_t kDbSnapshotsEachNblock
Definition storage.hpp:169
void loadSnapshots()
Definition storage.cpp:286
uint64_t earliest_block_number_
Definition storage.hpp:173
static auto const & toSlice(Slice const &s)
Definition storage.hpp:404
static Slice toSlice(std::string const &str)
Definition storage.hpp:402
static Slice toSlice(dev::FixedHash< N > const &h)
Definition storage.hpp:391
void removeFilesWithPattern(const std::string &directory, const std::regex &pattern) const
Definition storage.cpp:116
std::unique_ptr< rocksdb::Iterator > getColumnIterator(const Column &c)
Definition storage.cpp:422
void insert(Batch &batch, rocksdb::ColumnFamilyHandle *col, K const &k, V const &v)
Definition storage.hpp:471
void replaceTwoTPlusOneVotesToBatch(TwoTPlusOneVotedBlockType type, const std::vector< std::shared_ptr< PbftVote > > &votes, Batch &write_batch)
Definition storage.cpp:1194
Changing vrf params for DAG blocks generation.
Definition sortition_params_manager.hpp:18
PeriodData class is for block execution, that includes PBFT block, certify votes, DAG blocks,...
Definition period_data.hpp:26
Definition pillar_block.hpp:155
std::vector<::byte > bytes
Definition Common.h:46
std::hash for asio::adress
Definition FixedHash.h:483
std::shared_ptr< spdlog::logger > Logger
Definition logging.hpp:12
Definition app.hpp:16
rocksdb::WriteBatch Batch
Definition storage.hpp:70
std::shared_ptr< Transaction > SharedTransaction
Definition transaction.hpp:84
uint64_t EthBlockNumber
Definition types.hpp:24
std::shared_ptr< std::vector< TransactionReceipt > > SharedTransactionReceipts
Definition receipt.hpp:54
DBMetaKeys
Definition storage.hpp:52
std::vector< byte > bytes
Definition types.hpp:53
uint32_t PbftRound
Definition types.hpp:26
EthBlockNumber PbftPeriod
Definition types.hpp:25
std::vector< SharedTransaction > SharedTransactions
Definition transaction.hpp:86
rocksdb::Slice Slice
Definition storage.hpp:71
std::function< void(Slice const &, Slice const &)> OnEntry
Definition storage.hpp:72
PbftMgrStatus
Definition storage.hpp:45
@ NextVotedNullBlockHash
Definition storage.hpp:49
@ ExecutedInRound
Definition storage.hpp:47
@ NextVotedSoftValue
Definition storage.hpp:48
@ ExecutedBlock
Definition storage.hpp:46
uint64_t level_t
Definition types.hpp:47
StatusDbField
Definition storage.hpp:33
@ DbMinorVersion
Definition storage.hpp:40
@ DbMajorVersion
Definition storage.hpp:39
@ DagEdgeCount
Definition storage.hpp:38
@ ExecutedTrxCount
Definition storage.hpp:35
@ TrxCount
Definition storage.hpp:36
@ ExecutedBlkCount
Definition storage.hpp:34
@ DagBlkCount
Definition storage.hpp:37
PbftMgrField
Definition storage.hpp:43
TwoTPlusOneVotedBlockType
Definition verified_votes.hpp:11
Definition transaction.hpp:12