TARAXA
Loading...
Searching...
No Matches
storage.hpp
Go to the documentation of this file.
1#pragma once
2
3#include <rocksdb/db.h>
4#include <rocksdb/options.h>
5#include <rocksdb/slice.h>
6#include <rocksdb/write_batch.h>
7
8#include <filesystem>
9#include <functional>
10#include <regex>
11
12#include "common/types.hpp"
13#include "dag/dag_block.hpp"
14#include "logger/logging.hpp"
15#include "pbft/pbft_block.hpp"
16#include "pbft/period_data.hpp"
21#include "vote/pillar_vote.hpp"
23
24namespace taraxa {
25namespace fs = std::filesystem;
26struct SortitionParamsChange;
27
28namespace pillar_chain {
29struct PillarBlockData;
30class PillarBlock;
31} // namespace pillar_chain
32
42
43enum class PbftMgrField : uint8_t { Round = 0, Step };
44
51
52enum class DBMetaKeys { LAST_NUMBER = 1 };
53
54class DbException : public std::exception {
55 public:
56 explicit DbException(const std::string& desc) : desc_(desc) {}
57 virtual ~DbException() = default;
58
59 DbException(const DbException&) = default;
63
64 virtual const char* what() const noexcept { return desc_.c_str(); }
65
66 private:
67 const std::string desc_;
68};
69
70using Batch = rocksdb::WriteBatch;
71using Slice = rocksdb::Slice;
72using OnEntry = std::function<void(Slice const&, Slice const&)>;
73
74class DbStorage : public std::enable_shared_from_this<DbStorage> {
75 public:
76 class Column {
77 std::string const name_;
78
79 public:
80 size_t const ordinal_;
81 const rocksdb::Comparator* comparator_;
82
83 Column(std::string name, size_t ordinal, const rocksdb::Comparator* comparator)
84 : name_(std::move(name)), ordinal_(ordinal), comparator_(comparator) {}
85
86 Column(std::string name, size_t ordinal) : name_(std::move(name)), ordinal_(ordinal), comparator_(nullptr) {}
87
88 auto const& name() const { return ordinal_ ? name_ : rocksdb::kDefaultColumnFamilyName; }
89 };
90
91 class Columns {
92 static inline std::vector<Column> all_;
93
94 public:
95 static inline auto const& all = all_;
96
97#define COLUMN(__name__) static inline auto const __name__ = all_.emplace_back(#__name__, all_.size())
98#define COLUMN_W_COMP(__name__, ...) \
99 static inline auto const __name__ = all_.emplace_back(#__name__, all_.size(), __VA_ARGS__)
100
101 // do not change/move
102 COLUMN(default_column);
103 // migrations
104 COLUMN(migrations);
105 // Contains full data for an executed PBFT block including PBFT block, cert votes, dag blocks and transactions
106 COLUMN_W_COMP(period_data, getIntComparator<PbftPeriod>());
107 COLUMN(genesis);
108 COLUMN(dag_blocks);
109 COLUMN_W_COMP(dag_blocks_level, getIntComparator<uint64_t>());
110 COLUMN(transactions);
111 COLUMN(trx_period);
112 COLUMN(status);
113 COLUMN(pbft_mgr_round_step);
114 COLUMN(pbft_mgr_status);
115 COLUMN(cert_voted_block_in_round); // Cert voted block + round -> node voted for this block
116 COLUMN(proposed_pbft_blocks); // Proposed pbft blocks
117 COLUMN(pbft_head);
118 COLUMN(latest_round_own_votes); // own votes of any type for the latest round
119 COLUMN(latest_round_two_t_plus_one_votes); // 2t+1 votes bundles of any type for the latest round
120 COLUMN(extra_reward_votes); // extra reward votes on top of 2t+1 cert votes bundle from
121 // latest_round_two_t_plus_one_votes
122 COLUMN(pbft_block_period);
123 COLUMN(dag_block_period);
124 COLUMN_W_COMP(proposal_period_levels_map, getIntComparator<uint64_t>());
125 COLUMN(final_chain_meta);
126 COLUMN(final_chain_blk_by_number);
127 COLUMN(final_chain_blk_hash_by_number);
128 COLUMN(final_chain_blk_number_by_hash);
129 COLUMN(final_chain_receipt_by_trx_hash);
130 COLUMN(final_chain_log_blooms_index);
131 COLUMN_W_COMP(sortition_params_change, getIntComparator<PbftPeriod>());
132
133 COLUMN_W_COMP(block_rewards_stats, getIntComparator<uint64_t>());
134
135 // Finalized pillar blocks
136 COLUMN_W_COMP(pillar_block, getIntComparator<PbftPeriod>());
137 // Current pillar block data - current pillar block + current vote counts
138 COLUMN(current_pillar_block_data);
139 // Current pillar block own pillar vote
140 COLUMN(current_pillar_block_own_vote);
141 // system transactions that is not a part of the block
142 COLUMN(system_transaction);
143 // system transactions hashes by period
144 COLUMN(period_system_transactions);
145 // final chain receipts by period
146 COLUMN_W_COMP(final_chain_receipt_by_period, getIntComparator<PbftPeriod>());
147
148#undef COLUMN
149#undef COLUMN_W_COMP
150 };
151
152 auto handle(Column const& col) const { return handles_[col.ordinal_]; }
153 rocksdb::ReadOptions read_options_;
154
155 rocksdb::WriteOptions async_write_;
156 rocksdb::WriteOptions sync_write_;
157
158 private:
159 fs::path path_;
160 fs::path db_path_;
162 const std::string kDbDir = "db";
163 const std::string kStateDbDir = "state_db";
164 std::unique_ptr<rocksdb::DB> db_;
165 std::vector<rocksdb::ColumnFamilyHandle*> handles_;
167 std::atomic<uint64_t> dag_blocks_count_;
168 std::atomic<uint64_t> dag_edge_count_;
169 const uint32_t kDbSnapshotsEachNblock = 0;
170 std::atomic<bool> snapshots_enabled_ = true;
171 const uint32_t kDbSnapshotsMaxCount = 0;
172 std::set<PbftPeriod> snapshots_;
173
177
179
180 public:
181 explicit DbStorage(fs::path const& base_path, uint32_t db_snapshot_each_n_pbft_block = 0, uint32_t max_open_files = 0,
182 uint32_t db_max_snapshots = 0, PbftPeriod db_revert_to_period = 0, bool rebuild = false);
183 ~DbStorage();
184
185 DbStorage(const DbStorage&) = delete;
186 DbStorage(DbStorage&&) = delete;
187 DbStorage& operator=(const DbStorage&) = delete;
189
190 auto const& path() const { return path_; }
191 auto dbStoragePath() const { return db_path_; }
192 auto stateDbStoragePath() const { return state_db_path_; }
193 static Batch createWriteBatch();
194 void commitWriteBatch(Batch& write_batch, const rocksdb::WriteOptions& opts);
195 void commitWriteBatch(Batch& write_batch) { commitWriteBatch(write_batch, async_write_); }
196
197 void rebuildColumns(const rocksdb::Options& options);
198 bool createSnapshot(PbftPeriod period);
199 void deleteSnapshot(PbftPeriod period);
200 void recoverToPeriod(PbftPeriod period);
201 void loadSnapshots();
202 void disableSnapshots();
203 void enableSnapshots();
204 void updateDbVersions();
205 void deleteColumnData(const Column& c);
206
207 void replaceColumn(const Column& to_be_replaced_col, std::unique_ptr<rocksdb::ColumnFamilyHandle>&& replacing_col);
208 std::unique_ptr<rocksdb::ColumnFamilyHandle> copyColumn(rocksdb::ColumnFamilyHandle* orig_column,
209 const std::string& new_col_name, bool move_data = false);
210
211 // For removal of LOG.old.* files in the database
212 void removeTempFiles() const;
213 void removeFilesWithPattern(const std::string& directory, const std::regex& pattern) const;
214 void deleteTmpDirectories(const std::string& path) const;
215
216 uint32_t getMajorVersion() const;
217 std::unique_ptr<rocksdb::Iterator> getColumnIterator(const Column& c);
218 std::unique_ptr<rocksdb::Iterator> getColumnIterator(rocksdb::ColumnFamilyHandle* c);
219
220 // Genesis
221 void setGenesisHash(const h256& genesis_hash);
222 std::optional<h256> getGenesisHash();
223
224 // Period data
225 void savePeriodData(const PeriodData& period_data, Batch& write_batch);
226 void clearPeriodDataHistory(PbftPeriod period, uint64_t dag_level_to_keep, PbftPeriod last_block_number);
228 std::optional<PeriodData> getPeriodData(PbftPeriod period) const;
229 std::optional<PbftBlock> getPbftBlock(PbftPeriod period) const;
230 std::vector<std::shared_ptr<PbftVote>> getPeriodCertVotes(PbftPeriod period) const;
232 SharedTransactions transactionsFromPeriodDataRlp(PbftPeriod period, const dev::RLP& period_data_rlp) const;
233 std::optional<SharedTransactions> getPeriodTransactions(PbftPeriod period) const;
234 std::vector<std::shared_ptr<PillarVote>> getPeriodPillarVotes(PbftPeriod period) const;
235
236 // Pillar chain
237 void savePillarBlock(const std::shared_ptr<pillar_chain::PillarBlock>& pillar_block);
238 std::shared_ptr<pillar_chain::PillarBlock> getPillarBlock(PbftPeriod period) const;
239 std::shared_ptr<pillar_chain::PillarBlock> getLatestPillarBlock() const;
240 void saveOwnPillarBlockVote(const std::shared_ptr<PillarVote>& vote);
241 std::shared_ptr<PillarVote> getOwnPillarBlockVote() const;
242 void saveCurrentPillarBlockData(const pillar_chain::CurrentPillarBlockDataDb& current_pillar_block_data);
243 std::optional<pillar_chain::CurrentPillarBlockDataDb> getCurrentPillarBlockData() const;
244
245 // DAG
246 void saveDagBlock(const std::shared_ptr<DagBlock>& blk, Batch* write_batch_p = nullptr);
247 std::shared_ptr<DagBlock> getDagBlock(blk_hash_t const& hash);
248 bool dagBlockInDb(blk_hash_t const& hash);
249 std::set<blk_hash_t> getBlocksByLevel(level_t level);
251 std::vector<std::shared_ptr<DagBlock>> getDagBlocksAtLevel(level_t level, int number_of_levels);
252 void updateDagBlockCounters(std::vector<std::shared_ptr<DagBlock>> blks);
253 std::map<level_t, std::vector<std::shared_ptr<DagBlock>>> getNonfinalizedDagBlocks();
254 void removeDagBlockBatch(Batch& write_batch, blk_hash_t const& hash);
255 void removeDagBlock(blk_hash_t const& hash);
256 // Sortition params
257 void saveSortitionParamsChange(PbftPeriod period, const SortitionParamsChange& params, Batch& batch);
258 std::deque<SortitionParamsChange> getLastSortitionParams(size_t count);
259 std::optional<SortitionParamsChange> getParamsChangeForPeriod(PbftPeriod period);
260
261 // Transaction
262 std::shared_ptr<Transaction> getTransaction(trx_hash_t const& hash) const;
263 std::shared_ptr<Transaction> getTransaction(PbftPeriod period, uint32_t position) const;
264
266 bool transactionInDb(trx_hash_t const& hash);
267 bool transactionFinalized(trx_hash_t const& hash);
268 std::vector<bool> transactionsInDb(std::vector<trx_hash_t> const& trx_hashes);
269 std::vector<bool> transactionsFinalized(std::vector<trx_hash_t> const& trx_hashes);
270 void addTransactionToBatch(Transaction const& trx, Batch& write_batch);
271 void removeTransactionToBatch(trx_hash_t const& trx, Batch& write_batch);
272
273 void addTransactionLocationToBatch(Batch& write_batch, trx_hash_t const& trx, PbftPeriod period, uint32_t position,
274 bool is_system = false);
275 std::optional<TransactionLocation> getTransactionLocation(trx_hash_t const& hash) const;
276 std::unordered_map<trx_hash_t, PbftPeriod> getAllTransactionPeriod();
277 uint64_t getTransactionCount(PbftPeriod period) const;
279 std::optional<TransactionReceipt> getTransactionReceipt(EthBlockNumber blk_n, uint64_t position) const;
280
288 SharedTransactions getFinalizedTransactions(std::vector<trx_hash_t> const& trx_hashes) const;
289
290 // System transaction
292 std::shared_ptr<Transaction> getSystemTransaction(const trx_hash_t& hash) const;
293 void addPeriodSystemTransactions(Batch& write_batch, SharedTransactions trxs, PbftPeriod period);
294 std::vector<trx_hash_t> getPeriodSystemTransactionsHashes(PbftPeriod period) const;
296
297 // PBFT manager
298 uint32_t getPbftMgrField(PbftMgrField field);
299 void savePbftMgrField(PbftMgrField field, uint32_t value);
300 void addPbftMgrFieldToBatch(PbftMgrField field, uint32_t value, Batch& write_batch);
301
303 void savePbftMgrStatus(PbftMgrStatus field, bool const& value);
304 void addPbftMgrStatusToBatch(PbftMgrStatus field, bool const& value, Batch& write_batch);
305
306 void saveCertVotedBlockInRound(PbftRound round, const std::shared_ptr<PbftBlock>& block);
307 std::optional<std::pair<PbftRound, std::shared_ptr<PbftBlock>>> getCertVotedBlockInRound() const;
308 void removeCertVotedBlockInRound(Batch& write_batch);
309
310 // pbft_blocks
311 std::optional<PbftBlock> getPbftBlock(blk_hash_t const& hash);
312 bool pbftBlockInDb(blk_hash_t const& hash);
313
314 // Proposed pbft blocks
315 void saveProposedPbftBlock(const std::shared_ptr<PbftBlock>& block);
316 void removeProposedPbftBlock(const blk_hash_t& block_hash, Batch& write_batch);
317 std::vector<std::shared_ptr<PbftBlock>> getProposedPbftBlocks();
318
319 // pbft_blocks (head)
320 std::string getPbftHead(blk_hash_t const& hash);
321 void savePbftHead(blk_hash_t const& hash, std::string const& pbft_chain_head_str);
322 void addPbftHeadToBatch(taraxa::blk_hash_t const& head_hash, std::string const& head_str, Batch& write_batch);
323
324 // status
325 uint64_t getStatusField(StatusDbField const& field);
326 void saveStatusField(StatusDbField const& field, uint64_t value);
327 void addStatusFieldToBatch(StatusDbField const& field, uint64_t value, Batch& write_batch);
328
329 // Own votes for the latest round
330 void saveOwnVerifiedVote(const std::shared_ptr<PbftVote>& vote);
331 std::vector<std::shared_ptr<PbftVote>> getOwnVerifiedVotes();
332 void clearOwnVerifiedVotes(Batch& write_batch, const std::vector<std::shared_ptr<PbftVote>>& own_verified_votes);
333
334 // 2t+1 votes bundles for the latest round
335 void replaceTwoTPlusOneVotes(TwoTPlusOneVotedBlockType type, const std::vector<std::shared_ptr<PbftVote>>& votes);
337 const std::vector<std::shared_ptr<PbftVote>>& votes, Batch& write_batch);
338 std::vector<std::shared_ptr<PbftVote>> getAllTwoTPlusOneVotes();
339
340 // Reward votes - cert votes for the latest finalized block
341 void removeExtraRewardVotes(const std::vector<vote_hash_t>& votes, Batch& write_batch);
342 void saveExtraRewardVote(const std::shared_ptr<PbftVote>& vote);
343 std::vector<std::shared_ptr<PbftVote>> getRewardVotes();
344
345 // period_pbft_block
346 void addPbftBlockPeriodToBatch(PbftPeriod period, taraxa::blk_hash_t const& pbft_block_hash, Batch& write_batch);
347 std::pair<bool, PbftPeriod> getPeriodFromPbftHash(taraxa::blk_hash_t const& pbft_block_hash);
348 // dag_block_period
349 std::shared_ptr<std::pair<PbftPeriod, uint32_t>> getDagBlockPeriod(blk_hash_t const& hash);
350 void addDagBlockPeriodToBatch(blk_hash_t const& hash, PbftPeriod period, uint32_t position, Batch& write_batch);
351
352 uint64_t getDagBlocksCount() const { return dag_blocks_count_.load(); }
353 uint64_t getDagEdgeCount() const { return dag_edge_count_.load(); }
354
358
359 std::vector<blk_hash_t> getFinalizedDagBlockHashesByPeriod(PbftPeriod period);
360 std::vector<std::shared_ptr<DagBlock>> getFinalizedDagBlockByPeriod(PbftPeriod period);
361 std::pair<blk_hash_t, std::vector<std::shared_ptr<DagBlock>>> getLastPbftBlockHashAndFinalizedDagBlockByPeriod(
362 PbftPeriod period);
363
364 // DPOS level to proposal period map
365 std::optional<uint64_t> getProposalPeriodForDagLevel(uint64_t level);
366 void saveProposalPeriodDagLevelsMap(uint64_t level, PbftPeriod period);
367 void addProposalPeriodDagLevelsMapToBatch(uint64_t level, PbftPeriod period, Batch& write_batch);
368
371
372 void compactColumn(Column const& column) { db_->CompactRange({}, handle(column), nullptr, nullptr); }
373
374 inline static bytes asBytes(std::string const& b) {
375 return bytes((byte const*)b.data(), (byte const*)(b.data() + b.size()));
376 }
377
378 template <typename T>
379 inline static Slice make_slice(T const* begin, size_t size) {
380 if (!size) {
381 return {};
382 }
383 return {reinterpret_cast<char const*>(begin), size};
384 }
385
386 inline static Slice toSlice(dev::bytesConstRef const& b) { return make_slice(b.data(), b.size()); }
387
388 template <unsigned N>
389 inline static Slice toSlice(dev::FixedHash<N> const& h) {
390 return make_slice(h.data(), N);
391 }
392
393 inline static Slice toSlice(dev::bytes const& b) { return make_slice(b.data(), b.size()); }
394
395 template <class N>
396 inline static auto toSlice(N const& n) -> std::enable_if_t<std::is_integral_v<N> || std::is_enum_v<N>, Slice> {
397 return make_slice(&n, sizeof(N));
398 }
399
400 inline static Slice toSlice(std::string const& str) { return make_slice(str.data(), str.size()); }
401
402 inline static auto const& toSlice(Slice const& s) { return s; }
403
404 template <typename T>
405 inline static auto toSlices(std::vector<T> const& keys) {
406 std::vector<Slice> ret;
407 ret.reserve(keys.size());
408 for (auto const& k : keys) {
409 ret.emplace_back(toSlice(k));
410 }
411 return ret;
412 }
413
414 inline static auto const& toSlices(std::vector<Slice> const& ss) { return ss; }
415
416 template <typename K>
417 std::string lookup(K const& key, Column const& column) const {
418 std::string value;
419 auto status = db_->Get(read_options_, handle(column), toSlice(key), &value);
420 if (status.IsNotFound()) {
421 return value;
422 }
423 checkStatus(status);
424 return value;
425 }
426
427 template <typename Int, typename K>
428 auto lookup_int(K const& key, Column const& column) -> std::enable_if_t<std::is_integral_v<Int>, std::optional<Int>> {
429 auto str = lookup(key, column);
430 if (str.empty()) {
431 return std::nullopt;
432 }
433 return *reinterpret_cast<Int*>(str.data());
434 }
435
436 template <typename K>
437 bool exist(K const& key, Column const& column) {
438 std::string value;
439 // KeyMayExist can lead to a few false positives, but not false negatives.
440 if (db_->KeyMayExist(read_options_, handle(column), toSlice(key), &value)) {
441 auto status = db_->Get(read_options_, handle(column), toSlice(key), &value);
442 if (status.IsNotFound()) {
443 return false;
444 }
445 checkStatus(status);
446 return !value.empty();
447 }
448 return false;
449 }
450
451 static void checkStatus(rocksdb::Status const& status);
452
453 template <typename K, typename V>
454 void insert(rocksdb::ColumnFamilyHandle* col, const K& k, const V& v) {
455 checkStatus(db_->Put(async_write_, col, toSlice(k), toSlice(v)));
456 }
457
458 template <typename K, typename V>
459 void insert(Column const& col, K const& k, V const& v) {
460 checkStatus(db_->Put(async_write_, handle(col), toSlice(k), toSlice(v)));
461 }
462
463 template <typename K, typename V>
464 void insert(Batch& batch, Column const& col, K const& k, V const& v) {
465 checkStatus(batch.Put(handle(col), toSlice(k), toSlice(v)));
466 }
467
468 template <typename K, typename V>
469 void insert(Batch& batch, rocksdb::ColumnFamilyHandle* col, K const& k, V const& v) {
470 checkStatus(batch.Put(col, toSlice(k), toSlice(v)));
471 }
472
473 template <typename K>
474 void remove(Column const& col, K const& k) {
475 checkStatus(db_->Delete(async_write_, handle(col), toSlice(k)));
476 }
477
478 template <typename K>
479 void remove(Batch& batch, Column const& col, K const& k) {
480 checkStatus(batch.Delete(handle(col), toSlice(k)));
481 }
482
483 void forEach(Column const& col, OnEntry const& f);
484};
485
486} // namespace taraxa
::byte * data()
Definition FixedHash.h:189
Definition RLP.h:56
Definition vector_ref.h:23
_T * data() const
Definition vector_ref.h:76
size_t size() const
Definition vector_ref.h:82
Definition storage.hpp:54
virtual const char * what() const noexcept
Definition storage.hpp:64
virtual ~DbException()=default
const std::string desc_
Definition storage.hpp:67
DbException(const std::string &desc)
Definition storage.hpp:56
DbException(const DbException &)=default
DbException(DbException &&)=default
DbException & operator=(DbException &&)=delete
DbException & operator=(const DbException &)=delete
Definition storage.hpp:76
Column(std::string name, size_t ordinal, const rocksdb::Comparator *comparator)
Definition storage.hpp:83
const rocksdb::Comparator * comparator_
Definition storage.hpp:81
Column(std::string name, size_t ordinal)
Definition storage.hpp:86
auto const & name() const
Definition storage.hpp:88
size_t const ordinal_
Definition storage.hpp:80
std::string const name_
Definition storage.hpp:77
Definition storage.hpp:91
static auto const & all
Definition storage.hpp:95
COLUMN(period_system_transactions)
COLUMN(cert_voted_block_in_round)
COLUMN_W_COMP(dag_blocks_level, getIntComparator< uint64_t >())
COLUMN(current_pillar_block_own_vote)
COLUMN(final_chain_meta)
COLUMN(dag_block_period)
COLUMN_W_COMP(pillar_block, getIntComparator< PbftPeriod >())
COLUMN(final_chain_receipt_by_trx_hash)
COLUMN_W_COMP(block_rewards_stats, getIntComparator< uint64_t >())
COLUMN(final_chain_blk_by_number)
COLUMN(extra_reward_votes)
COLUMN(final_chain_blk_hash_by_number)
COLUMN(pbft_block_period)
COLUMN(latest_round_two_t_plus_one_votes)
COLUMN(system_transaction)
COLUMN(final_chain_log_blooms_index)
COLUMN(latest_round_own_votes)
COLUMN_W_COMP(proposal_period_levels_map, getIntComparator< uint64_t >())
COLUMN(pbft_mgr_round_step)
static std::vector< Column > all_
Definition storage.hpp:92
COLUMN_W_COMP(period_data, getIntComparator< PbftPeriod >())
COLUMN(pbft_mgr_status)
COLUMN(proposed_pbft_blocks)
COLUMN_W_COMP(sortition_params_change, getIntComparator< PbftPeriod >())
COLUMN(current_pillar_block_data)
COLUMN(final_chain_blk_number_by_hash)
COLUMN_W_COMP(final_chain_receipt_by_period, getIntComparator< PbftPeriod >())
Definition storage.hpp:74
void addStatusFieldToBatch(StatusDbField const &field, uint64_t value, Batch &write_batch)
Definition storage.cpp:1060
void addDagBlockPeriodToBatch(blk_hash_t const &hash, PbftPeriod period, uint32_t position, Batch &write_batch)
Definition storage.cpp:1273
auto const & path() const
Definition storage.hpp:190
uint64_t getTransactionCount(PbftPeriod period) const
Definition storage.cpp:871
std::atomic< uint64_t > dag_blocks_count_
Definition storage.hpp:167
SharedTransactions getPeriodSystemTransactions(PbftPeriod period) const
Definition storage.cpp:936
static auto toSlices(std::vector< T > const &keys)
Definition storage.hpp:405
bool dagBlockInDb(blk_hash_t const &hash)
Definition storage.cpp:461
void remove(Batch &batch, Column const &col, K const &k)
Definition storage.hpp:479
std::shared_ptr< DagBlock > getDagBlock(blk_hash_t const &hash)
Definition storage.cpp:444
fs::path db_path_
Definition storage.hpp:160
std::shared_ptr< Transaction > getTransaction(trx_hash_t const &hash) const
Definition storage.cpp:846
void disableSnapshots()
Definition storage.cpp:393
std::vector< std::shared_ptr< PbftVote > > getAllTwoTPlusOneVotes()
Definition storage.cpp:1202
std::optional< uint64_t > getProposalPeriodForDagLevel(uint64_t level)
Definition storage.cpp:1320
std::mutex dag_blocks_mutex_
Definition storage.hpp:166
uint32_t getPbftMgrField(PbftMgrField field)
Definition storage.cpp:1066
std::optional< PeriodData > getPeriodData(PbftPeriod period) const
Definition storage.cpp:710
void saveProposedPbftBlock(const std::shared_ptr< PbftBlock > &block)
Definition storage.cpp:811
void removeProposedPbftBlock(const blk_hash_t &block_hash, Batch &write_batch)
Definition storage.cpp:815
void addProposalPeriodDagLevelsMapToBatch(uint64_t level, PbftPeriod period, Batch &write_batch)
Definition storage.cpp:1339
void saveCertVotedBlockInRound(PbftRound round, const std::shared_ptr< PbftBlock > &block)
Definition storage.cpp:1103
std::unordered_map< trx_hash_t, PbftPeriod > getAllTransactionPeriod()
Definition storage.cpp:798
void saveOwnPillarBlockVote(const std::shared_ptr< PillarVote > &vote)
Definition storage.cpp:742
rocksdb::WriteOptions async_write_
Definition storage.hpp:155
std::shared_ptr< pillar_chain::PillarBlock > getPillarBlock(PbftPeriod period) const
Definition storage.cpp:723
void addPbftMgrStatusToBatch(PbftMgrStatus field, bool const &value, Batch &write_batch)
Definition storage.cpp:1099
void removeExtraRewardVotes(const std::vector< vote_hash_t > &votes, Batch &write_batch)
Definition storage.cpp:1222
void rebuildColumns(const rocksdb::Options &options)
Definition storage.cpp:229
void addTransactionLocationToBatch(Batch &write_batch, trx_hash_t const &trx, PbftPeriod period, uint32_t position, bool is_system=false)
Definition storage.cpp:768
SharedTransactions transactionsFromPeriodDataRlp(PbftPeriod period, const dev::RLP &period_data_rlp) const
Definition storage.cpp:962
blk_hash_t getPeriodBlockHash(PbftPeriod period) const
Definition storage.cpp:838
std::vector< std::shared_ptr< PbftBlock > > getProposedPbftBlocks()
Definition storage.cpp:819
std::vector< std::shared_ptr< DagBlock > > getDagBlocksAtLevel(level_t level, int number_of_levels)
Definition storage.cpp:485
std::shared_ptr< PillarVote > getOwnPillarBlockVote() const
Definition storage.cpp:746
std::atomic< bool > snapshots_enabled_
Definition storage.hpp:170
DbStorage & operator=(DbStorage &&)=delete
uint32_t kMajorVersion_
Definition storage.hpp:174
const uint32_t kDbSnapshotsMaxCount
Definition storage.hpp:171
void addPeriodSystemTransactions(Batch &write_batch, SharedTransactions trxs, PbftPeriod period)
Definition storage.cpp:919
static auto const & toSlices(std::vector< Slice > const &ss)
Definition storage.hpp:414
std::optional< h256 > getGenesisHash()
Definition storage.cpp:403
void compactColumn(Column const &column)
Definition storage.hpp:372
SharedTransactions getAllNonfinalizedTransactions()
Definition storage.cpp:510
void insert(Batch &batch, Column const &col, K const &k, V const &v)
Definition storage.hpp:464
auto stateDbStoragePath() const
Definition storage.hpp:192
uint64_t getDagBlocksCount() const
Definition storage.hpp:352
std::vector< blk_hash_t > getFinalizedDagBlockHashesByPeriod(PbftPeriod period)
Definition storage.cpp:1282
bool transactionInDb(trx_hash_t const &hash)
Definition storage.cpp:1026
void saveOwnVerifiedVote(const std::shared_ptr< PbftVote > &vote)
Definition storage.cpp:1157
bool minor_version_changed_
Definition storage.hpp:176
bool createSnapshot(PbftPeriod period)
Definition storage.cpp:312
SharedTransactions getFinalizedTransactions(std::vector< trx_hash_t > const &trx_hashes) const
Gets finalized transactions from provided hashes.
Definition storage.cpp:880
std::map< level_t, std::vector< std::shared_ptr< DagBlock > > > getNonfinalizedDagBlocks()
Definition storage.cpp:500
void saveStatusField(StatusDbField const &field, uint64_t value)
Definition storage.cpp:1056
void saveCurrentPillarBlockData(const pillar_chain::CurrentPillarBlockDataDb &current_pillar_block_data)
Definition storage.cpp:755
rocksdb::ReadOptions read_options_
Definition storage.hpp:153
std::unique_ptr< rocksdb::DB > db_
Definition storage.hpp:164
void commitWriteBatch(Batch &write_batch, const rocksdb::WriteOptions &opts)
Definition storage.cpp:438
bool exist(K const &key, Column const &column)
Definition storage.hpp:437
void saveExtraRewardVote(const std::shared_ptr< PbftVote > &vote)
Definition storage.cpp:1228
void removeDagBlock(blk_hash_t const &hash)
Definition storage.cpp:523
fs::path path_
Definition storage.hpp:159
void insert(Column const &col, K const &k, V const &v)
Definition storage.hpp:459
std::vector< bool > transactionsInDb(std::vector< trx_hash_t > const &trx_hashes)
Definition storage.cpp:1034
std::pair< blk_hash_t, std::vector< std::shared_ptr< DagBlock > > > getLastPbftBlockHashAndFinalizedDagBlockByPeriod(PbftPeriod period)
Definition storage.cpp:1306
void savePeriodData(const PeriodData &period_data, Batch &write_batch)
Definition storage.cpp:683
void deleteColumnData(const Column &c)
Definition storage.cpp:218
void recoverToPeriod(PbftPeriod period)
Definition storage.cpp:345
fs::path state_db_path_
Definition storage.hpp:161
std::shared_ptr< pillar_chain::PillarBlock > getLatestPillarBlock() const
Definition storage.cpp:732
std::optional< TransactionLocation > getTransactionLocation(trx_hash_t const &hash) const
Definition storage.cpp:780
bool getPbftMgrStatus(PbftMgrStatus field)
Definition storage.cpp:1085
bool major_version_changed_
Definition storage.hpp:175
void savePillarBlock(const std::shared_ptr< pillar_chain::PillarBlock > &pillar_block)
Definition storage.cpp:719
void savePbftMgrField(PbftMgrField field, uint32_t value)
Definition storage.cpp:1077
logger::Logger logger_
Definition storage.hpp:178
std::optional< std::pair< PbftRound, std::shared_ptr< PbftBlock > > > getCertVotedBlockInRound() const
Definition storage.cpp:1112
static Slice make_slice(T const *begin, size_t size)
Definition storage.hpp:379
std::vector< bool > transactionsFinalized(std::vector< trx_hash_t > const &trx_hashes)
Definition storage.cpp:788
bool transactionFinalized(trx_hash_t const &hash)
Definition storage.cpp:1030
std::deque< SortitionParamsChange > getLastSortitionParams(size_t count)
Definition storage.cpp:578
static auto toSlice(N const &n) -> std::enable_if_t< std::is_integral_v< N >||std::is_enum_v< N >, Slice >
Definition storage.hpp:396
void deleteSnapshot(PbftPeriod period)
Definition storage.cpp:377
std::shared_ptr< std::pair< PbftPeriod, uint32_t > > getDagBlockPeriod(blk_hash_t const &hash)
Definition storage.cpp:1260
void forEach(Column const &col, OnEntry const &f)
Definition storage.cpp:1343
void remove(Column const &col, K const &k)
Definition storage.hpp:474
DbStorage(DbStorage &&)=delete
std::set< PbftPeriod > snapshots_
Definition storage.hpp:172
void removeCertVotedBlockInRound(Batch &write_batch)
Definition storage.cpp:1128
void savePbftMgrStatus(PbftMgrStatus field, bool const &value)
Definition storage.cpp:1095
void replaceColumn(const Column &to_be_replaced_col, std::unique_ptr< rocksdb::ColumnFamilyHandle > &&replacing_col)
Definition storage.cpp:200
std::optional< PbftBlock > getPbftBlock(PbftPeriod period) const
Definition storage.cpp:828
void addPbftBlockPeriodToBatch(PbftPeriod period, taraxa::blk_hash_t const &pbft_block_hash, Batch &write_batch)
Definition storage.cpp:1243
void addSystemTransactionToBatch(Batch &write_batch, SharedTransaction trx)
Definition storage.cpp:906
auto getNumTransactionExecuted()
Definition storage.hpp:355
void commitWriteBatch(Batch &write_batch)
Definition storage.hpp:195
static void checkStatus(rocksdb::Status const &status)
Definition storage.cpp:430
std::optional< SharedTransactions > getPeriodTransactions(PbftPeriod period) const
Definition storage.cpp:973
static Batch createWriteBatch()
Definition storage.cpp:436
void clearOwnVerifiedVotes(Batch &write_batch, const std::vector< std::shared_ptr< PbftVote > > &own_verified_votes)
Definition storage.cpp:1172
void clearPeriodDataHistory(PbftPeriod period, uint64_t dag_level_to_keep, PbftPeriod last_block_number)
Definition storage.cpp:603
rocksdb::WriteOptions sync_write_
Definition storage.hpp:156
level_t getLastBlocksLevel() const
Definition storage.cpp:475
void addTransactionToBatch(Transaction const &trx, Batch &write_batch)
Definition storage.cpp:1018
std::set< blk_hash_t > getBlocksByLevel(level_t level)
Definition storage.cpp:469
void setGenesisHash(const h256 &genesis_hash)
Definition storage.cpp:397
auto dbStoragePath() const
Definition storage.hpp:191
std::string lookup(K const &key, Column const &column) const
Definition storage.hpp:417
uint32_t getMajorVersion() const
Definition storage.cpp:420
std::vector< std::shared_ptr< DagBlock > > getFinalizedDagBlockByPeriod(PbftPeriod period)
Definition storage.cpp:1295
const std::string kStateDbDir
Definition storage.hpp:163
void saveDagBlock(const std::shared_ptr< DagBlock > &blk, Batch *write_batch_p=nullptr)
Definition storage.cpp:546
~DbStorage()
Definition storage.cpp:411
std::vector< std::shared_ptr< PbftVote > > getOwnVerifiedVotes()
Definition storage.cpp:1161
const std::string kDbDir
Definition storage.hpp:162
void deleteTmpDirectories(const std::string &path) const
Definition storage.cpp:134
std::vector< std::shared_ptr< PillarVote > > getPeriodPillarVotes(PbftPeriod period) const
Definition storage.cpp:1003
dev::bytes getPeriodDataRaw(PbftPeriod period) const
Definition storage.cpp:706
std::unique_ptr< rocksdb::ColumnFamilyHandle > copyColumn(rocksdb::ColumnFamilyHandle *orig_column, const std::string &new_col_name, bool move_data=false)
Definition storage.cpp:156
std::pair< bool, PbftPeriod > getPeriodFromPbftHash(taraxa::blk_hash_t const &pbft_block_hash)
Definition storage.cpp:1248
void insert(rocksdb::ColumnFamilyHandle *col, const K &k, const V &v)
Definition storage.hpp:454
auto getNumTransactionInDag()
Definition storage.hpp:356
void enableSnapshots()
Definition storage.cpp:395
void saveProposalPeriodDagLevelsMap(uint64_t level, PbftPeriod period)
Definition storage.cpp:1335
std::string getPbftHead(blk_hash_t const &hash)
Definition storage.cpp:1144
void updateDagBlockCounters(std::vector< std::shared_ptr< DagBlock > > blks)
Definition storage.cpp:525
void replaceTwoTPlusOneVotes(TwoTPlusOneVotedBlockType type, const std::vector< std::shared_ptr< PbftVote > > &votes)
Definition storage.cpp:1179
void removeTempFiles() const
Definition storage.cpp:109
bool hasMinorVersionChanged()
Definition storage.hpp:369
uint64_t getStatusField(StatusDbField const &field)
Definition storage.cpp:1045
auto handle(Column const &col) const
Definition storage.hpp:152
std::optional< pillar_chain::CurrentPillarBlockDataDb > getCurrentPillarBlockData() const
Definition storage.cpp:759
std::atomic< uint64_t > dag_edge_count_
Definition storage.hpp:168
std::vector< rocksdb::ColumnFamilyHandle * > handles_
Definition storage.hpp:165
SharedTransactionReceipts getBlockReceipts(PbftPeriod period) const
Definition storage.cpp:994
static Slice toSlice(dev::bytesConstRef const &b)
Definition storage.hpp:386
static bytes asBytes(std::string const &b)
Definition storage.hpp:374
std::vector< std::shared_ptr< PbftVote > > getRewardVotes()
Definition storage.cpp:1232
uint64_t getDagEdgeCount() const
Definition storage.hpp:353
std::optional< TransactionReceipt > getTransactionReceipt(EthBlockNumber blk_n, uint64_t position) const
Definition storage.cpp:982
void updateDbVersions()
Definition storage.cpp:150
void addPbftHeadToBatch(taraxa::blk_hash_t const &head_hash, std::string const &head_str, Batch &write_batch)
Definition storage.cpp:1152
void savePbftHead(blk_hash_t const &hash, std::string const &pbft_chain_head_str)
Definition storage.cpp:1148
auto lookup_int(K const &key, Column const &column) -> std::enable_if_t< std::is_integral_v< Int >, std::optional< Int > >
Definition storage.hpp:428
void saveSortitionParamsChange(PbftPeriod period, const SortitionParamsChange &params, Batch &batch)
Definition storage.cpp:574
void addPbftMgrFieldToBatch(PbftMgrField field, uint32_t value, Batch &write_batch)
Definition storage.cpp:1081
std::vector< std::shared_ptr< PbftVote > > getPeriodCertVotes(PbftPeriod period) const
Definition storage.cpp:948
DbStorage & operator=(const DbStorage &)=delete
auto getNumBlockExecuted()
Definition storage.hpp:357
bool hasMajorVersionChanged()
Definition storage.hpp:370
std::vector< trx_hash_t > getPeriodSystemTransactionsHashes(PbftPeriod period) const
Definition storage.cpp:928
bool pbftBlockInDb(blk_hash_t const &hash)
Definition storage.cpp:1140
void removeDagBlockBatch(Batch &write_batch, blk_hash_t const &hash)
Definition storage.cpp:519
std::shared_ptr< Transaction > getSystemTransaction(const trx_hash_t &hash) const
Definition storage.cpp:910
void removeTransactionToBatch(trx_hash_t const &trx, Batch &write_batch)
Definition storage.cpp:1022
static Slice toSlice(dev::bytes const &b)
Definition storage.hpp:393
std::optional< SortitionParamsChange > getParamsChangeForPeriod(PbftPeriod period)
Definition storage.cpp:590
DbStorage(const DbStorage &)=delete
const uint32_t kDbSnapshotsEachNblock
Definition storage.hpp:169
void loadSnapshots()
Definition storage.cpp:286
static auto const & toSlice(Slice const &s)
Definition storage.hpp:402
static Slice toSlice(std::string const &str)
Definition storage.hpp:400
static Slice toSlice(dev::FixedHash< N > const &h)
Definition storage.hpp:389
void removeFilesWithPattern(const std::string &directory, const std::regex &pattern) const
Definition storage.cpp:116
std::unique_ptr< rocksdb::Iterator > getColumnIterator(const Column &c)
Definition storage.cpp:422
void insert(Batch &batch, rocksdb::ColumnFamilyHandle *col, K const &k, V const &v)
Definition storage.hpp:469
void replaceTwoTPlusOneVotesToBatch(TwoTPlusOneVotedBlockType type, const std::vector< std::shared_ptr< PbftVote > > &votes, Batch &write_batch)
Definition storage.cpp:1190
Changing vrf params for DAG blocks generation.
Definition sortition_params_manager.hpp:18
PeriodData class is for block execution, that includes PBFT block, certify votes, DAG blocks,...
Definition period_data.hpp:26
Definition pillar_block.hpp:155
std::vector<::byte > bytes
Definition Common.h:46
std::hash for asio::adress
Definition FixedHash.h:483
std::shared_ptr< spdlog::logger > Logger
Definition logging.hpp:12
Definition app.hpp:16
rocksdb::WriteBatch Batch
Definition storage.hpp:70
std::shared_ptr< Transaction > SharedTransaction
Definition transaction.hpp:84
uint64_t EthBlockNumber
Definition types.hpp:24
std::shared_ptr< std::vector< TransactionReceipt > > SharedTransactionReceipts
Definition receipt.hpp:54
DBMetaKeys
Definition storage.hpp:52
std::vector< byte > bytes
Definition types.hpp:53
uint32_t PbftRound
Definition types.hpp:26
EthBlockNumber PbftPeriod
Definition types.hpp:25
std::vector< SharedTransaction > SharedTransactions
Definition transaction.hpp:86
rocksdb::Slice Slice
Definition storage.hpp:71
std::function< void(Slice const &, Slice const &)> OnEntry
Definition storage.hpp:72
PbftMgrStatus
Definition storage.hpp:45
@ NextVotedNullBlockHash
Definition storage.hpp:49
@ ExecutedInRound
Definition storage.hpp:47
@ NextVotedSoftValue
Definition storage.hpp:48
@ ExecutedBlock
Definition storage.hpp:46
uint64_t level_t
Definition types.hpp:47
StatusDbField
Definition storage.hpp:33
@ DbMinorVersion
Definition storage.hpp:40
@ DbMajorVersion
Definition storage.hpp:39
@ DagEdgeCount
Definition storage.hpp:38
@ ExecutedTrxCount
Definition storage.hpp:35
@ TrxCount
Definition storage.hpp:36
@ ExecutedBlkCount
Definition storage.hpp:34
@ DagBlkCount
Definition storage.hpp:37
PbftMgrField
Definition storage.hpp:43
TwoTPlusOneVotedBlockType
Definition verified_votes.hpp:11
Definition transaction.hpp:12