TARAXA
Loading...
Searching...
No Matches
util.hpp
Go to the documentation of this file.
1#pragma once
2
3#include <execinfo.h>
4#include <json/json.h>
5#include <libdevcore/RLP.h>
6
7#include <boost/asio.hpp>
8#include <boost/format.hpp>
9#include <boost/iostreams/device/back_inserter.hpp>
10#include <boost/iostreams/stream.hpp>
11#include <boost/iostreams/stream_buffer.hpp>
12#include <boost/thread.hpp>
13#include <iostream>
14#include <list>
15#include <shared_mutex>
16#include <streambuf>
17#include <string>
18#include <unordered_set>
19
20namespace taraxa {
21
26std::string jsonToUnstyledString(const Json::Value &value);
27
28template <typename T>
29std::weak_ptr<T> as_weak(std::shared_ptr<T> sp) {
30 return std::weak_ptr<T>(sp);
31}
32
33template <typename Int1, typename Int2>
34auto int_pow(Int1 x, Int2 y) {
35 if (!y) {
36 return 1;
37 }
38 while (--y > 0) {
39 x *= x;
40 }
41 return x;
42}
43
44template <typename T>
45std::vector<T> asVector(const Json::Value &json) {
46 std::vector<T> v;
47 v.reserve(json.size());
48 std::transform(json.begin(), json.end(), std::back_inserter(v),
49 [](const Json::Value &item) { return T(item.asString()); });
50 return v;
51}
52
53std::vector<uint64_t> asUInt64Vector(const Json::Value &json);
54
55using stream = std::basic_streambuf<uint8_t>;
56using bufferstream = boost::iostreams::stream_buffer<boost::iostreams::basic_array_source<uint8_t>>;
57using vectorstream = boost::iostreams::stream_buffer<boost::iostreams::back_insert_device<std::vector<uint8_t>>>;
58
59// Read a raw byte stream the size of T and fill value
60// return true if success
61template <typename T>
62bool read(stream &stm, T &value) {
63 static_assert(std::is_standard_layout<T>::value, "Cannot stream read non-standard layout types");
64 auto bytes(stm.sgetn(reinterpret_cast<uint8_t *>(&value), sizeof(value)));
65 return bytes == sizeof(value);
66}
67
68template <typename T>
69bool write(stream &stm, T const &value) {
70 static_assert(std::is_standard_layout<T>::value, "Cannot stream write non-standard layout types");
71 auto bytes(stm.sputn(reinterpret_cast<uint8_t const *>(&value), sizeof(value)));
72 assert(bytes == sizeof(value));
73 return bytes == sizeof(value);
74}
75
76void thisThreadSleepForSeconds(unsigned sec);
77void thisThreadSleepForMilliSeconds(unsigned millisec);
78void thisThreadSleepForMicroSeconds(unsigned microsec);
79
80unsigned long getCurrentTimeMilliSeconds();
81
82std::string getFormattedVersion(std::initializer_list<uint32_t> list);
83
89template <typename K, typename V>
91 public:
92 using UnsafeStatusTable = std::unordered_map<K, V>;
93 StatusTable(size_t capacity = 10000) : capacity_(capacity) {}
94 std::pair<V, bool> get(K const &hash) {
95 std::scoped_lock lock(shared_mutex_);
96 auto iter = status_.find(hash);
97 if (iter != status_.end()) {
98 auto kv = *(iter->second);
99 auto &k = kv.first;
100 auto &v = kv.second;
101 lru_.erase(iter->second);
102 lru_.push_front(kv);
103 status_[k] = lru_.begin();
104 return {v, true};
105 } else {
106 return {V(), false};
107 }
108 }
109 unsigned long size() const {
110 std::shared_lock lock(shared_mutex_);
111 return status_.size();
112 }
113 bool insert(K const &hash, V status) {
114 std::scoped_lock lock(shared_mutex_);
115 bool ret = false;
116 if (status_.count(hash)) {
117 ret = false;
118 } else {
119 if (status_.size() == capacity_) {
120 clearOldData();
121 }
122 lru_.push_front({hash, status});
123 status_[hash] = lru_.begin();
124 ret = true;
125 }
126 assert(status_.size() == lru_.size());
127 return ret;
128 }
129 void update(K const &hash, V status) {
130 std::scoped_lock lock(shared_mutex_);
131 auto iter = status_.find(hash);
132 if (iter != status_.end()) {
133 lru_.erase(iter->second);
134 lru_.push_front({hash, status});
135 status_[hash] = lru_.begin();
136 }
137 assert(status_.size() == lru_.size());
138 }
139
140 bool update(K const &hash, V status, V expected_status) {
141 bool ret = false;
142 std::scoped_lock lock(shared_mutex_);
143 auto iter = status_.find(hash);
144 if (iter != status_.end() && iter->second == expected_status) {
145 lru_.erase(iter->second);
146 lru_.push_front({hash, status});
147 status_[hash] = lru_.begin();
148 ret = true;
149 }
150 assert(status_.size() == lru_.size());
151 return ret;
152 }
153 // clear everything
154 void clear() {
155 std::scoped_lock lock(shared_mutex_);
156 status_.clear();
157 lru_.clear();
158 }
159 bool erase(K const &hash) {
160 bool ret = false;
161 std::scoped_lock lock(shared_mutex_);
162 auto iter = status_.find(hash);
163 if (iter != status_.end()) {
164 lru_.erase(iter->second);
165 status_.erase(hash);
166 }
167 assert(status_.size() == lru_.size());
168 return ret;
169 }
170
172 std::shared_lock lock(shared_mutex_);
173 return status_;
174 }
175
176 private:
178 int sz = capacity_ / 10;
179 for (int i = 0; i < sz; ++i) {
180 auto kv = lru_.rbegin();
181 auto &k = kv->first;
182 status_.erase(k);
183 lru_.pop_back();
184 assert(!lru_.empty()); // should not happen
185 }
186 }
187 using Element = std::list<std::pair<K, V>>;
188 size_t capacity_;
189 mutable std::shared_mutex shared_mutex_;
190 std::unordered_map<K, typename std::list<std::pair<K, V>>::iterator> status_;
191 std::list<std::pair<K, V>> lru_;
192}; // namespace taraxa
193
194template <typename... TS>
195std::string fmt(const std::string &pattern, const TS &...args) {
196 return (boost::format(pattern) % ... % args).str();
197}
198
199} // namespace taraxa
200
201template <class Key>
203 public:
204 ExpirationCache(uint32_t max_size, uint32_t delete_step) : kMaxSize(max_size), kDeleteStep(delete_step) {}
205
214 bool insert(Key const &key) {
215 if (contains(key)) {
216 return false;
217 }
218
219 // There must be double check if key is not already in cache due to possible race condition
220 std::unique_lock lock(mtx_);
221 if (!cache_.insert(key).second) {
222 return false;
223 }
224
225 expiration_.push_back(key);
226 if (cache_.size() > kMaxSize) {
227 for (uint32_t i = 0; i < kDeleteStep; i++) {
228 cache_.erase(expiration_.front());
229 expiration_.pop_front();
230 }
231 }
232
233 return true;
234 }
235
236 bool contains(Key const &key) const {
237 std::shared_lock lck(mtx_);
238 return cache_.contains(key);
239 }
240
241 void erase(Key const &key) {
242 std::unique_lock lck(mtx_);
243 cache_.erase(key);
244 }
245
246 std::size_t count(Key const &key) const {
247 std::shared_lock lck(mtx_);
248 return cache_.count(key);
249 }
250
251 std::size_t size() const {
252 std::shared_lock lck(mtx_);
253 return cache_.size();
254 }
255
256 void clear() {
257 std::unique_lock lck(mtx_);
258 cache_.clear();
259 expiration_.clear();
260 }
261
262 protected:
263 std::unordered_set<Key> cache_;
264 const uint32_t kMaxSize;
265 const uint32_t kDeleteStep;
266 mutable std::shared_mutex mtx_;
267
268 private:
269 std::deque<Key> expiration_;
270};
271
272template <class Key>
274 public:
275 ExpirationBlockNumberCache(uint32_t max_size, uint32_t delete_step, uint32_t blocks_to_keep)
276 : ExpirationCache<Key>(max_size, delete_step), kBlocksToKeep(blocks_to_keep) {}
277
287 bool insert(Key const &key, uint64_t block_number) {
288 if (this->contains(key)) {
289 return false;
290 }
291 // There must be double check if key is not already in cache due to possible race condition
292 std::unique_lock lock(this->mtx_);
293 if (!this->cache_.insert(key).second) {
294 return false;
295 }
296
297 if (block_number > last_block_number_) {
298 last_block_number_ = block_number;
299 while (block_expiration_.size() > 0) {
300 const auto &exp = block_expiration_.front();
301 if (block_number > kBlocksToKeep && exp.second < block_number - kBlocksToKeep) {
302 this->cache_.erase(exp.first);
303 block_expiration_.pop_front();
304 } else {
305 break;
306 }
307 }
308 }
309 block_expiration_.push_back({key, block_number});
310 if (this->cache_.size() > this->kMaxSize) {
311 for (uint32_t i = 0; i < this->kDeleteStep; i++) {
312 this->cache_.erase(block_expiration_.front().first);
313 block_expiration_.pop_front();
314 }
315 }
316
317 return true;
318 }
319
320 void clear() {
321 std::shared_lock lck(this->mtx_);
322 this->cache_.clear();
323 block_expiration_.clear();
324 }
325
326 private:
327 std::deque<std::pair<Key, uint64_t>> block_expiration_;
328 const uint32_t kBlocksToKeep;
329 uint64_t last_block_number_ = 0;
330};
331
332template <typename T>
333auto slice(std::vector<T> const &v, std::size_t from = -1, std::size_t to = -1) {
334 auto b = v.begin();
335 return std::vector<T>(from == (std::size_t)-1 ? b : b + from, to == (std::size_t)-1 ? v.end() : b + to);
336}
337
338template <typename T>
339auto getRlpBytes(T const &t) {
341 s << t;
342 return s.invalidate();
343}
344
345template <class Key, class Value>
347 public:
348 ExpirationCacheMap(uint32_t max_size, uint32_t delete_step) : kMaxSize(max_size), kDeleteStep(delete_step) {}
349
359 bool insert(Key const &key, Value const &value) {
360 {
361 std::shared_lock lock(mtx_);
362 if (cache_.count(key)) {
363 return false;
364 }
365 }
366
367 std::unique_lock lock(mtx_);
368
369 // There must be double check if key is not already in cache due to possible race condition
370 if (!cache_.emplace(key, value).second) {
371 return false;
372 }
373
374 expiration_.push_back(key);
375 if (cache_.size() > kMaxSize) {
376 eraseOldest();
377 }
378 return true;
379 }
380
381 std::size_t count(Key const &key) const {
382 std::shared_lock lck(mtx_);
383 return cache_.count(key);
384 }
385
386 std::size_t size() const {
387 std::shared_lock lck(mtx_);
388 return cache_.size();
389 }
390
391 std::pair<Value, bool> get(Key const &key) const {
392 std::shared_lock lck(mtx_);
393 auto it = cache_.find(key);
394 if (it == cache_.end()) return std::make_pair(Value(), false);
395 return std::make_pair(it->second, true);
396 }
397
398 void clear() {
399 std::unique_lock lck(mtx_);
400 cache_.clear();
401 expiration_.clear();
402 }
403
404 void update(Key const &key, Value value) {
405 std::unique_lock lck(mtx_);
406
407 if (cache_.find(key) != cache_.end()) {
408 expiration_.erase(std::remove(expiration_.begin(), expiration_.end(), key), expiration_.end());
409 }
410
411 cache_[key] = value;
412 expiration_.push_back(key);
413
414 if (cache_.size() > kMaxSize) {
415 for (uint32_t i = 0; i < kDeleteStep; i++) {
416 cache_.erase(expiration_.front());
417 expiration_.pop_front();
418 }
419 }
420 }
421
422 void erase(const Key &key) {
423 std::unique_lock lck(mtx_);
424 cache_.erase(key);
425 expiration_.erase(std::remove(expiration_.begin(), expiration_.end(), key), expiration_.end());
426 }
427
428 std::pair<Value, bool> updateWithGet(Key const &key, Value value) {
429 std::unique_lock lck(mtx_);
430 std::pair<Value, bool> ret;
431 auto it = cache_.find(key);
432 if (it == cache_.end()) {
433 ret = std::make_pair(Value(), false);
434 } else {
435 ret = std::make_pair(it->second, true);
436 }
437 cache_[key] = value;
438 expiration_.push_back(key);
439 if (cache_.size() > kMaxSize) {
440 for (uint32_t i = 0; i < kDeleteStep; i++) {
441 cache_.erase(expiration_.front());
442 expiration_.pop_front();
443 }
444 }
445 return ret;
446 }
447
448 std::unordered_map<Key, Value> getRawMap() {
449 std::shared_lock lck(mtx_);
450 return cache_;
451 }
452
453 bool update(Key const &key, Value value, Value expected_value) {
454 std::unique_lock lck(mtx_);
455 auto it = cache_.find(key);
456 if (it != cache_.end() && it->second == expected_value) {
457 it->second = value;
458 expiration_.push_back(key);
459 if (cache_.size() > kMaxSize) {
460 for (auto i = 0; i < kDeleteStep; i++) {
461 cache_.erase(expiration_.front());
462 expiration_.pop_front();
463 }
464 }
465 return true;
466 }
467 return false;
468 }
469
470 protected:
471 virtual void eraseOldest() {
472 for (uint32_t i = 0; i < kDeleteStep; i++) {
473 cache_.erase(expiration_.front());
474 expiration_.pop_front();
475 }
476 }
477
478 std::unordered_map<Key, Value> cache_;
479 std::deque<Key> expiration_;
480 const uint32_t kMaxSize;
481 const uint32_t kDeleteStep;
482 mutable std::shared_mutex mtx_;
483};
484
485template <class Key, class Value>
487 public:
488 bool emplace(Key const &key, Value const &value) {
489 std::unique_lock lock(mtx_);
490 return map_.emplace(key, value).second;
491 }
492
493 std::size_t count(Key const &key) const {
494 std::shared_lock lck(mtx_);
495 return map_.count(key);
496 }
497
498 std::size_t size() const {
499 std::shared_lock lck(mtx_);
500 return map_.size();
501 }
502
503 std::pair<Value, bool> get(Key const &key) const {
504 std::shared_lock lck(mtx_);
505 auto it = map_.find(key);
506 if (it == map_.end()) return std::make_pair(Value(), false);
507 return std::make_pair(it->second, true);
508 }
509
510 std::vector<Value> getValues(uint32_t count = 0) const {
511 std::vector<Value> values;
512 uint32_t counter = 0;
513 std::shared_lock lck(mtx_);
514 values.reserve(map_.size());
515 for (auto const &t : map_) {
516 values.emplace_back(t.second);
517 if (count != 0) {
518 counter++;
519 if (counter == count) {
520 break;
521 }
522 }
523 }
524 return values;
525 }
526
527 void erase(std::function<bool(Value)> condition) {
528 std::unique_lock lck(mtx_);
529 for (auto it = map_.cbegin(), next_it = it; it != map_.cend(); it = next_it) {
530 ++next_it;
531 if (condition(it->second)) {
532 map_.erase(it);
533 }
534 }
535 }
536
537 void clear() {
538 std::unique_lock lck(mtx_);
539 map_.clear();
540 }
541
542 bool erase(const Key &key) {
543 std::unique_lock lck(mtx_);
544 return map_.erase(key);
545 }
546
547 protected:
548 std::unordered_map<Key, Value> map_;
549 mutable std::shared_mutex mtx_;
550};
551
552template <class Key>
554 public:
555 bool emplace(Key const &key) {
556 std::unique_lock lock(mtx_);
557 return set_.insert(key).second;
558 }
559
560 std::size_t count(Key const &key) const {
561 std::shared_lock lck(mtx_);
562 return set_.count(key);
563 }
564
565 void clear() {
566 std::unique_lock lck(mtx_);
567 set_.clear();
568 }
569
570 bool erase(const Key &key) {
571 std::unique_lock lck(mtx_);
572 return set_.erase(key);
573 }
574
575 std::size_t size() const {
576 std::shared_lock lck(mtx_);
577 return set_.size();
578 }
579
580 private:
581 std::unordered_set<Key> set_;
582 mutable std::shared_mutex mtx_;
583};
Definition util.hpp:273
ExpirationBlockNumberCache(uint32_t max_size, uint32_t delete_step, uint32_t blocks_to_keep)
Definition util.hpp:275
void clear()
Definition util.hpp:320
std::deque< std::pair< Key, uint64_t > > block_expiration_
Definition util.hpp:327
const uint32_t kBlocksToKeep
Definition util.hpp:328
bool insert(Key const &key, uint64_t block_number)
Inserts key into the cache map. In case provided key is already in cache, only shared lock is acquire...
Definition util.hpp:287
uint64_t last_block_number_
Definition util.hpp:329
Definition util.hpp:202
const uint32_t kDeleteStep
Definition util.hpp:265
bool insert(Key const &key)
Inserts key into the cache map. In case provided key is already in cache, only shared lock is acquire...
Definition util.hpp:214
std::size_t count(Key const &key) const
Definition util.hpp:246
const uint32_t kMaxSize
Definition util.hpp:264
std::shared_mutex mtx_
Definition util.hpp:266
ExpirationCache(uint32_t max_size, uint32_t delete_step)
Definition util.hpp:204
void clear()
Definition util.hpp:256
bool contains(Key const &key) const
Definition util.hpp:236
void erase(Key const &key)
Definition util.hpp:241
std::unordered_set< Key > cache_
Definition util.hpp:263
std::size_t size() const
Definition util.hpp:251
std::deque< Key > expiration_
Definition util.hpp:269
Definition util.hpp:346
virtual void eraseOldest()
Definition util.hpp:471
bool insert(Key const &key, Value const &value)
Inserts <key,value> pair into the cache map. In case provided key is already in cache,...
Definition util.hpp:359
std::size_t size() const
Definition util.hpp:386
std::unordered_map< Key, Value > getRawMap()
Definition util.hpp:448
std::size_t count(Key const &key) const
Definition util.hpp:381
std::pair< Value, bool > get(Key const &key) const
Definition util.hpp:391
std::deque< Key > expiration_
Definition util.hpp:479
std::unordered_map< Key, Value > cache_
Definition util.hpp:478
ExpirationCacheMap(uint32_t max_size, uint32_t delete_step)
Definition util.hpp:348
void erase(const Key &key)
Definition util.hpp:422
const uint32_t kDeleteStep
Definition util.hpp:481
void update(Key const &key, Value value)
Definition util.hpp:404
std::shared_mutex mtx_
Definition util.hpp:482
std::pair< Value, bool > updateWithGet(Key const &key, Value value)
Definition util.hpp:428
const uint32_t kMaxSize
Definition util.hpp:480
bool update(Key const &key, Value value, Value expected_value)
Definition util.hpp:453
void clear()
Definition util.hpp:398
Definition util.hpp:486
std::shared_mutex mtx_
Definition util.hpp:549
std::unordered_map< Key, Value > map_
Definition util.hpp:548
bool erase(const Key &key)
Definition util.hpp:542
bool emplace(Key const &key, Value const &value)
Definition util.hpp:488
std::size_t count(Key const &key) const
Definition util.hpp:493
std::vector< Value > getValues(uint32_t count=0) const
Definition util.hpp:510
std::size_t size() const
Definition util.hpp:498
std::pair< Value, bool > get(Key const &key) const
Definition util.hpp:503
void clear()
Definition util.hpp:537
void erase(std::function< bool(Value)> condition)
Definition util.hpp:527
Definition util.hpp:553
void clear()
Definition util.hpp:565
bool emplace(Key const &key)
Definition util.hpp:555
bool erase(const Key &key)
Definition util.hpp:570
std::unordered_set< Key > set_
Definition util.hpp:581
std::size_t size() const
Definition util.hpp:575
std::size_t count(Key const &key) const
Definition util.hpp:560
std::shared_mutex mtx_
Definition util.hpp:582
Class for writing to an RLP bytestream.
Definition RLP.h:484
bytes && invalidate()
Invalidate the object and steal the output byte stream.
Definition RLP.h:620
Definition util.hpp:90
void clearOldData()
Definition util.hpp:177
size_t capacity_
Definition util.hpp:188
bool update(K const &hash, V status, V expected_status)
Definition util.hpp:140
StatusTable(size_t capacity=10000)
Definition util.hpp:93
bool erase(K const &hash)
Definition util.hpp:159
std::list< std::pair< K, V > > lru_
Definition util.hpp:191
void update(K const &hash, V status)
Definition util.hpp:129
std::shared_mutex shared_mutex_
Definition util.hpp:189
bool insert(K const &hash, V status)
Definition util.hpp:113
std::unordered_map< K, typename std::list< std::pair< K, V > >::iterator > status_
Definition util.hpp:190
void clear()
Definition util.hpp:154
std::unordered_map< K, V > UnsafeStatusTable
Definition util.hpp:92
std::pair< V, bool > get(K const &hash)
Definition util.hpp:94
unsigned long size() const
Definition util.hpp:109
std::list< std::pair< K, V > > Element
Definition util.hpp:187
UnsafeStatusTable getThreadUnsafeCopy() const
Definition util.hpp:171
Definition app.hpp:16
std::basic_streambuf< uint8_t > stream
Definition util.hpp:55
void thisThreadSleepForMicroSeconds(unsigned microsec)
Definition util.cpp:17
bool read(stream &stm, T &value)
Definition util.hpp:62
void thisThreadSleepForMilliSeconds(unsigned millisec)
Definition util.cpp:13
std::weak_ptr< T > as_weak(std::shared_ptr< T > sp)
Definition util.hpp:29
boost::iostreams::stream_buffer< boost::iostreams::back_insert_device< std::vector< uint8_t > > > vectorstream
Definition util.hpp:57
std::vector< byte > bytes
Definition types.hpp:53
std::string getFormattedVersion(std::initializer_list< uint32_t > list)
Definition util.cpp:26
void thisThreadSleepForSeconds(unsigned sec)
Definition util.cpp:11
std::string fmt(const std::string &pattern, const TS &...args)
Definition util.hpp:195
bool write(stream &stm, T const &value)
Definition util.hpp:69
std::vector< T > asVector(const Json::Value &json)
Definition util.hpp:45
std::vector< uint64_t > asUInt64Vector(const Json::Value &json)
Definition util.cpp:34
std::string jsonToUnstyledString(const Json::Value &value)
Definition util.cpp:5
boost::iostreams::stream_buffer< boost::iostreams::basic_array_source< uint8_t > > bufferstream
Definition util.hpp:56
auto int_pow(Int1 x, Int2 y)
Definition util.hpp:34
unsigned long getCurrentTimeMilliSeconds()
Definition util.cpp:21
auto slice(std::vector< T > const &v, std::size_t from=-1, std::size_t to=-1)
Definition util.hpp:333
auto getRlpBytes(T const &t)
Definition util.hpp:339