Add storage spaces support

core/blockchain/impl/block_header_repository_impl.cpp

@@ -11,23 +11,23 @@
 #include "blockchain/impl/storage_util.hpp"
 #include "scale/scale.hpp"
 
-using kagome::blockchain::prefix::Prefix;
 using kagome::common::Hash256;
 using kagome::primitives::BlockId;
 using kagome::primitives::BlockNumber;
+using kagome::storage::Space;
 
 namespace kagome::blockchain {
 
   BlockHeaderRepositoryImpl::BlockHeaderRepositoryImpl(
-      std::shared_ptr<storage::BufferStorage> map,
+      std::shared_ptr<storage::SpacedStorage> storage,
       std::shared_ptr<crypto::Hasher> hasher)
-      : map_{std::move(map)}, hasher_{std::move(hasher)} {
+      : storage_{std::move(storage)}, hasher_{std::move(hasher)} {
     BOOST_ASSERT(hasher_);
   }
 
   outcome::result<BlockNumber> BlockHeaderRepositoryImpl::getNumberByHash(
       const Hash256 &hash) const {
-    OUTCOME_TRY(key, idToLookupKey(*map_, hash));
+    OUTCOME_TRY(key, idToLookupKey(*storage_, hash));
     if (!key.has_value()) return BlockTreeError::HEADER_NOT_FOUND;
     auto maybe_number = lookupKeyToNumber(key.value());
 
@@ -43,7 +43,7 @@ namespace kagome::blockchain {
 
   outcome::result<primitives::BlockHeader>
   BlockHeaderRepositoryImpl::getBlockHeader(const BlockId &id) const {
-    OUTCOME_TRY(header_opt, getWithPrefix(*map_, Prefix::HEADER, id));
+    OUTCOME_TRY(header_opt, getFromSpace(*storage_, Space::kHeader, id));
     if (header_opt.has_value()) {
       return scale::decode<primitives::BlockHeader>(header_opt.value());
     }

core/blockchain/impl/block_header_repository_impl.hpp

@@ -10,12 +10,13 @@
 
 #include "blockchain/impl/common.hpp"
 #include "crypto/hasher.hpp"
+#include "storage/spaced_storage.hpp"
 
 namespace kagome::blockchain {
 
   class BlockHeaderRepositoryImpl : public BlockHeaderRepository {
    public:
-    BlockHeaderRepositoryImpl(std::shared_ptr<storage::BufferStorage> map,
+    BlockHeaderRepositoryImpl(std::shared_ptr<storage::SpacedStorage> storage,
                               std::shared_ptr<crypto::Hasher> hasher);
 
     ~BlockHeaderRepositoryImpl() override = default;
@@ -33,7 +34,7 @@ namespace kagome::blockchain {
         const primitives::BlockId &id) const override;
 
    private:
-    std::shared_ptr<storage::BufferStorage> map_;
+    std::shared_ptr<storage::SpacedStorage> storage_;
     std::shared_ptr<crypto::Hasher> hasher_;
   };
 

core/blockchain/impl/block_storage_impl.cpp

@@ -12,11 +12,11 @@
 namespace kagome::blockchain {
   using primitives::Block;
   using primitives::BlockId;
+  using storage::Space;
   using Buffer = common::Buffer;
-  using Prefix = prefix::Prefix;
 
   BlockStorageImpl::BlockStorageImpl(
-      std::shared_ptr<storage::BufferStorage> storage,
+      std::shared_ptr<storage::SpacedStorage> storage,
       std::shared_ptr<crypto::Hasher> hasher)
       : storage_{std::move(storage)},
         hasher_{std::move(hasher)},
@@ -27,7 +27,7 @@ namespace kagome::blockchain {
 
   outcome::result<std::shared_ptr<BlockStorageImpl>> BlockStorageImpl::create(
       storage::trie::RootHash state_root,
-      const std::shared_ptr<storage::BufferStorage> &storage,
+      const std::shared_ptr<storage::SpacedStorage> &storage,
       const std::shared_ptr<crypto::Hasher> &hasher) {
     auto block_storage = std::shared_ptr<BlockStorageImpl>(
         new BlockStorageImpl(storage, hasher));
@@ -59,13 +59,13 @@ namespace kagome::blockchain {
 
   outcome::result<bool> BlockStorageImpl::hasBlockHeader(
       const primitives::BlockId &id) const {
-    return hasWithPrefix(*storage_, Prefix::HEADER, id);
+    return hasInSpace(*storage_, Space::kHeader, id);
   }
 
   outcome::result<std::optional<primitives::BlockHeader>>
   BlockStorageImpl::getBlockHeader(const primitives::BlockId &id) const {
     OUTCOME_TRY(encoded_header_opt,
-                getWithPrefix(*storage_, Prefix::HEADER, id));
+                getFromSpace(*storage_, Space::kHeader, id));
     if (encoded_header_opt.has_value()) {
       OUTCOME_TRY(
           header,
@@ -87,7 +87,7 @@ namespace kagome::blockchain {
   outcome::result<std::optional<primitives::BlockData>>
   BlockStorageImpl::getBlockData(const primitives::BlockId &id) const {
     OUTCOME_TRY(encoded_block_data_opt,
-                getWithPrefix(*storage_, Prefix::BLOCK_DATA, id));
+                getFromSpace(*storage_, Space::kBlockData, id));
     if (encoded_block_data_opt.has_value()) {
       OUTCOME_TRY(
           block_data,
@@ -99,10 +99,13 @@ namespace kagome::blockchain {
 
   outcome::result<std::optional<primitives::Justification>>
   BlockStorageImpl::getJustification(const primitives::BlockId &block) const {
-    OUTCOME_TRY(block_data, getBlockData(block));
-    if (block_data.has_value()
-        && block_data.value().justification.has_value()) {
-      return block_data.value().justification.value();
+    OUTCOME_TRY(encoded_justification_opt,
+                getFromSpace(*storage_, Space::kJustification, block));
+    if (encoded_justification_opt.has_value()) {
+      OUTCOME_TRY(justification,
+                  scale::decode<primitives::Justification>(
+                      encoded_justification_opt.value()));
+      return std::move(justification);
     }
     return std::nullopt;
   }
@@ -117,11 +120,11 @@ namespace kagome::blockchain {
       const primitives::BlockHeader &header) {
     OUTCOME_TRY(encoded_header, scale::encode(header));
     auto block_hash = hasher_->blake2b_256(encoded_header);
-    OUTCOME_TRY(putWithPrefix(*storage_,
-                              Prefix::HEADER,
-                              header.number,
-                              block_hash,
-                              Buffer{std::move(encoded_header)}));
+    OUTCOME_TRY(putToSpace(*storage_,
+                           Space::kHeader,
+                           header.number,
+                           block_hash,
+                           Buffer{std::move(encoded_header)}));
     return block_hash;
   }
 
@@ -154,11 +157,11 @@ namespace kagome::blockchain {
     }
 
     OUTCOME_TRY(encoded_block_data, scale::encode(to_insert));
-    OUTCOME_TRY(putWithPrefix(*storage_,
-                              Prefix::BLOCK_DATA,
-                              block_number,
-                              block_data.hash,
-                              Buffer{encoded_block_data}));
+    OUTCOME_TRY(putToSpace(*storage_,
+                           Space::kBlockData,
+                           block_number,
+                           block_data.hash,
+                           Buffer{encoded_block_data}));
     return outcome::success();
   }
 
@@ -194,11 +197,11 @@ namespace kagome::blockchain {
         move_if_flag(!remove_flags.receipt, std::move(existing_data.receipt));
 
     OUTCOME_TRY(encoded_block_data, scale::encode(to_insert));
-    OUTCOME_TRY(putWithPrefix(*storage_,
-                              Prefix::BLOCK_DATA,
-                              block_number,
-                              remove_flags.hash,
-                              Buffer{encoded_block_data}));
+    OUTCOME_TRY(putToSpace(*storage_,
+                           Space::kBlockData,
+                           block_number,
+                           remove_flags.hash,
+                           Buffer{encoded_block_data}));
     return outcome::success();
   }
 
@@ -225,18 +228,26 @@ namespace kagome::blockchain {
       const primitives::BlockHash &hash,
       primitives::BlockNumber block_number) {
     BOOST_ASSERT(not j.data.empty());
-    // insert justification into the database as a part of BlockData
-    primitives::BlockData block_data{.hash = hash, .justification = j};
+
+    OUTCOME_TRY(justification, scale::encode(j));
+    OUTCOME_TRY(putToSpace(*storage_,
+                           Space::kJustification,
+                           block_number,
+                           hash,
+                           Buffer{justification}));
+
+    // the following is still required
+    primitives::BlockData block_data{.hash = hash};
     OUTCOME_TRY(putBlockData(block_number, block_data));
     return outcome::success();
   }
 
   outcome::result<void> BlockStorageImpl::removeJustification(
       const primitives::BlockHash &hash, primitives::BlockNumber number) {
-    primitives::BlockDataFlags flags =
-        primitives::BlockDataFlags::allUnset(hash);
-    flags.justification = true;
-    OUTCOME_TRY(removeBlockData(number, flags));
+    auto key = numberAndHashToLookupKey(number, hash);
+    auto space = storage_->getSpace(Space::kJustification);
+
+    OUTCOME_TRY(space->remove(key));
     return outcome::success();
   }
 
@@ -245,19 +256,18 @@ namespace kagome::blockchain {
     auto block_lookup_key = numberAndHashToLookupKey(block.number, block.hash);
 
     SL_TRACE(logger_, "Removing block {}...", block);
+    auto key_space = storage_->getSpace(Space::kLookupKey);
 
-    auto hash_to_idx_key = prependPrefix(block.hash, Prefix::ID_TO_LOOKUP_KEY);
-    if (auto res = storage_->remove(hash_to_idx_key); res.has_error()) {
+    if (auto res = key_space->remove(block.hash); res.has_error()) {
       logger_->error("could not remove hash-to-idx from the storage: {}",
                      res.error());
       return res;
     }
 
-    auto num_to_idx_key =
-        prependPrefix(numberToIndexKey(block.number), Prefix::ID_TO_LOOKUP_KEY);
-    OUTCOME_TRY(num_to_idx_val_opt, storage_->tryGet(num_to_idx_key.view()));
+    auto num_to_idx_key = numberToIndexKey(block.number);
+    OUTCOME_TRY(num_to_idx_val_opt, key_space->tryGet(num_to_idx_key.view()));
     if (num_to_idx_val_opt == block_lookup_key) {
-      if (auto res = storage_->remove(num_to_idx_key); res.has_error()) {
+      if (auto res = key_space->remove(num_to_idx_key); res.has_error()) {
         SL_ERROR(logger_,
                  "could not remove num-to-idx from the storage: {}",
                  block,
@@ -270,24 +280,35 @@ namespace kagome::blockchain {
     // TODO(xDimon): needed to clean up trie storage if block deleted
     //  issue: https://github.com/soramitsu/kagome/issues/1128
 
-    auto body_key = prependPrefix(block_lookup_key, Prefix::BLOCK_DATA);
-    if (auto res = storage_->remove(body_key); res.has_error()) {
+    auto block_data_space = storage_->getSpace(Space::kBlockData);
+    if (auto res = block_data_space->remove(block_lookup_key);
+        res.has_error()) {
       SL_ERROR(logger_,
                "could not remove body of block {} from the storage: {}",
                block,
                res.error());
       return res;
     }
 
-    auto header_key = prependPrefix(block_lookup_key, Prefix::HEADER);
-    if (auto res = storage_->remove(header_key); res.has_error()) {
+    auto header_space = storage_->getSpace(Space::kHeader);
+    if (auto res = header_space->remove(block_lookup_key); res.has_error()) {
       SL_ERROR(logger_,
                "could not remove header of block {} from the storage: {}",
                block,
                res.error());
       return res;
     }
 
+    if (auto res = removeJustification(block.hash, block.number);
+        res.has_error()) {
+      SL_ERROR(
+          logger_,
+          "could not remove justification for block {} from the storage: {}",
+          block,
+          res.error());
+      return res;
+    }
+
     logger_->info("Removed block {}", block);
 
     return outcome::success();
@@ -299,8 +320,9 @@ namespace kagome::blockchain {
       return block_tree_leaves_.value();
     }
 
+    auto default_space = storage_->getSpace(Space::kDefault);
     OUTCOME_TRY(leaves_opt,
-                storage_->tryGet(storage::kBlockTreeLeavesLookupKey));
+                default_space->tryGet(storage::kBlockTreeLeavesLookupKey));
     if (not leaves_opt.has_value()) {
       return BlockStorageError::BLOCK_TREE_LEAVES_NOT_FOUND;
     }
@@ -321,9 +343,10 @@ namespace kagome::blockchain {
       return outcome::success();
     }
 
+    auto default_space = storage_->getSpace(Space::kDefault);
     OUTCOME_TRY(encoded_leaves, scale::encode(leaves));
-    OUTCOME_TRY(storage_->put(storage::kBlockTreeLeavesLookupKey,
-                              Buffer{std::move(encoded_leaves)}));
+    OUTCOME_TRY(default_space->put(storage::kBlockTreeLeavesLookupKey,
+                                   Buffer{std::move(encoded_leaves)}));
 
     block_tree_leaves_.emplace(std::move(leaves));
 

core/blockchain/impl/block_storage_impl.hpp

@@ -12,6 +12,7 @@
 #include "crypto/hasher.hpp"
 #include "log/logger.hpp"
 #include "storage/predefined_keys.hpp"
+#include "storage/spaced_storage.hpp"
 
 namespace kagome::blockchain {
 
@@ -28,7 +29,7 @@ namespace kagome::blockchain {
      */
     static outcome::result<std::shared_ptr<BlockStorageImpl>> create(
         storage::trie::RootHash state_root,
-        const std::shared_ptr<storage::BufferStorage> &storage,
+        const std::shared_ptr<storage::SpacedStorage> &storage,
         const std::shared_ptr<crypto::Hasher> &hasher);
 
     outcome::result<std::vector<primitives::BlockHash>> getBlockTreeLeaves()
@@ -76,10 +77,11 @@ namespace kagome::blockchain {
     outcome::result<primitives::BlockInfo> getLastFinalized() const override;
 
    private:
-    BlockStorageImpl(std::shared_ptr<storage::BufferStorage> storage,
+    BlockStorageImpl(std::shared_ptr<storage::SpacedStorage> storage,
                      std::shared_ptr<crypto::Hasher> hasher);
 
-    std::shared_ptr<storage::BufferStorage> storage_;
+    std::shared_ptr<storage::SpacedStorage> storage_;
+
     std::shared_ptr<crypto::Hasher> hasher_;
     log::Logger logger_;
 

core/blockchain/impl/block_tree_impl.cpp

@@ -27,7 +27,6 @@ namespace {
 
 namespace kagome::blockchain {
   using Buffer = common::Buffer;
-  using Prefix = prefix::Prefix;
   using DatabaseError = kagome::storage::DatabaseError;
   using consensus::babe::isPrimary;
 

core/blockchain/impl/common.cpp

@@ -11,18 +11,14 @@
 namespace kagome::blockchain {
 
   outcome::result<std::optional<common::BufferOrView>> idToLookupKey(
-      const ReadableBufferStorage &map, const primitives::BlockId &id) {
+      storage::SpacedStorage &storage, const primitives::BlockId &id) {
     auto key = visit_in_place(
         id,
-        [](const primitives::BlockNumber &n) {
-          return prependPrefix(numberToIndexKey(n),
-                               prefix::Prefix::ID_TO_LOOKUP_KEY);
-        },
-        [](const common::Hash256 &hash) {
-          return prependPrefix(hash, prefix::Prefix::ID_TO_LOOKUP_KEY);
-        });
+        [](const primitives::BlockNumber &n) { return numberToIndexKey(n); },
+        [](const common::Hash256 &hash) { return hash; });
 
-    OUTCOME_TRY(key_opt, map.tryGet(key));
+    auto key_space = storage.getSpace(storage::Space::kLookupKey);
+    OUTCOME_TRY(key_opt, key_space->tryGet(key));
 
     return std::move(key_opt);
   }

core/blockchain/impl/common.hpp

@@ -9,20 +9,17 @@
 #include "common/buffer.hpp"
 #include "outcome/outcome.hpp"
 #include "primitives/block_id.hpp"
-#include "storage/buffer_map_types.hpp"
+#include "storage/spaced_storage.hpp"
 #include "storage/trie/types.hpp"
 
 namespace kagome::blockchain {
 
-  using ReadableBufferStorage =
-      storage::face::Readable<common::Buffer, common::Buffer>;
-
   /**
-   * Convert a block ID into a key, which is a first part of a key, by which the
-   * columns are stored in the database
+   * Convert a block ID (which is either hash or number) into a key, which is a
+   * first part of a key, by which the columns are stored in the database
    */
   outcome::result<std::optional<common::BufferOrView>> idToLookupKey(
-      const ReadableBufferStorage &map, const primitives::BlockId &id);
+      storage::SpacedStorage &storage, const primitives::BlockId &id);
 }  // namespace kagome::blockchain
 
 #endif  // KAGOME_BLOCKCHAIN_COMMON_HPP