Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .changes/breaking/3053.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Disabled merkleization of compression tables. New tables are located separately.
1 change: 0 additions & 1 deletion benches/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ default = ["fuel-core/rocksdb", "fuel-core/rocksdb-production"]
fault-proving = [
"fuel-core-types/fault-proving",
"fuel-core-chain-config/fault-proving",
"fuel-core/fault-proving",
"fuel-core-storage/fault-proving",
"fuel-core-database/fault-proving",
"fuel-core-sync/fault-proving",
Expand Down
1 change: 0 additions & 1 deletion bin/fuel-core/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ fault-proving = [
"fuel-core-storage/fault-proving",
"fuel-core-types/fault-proving",
"fuel-core-chain-config/fault-proving",
"fuel-core/fault-proving",
]

[dependencies]
Expand Down
2 changes: 1 addition & 1 deletion crates/compression/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ test-helpers = [
"fuel-core-types/random",
"fuel-core-types/std",
]
fault-proving = ["fuel-core-types/fault-proving"]
# fault-proving = ["fuel-core-types/fault-proving"]

[dependencies]
anyhow = { workspace = true }
Expand Down
1 change: 1 addition & 0 deletions crates/compression/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
#![deny(clippy::cast_possible_truncation)]
#![deny(unused_crate_dependencies)]
#![deny(warnings)]
#![allow(unexpected_cfgs)]

pub mod compress;
mod compressed_block_payload;
Expand Down
1 change: 0 additions & 1 deletion crates/fuel-core/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ fault-proving = [
"fuel-core-sync?/fault-proving",
"fuel-core-importer/fault-proving",
"fuel-core-poa/fault-proving",
"fuel-core-compression-service/fault-proving",
"fuel-core-upgradable-executor/fault-proving",
]

Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
use crate::database::database_description::DatabaseDescription;
use fuel_core_compression_service::storage::column::CompressionColumn;
use fuel_core_storage::merkle::column::MerkleizedColumn;
use fuel_core_types::fuel_types::BlockHeight;

#[derive(Clone, Copy, Debug)]
pub struct CompressionDatabase;

impl DatabaseDescription for CompressionDatabase {
type Column = MerkleizedColumn<CompressionColumn>;
type Column = CompressionColumn;
type Height = BlockHeight;

fn version() -> u32 {
Expand Down
12 changes: 6 additions & 6 deletions crates/services/compression/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@ test-helpers = [
"dep:rand",
"dep:fuel-core-chain-config",
]
fault-proving = [
"fuel-core-compression/fault-proving",
"fuel-core-chain-config?/fault-proving",
"fuel-core-types/fault-proving",
"fuel-core-storage/fault-proving",
]
# fault-proving = [
# "fuel-core-compression/fault-proving",
# "fuel-core-chain-config?/fault-proving",
# "fuel-core-types/fault-proving",
# "fuel-core-storage/fault-proving",
# ]

[dependencies]
anyhow = { workspace = true }
Expand Down
1 change: 1 addition & 0 deletions crates/services/compression/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
#![deny(unused_crate_dependencies)]
#![deny(missing_docs)]
#![deny(warnings)]
#![allow(unexpected_cfgs)]

/// Configuration for the compression service
pub mod config;
Expand Down
11 changes: 3 additions & 8 deletions crates/services/compression/src/ports/compression_storage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ use fuel_core_storage::{
StorageAsMut,
StorageSize,
kv_store::KeyValueInspect,
merkle::column::MerkleizedColumn,
not_found,
transactional::{
Modifiable,
Expand All @@ -29,15 +28,12 @@ pub trait LatestHeight {

/// Trait for interacting with storage that supports compression
pub trait CompressionStorage:
KeyValueInspect<Column = MerkleizedColumn<storage::column::CompressionColumn>>
+ Modifiable
+ Send
+ Sync
KeyValueInspect<Column = storage::column::CompressionColumn> + Modifiable + Send + Sync
{
}

impl<T> CompressionStorage for T where
T: KeyValueInspect<Column = MerkleizedColumn<storage::column::CompressionColumn>>
T: KeyValueInspect<Column = storage::column::CompressionColumn>
+ Modifiable
+ Send
+ Sync
Expand All @@ -54,8 +50,7 @@ pub(crate) trait WriteCompressedBlock {

impl<Storage> WriteCompressedBlock for StorageTransaction<Storage>
where
Storage:
KeyValueInspect<Column = MerkleizedColumn<storage::column::CompressionColumn>>,
Storage: KeyValueInspect<Column = storage::column::CompressionColumn>,
{
fn write_compressed_block(
&mut self,
Expand Down
6 changes: 2 additions & 4 deletions crates/services/compression/src/service.rs
Original file line number Diff line number Diff line change
Expand Up @@ -468,7 +468,6 @@ mod tests {
IteratorOverTable,
changes_iterator::ChangesIterator,
},
merkle::column::MerkleizedColumn,
structured_storage::test::InMemoryStorage,
transactional::{
IntoTransaction,
Expand All @@ -492,9 +491,8 @@ mod tests {
}
}

type MockStorage = StorageTransaction<
InMemoryStorage<MerkleizedColumn<crate::storage::column::CompressionColumn>>,
>;
type MockStorage =
StorageTransaction<InMemoryStorage<crate::storage::column::CompressionColumn>>;

fn test_storage() -> MockStorage {
InMemoryStorage::default().into_transaction()
Expand Down
20 changes: 9 additions & 11 deletions crates/services/compression/src/storage.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
use fuel_core_storage::merkle::sparse::Merkleized;

/// Generates a random RegistryKey
#[cfg(feature = "test-helpers")]
#[allow(clippy::arithmetic_side_effects)]
Expand All @@ -25,32 +23,32 @@ pub mod registrations;
pub mod timestamps;

/// Merkleized Address table type alias
pub type Address = Merkleized<address::Address>;
pub type Address = address::Address;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, curious how it will affect the storage. It looks like we still will have trash in the Merkle-related tables. Maybe we need to come up with the solution how to cleanup them

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah we can wipe compression database manually I think?


/// Merkleized AssetId table type alias
pub type AssetId = Merkleized<asset_id::AssetId>;
pub type AssetId = asset_id::AssetId;

/// Merkleized ContractId table type alias
pub type ContractId = Merkleized<contract_id::ContractId>;
pub type ContractId = contract_id::ContractId;

/// Merkleized EvictorCache table type alias
pub type EvictorCache = Merkleized<evictor_cache::EvictorCache>;
pub type EvictorCache = evictor_cache::EvictorCache;

/// Merkleized PredicateCode table type alias
pub type PredicateCode = Merkleized<predicate_code::PredicateCode>;
pub type PredicateCode = predicate_code::PredicateCode;

/// Merkleized ScriptCode table type alias
pub type ScriptCode = Merkleized<script_code::ScriptCode>;
pub type ScriptCode = script_code::ScriptCode;

/// Merkleized RegistryIndex table type alias
pub type RegistryIndex = Merkleized<registry_index::RegistryIndex>;
pub type RegistryIndex = registry_index::RegistryIndex;

/// Merkleized Timestamps table type alias
pub type Timestamps = Merkleized<timestamps::Timestamps>;
pub type Timestamps = timestamps::Timestamps;

/// Re-export to match api
pub use compressed_blocks::CompressedBlocks;

/// Merkleized Registrations table type alias
#[cfg(feature = "fault-proving")]
pub type Registrations = Merkleized<registrations::Registrations>;
pub type Registrations = registrations::Registrations;
18 changes: 3 additions & 15 deletions crates/services/compression/src/storage/address.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,27 +7,15 @@ use fuel_core_storage::{
postcard::Postcard,
raw::Raw,
},
merkle::sparse::MerkleizedTableColumn,
structured_storage::TableWithBlueprint,
};
use fuel_core_types::fuel_compression::RegistryKey;

use super::column::{
CompressionColumn,
MerkleizedColumnOf,
};
use super::column::CompressionColumn;

/// Table that indexes the addresses.
pub struct Address;

impl MerkleizedTableColumn for Address {
type TableColumn = CompressionColumn;

fn table_column() -> Self::TableColumn {
Self::TableColumn::Address
}
}

impl Mappable for Address {
type Key = Self::OwnedKey;
type OwnedKey = RegistryKey;
Expand All @@ -37,10 +25,10 @@ impl Mappable for Address {

impl TableWithBlueprint for Address {
type Blueprint = Plain<Postcard, Raw>;
type Column = MerkleizedColumnOf<Self>;
type Column = CompressionColumn;

fn column() -> Self::Column {
Self::Column::TableColumn(Self::table_column())
Self::Column::Address
}
}

Expand Down
18 changes: 3 additions & 15 deletions crates/services/compression/src/storage/asset_id.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,27 +7,15 @@ use fuel_core_storage::{
postcard::Postcard,
raw::Raw,
},
merkle::sparse::MerkleizedTableColumn,
structured_storage::TableWithBlueprint,
};
use fuel_core_types::fuel_compression::RegistryKey;

use super::column::{
CompressionColumn,
MerkleizedColumnOf,
};
use super::column::CompressionColumn;

/// Table that indexes the asset ids.
pub struct AssetId;

impl MerkleizedTableColumn for AssetId {
type TableColumn = CompressionColumn;

fn table_column() -> Self::TableColumn {
Self::TableColumn::AssetId
}
}

impl Mappable for AssetId {
type Key = Self::OwnedKey;
type OwnedKey = RegistryKey;
Expand All @@ -37,10 +25,10 @@ impl Mappable for AssetId {

impl TableWithBlueprint for AssetId {
type Blueprint = Plain<Postcard, Raw>;
type Column = MerkleizedColumnOf<Self>;
type Column = CompressionColumn;

fn column() -> Self::Column {
Self::Column::TableColumn(Self::table_column())
Self::Column::AssetId
}
}

Expand Down
44 changes: 25 additions & 19 deletions crates/services/compression/src/storage/column.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,8 @@
//! Column enum for the database.

use fuel_core_storage::merkle::{
column::{
AsU32,
MerkleizedColumn,
},
sparse::MerkleizedTableColumn,
use fuel_core_storage::{
kv_store::StorageColumn,
merkle::column::AsU32,
};

/// Enum representing the columns in the storage.
Expand All @@ -22,27 +19,29 @@ use fuel_core_storage::merkle::{
Hash,
)]
pub enum CompressionColumn {
/// Metadata table
Metadata = 0,
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it a good idea to add metadata and change the table numbers now? Doing that over an existing database means database will be corrupted

Copy link
Member Author

@rymnc rymnc Jul 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fairly certain it will reuse metadata table from before, we will have to prune the compression database first

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should get confirmation on this. @rymnc are you saying that since we didn't put the SMT's into the mainnet compression yet, this is essentially reverting a breaking change that hasn't been deployed yet?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it's already in mainnet, I can clone a devnet database and see how it works

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i checked locally running a testnet node with 0.43.2 with compression enabled and then after with this branch. queries for compressed blocks that were processed with 0.43.2 yield null, but the rest come through as normal, no errors in syncing or on graphql. this would be the expected behaviour, so it's fine in my opinion.

  1. scale down compression sentry
  2. delete compression database
  3. configure --da-compression-starting-height and new image
  4. scale up compression sentry

it will work without errors.

/// CompressedBlocks, see [`CompressedBlocks`](crate::storage::compressed_blocks::CompressedBlocks)
CompressedBlocks = 0,
CompressedBlocks = 1,
/// RegistryKey to Address index, see [`Address`](crate::storage::address::Address)
Address = 1,
Address = 2,
/// RegistryKey to AssetId index, see [`AssetId`](crate::storage::asset_id::AssetId)
AssetId = 2,
AssetId = 3,
/// RegistryKey to ContractId index, see [`ContractId`](crate::storage::contract_id::ContractId)
ContractId = 3,
ContractId = 4,
/// RegistryKey to ScriptCode index, see [`ScriptCode`](crate::storage::script_code::ScriptCode)
ScriptCode = 4,
ScriptCode = 5,
/// RegistryKey to PredicateCode index, see [`PredicateCode`](crate::storage::predicate_code::PredicateCode)
PredicateCode = 5,
PredicateCode = 6,
/// RegistryKey to ReverseKey index, see [`RegistryIndex`](crate::storage::registry_index::RegistryIndex)
RegistryIndex = 6,
RegistryIndex = 7,
/// Keeps track of keys to remove, see [`EvictorCache`](crate::storage::evictor_cache::EvictorCache)
EvictorCache = 7,
EvictorCache = 8,
/// Keeps track of timestamps, will be removed eventually, see [`Timestamps`](crate::storage::timestamps::Timestamps)
Timestamps = 8,
Timestamps = 9,
#[cfg(feature = "fault-proving")]
/// Keeps track of registrations per table associated with a compressed block, see [`Registrations`](crate::storage::registrations::Registrations)
Registrations = 9,
Registrations = 10,
}

impl AsU32 for CompressionColumn {
Expand All @@ -51,6 +50,13 @@ impl AsU32 for CompressionColumn {
}
}

/// Type alias to get the `MerkleizedColumn` for some type that implements `MerkleizedTableColumn`.
pub type MerkleizedColumnOf<TC> =
MerkleizedColumn<<TC as MerkleizedTableColumn>::TableColumn>;
impl StorageColumn for CompressionColumn {
fn name(&self) -> String {
let str: &str = self.into();
str.to_string()
}

fn id(&self) -> u32 {
self.as_u32()
}
}
18 changes: 3 additions & 15 deletions crates/services/compression/src/storage/compressed_blocks.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
//! The table for the compressed blocks sent to DA.

use super::column::{
CompressionColumn,
MerkleizedColumnOf,
};
use super::column::CompressionColumn;
use fuel_core_compression::VersionedCompressedBlock;
use fuel_core_storage::{
Mappable,
Expand All @@ -12,7 +9,6 @@ use fuel_core_storage::{
postcard::Postcard,
primitive::Primitive,
},
merkle::sparse::MerkleizedTableColumn,
structured_storage::TableWithBlueprint,
};

Expand All @@ -28,21 +24,13 @@ impl Mappable for CompressedBlocks {
type OwnedValue = VersionedCompressedBlock;
}

impl MerkleizedTableColumn for CompressedBlocks {
type TableColumn = CompressionColumn;

fn table_column() -> Self::TableColumn {
Self::TableColumn::CompressedBlocks
}
}

impl TableWithBlueprint for CompressedBlocks {
// we don't use the Merkleized blueprint because we don't need it for this table
type Blueprint = Plain<Primitive<4>, Postcard>;
type Column = MerkleizedColumnOf<Self>;
type Column = CompressionColumn;

fn column() -> Self::Column {
Self::Column::TableColumn(Self::table_column())
Self::Column::CompressedBlocks
}
}

Expand Down
Loading
Loading