cfx_storage/
lib.rs

1// Copyright 2019 Conflux Foundation. All rights reserved.
2// Conflux is free software and distributed under GNU General Public License.
3// See http://www.gnu.org/licenses/
4
5// TODO: check them again and reason about the safety of each usage.
6#![allow(clippy::mut_from_ref, clippy::cast_ref_to_mut, clippy::drop_ref)]
7#![allow(deprecated)]
8
9#[macro_use]
10extern crate cfx_util_macros;
11#[macro_use]
12extern crate lazy_static;
13#[macro_use]
14extern crate log;
15
16#[macro_use]
17pub mod utils;
18
19pub(self) mod snapshot_manager;
20pub mod state;
21pub mod state_manager;
22#[macro_use]
23pub mod storage_db;
24
25pub mod tests;
26
27mod impls;
28
29pub mod storage_dir {
30    use std::path::PathBuf;
31    lazy_static! {
32        pub static ref DELTA_MPTS_DIR: PathBuf =
33            ["storage_db", "delta_mpts"].iter().collect::<PathBuf>();
34        pub static ref SNAPSHOT_DIR: PathBuf =
35            ["storage_db", "snapshot"].iter().collect::<PathBuf>();
36        pub static ref SNAPSHOT_INFO_DB_NAME: &'static str = "snapshot_info";
37        pub static ref SNAPSHOT_INFO_DB_PATH: PathBuf =
38            ["storage_db", "snapshot_info_db"]
39                .iter()
40                .collect::<PathBuf>();
41        pub static ref STORAGE_DIR: PathBuf = "storage_db".into();
42    }
43}
44
45/// Consensus parameter is only configurable in test mode.
46#[derive(Debug, Clone)]
47pub struct ConsensusParam {
48    // The current delta switching rule is simply split by height at
49    // multiple of SNAPSHOT_EPOCHS_CAPACITY.
50    //
51    // Only if we see problem dealing with attacks, consider rules like the
52    // size of delta trie.
53    pub snapshot_epoch_count: u32,
54
55    pub era_epoch_count: u64,
56}
57
58#[derive(Debug, Clone, Eq, PartialEq)]
59pub enum ProvideExtraSnapshotSyncConfig {
60    /// Keep the snapshot at the same epoch as the checkpoint.
61    /// TODO:
62    ///  This config will be removed when there is a more reasonable
63    ///  snapshot sync point.
64    StableCheckpoint,
65    EpochNearestMultipleOf(u32),
66}
67
68impl ProvideExtraSnapshotSyncConfig {
69    pub fn from_str(config: &str) -> Option<Self> {
70        const MULTIPLE_OF_PREFIX: &'static str = "multiple_of_";
71        if config == "checkpoint" {
72            Some(Self::StableCheckpoint)
73        } else if config.starts_with(MULTIPLE_OF_PREFIX) {
74            let number_str = &config[MULTIPLE_OF_PREFIX.len()..];
75            match number_str.parse::<u32>() {
76                Err(_) => None,
77                Ok(num) => Some(Self::EpochNearestMultipleOf(num)),
78            }
79        } else {
80            None
81        }
82    }
83
84    pub fn parse_config_list(
85        config: &str,
86    ) -> std::result::Result<Vec<Self>, String> {
87        let mut list = vec![];
88        for item in config.split(",") {
89            if item.len() > 0 {
90                list.push(Self::from_str(item).ok_or_else(|| {
91                    format!(
92                        "{} is not a valid ProvideExtraSnapshotSyncConfig",
93                        item
94                    )
95                })?);
96            }
97        }
98        Ok(list)
99    }
100}
101
102#[derive(Debug, Clone)]
103pub struct StorageConfiguration {
104    pub additional_maintained_snapshot_count: u32,
105    pub consensus_param: ConsensusParam,
106    pub debug_snapshot_checker_threads: u16,
107    pub delta_mpts_cache_recent_lfu_factor: f64,
108    pub delta_mpts_cache_start_size: u32,
109    pub delta_mpts_cache_size: u32,
110    pub delta_mpts_node_map_vec_size: u32,
111    pub delta_mpts_slab_idle_size: u32,
112    pub single_mpt_cache_start_size: u32,
113    pub single_mpt_cache_size: u32,
114    pub single_mpt_slab_idle_size: u32,
115    pub max_open_snapshots: u16,
116    pub path_delta_mpts_dir: PathBuf,
117    pub path_storage_dir: PathBuf,
118    pub path_snapshot_dir: PathBuf,
119    pub path_snapshot_info_db: PathBuf,
120    pub provide_more_snapshot_for_sync: Vec<ProvideExtraSnapshotSyncConfig>,
121    pub max_open_mpt_count: u32,
122    pub enable_single_mpt_storage: bool,
123    pub single_mpt_space: Option<Space>,
124    pub cip90a: u64,
125    pub keep_snapshot_before_stable_checkpoint: bool,
126    pub use_isolated_db_for_mpt_table: bool,
127    pub use_isolated_db_for_mpt_table_height: Option<u64>,
128    pub keep_era_genesis_snapshot: bool,
129    pub backup_mpt_snapshot: bool,
130}
131
132impl StorageConfiguration {
133    pub fn new_default(
134        conflux_data_dir: &str, snapshot_epoch_count: u32, era_epoch_count: u64,
135    ) -> Self {
136        let conflux_data_path = Path::new(conflux_data_dir);
137        StorageConfiguration {
138            additional_maintained_snapshot_count: 0,
139            consensus_param: ConsensusParam {
140                snapshot_epoch_count,
141                era_epoch_count,
142            },
143            debug_snapshot_checker_threads:
144                defaults::DEFAULT_DEBUG_SNAPSHOT_CHECKER_THREADS,
145            delta_mpts_cache_recent_lfu_factor:
146                defaults::DEFAULT_DELTA_MPTS_CACHE_RECENT_LFU_FACTOR,
147            delta_mpts_cache_size: defaults::DEFAULT_DELTA_MPTS_CACHE_SIZE,
148            delta_mpts_cache_start_size:
149                defaults::DEFAULT_DELTA_MPTS_CACHE_START_SIZE,
150            delta_mpts_node_map_vec_size: defaults::DEFAULT_NODE_MAP_SIZE,
151            delta_mpts_slab_idle_size:
152                defaults::DEFAULT_DELTA_MPTS_SLAB_IDLE_SIZE,
153            single_mpt_cache_start_size:
154                defaults::DEFAULT_DELTA_MPTS_CACHE_START_SIZE * 2,
155            single_mpt_cache_size: defaults::DEFAULT_DELTA_MPTS_CACHE_SIZE * 2,
156            single_mpt_slab_idle_size:
157                defaults::DEFAULT_DELTA_MPTS_SLAB_IDLE_SIZE * 2,
158            max_open_snapshots: defaults::DEFAULT_MAX_OPEN_SNAPSHOTS,
159            path_delta_mpts_dir: conflux_data_path
160                .join(&*storage_dir::DELTA_MPTS_DIR),
161            path_snapshot_dir: conflux_data_path
162                .join(&*storage_dir::SNAPSHOT_DIR),
163            path_snapshot_info_db: conflux_data_path
164                .join(&*storage_dir::SNAPSHOT_INFO_DB_PATH),
165            path_storage_dir: conflux_data_path
166                .join(&*storage_dir::STORAGE_DIR),
167            provide_more_snapshot_for_sync: vec![
168                ProvideExtraSnapshotSyncConfig::StableCheckpoint,
169            ],
170            max_open_mpt_count: defaults::DEFAULT_MAX_OPEN_MPT,
171            enable_single_mpt_storage: false,
172            single_mpt_space: None,
173            cip90a: 0,
174            keep_snapshot_before_stable_checkpoint: true,
175            use_isolated_db_for_mpt_table: false,
176            use_isolated_db_for_mpt_table_height: None,
177            keep_era_genesis_snapshot: false,
178            backup_mpt_snapshot: true,
179        }
180    }
181
182    pub fn full_state_start_height(&self) -> Option<u64> {
183        if self.enable_single_mpt_storage {
184            let height = if self.single_mpt_space == Some(Space::Ethereum) {
185                // The eSpace state is only available after cip90 is
186                // enabled.
187                self.cip90a
188            } else {
189                0
190            };
191            Some(height)
192        } else {
193            None
194        }
195    }
196}
197
198pub use self::{
199    impls::{
200        defaults,
201        delta_mpt::*,
202        errors::{Error, Result},
203        merkle_patricia_trie::{
204            mpt_cursor::rlp_key_value_len, simple_mpt::*,
205            trie_proof::TrieProofNode, CompressedPathRaw, KVInserter,
206            MptKeyValue, TrieProof, VanillaChildrenTable,
207        },
208        node_merkle_proof::{NodeMerkleProof, StorageRootProof},
209        proof_merger::StateProofMerger,
210        recording_storage::RecordingStorage,
211        snapshot_sync::{FullSyncVerifier, MptSlicer},
212        state_proof::StateProof,
213        storage_db::{
214            kvdb_rocksdb::KvdbRocksdb,
215            kvdb_sqlite::{KvdbSqlite, KvdbSqliteStatements},
216            kvdb_sqlite_sharded::KvdbSqliteSharded,
217            snapshot_db_manager_sqlite::SnapshotDbManagerSqlite,
218            sqlite::SqliteConnection,
219        },
220    },
221    replicated_state::ReplicatedState,
222    state::{
223        State as StorageState, StateTrait as StorageStateTrait,
224        StateTraitExt as StorageStateTraitExt,
225    },
226    state_manager::{
227        StateIndex, StateManager as StorageManager,
228        StateManagerTrait as StorageManagerTrait,
229    },
230    storage_db::{KeyValueDbIterableTrait, KeyValueDbTrait},
231};
232
233#[cfg(any(test, feature = "testonly_code"))]
234pub use self::tests::new_state_manager_for_unit_test as new_storage_manager_for_testing;
235use crate::impls::replicated_state;
236use cfx_internal_common::StateRootWithAuxInfo;
237use cfx_types::Space;
238use std::path::{Path, PathBuf};
cfx_storage/lib.rs

cfx_storage/
lib.rs
