cfx_storage/
lib.rs

1// Copyright 2019 Conflux Foundation. All rights reserved.
2// Conflux is free software and distributed under GNU General Public License.
3// See http://www.gnu.org/licenses/
4
5// TODO: check them again and reason about the safety of each usage.
6#![allow(clippy::mut_from_ref)]
7
8#[macro_use]
9extern crate cfx_util_macros;
10#[macro_use]
11extern crate lazy_static;
12#[macro_use]
13extern crate log;
14
15#[macro_use]
16pub mod utils;
17
18pub(self) mod snapshot_manager;
19pub mod state;
20pub mod state_manager;
21#[macro_use]
22pub mod storage_db;
23
24pub mod tests;
25
26mod impls;
27
28pub mod storage_dir {
29    use std::path::PathBuf;
30    lazy_static! {
31        pub static ref DELTA_MPTS_DIR: PathBuf =
32            ["storage_db", "delta_mpts"].iter().collect::<PathBuf>();
33        pub static ref SNAPSHOT_DIR: PathBuf =
34            ["storage_db", "snapshot"].iter().collect::<PathBuf>();
35        pub static ref SNAPSHOT_INFO_DB_NAME: &'static str = "snapshot_info";
36        pub static ref SNAPSHOT_INFO_DB_PATH: PathBuf =
37            ["storage_db", "snapshot_info_db"]
38                .iter()
39                .collect::<PathBuf>();
40        pub static ref STORAGE_DIR: PathBuf = "storage_db".into();
41    }
42}
43
44/// Consensus parameter is only configurable in test mode.
45#[derive(Debug, Clone)]
46pub struct ConsensusParam {
47    // The current delta switching rule is simply split by height at
48    // multiple of SNAPSHOT_EPOCHS_CAPACITY.
49    //
50    // Only if we see problem dealing with attacks, consider rules like the
51    // size of delta trie.
52    pub snapshot_epoch_count: u32,
53
54    pub era_epoch_count: u64,
55}
56
57#[derive(Debug, Clone, Eq, PartialEq)]
58pub enum ProvideExtraSnapshotSyncConfig {
59    /// Keep the snapshot at the same epoch as the checkpoint.
60    /// TODO:
61    ///  This config will be removed when there is a more reasonable
62    ///  snapshot sync point.
63    StableCheckpoint,
64    EpochNearestMultipleOf(u32),
65}
66
67impl ProvideExtraSnapshotSyncConfig {
68    pub fn from_str(config: &str) -> Option<Self> {
69        const MULTIPLE_OF_PREFIX: &'static str = "multiple_of_";
70        if config == "checkpoint" {
71            Some(Self::StableCheckpoint)
72        } else if config.starts_with(MULTIPLE_OF_PREFIX) {
73            let number_str = &config[MULTIPLE_OF_PREFIX.len()..];
74            match number_str.parse::<u32>() {
75                Err(_) => None,
76                Ok(num) => Some(Self::EpochNearestMultipleOf(num)),
77            }
78        } else {
79            None
80        }
81    }
82
83    pub fn parse_config_list(
84        config: &str,
85    ) -> std::result::Result<Vec<Self>, String> {
86        let mut list = vec![];
87        for item in config.split(",") {
88            if item.len() > 0 {
89                list.push(Self::from_str(item).ok_or_else(|| {
90                    format!(
91                        "{} is not a valid ProvideExtraSnapshotSyncConfig",
92                        item
93                    )
94                })?);
95            }
96        }
97        Ok(list)
98    }
99}
100
101#[derive(Debug, Clone)]
102pub struct StorageConfiguration {
103    pub additional_maintained_snapshot_count: u32,
104    pub consensus_param: ConsensusParam,
105    pub debug_snapshot_checker_threads: u16,
106    pub delta_mpts_cache_recent_lfu_factor: f64,
107    pub delta_mpts_cache_start_size: u32,
108    pub delta_mpts_cache_size: u32,
109    pub delta_mpts_node_map_vec_size: u32,
110    pub delta_mpts_slab_idle_size: u32,
111    pub single_mpt_cache_start_size: u32,
112    pub single_mpt_cache_size: u32,
113    pub single_mpt_slab_idle_size: u32,
114    pub max_open_snapshots: u16,
115    pub path_delta_mpts_dir: PathBuf,
116    pub path_storage_dir: PathBuf,
117    pub path_snapshot_dir: PathBuf,
118    pub path_snapshot_info_db: PathBuf,
119    pub provide_more_snapshot_for_sync: Vec<ProvideExtraSnapshotSyncConfig>,
120    pub max_open_mpt_count: u32,
121    pub enable_single_mpt_storage: bool,
122    pub single_mpt_space: Option<Space>,
123    pub cip90a: u64,
124    pub keep_snapshot_before_stable_checkpoint: bool,
125    pub use_isolated_db_for_mpt_table: bool,
126    pub use_isolated_db_for_mpt_table_height: Option<u64>,
127    pub keep_era_genesis_snapshot: bool,
128    pub backup_mpt_snapshot: bool,
129}
130
131impl StorageConfiguration {
132    pub fn new_default(
133        conflux_data_dir: &str, snapshot_epoch_count: u32, era_epoch_count: u64,
134    ) -> Self {
135        let conflux_data_path = Path::new(conflux_data_dir);
136        StorageConfiguration {
137            additional_maintained_snapshot_count: 0,
138            consensus_param: ConsensusParam {
139                snapshot_epoch_count,
140                era_epoch_count,
141            },
142            debug_snapshot_checker_threads:
143                defaults::DEFAULT_DEBUG_SNAPSHOT_CHECKER_THREADS,
144            delta_mpts_cache_recent_lfu_factor:
145                defaults::DEFAULT_DELTA_MPTS_CACHE_RECENT_LFU_FACTOR,
146            delta_mpts_cache_size: defaults::DEFAULT_DELTA_MPTS_CACHE_SIZE,
147            delta_mpts_cache_start_size:
148                defaults::DEFAULT_DELTA_MPTS_CACHE_START_SIZE,
149            delta_mpts_node_map_vec_size: defaults::DEFAULT_NODE_MAP_SIZE,
150            delta_mpts_slab_idle_size:
151                defaults::DEFAULT_DELTA_MPTS_SLAB_IDLE_SIZE,
152            single_mpt_cache_start_size:
153                defaults::DEFAULT_DELTA_MPTS_CACHE_START_SIZE * 2,
154            single_mpt_cache_size: defaults::DEFAULT_DELTA_MPTS_CACHE_SIZE * 2,
155            single_mpt_slab_idle_size:
156                defaults::DEFAULT_DELTA_MPTS_SLAB_IDLE_SIZE * 2,
157            max_open_snapshots: defaults::DEFAULT_MAX_OPEN_SNAPSHOTS,
158            path_delta_mpts_dir: conflux_data_path
159                .join(&*storage_dir::DELTA_MPTS_DIR),
160            path_snapshot_dir: conflux_data_path
161                .join(&*storage_dir::SNAPSHOT_DIR),
162            path_snapshot_info_db: conflux_data_path
163                .join(&*storage_dir::SNAPSHOT_INFO_DB_PATH),
164            path_storage_dir: conflux_data_path
165                .join(&*storage_dir::STORAGE_DIR),
166            provide_more_snapshot_for_sync: vec![
167                ProvideExtraSnapshotSyncConfig::StableCheckpoint,
168            ],
169            max_open_mpt_count: defaults::DEFAULT_MAX_OPEN_MPT,
170            enable_single_mpt_storage: false,
171            single_mpt_space: None,
172            cip90a: 0,
173            keep_snapshot_before_stable_checkpoint: true,
174            use_isolated_db_for_mpt_table: false,
175            use_isolated_db_for_mpt_table_height: None,
176            keep_era_genesis_snapshot: false,
177            backup_mpt_snapshot: true,
178        }
179    }
180
181    pub fn full_state_start_height(&self) -> Option<u64> {
182        if self.enable_single_mpt_storage {
183            let height = if self.single_mpt_space == Some(Space::Ethereum) {
184                // The eSpace state is only available after cip90 is
185                // enabled.
186                self.cip90a
187            } else {
188                0
189            };
190            Some(height)
191        } else {
192            None
193        }
194    }
195}
196
197pub use self::{
198    impls::{
199        defaults,
200        delta_mpt::*,
201        errors::{Error, Result},
202        merkle_patricia_trie::{
203            mpt_cursor::rlp_key_value_len, simple_mpt::*,
204            trie_proof::TrieProofNode, CompressedPathRaw, KVInserter,
205            MptKeyValue, TrieProof, VanillaChildrenTable,
206        },
207        node_merkle_proof::{NodeMerkleProof, StorageRootProof},
208        proof_merger::StateProofMerger,
209        recording_storage::RecordingStorage,
210        snapshot_sync::{FullSyncVerifier, MptSlicer},
211        state_proof::StateProof,
212        storage_db::{
213            kvdb_rocksdb::KvdbRocksdb,
214            kvdb_sqlite::{KvdbSqlite, KvdbSqliteStatements},
215            kvdb_sqlite_sharded::KvdbSqliteSharded,
216            snapshot_db_manager_sqlite::SnapshotDbManagerSqlite,
217            sqlite::SqliteConnection,
218        },
219    },
220    replicated_state::ReplicatedState,
221    state::{
222        State as StorageState, StateTrait as StorageStateTrait,
223        StateTraitExt as StorageStateTraitExt,
224    },
225    state_manager::{
226        StateIndex, StateManager as StorageManager,
227        StateManagerTrait as StorageManagerTrait,
228    },
229    storage_db::{KeyValueDbIterableTrait, KeyValueDbTrait},
230};
231
232#[cfg(any(test, feature = "testonly_code"))]
233pub use self::tests::new_state_manager_for_unit_test as new_storage_manager_for_testing;
234use crate::impls::replicated_state;
235use cfx_internal_common::StateRootWithAuxInfo;
236use cfx_types::Space;
237use std::path::{Path, PathBuf};
cfx_storage/lib.rs

cfx_storage/
lib.rs
