diem_config/config/
mod.rs

1// Copyright (c) The Diem Core Contributors
2// SPDX-License-Identifier: Apache-2.0
3
4// Copyright 2021 Conflux Foundation. All rights reserved.
5// Conflux is free software and distributed under GNU General Public License.
6// See http://www.gnu.org/licenses/
7
8use rand::{rngs::StdRng, SeedableRng};
9use serde::{de::DeserializeOwned, Deserialize, Serialize};
10use std::{
11    collections::HashMap,
12    fmt,
13    fs::{self, File},
14    io::{Read, Write},
15    path::{Path, PathBuf},
16    str::FromStr,
17};
18use thiserror::Error;
19
20mod consensus_config;
21pub use consensus_config::*;
22mod debug_interface_config;
23pub use debug_interface_config::*;
24mod error;
25pub use error::*;
26mod execution_config;
27pub use execution_config::*;
28mod logger_config;
29pub use logger_config::*;
30mod metrics_config;
31pub use metrics_config::*;
32mod mempool_config;
33pub use mempool_config::*;
34mod network_config;
35pub use network_config::*;
36mod json_rpc_config;
37pub use json_rpc_config::*;
38mod secure_backend_config;
39pub use secure_backend_config::*;
40mod state_sync_config;
41pub use state_sync_config::*;
42mod storage_config;
43pub use storage_config::*;
44mod safety_rules_config;
45pub use safety_rules_config::*;
46mod upstream_config;
47pub use upstream_config::*;
48mod test_config;
49use diem_secure_storage::{KVStorage, Storage};
50use diem_types::waypoint::Waypoint;
51pub use test_config::*;
52
53/// Config pulls in configuration information from the config file.
54/// This is used to set up the nodes and configure various parameters.
55/// The config file is broken up into sections for each module
56/// so that only that module can be passed around
57#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
58#[serde(deny_unknown_fields)]
59pub struct NodeConfig {
60    #[serde(default)]
61    pub base: BaseConfig,
62    #[serde(default)]
63    pub consensus: ConsensusConfig,
64    #[serde(default)]
65    pub debug_interface: DebugInterfaceConfig,
66    #[serde(default)]
67    pub execution: ExecutionConfig,
68    //#[serde(default, skip_serializing_if = "Vec::is_empty")]
69    //pub full_node_networks: Vec<NetworkConfig>,
70    #[serde(default)]
71    pub logger: LoggerConfig,
72    #[serde(default)]
73    pub metrics: MetricsConfig,
74    #[serde(default)]
75    pub mempool: MempoolConfig,
76    #[serde(default)]
77    pub json_rpc: JsonRpcConfig,
78    #[serde(default)]
79    pub state_sync: StateSyncConfig,
80    #[serde(default)]
81    pub storage: StorageConfig,
82    #[serde(default)]
83    pub test: Option<TestConfig>,
84    #[serde(default)]
85    pub upstream: UpstreamConfig,
86    //#[serde(default)]
87    //pub validator_network: Option<NetworkConfig>,
88    #[serde(default)]
89    pub failpoints: Option<HashMap<String, String>>,
90}
91
92#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
93#[serde(default, deny_unknown_fields)]
94pub struct BaseConfig {
95    data_dir: PathBuf,
96    pub role: RoleType,
97    #[serde(with = "yaml_serde::with::singleton_map")]
98    pub waypoint: WaypointConfig,
99}
100
101impl Default for BaseConfig {
102    fn default() -> BaseConfig {
103        BaseConfig {
104            data_dir: PathBuf::from("./pos_db"),
105            role: RoleType::Validator,
106            waypoint: WaypointConfig::None,
107        }
108    }
109}
110
111#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
112#[serde(rename_all = "snake_case")]
113pub enum WaypointConfig {
114    FromConfig(Waypoint),
115    FromFile(PathBuf),
116    FromStorage(SecureBackend),
117    None,
118}
119
120impl WaypointConfig {
121    pub fn waypoint_from_config(&self) -> Option<Waypoint> {
122        if let WaypointConfig::FromConfig(waypoint) = self {
123            Some(*waypoint)
124        } else {
125            None
126        }
127    }
128
129    pub fn waypoint(&self) -> Waypoint {
130        let waypoint = match &self {
131            WaypointConfig::FromConfig(waypoint) => Some(*waypoint),
132            WaypointConfig::FromFile(path) => {
133                let content = fs::read_to_string(path).unwrap_or_else(|_| {
134                    panic!("Failed to read waypoint file {}", path.display())
135                });
136                Some(Waypoint::from_str(&content.trim()).unwrap_or_else(|_| {
137                    panic!("Failed to parse waypoint: {}", content.trim())
138                }))
139            }
140            WaypointConfig::FromStorage(backend) => {
141                let storage: Storage = backend.into();
142                let waypoint = storage
143                    .get::<Waypoint>(diem_global_constants::WAYPOINT)
144                    .expect("Unable to read waypoint")
145                    .value;
146                Some(waypoint)
147            }
148            WaypointConfig::None => None,
149        };
150        waypoint.expect("waypoint should be present")
151    }
152
153    pub fn genesis_waypoint(&self) -> Waypoint {
154        match &self {
155            WaypointConfig::FromStorage(backend) => {
156                let storage: Storage = backend.into();
157                storage
158                    .get::<Waypoint>(diem_global_constants::GENESIS_WAYPOINT)
159                    .expect("Unable to read waypoint")
160                    .value
161            }
162            _ => self.waypoint(),
163        }
164    }
165}
166
167#[derive(Clone, Copy, Deserialize, Eq, PartialEq, Serialize)]
168#[serde(rename_all = "snake_case")]
169pub enum RoleType {
170    Validator,
171    FullNode,
172}
173
174impl RoleType {
175    pub fn is_validator(self) -> bool { self == RoleType::Validator }
176
177    pub fn as_str(self) -> &'static str {
178        match self {
179            RoleType::Validator => "validator",
180            RoleType::FullNode => "full_node",
181        }
182    }
183}
184
185impl FromStr for RoleType {
186    type Err = ParseRoleError;
187
188    fn from_str(s: &str) -> Result<Self, Self::Err> {
189        match s {
190            "validator" => Ok(RoleType::Validator),
191            "full_node" => Ok(RoleType::FullNode),
192            _ => Err(ParseRoleError(s.to_string())),
193        }
194    }
195}
196
197impl fmt::Debug for RoleType {
198    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
199        write!(f, "{}", self)
200    }
201}
202
203impl fmt::Display for RoleType {
204    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
205        write!(f, "{}", self.as_str())
206    }
207}
208
209#[derive(Debug, Error)]
210#[error("Invalid node role: {0}")]
211pub struct ParseRoleError(String);
212
213impl NodeConfig {
214    pub fn data_dir(&self) -> &Path { &self.base.data_dir }
215
216    pub fn set_data_dir(&mut self, data_dir: PathBuf) {
217        self.base.data_dir = data_dir.clone();
218        self.consensus.set_data_dir(data_dir.clone());
219        self.execution.set_data_dir(data_dir.clone());
220        self.metrics.set_data_dir(data_dir.clone());
221        self.storage.set_data_dir(data_dir);
222    }
223
224    /// Reads the config file and returns the configuration object in addition
225    /// to doing some post-processing of the config
226    /// Paths used in the config are either absolute or relative to the config
227    /// location
228    pub fn load<P: AsRef<Path>>(input_path: P) -> Result<Self, Error> {
229        let mut config = Self::load_config(&input_path)?;
230
231        let input_dir = RootPath::new(input_path);
232        config.execution.load(&input_dir)?;
233
234        //let mut config = config.validate_network_configs()?;
235        //config.set_data_dir(config.data_dir().to_path_buf());
236        Ok(config)
237    }
238
239    /// Checks `NetworkConfig` setups so that they exist on proper networks
240    /// Additionally, handles any strange missing default cases
241    /*fn validate_network_configs(mut self) -> Result<NodeConfig, Error> {
242        if self.base.role.is_validator() {
243            invariant(
244                self.validator_network.is_some(),
245                "Missing a validator network config for a validator node"
246                    .into(),
247            )?;
248        } else {
249            invariant(
250                self.validator_network.is_none(),
251                "Provided a validator network config for a full_node node"
252                    .into(),
253            )?;
254        }
255
256        let mut network_ids = HashSet::new();
257        if let Some(network) = &mut self.validator_network {
258            network.load_validator_network()?;
259            network_ids.insert(network.network_id.clone());
260        }
261        for network in &mut self.full_node_networks {
262            network.load_fullnode_network()?;
263
264            // Check a validator network is not included in a list of full-node
265            // networks
266            let network_id = &network.network_id;
267            invariant(
268                !matches!(network_id, NetworkId::Validator),
269                "Included a validator network in full_node_networks".into(),
270            )?;
271            network_ids.insert(network_id.clone());
272        }
273        Ok(self)
274    }*/
275
276    pub fn save<P: AsRef<Path>>(
277        &mut self, output_path: P,
278    ) -> Result<(), Error> {
279        let output_dir = RootPath::new(&output_path);
280        self.execution.save(&output_dir)?;
281        // This must be last as calling save on subconfigs may change their
282        // fields
283        self.save_config(&output_path)?;
284        Ok(())
285    }
286
287    /*pub fn randomize_ports(&mut self) {
288        self.debug_interface.randomize_ports();
289        self.json_rpc.randomize_ports();
290        self.storage.randomize_ports();
291
292        if let Some(network) = self.validator_network.as_mut() {
293            network.listen_address =
294                crate::utils::get_available_port_in_multiaddr(true);
295        }
296
297        for network in self.full_node_networks.iter_mut() {
298            network.listen_address =
299                crate::utils::get_available_port_in_multiaddr(true);
300        }
301    }*/
302
303    pub fn random() -> Self {
304        let mut rng = StdRng::from_seed([0u8; 32]);
305        Self::random_with_template(0, &NodeConfig::default(), &mut rng)
306    }
307
308    pub fn random_with_template(
309        idx: u32, template: &Self, rng: &mut StdRng,
310    ) -> Self {
311        let mut config = template.clone();
312        config.random_internal(idx, rng);
313        config
314    }
315
316    fn random_internal(&mut self, idx: u32, rng: &mut StdRng) {
317        let mut test = TestConfig::new_with_temp_dir(None);
318
319        if self.base.role == RoleType::Validator {
320            test.random_account_key(rng);
321            let peer_id = crate::utils::validator_owner_account_from_name(
322                idx.to_string().as_bytes(),
323            );
324
325            /*if self.validator_network.is_none() {
326                let network_config =
327                    NetworkConfig::network_with_id(NetworkId::Validator);
328                self.validator_network = Some(network_config);
329            }
330
331            let validator_network = self.validator_network.as_mut().unwrap();
332            validator_network.random_with_peer_id(rng, Some(peer_id));*/
333            // We want to produce this key twice
334            let mut cloned_rng = rng.clone();
335            test.random_execution_key(rng);
336
337            let mut safety_rules_test_config =
338                SafetyRulesTestConfig::new(peer_id);
339            safety_rules_test_config.random_consensus_key(rng);
340            safety_rules_test_config.random_execution_key(&mut cloned_rng);
341            self.consensus.safety_rules.test = Some(safety_rules_test_config);
342        } else {
343            /*self.validator_network = None;
344            if self.full_node_networks.is_empty() {
345                let network_config =
346                    NetworkConfig::network_with_id(NetworkId::Public);
347                self.full_node_networks.push(network_config);
348            }
349            for network in &mut self.full_node_networks {
350                network.random(rng);
351            }*/
352        }
353        self.set_data_dir(test.temp_dir().unwrap().to_path_buf());
354        self.test = Some(test);
355    }
356
357    fn default_config(serialized: &str, path: &'static str) -> Self {
358        let config = Self::parse(serialized)
359            .unwrap_or_else(|e| panic!("Error in {}: {}", path, e));
360        config
361        /*
362        config
363            .validate_network_configs()
364            .unwrap_or_else(|e| panic!("Error in {}: {}", path, e))*/
365    }
366
367    pub fn default_for_public_full_node() -> Self {
368        let contents = std::include_str!("test_data/public_full_node.yaml");
369        Self::default_config(contents, "default_for_public_full_node")
370    }
371
372    pub fn default_for_validator() -> Self {
373        let contents = std::include_str!("test_data/validator.yaml");
374        Self::default_config(contents, "default_for_validator")
375    }
376
377    pub fn default_for_validator_full_node() -> Self {
378        let contents = std::include_str!("test_data/validator_full_node.yaml");
379        Self::default_config(contents, "default_for_validator_full_node")
380    }
381}
382
383pub trait PersistableConfig: Serialize + DeserializeOwned {
384    fn load_config<P: AsRef<Path>>(path: P) -> Result<Self, Error> {
385        let mut file = File::open(&path).map_err(|e| {
386            Error::IO(path.as_ref().to_str().unwrap().to_string(), e)
387        })?;
388        let mut contents = String::new();
389        file.read_to_string(&mut contents).map_err(|e| {
390            Error::IO(path.as_ref().to_str().unwrap().to_string(), e)
391        })?;
392        Self::parse(&contents)
393    }
394
395    fn save_config<P: AsRef<Path>>(&self, output_file: P) -> Result<(), Error> {
396        let contents = yaml_serde::to_string(&self)
397            .map_err(|e| {
398                Error::Yaml(
399                    output_file.as_ref().to_str().unwrap().to_string(),
400                    e,
401                )
402            })?
403            .into_bytes();
404        let mut file = File::create(output_file.as_ref()).map_err(|e| {
405            Error::IO(output_file.as_ref().to_str().unwrap().to_string(), e)
406        })?;
407        file.write_all(&contents).map_err(|e| {
408            Error::IO(output_file.as_ref().to_str().unwrap().to_string(), e)
409        })?;
410        Ok(())
411    }
412
413    fn parse(serialized: &str) -> Result<Self, Error> {
414        yaml_serde::from_str(&serialized)
415            .map_err(|e| Error::Yaml("config".to_string(), e))
416    }
417}
418
419impl<T: ?Sized> PersistableConfig for T where T: Serialize + DeserializeOwned {}
420
421#[derive(Debug)]
422pub struct RootPath {
423    root_path: PathBuf,
424}
425
426impl RootPath {
427    pub fn new<P: AsRef<Path>>(path: P) -> Self {
428        let root_path = if let Some(parent) = path.as_ref().parent() {
429            parent.to_path_buf()
430        } else {
431            PathBuf::from("")
432        };
433
434        Self { root_path }
435    }
436
437    /// This function assumes that the path is already a directory
438    pub fn new_path<P: AsRef<Path>>(path: P) -> Self {
439        let root_path = path.as_ref().to_path_buf();
440        Self { root_path }
441    }
442
443    /// This adds a full path when loading / storing if one is not specified
444    pub fn full_path(&self, file_path: &Path) -> PathBuf {
445        if file_path.is_relative() {
446            self.root_path.join(file_path)
447        } else {
448            file_path.to_path_buf()
449        }
450    }
451}
452
453#[cfg(test)]
454mod test {
455    use super::*;
456
457    #[test]
458    fn verify_role_type_conversion() {
459        // Verify relationship between RoleType and as_string() is reflexive
460        let validator = RoleType::Validator;
461        let full_node = RoleType::FullNode;
462        let converted_validator =
463            RoleType::from_str(validator.as_str()).unwrap();
464        let converted_full_node =
465            RoleType::from_str(full_node.as_str()).unwrap();
466        assert_eq!(converted_validator, validator);
467        assert_eq!(converted_full_node, full_node);
468    }
469
470    #[test]
471    // TODO(joshlind): once the 'matches' crate becomes stable, clean this test
472    // up!
473    fn verify_parse_role_error_on_invalid_role() {
474        let invalid_role_type = "this is not a valid role type";
475        match RoleType::from_str(invalid_role_type) {
476            Err(ParseRoleError(_)) => { /* the expected error was thrown! */ }
477            _ => panic!("A ParseRoleError should have been thrown on the invalid role type!"),
478        }
479    }
480
481    /*#[test]
482    fn verify_configs() {
483        NodeConfig::default_for_public_full_node();
484        NodeConfig::default_for_validator();
485        NodeConfig::default_for_validator_full_node();
486
487        let docker_public_full_node =
488            std::include_str!("../../../docker/compose/public_full_node/public_full_node.yaml");
489        // Only verify it is in the correct format as the values cannot be loaded for this config
490        NodeConfig::parse(docker_public_full_node).unwrap();
491
492        let contents = std::include_str!("test_data/safety_rules.yaml");
493        SafetyRulesConfig::parse(&contents)
494            .unwrap_or_else(|e| panic!("Error in safety_rules.yaml: {}", e));
495    }*/
496}
diem_config/config/mod.rs

diem_config/config/
mod.rs
