1#[cfg(test)]
6mod slicer;
7#[cfg(test)]
8mod verifier;
9
10#[derive(Clone, Default)]
11pub struct FakeSnapshotMptDb {
12 db: BTreeMap<Vec<u8>, SnapshotMptNode>,
13 in_place_mode: bool,
14 already_written: HashSet<Vec<u8>>,
15 discard_write: bool,
16}
17
18impl FakeSnapshotMptDb {
19 pub fn new_discard_write() -> Self {
23 Self {
24 discard_write: true,
25 ..Default::default()
26 }
27 }
28
29 #[cfg(test)]
30 fn reset(&mut self, in_place_mode: bool) {
31 self.in_place_mode = in_place_mode;
32 self.already_written.clear();
33 }
34
35 #[cfg(test)]
36 fn assert_eq(&self, expected: &Self) {
37 for (k, node) in &expected.db {
43 let maybe_got_node = self.db.get(k);
44 match maybe_got_node {
45 None => panic!("key {:?} not found in resulting mpt.", k),
46 Some(got_node) => {
47 for (child_index, child_ref) in
48 node.get_children_table_ref().iter()
49 {
50 match got_node
51 .get_children_table_ref()
52 .get_child(child_index)
53 {
54 None => {
55 panic!(
56 "Child {} not found. Expected\n\t{:?}\n\
57 got\n\t{:?}\nkey {:?}",
58 child_index, node, got_node, k,
59 )
60 }
61 Some(got_child_ref) => {
62 assert_eq!(
63 child_ref.subtree_size, got_child_ref.subtree_size,
64 "Subtree size of child {} mismatch. Expected\n\t{:?}\n\
65 got\n\t{:?}\nkey {:?}",
66 child_index, node, got_node, k,
67 )
68 }
69 }
70 }
71 }
72 }
73 }
74 }
75}
76
77struct FakeSnapshotMptDbIter<'a>(
78 btree_map::Range<'a, Vec<u8>, SnapshotMptNode>,
79);
80
81impl SnapshotMptTraitRead for FakeSnapshotMptDb {
82 fn get_merkle_root(&self) -> MerkleHash { unimplemented!() }
83
84 fn load_node(
85 &mut self, path: &dyn CompressedPathTrait,
86 ) -> Result<Option<SnapshotMptNode>> {
87 let result = Ok(self.db.get(&mpt_node_path_to_db_key(path)).cloned());
88 if CHECK_LOADED_SNAPSHOT_MPT_NODE {
89 if let Ok(Some(node)) = &result {
90 node.is_valid(path);
91 }
92 }
93 result
94 }
95}
96
97impl SnapshotMptTraitReadAndIterate for FakeSnapshotMptDb {
98 fn iterate_subtree_trie_nodes_without_root(
99 &mut self, path: &dyn CompressedPathTrait,
100 ) -> Result<Box<dyn SnapshotMptIteraterTrait + '_>> {
101 let begin_key_excl = mpt_node_path_to_db_key(path);
102
103 let mut end_key_excl = begin_key_excl.clone();
104 *end_key_excl.last_mut().unwrap() += 1;
106
107 Ok(Box::new(FakeSnapshotMptDbIter(self.db.range((
108 Excluded(begin_key_excl),
109 Excluded(end_key_excl),
110 )))))
111 }
112}
113
114impl SnapshotMptTraitRw for FakeSnapshotMptDb {
115 fn delete_node(&mut self, path: &dyn CompressedPathTrait) -> Result<()> {
116 if self.discard_write {
117 return Ok(());
118 }
119 let key = mpt_node_path_to_db_key(path);
120 let old_value = self.db.remove(&key);
121 if self.in_place_mode {
122 assert!(
123 old_value.is_some(),
124 "Shouldn't delete node {:?} in in-place mode",
125 key
126 );
127 } else {
128 panic!("Shouldn't call delete_node in save-as mode. key {:?}", key);
129 }
130 if self.already_written.contains(&key) {
131 panic!("Shouldn't delete a newly written node. key {:?}", key);
132 }
133 Ok(())
134 }
135
136 fn write_node(
137 &mut self, path: &dyn CompressedPathTrait, trie_node: &SnapshotMptNode,
138 ) -> Result<()> {
139 if self.discard_write {
140 return Ok(());
141 }
142 let key = mpt_node_path_to_db_key(path);
143 self.db.insert(key.clone(), trie_node.clone());
144 if !self.already_written.insert(key.clone()) {
145 panic!("Shouldn't write a node more than one time. key {:?}", key);
146 }
147 Ok(())
148 }
149}
150
151impl FallibleIterator for FakeSnapshotMptDbIter<'_> {
152 type Error = Error;
153 type Item = (CompressedPathRaw, SnapshotMptNode);
154
155 fn next(&mut self) -> Result<Option<Self::Item>> {
156 match self.0.next() {
157 Some((k, v)) => {
158 Ok(Some((mpt_node_path_from_db_key(k)?, v.clone())))
159 }
160 None => Ok(None),
161 }
162 }
163}
164
165#[cfg(test)]
166fn assert_snapshot_mpt_formation(mpt_kv_iter: &DumpedMptKvIterator) {
167 let snapshot_mpt_nodes;
168 let delta_mpt_root = {
169 let state_manager = new_state_manager_for_unit_test();
170 let mut state = state_manager.get_state_for_genesis_write();
171 for (key, value) in &mpt_kv_iter.kv {
172 state
173 .set(
174 StorageKey::AccountKey(key).with_native_space(),
175 value.clone(),
176 )
177 .expect("Failed to insert key.");
178 }
179
180 let mut epoch_id = EpochId::default();
181 epoch_id.as_bytes_mut()[0] = 1;
182 let root = state.compute_state_root().unwrap().state_root;
183 state.commit(epoch_id).unwrap();
184
185 snapshot_mpt_nodes =
186 state_manager.number_committed_nodes.load(Ordering::Relaxed);
187
188 root.delta_root
189 };
190
191 println!(
192 "Checking snapshot mpt formation of {} keys with expected \
193 merkle_root {:?} and number of nodes {}",
194 mpt_kv_iter.kv.len(),
195 delta_mpt_root,
196 snapshot_mpt_nodes
197 );
198
199 let mut empty_snapshot_mpt = FakeSnapshotMptDb::default();
200 let mut new_snapshot_mpt = FakeSnapshotMptDb::default();
201
202 new_snapshot_mpt.reset(true);
204 let snapshot_merkle_root =
205 MptMerger::new(Some(&mut empty_snapshot_mpt), &mut new_snapshot_mpt)
206 .merge(&mpt_kv_iter)
207 .unwrap();
208 assert_eq!(delta_mpt_root, snapshot_merkle_root);
209 assert_eq!(new_snapshot_mpt.db.len(), snapshot_mpt_nodes);
210
211 empty_snapshot_mpt.reset(true);
212 let snapshot_merkle_root = MptMerger::new(None, &mut empty_snapshot_mpt)
213 .merge(&mpt_kv_iter)
214 .unwrap();
215 assert_eq!(delta_mpt_root, snapshot_merkle_root);
216 assert_eq!(empty_snapshot_mpt.db.len(), snapshot_mpt_nodes);
217}
218
219#[cfg(test)]
220#[test]
221fn test_mpt_node_path_to_from_db_key() {
222 let mpt_kv = [
224 (vec![0x00, 0x10, 0x00, 0x00], vec![0x00]),
225 (vec![0x00, 0x01, 0x00, 0x00, 0x02], vec![0x00]),
226 (vec![0x00, 0x01, 0x00, 0x00, 0x03], vec![0x00]),
227 (vec![0x00, 0x01, 0x02, 0x00], vec![0x00]),
228 ];
229 let state_manager = new_state_manager_for_unit_test();
236 let mut state = state_manager.get_state_for_genesis_write();
237 for (key, value) in &mpt_kv {
238 state
239 .set(
240 StorageKey::AccountKey(key).with_native_space(),
241 value.clone().into_boxed_slice(),
242 )
243 .expect("Failed to insert key.");
244 }
245 let mut epoch_id = EpochId::default();
246 epoch_id.as_bytes_mut()[0] = 1;
247 state.compute_state_root().unwrap();
248 let state_root_with_aux_info = state.commit(epoch_id).unwrap();
249
250 let state = state_manager
251 .get_state_no_commit_inner(
252 StateIndex::new_for_readonly(&epoch_id, &state_root_with_aux_info),
253 false,
254 true,
255 )
256 .unwrap()
257 .unwrap();
258
259 for (key, value) in &mpt_kv {
262 let (v, proof) = state
263 .get_with_proof(StorageKey::AccountKey(key).with_native_space())
264 .unwrap();
265 assert_eq!(v, Some(value.clone().into_boxed_slice()));
266 for node in proof.delta_proof.unwrap().get_proof_nodes() {
267 let compressed_path = node.compressed_path_ref();
268 if CompressedPathRaw::second_nibble(compressed_path.path_mask())
271 == CompressedPathRaw::NO_MISSING_NIBBLE
272 {
273 let db_key = mpt_node_path_to_db_key(&compressed_path);
274 let loaded_compressed_path =
275 mpt_node_path_from_db_key(&db_key).unwrap();
276 assert_eq!(
277 &compressed_path as &dyn CompressedPathTrait,
278 &loaded_compressed_path as &dyn CompressedPathTrait,
279 );
280 }
281 }
282 }
283}
284
285#[cfg(test)]
286#[test]
287fn test_merkle_root() {
288 assert_snapshot_mpt_formation(&DumpedMptKvIterator::default());
290
291 let mut rng = get_rng_for_test();
293 for _i in 0..5 {
294 let keys: Vec<Vec<u8>> = generate_keys(TEST_NUMBER_OF_KEYS)
295 .iter()
296 .filter(|_| rng.gen_bool(0.1))
297 .cloned()
298 .collect();
299 let mpt_kv_iter = DumpedMptKvIterator {
300 kv: keys.iter().map(|k| (k[..].into(), k[..].into())).collect(),
301 };
302 assert_snapshot_mpt_formation(&mpt_kv_iter);
303 }
304}
305
306#[cfg(test)]
307#[test]
308fn test_delete_all() {
309 let mut rng = get_rng_for_test();
310 let keys: Vec<Vec<u8>> = generate_keys(TEST_NUMBER_OF_KEYS)
311 .iter()
312 .filter(|_| rng.gen_bool(0.5))
313 .cloned()
314 .collect();
315 let mpt_kv_iter = DumpedMptKvIterator {
316 kv: keys.iter().map(|k| (k[..].into(), k[..].into())).collect(),
317 };
318
319 let mut empty_snapshot_mpt = FakeSnapshotMptDb::default();
320 let mut snapshot_mpt = FakeSnapshotMptDb::default();
321
322 snapshot_mpt.reset(false);
324 MptMerger::new(Some(&mut empty_snapshot_mpt), &mut snapshot_mpt)
325 .merge(&mpt_kv_iter)
326 .unwrap();
327
328 let mpt_deleter = DumpedMptKvIterator {
329 kv: keys
330 .iter()
331 .map(|k| (k[..].into(), Default::default()))
332 .collect(),
333 };
334
335 empty_snapshot_mpt.reset(false);
337 let merkle_root =
338 MptMerger::new(Some(&mut snapshot_mpt), &mut empty_snapshot_mpt)
339 .merge(&mpt_deleter)
340 .unwrap();
341 assert_eq!(MERKLE_NULL_NODE, merkle_root);
342 assert_eq!(0, empty_snapshot_mpt.db.len());
343
344 snapshot_mpt.reset(true);
346 let merkle_root = MptMerger::new(None, &mut snapshot_mpt)
347 .merge(&mpt_deleter)
348 .unwrap();
349 assert_eq!(MERKLE_NULL_NODE, merkle_root);
350 assert_eq!(0, snapshot_mpt.db.len());
351}
352
353#[cfg(test)]
354#[test]
355fn test_inserts_deletes_and_subtree_size() {
356 let keys: Vec<Vec<u8>> = generate_keys(TEST_NUMBER_OF_KEYS);
357 let set_size = TEST_NUMBER_OF_KEYS / 10;
358 let (keys_unchanged, keys_overwritten, keys_delete, keys_new) = (
359 &keys[0..set_size],
360 &keys[set_size..set_size * 2],
361 &keys[set_size * 2..set_size * 3],
362 &keys[set_size * 3..set_size * 4],
363 );
364
365 let mpt_kv_iter = DumpedMptKvIterator {
368 kv: keys_delete
369 .iter()
370 .map(|k| (k[..].into(), k[..].into()))
371 .collect(),
372 };
373
374 let mut in_place_mod_mpt = FakeSnapshotMptDb::default();
375 in_place_mod_mpt.reset(true);
376 let init_merkle_root = MptMerger::new(None, &mut in_place_mod_mpt)
377 .merge(&mpt_kv_iter)
378 .unwrap();
379
380 let mut in_place_mod_mpt_discard_write =
381 FakeSnapshotMptDb::new_discard_write();
382 in_place_mod_mpt_discard_write.reset(true);
383 let merkle_root = MptMerger::new(None, &mut in_place_mod_mpt_discard_write)
384 .merge(&mpt_kv_iter)
385 .unwrap();
386 assert_eq!(init_merkle_root, merkle_root);
387
388 let mpt_kv_iter = DumpedMptKvIterator {
389 kv: keys_new
390 .iter()
391 .map(|k| (k[..].into(), k[..].into()))
392 .collect(),
393 };
394 let mut new_snapshot_mpt = FakeSnapshotMptDb::default();
395 new_snapshot_mpt.reset(true);
396 let supposed_merkle_root = MptMerger::new(None, &mut new_snapshot_mpt)
397 .merge(&mpt_kv_iter)
398 .unwrap();
399
400 let delta_mpt_iter = DumpedMptKvIterator {
401 kv: [
402 keys_delete
403 .iter()
404 .map(|k| (Vec::<u8>::from(&k[..]), Box::<[u8]>::default()))
405 .collect::<Vec<_>>(),
406 keys_new
407 .iter()
408 .map(|k| (Vec::<u8>::from(&k[..]), Box::<[u8]>::from(&k[..])))
409 .collect::<Vec<_>>(),
410 ]
411 .concat(),
412 };
413 let mut save_as_mode_mpt = FakeSnapshotMptDb::default();
414 let new_merkle_root =
415 MptMerger::new(Some(&mut in_place_mod_mpt), &mut save_as_mode_mpt)
416 .merge(&delta_mpt_iter)
417 .unwrap();
418 assert_eq!(new_merkle_root, supposed_merkle_root);
419 save_as_mode_mpt.assert_eq(&new_snapshot_mpt);
420
421 in_place_mod_mpt.reset(true);
422 let new_merkle_root = MptMerger::new(None, &mut in_place_mod_mpt)
423 .merge(&delta_mpt_iter)
424 .unwrap();
425 assert_eq!(new_merkle_root, supposed_merkle_root);
426 in_place_mod_mpt.assert_eq(&new_snapshot_mpt);
427
428 let mpt_kv_iter = DumpedMptKvIterator {
432 kv: [
433 keys_unchanged
434 .iter()
435 .map(|k| (Vec::<u8>::from(&k[..]), Box::<[u8]>::from(&k[..])))
436 .collect::<Vec<_>>(),
437 keys_delete
438 .iter()
439 .map(|k| (Vec::<u8>::from(&k[..]), Box::<[u8]>::from(&k[..])))
440 .collect::<Vec<_>>(),
441 keys_overwritten
442 .iter()
443 .map(|k| (Vec::<u8>::from(&k[..]), Box::<[u8]>::from(&k[0..2])))
444 .collect::<Vec<_>>(),
445 ]
446 .concat(),
447 };
448
449 let mut in_place_mod_mpt = FakeSnapshotMptDb::default();
450 in_place_mod_mpt.reset(true);
451 MptMerger::new(None, &mut in_place_mod_mpt)
452 .merge(&mpt_kv_iter)
453 .unwrap();
454
455 let mpt_kv_iter = DumpedMptKvIterator {
456 kv: [
457 keys_unchanged
458 .iter()
459 .map(|k| (Vec::<u8>::from(&k[..]), Box::<[u8]>::from(&k[..])))
460 .collect::<Vec<_>>(),
461 keys_new
462 .iter()
463 .map(|k| (Vec::<u8>::from(&k[..]), Box::<[u8]>::from(&k[..])))
464 .collect::<Vec<_>>(),
465 keys_overwritten
466 .iter()
467 .map(|k| (Vec::<u8>::from(&k[..]), Box::<[u8]>::from(&k[..])))
468 .collect::<Vec<_>>(),
469 ]
470 .concat(),
471 };
472 let mut new_snapshot_mpt = FakeSnapshotMptDb::default();
473 new_snapshot_mpt.reset(true);
474 let supposed_merkle_root = MptMerger::new(None, &mut new_snapshot_mpt)
475 .merge(&mpt_kv_iter)
476 .unwrap();
477 let new_snapshot_db = Arc::new(Mutex::new(FakeSnapshotDb::new(
478 &mpt_kv_iter.kv,
479 &new_snapshot_mpt,
480 )));
481 verify_snapshot_db(&new_snapshot_db);
482
483 let delta_mpt_iter = DumpedMptKvIterator {
484 kv: [
485 keys_delete
486 .iter()
487 .map(|k| (Vec::<u8>::from(&k[..]), Box::<[u8]>::default()))
488 .collect::<Vec<_>>(),
489 keys_new
490 .iter()
491 .map(|k| (Vec::<u8>::from(&k[..]), Box::<[u8]>::from(&k[..])))
492 .collect::<Vec<_>>(),
493 keys_overwritten
494 .iter()
495 .map(|k| (Vec::<u8>::from(&k[..]), Box::<[u8]>::from(&k[..])))
496 .collect::<Vec<_>>(),
497 ]
498 .concat(),
499 };
500 let mut save_as_mode_mpt = FakeSnapshotMptDb::default();
501 let new_merkle_root =
502 MptMerger::new(Some(&mut in_place_mod_mpt), &mut save_as_mode_mpt)
503 .merge(&delta_mpt_iter)
504 .unwrap();
505 assert_eq!(new_merkle_root, supposed_merkle_root);
515 new_snapshot_mpt.assert_eq(&save_as_mode_mpt);
516
517 in_place_mod_mpt.reset(true);
518 let new_merkle_root = MptMerger::new(None, &mut in_place_mod_mpt)
519 .merge(&delta_mpt_iter)
520 .unwrap();
521 let in_place_mode_snapshot_db = Arc::new(Mutex::new(FakeSnapshotDb::new(
522 &mpt_kv_iter.kv,
523 &in_place_mod_mpt,
524 )));
525 verify_snapshot_db(&in_place_mode_snapshot_db);
526 assert_eq!(new_merkle_root, supposed_merkle_root);
527 in_place_mod_mpt.assert_eq(&new_snapshot_mpt);
528}
529
530#[cfg(test)]
531#[test]
532fn test_two_way_merge() {
533 let keys: Vec<Vec<u8>> = generate_keys(TEST_NUMBER_OF_KEYS);
534 let set_size = TEST_NUMBER_OF_KEYS / 10;
535 let (keys_unchanged, keys_overwritten, keys_delete, keys_new) = (
536 &keys[0..set_size],
537 &keys[set_size..set_size * 2],
538 &keys[set_size * 2..set_size * 3],
539 &keys[set_size * 3..set_size * 4],
540 );
541
542 let mpt_kv_iter = DumpedMptKvIterator {
543 kv: [
544 keys_unchanged
545 .iter()
546 .map(|k| (Vec::<u8>::from(&k[..]), Box::<[u8]>::from(&k[..])))
547 .collect::<Vec<_>>(),
548 keys_delete
549 .iter()
550 .map(|k| (Vec::<u8>::from(&k[..]), Box::<[u8]>::from(&k[..])))
551 .collect::<Vec<_>>(),
552 keys_overwritten
553 .iter()
554 .map(|k| (Vec::<u8>::from(&k[..]), Box::<[u8]>::from(&k[0..2])))
555 .collect::<Vec<_>>(),
556 ]
557 .concat(),
558 };
559
560 let mut in_place_mod_mpt = FakeSnapshotMptDb::default();
561 in_place_mod_mpt.reset(true);
562 MptMerger::new(None, &mut in_place_mod_mpt)
563 .merge(&mpt_kv_iter)
564 .unwrap();
565
566 let delta_mpt_iter = DumpedMptKvIterator {
568 kv: [
569 keys_delete
570 .iter()
571 .map(|k| (Vec::<u8>::from(&k[..]), Box::<[u8]>::default()))
572 .collect::<Vec<_>>(),
573 keys_new
574 .iter()
575 .map(|k| (Vec::<u8>::from(&k[..]), Box::<[u8]>::from(&k[..])))
576 .collect::<Vec<_>>(),
577 keys_overwritten
578 .iter()
579 .map(|k| (Vec::<u8>::from(&k[..]), Box::<[u8]>::from(&k[..])))
580 .collect::<Vec<_>>(),
581 ]
582 .concat(),
583 };
584 let mut save_as_mode_mpt = FakeSnapshotMptDb::default();
585 let supposed_merkle_root =
586 MptMerger::new(Some(&mut in_place_mod_mpt), &mut save_as_mode_mpt)
587 .merge(&delta_mpt_iter)
588 .unwrap();
589
590 in_place_mod_mpt.reset(true);
592 let mut keys_deletion = Vec::from(keys_delete);
593 keys_deletion.sort();
594 let deletion = keys_deletion
595 .iter()
596 .map(|k| Ok((Vec::<u8>::from(&k[..]), ())))
597 .collect::<Vec<_>>();
598 let mut keys_insertion = [keys_new, keys_overwritten].concat();
599 keys_insertion.sort();
600 let insertion = keys_insertion
601 .iter()
602 .map(|k| Ok((Vec::<u8>::from(&k[..]), Box::<[u8]>::from(&k[..]))))
603 .collect::<Vec<_>>();
604
605 let new_merkle_root = MptMerger::new(None, &mut in_place_mod_mpt)
606 .merge_insertion_deletion_separated(
607 fallible_iterator::convert(deletion.into_iter()),
608 fallible_iterator::convert(insertion.into_iter()),
609 false,
610 )
611 .unwrap();
612
613 assert_eq!(new_merkle_root, supposed_merkle_root);
615 in_place_mod_mpt.assert_eq(&save_as_mode_mpt);
616}
617
618#[allow(unused)]
619fn test_delta_subtree_size() {
620 unimplemented!()
622}
623
624use crate::{
625 impls::{
626 errors::*,
627 merkle_patricia_trie::{CompressedPathRaw, CompressedPathTrait},
628 storage_db::snapshot_mpt::{
629 mpt_node_path_from_db_key, mpt_node_path_to_db_key,
630 },
631 },
632 storage_db::{
633 snapshot_mpt::CHECK_LOADED_SNAPSHOT_MPT_NODE, SnapshotMptIteraterTrait,
634 SnapshotMptNode, SnapshotMptTraitRead, SnapshotMptTraitReadAndIterate,
635 SnapshotMptTraitRw,
636 },
637};
638use fallible_iterator::FallibleIterator;
639use primitives::MerkleHash;
640use std::{
641 collections::{btree_map, BTreeMap, HashSet},
642 ops::Bound::Excluded,
643};
644
645#[cfg(test)]
646use crate::{
647 impls::merkle_patricia_trie::{MptMerger, TrieNodeTrait},
648 impls::storage_db::snapshot_mpt::tests::verify_snapshot_db,
649 state_manager::StateManagerTrait,
650 tests::{
651 generate_keys, get_rng_for_test, new_state_manager_for_unit_test,
652 snapshot::verifier::FakeSnapshotDb, DumpedMptKvIterator,
653 TEST_NUMBER_OF_KEYS,
654 },
655 StateIndex, StorageStateTraitExt,
656};
657#[cfg(test)]
658use parking_lot::Mutex;
659#[cfg(test)]
660use primitives::{EpochId, StorageKey, MERKLE_NULL_NODE};
661#[cfg(test)]
662use rand::Rng;
663#[cfg(test)]
664use std::sync::{atomic::Ordering, Arc};