Cleanup redundant AsFixedBytes and FixedLength traits (#3131)

* no need for AsFixedBytes we can just use AsRef<[u8]>

* cleanup FixedLength trait

* revert this change for now

* fix store tests

* cleanup and fix tests after rebase

* fix tests

* update based on PR review
less hard-coded values now

* cleanup
This commit is contained in:
Antioch Peverell 2020-01-29 13:41:50 +00:00 committed by GitHub
parent 83a2649946
commit 616dad43fd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 112 additions and 265 deletions

1
.gitignore vendored
View file

@ -12,3 +12,4 @@ test_output
wallet_data wallet_data
wallet/db wallet/db
.idea/ .idea/
/node*

View file

@ -179,7 +179,6 @@ impl Chain {
"header", "header",
"header_head", "header_head",
false, false,
true,
ProtocolVersion(1), ProtocolVersion(1),
None, None,
)?; )?;
@ -188,7 +187,6 @@ impl Chain {
"header", "header",
"sync_head", "sync_head",
false, false,
true,
ProtocolVersion(1), ProtocolVersion(1),
None, None,
)?; )?;

View file

@ -20,7 +20,7 @@ use croaring::Bitmap;
use crate::core::core::hash::{DefaultHashable, Hash}; use crate::core::core::hash::{DefaultHashable, Hash};
use crate::core::core::pmmr::{self, ReadonlyPMMR, VecBackend, PMMR}; use crate::core::core::pmmr::{self, ReadonlyPMMR, VecBackend, PMMR};
use crate::core::ser::{self, FixedLength, PMMRable, Readable, Reader, Writeable, Writer}; use crate::core::ser::{self, PMMRable, Readable, Reader, Writeable, Writer};
use crate::error::{Error, ErrorKind}; use crate::error::{Error, ErrorKind};
/// The "bitmap accumulator" allows us to commit to a specific bitmap by splitting it into /// The "bitmap accumulator" allows us to commit to a specific bitmap by splitting it into
@ -215,10 +215,10 @@ impl PMMRable for BitmapChunk {
fn as_elmt(&self) -> Self::E { fn as_elmt(&self) -> Self::E {
self.clone() self.clone()
} }
}
impl FixedLength for BitmapChunk { fn elmt_size() -> Option<u16> {
const LEN: usize = Self::LEN_BYTES; Some(Self::LEN_BYTES as u16)
}
} }
impl DefaultHashable for BitmapChunk {} impl DefaultHashable for BitmapChunk {}

View file

@ -60,7 +60,6 @@ impl<T: PMMRable> PMMRHandle<T> {
sub_dir: &str, sub_dir: &str,
file_name: &str, file_name: &str,
prunable: bool, prunable: bool,
fixed_size: bool,
version: ProtocolVersion, version: ProtocolVersion,
header: Option<&BlockHeader>, header: Option<&BlockHeader>,
) -> Result<PMMRHandle<T>, Error> { ) -> Result<PMMRHandle<T>, Error> {
@ -69,8 +68,7 @@ impl<T: PMMRable> PMMRHandle<T> {
let path_str = path.to_str().ok_or(Error::from(ErrorKind::Other( let path_str = path.to_str().ok_or(Error::from(ErrorKind::Other(
"invalid file path".to_owned(), "invalid file path".to_owned(),
)))?; )))?;
let backend = let backend = PMMRBackend::new(path_str.to_string(), prunable, version, header)?;
PMMRBackend::new(path_str.to_string(), prunable, fixed_size, version, header)?;
let last_pos = backend.unpruned_size(); let last_pos = backend.unpruned_size();
Ok(PMMRHandle { backend, last_pos }) Ok(PMMRHandle { backend, last_pos })
} }
@ -136,7 +134,6 @@ impl TxHashSet {
TXHASHSET_SUBDIR, TXHASHSET_SUBDIR,
OUTPUT_SUBDIR, OUTPUT_SUBDIR,
true, true,
true,
ProtocolVersion(1), ProtocolVersion(1),
header, header,
)?; )?;
@ -146,7 +143,6 @@ impl TxHashSet {
TXHASHSET_SUBDIR, TXHASHSET_SUBDIR,
RANGE_PROOF_SUBDIR, RANGE_PROOF_SUBDIR,
true, true,
true,
ProtocolVersion(1), ProtocolVersion(1),
header, header,
)?; )?;
@ -162,7 +158,6 @@ impl TxHashSet {
TXHASHSET_SUBDIR, TXHASHSET_SUBDIR,
KERNEL_SUBDIR, KERNEL_SUBDIR,
false, // not prunable false, // not prunable
false, // variable size kernel data file
version, version,
None, None,
)?; )?;

View file

@ -26,14 +26,14 @@ use crate::core::{
use crate::global; use crate::global;
use crate::pow::{verify_size, Difficulty, Proof, ProofOfWork}; use crate::pow::{verify_size, Difficulty, Proof, ProofOfWork};
use crate::ser::{ use crate::ser::{
self, deserialize_default, serialize_default, FixedLength, PMMRable, Readable, Reader, self, deserialize_default, serialize_default, PMMRable, Readable, Reader, Writeable, Writer,
Writeable, Writer,
}; };
use chrono::naive::{MAX_DATE, MIN_DATE}; use chrono::naive::{MAX_DATE, MIN_DATE};
use chrono::prelude::{DateTime, NaiveDateTime, Utc}; use chrono::prelude::{DateTime, NaiveDateTime, Utc};
use chrono::Duration; use chrono::Duration;
use keychain::{self, BlindingFactor}; use keychain::{self, BlindingFactor};
use std::collections::HashSet; use std::collections::HashSet;
use std::convert::TryInto;
use std::fmt; use std::fmt;
use std::iter::FromIterator; use std::iter::FromIterator;
use std::sync::Arc; use std::sync::Arc;
@ -168,10 +168,6 @@ impl Writeable for HeaderEntry {
} }
} }
impl FixedLength for HeaderEntry {
const LEN: usize = Hash::LEN + 8 + Difficulty::LEN + 4 + 1;
}
impl Hashed for HeaderEntry { impl Hashed for HeaderEntry {
/// The hash of the underlying block. /// The hash of the underlying block.
fn hash(&self) -> Hash { fn hash(&self) -> Hash {
@ -265,6 +261,12 @@ impl PMMRable for BlockHeader {
is_secondary: self.pow.is_secondary(), is_secondary: self.pow.is_secondary(),
} }
} }
// Size is hash + u64 + difficulty + u32 + u8.
fn elmt_size() -> Option<u16> {
const LEN: usize = Hash::LEN + 8 + Difficulty::LEN + 4 + 1;
Some(LEN.try_into().unwrap())
}
} }
/// Serialization of a block header /// Serialization of a block header

View file

@ -17,9 +17,7 @@
//! Primary hash function used in the protocol //! Primary hash function used in the protocol
//! //!
use crate::ser::{ use crate::ser::{self, Error, ProtocolVersion, Readable, Reader, Writeable, Writer};
self, AsFixedBytes, Error, FixedLength, ProtocolVersion, Readable, Reader, Writeable, Writer,
};
use blake2::blake2b::Blake2b; use blake2::blake2b::Blake2b;
use byteorder::{BigEndian, ByteOrder}; use byteorder::{BigEndian, ByteOrder};
use std::cmp::min; use std::cmp::min;
@ -52,12 +50,10 @@ impl fmt::Display for Hash {
} }
} }
impl FixedLength for Hash {
/// Size of a hash in bytes.
const LEN: usize = 32;
}
impl Hash { impl Hash {
/// A hash is 32 bytes.
pub const LEN: usize = 32;
/// Builds a Hash from a byte vector. If the vector is too short, it will be /// Builds a Hash from a byte vector. If the vector is too short, it will be
/// completed by zeroes. If it's too long, it will be truncated. /// completed by zeroes. If it's too long, it will be truncated.
pub fn from_vec(v: &[u8]) -> Hash { pub fn from_vec(v: &[u8]) -> Hash {
@ -196,8 +192,8 @@ impl ser::Writer for HashWriter {
ser::SerializationMode::Hash ser::SerializationMode::Hash
} }
fn write_fixed_bytes<T: AsFixedBytes>(&mut self, b32: &T) -> Result<(), ser::Error> { fn write_fixed_bytes<T: AsRef<[u8]>>(&mut self, bytes: T) -> Result<(), ser::Error> {
self.state.update(b32.as_ref()); self.state.update(bytes.as_ref());
Ok(()) Ok(())
} }

View file

@ -19,14 +19,15 @@ use crate::core::verifier_cache::VerifierCache;
use crate::core::{committed, Committed}; use crate::core::{committed, Committed};
use crate::libtx::secp_ser; use crate::libtx::secp_ser;
use crate::ser::{ use crate::ser::{
self, read_multi, FixedLength, PMMRable, ProtocolVersion, Readable, Reader, self, read_multi, PMMRable, ProtocolVersion, Readable, Reader, VerifySortedAndUnique,
VerifySortedAndUnique, Writeable, Writer, Writeable, Writer,
}; };
use crate::{consensus, global}; use crate::{consensus, global};
use enum_primitive::FromPrimitive; use enum_primitive::FromPrimitive;
use keychain::{self, BlindingFactor}; use keychain::{self, BlindingFactor};
use std::cmp::Ordering; use std::cmp::Ordering;
use std::cmp::{max, min}; use std::cmp::{max, min};
use std::convert::TryInto;
use std::sync::Arc; use std::sync::Arc;
use std::{error, fmt}; use std::{error, fmt};
use util; use util;
@ -352,12 +353,10 @@ impl PMMRable for TxKernel {
fn as_elmt(&self) -> Self::E { fn as_elmt(&self) -> Self::E {
self.clone() self.clone()
} }
}
/// Kernels are "variable size" but we need to implement FixedLength for legacy reasons. fn elmt_size() -> Option<u16> {
/// At some point we will refactor the MMR backend so this is no longer required. None
impl FixedLength for TxKernel { }
const LEN: usize = 0;
} }
impl KernelFeatures { impl KernelFeatures {
@ -1430,6 +1429,14 @@ impl PMMRable for Output {
fn as_elmt(&self) -> OutputIdentifier { fn as_elmt(&self) -> OutputIdentifier {
OutputIdentifier::from_output(self) OutputIdentifier::from_output(self)
} }
fn elmt_size() -> Option<u16> {
Some(
(1 + secp::constants::PEDERSEN_COMMITMENT_SIZE)
.try_into()
.unwrap(),
)
}
} }
impl OutputFeatures { impl OutputFeatures {
@ -1549,10 +1556,6 @@ impl OutputIdentifier {
} }
} }
impl FixedLength for OutputIdentifier {
const LEN: usize = 1 + secp::constants::PEDERSEN_COMMITMENT_SIZE;
}
impl Writeable for OutputIdentifier { impl Writeable for OutputIdentifier {
fn write<W: Writer>(&self, writer: &mut W) -> Result<(), ser::Error> { fn write<W: Writer>(&self, writer: &mut W) -> Result<(), ser::Error> {
self.features.write(writer)?; self.features.write(writer)?;

View file

@ -17,7 +17,7 @@ use crate::core::hash::{DefaultHashable, Hashed};
use crate::global; use crate::global;
use crate::pow::common::EdgeType; use crate::pow::common::EdgeType;
use crate::pow::error::Error; use crate::pow::error::Error;
use crate::ser::{self, FixedLength, Readable, Reader, Writeable, Writer}; use crate::ser::{self, Readable, Reader, Writeable, Writer};
use rand::{thread_rng, Rng}; use rand::{thread_rng, Rng};
use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
/// Types for a Cuck(at)oo proof of work and its encapsulation as a fully usable /// Types for a Cuck(at)oo proof of work and its encapsulation as a fully usable
@ -156,8 +156,9 @@ impl Readable for Difficulty {
} }
} }
impl FixedLength for Difficulty { impl Difficulty {
const LEN: usize = 8; /// Difficulty is 8 bytes.
pub const LEN: usize = 8;
} }
impl Serialize for Difficulty { impl Serialize for Difficulty {

View file

@ -23,10 +23,10 @@ use crate::core::hash::{DefaultHashable, Hash, Hashed};
use crate::global::PROTOCOL_VERSION; use crate::global::PROTOCOL_VERSION;
use byteorder::{BigEndian, ByteOrder, ReadBytesExt}; use byteorder::{BigEndian, ByteOrder, ReadBytesExt};
use keychain::{BlindingFactor, Identifier, IDENTIFIER_SIZE}; use keychain::{BlindingFactor, Identifier, IDENTIFIER_SIZE};
use std::convert::TryInto;
use std::fmt::{self, Debug}; use std::fmt::{self, Debug};
use std::io::{self, Read, Write}; use std::io::{self, Read, Write};
use std::marker; use std::{cmp, error, marker};
use std::{cmp, error};
use util::secp::constants::{ use util::secp::constants::{
AGG_SIGNATURE_SIZE, COMPRESSED_PUBLIC_KEY_SIZE, MAX_PROOF_SIZE, PEDERSEN_COMMITMENT_SIZE, AGG_SIGNATURE_SIZE, COMPRESSED_PUBLIC_KEY_SIZE, MAX_PROOF_SIZE, PEDERSEN_COMMITMENT_SIZE,
SECRET_KEY_SIZE, SECRET_KEY_SIZE,
@ -179,14 +179,13 @@ pub trait Writer {
/// Writes a variable number of bytes. The length is encoded as a 64-bit /// Writes a variable number of bytes. The length is encoded as a 64-bit
/// prefix. /// prefix.
fn write_bytes<T: AsFixedBytes>(&mut self, bytes: &T) -> Result<(), Error> { fn write_bytes<T: AsRef<[u8]>>(&mut self, bytes: T) -> Result<(), Error> {
self.write_u64(bytes.as_ref().len() as u64)?; self.write_u64(bytes.as_ref().len() as u64)?;
self.write_fixed_bytes(bytes) self.write_fixed_bytes(bytes)
} }
/// Writes a fixed number of bytes from something that can turn itself into /// Writes a fixed number of bytes. The reader is expected to know the actual length on read.
/// a `&[u8]`. The reader is expected to know the actual length on read. fn write_fixed_bytes<T: AsRef<[u8]>>(&mut self, bytes: T) -> Result<(), Error>;
fn write_fixed_bytes<T: AsFixedBytes>(&mut self, fixed: &T) -> Result<(), Error>;
} }
/// Implementations defined how different numbers and binary structures are /// Implementations defined how different numbers and binary structures are
@ -567,15 +566,11 @@ impl Writeable for BlindingFactor {
impl Readable for BlindingFactor { impl Readable for BlindingFactor {
fn read(reader: &mut dyn Reader) -> Result<BlindingFactor, Error> { fn read(reader: &mut dyn Reader) -> Result<BlindingFactor, Error> {
let bytes = reader.read_fixed_bytes(BlindingFactor::LEN)?; let bytes = reader.read_fixed_bytes(SECRET_KEY_SIZE)?;
Ok(BlindingFactor::from_slice(&bytes)) Ok(BlindingFactor::from_slice(&bytes))
} }
} }
impl FixedLength for BlindingFactor {
const LEN: usize = SECRET_KEY_SIZE;
}
impl Writeable for Identifier { impl Writeable for Identifier {
fn write<W: Writer>(&self, writer: &mut W) -> Result<(), Error> { fn write<W: Writer>(&self, writer: &mut W) -> Result<(), Error> {
writer.write_fixed_bytes(self) writer.write_fixed_bytes(self)
@ -609,24 +604,24 @@ impl Readable for RangeProof {
} }
} }
impl FixedLength for RangeProof {
const LEN: usize = 8 // length prefix
+ MAX_PROOF_SIZE;
}
impl PMMRable for RangeProof { impl PMMRable for RangeProof {
type E = Self; type E = Self;
fn as_elmt(&self) -> Self::E { fn as_elmt(&self) -> Self::E {
self.clone() self.clone()
} }
// Size is length prefix (8 bytes for u64) + MAX_PROOF_SIZE.
fn elmt_size() -> Option<u16> {
Some((8 + MAX_PROOF_SIZE).try_into().unwrap())
}
} }
impl Readable for Signature { impl Readable for Signature {
fn read(reader: &mut dyn Reader) -> Result<Signature, Error> { fn read(reader: &mut dyn Reader) -> Result<Signature, Error> {
let a = reader.read_fixed_bytes(Signature::LEN)?; let a = reader.read_fixed_bytes(AGG_SIGNATURE_SIZE)?;
let mut c = [0; Signature::LEN]; let mut c = [0; AGG_SIGNATURE_SIZE];
c[..Signature::LEN].clone_from_slice(&a[..Signature::LEN]); c[..AGG_SIGNATURE_SIZE].clone_from_slice(&a[..AGG_SIGNATURE_SIZE]);
Ok(Signature::from_raw_data(&c).unwrap()) Ok(Signature::from_raw_data(&c).unwrap())
} }
} }
@ -637,19 +632,11 @@ impl Writeable for Signature {
} }
} }
impl FixedLength for Signature {
const LEN: usize = AGG_SIGNATURE_SIZE;
}
impl FixedLength for PublicKey {
const LEN: usize = COMPRESSED_PUBLIC_KEY_SIZE;
}
impl Writeable for PublicKey { impl Writeable for PublicKey {
// Write the public key in compressed form // Write the public key in compressed form
fn write<W: Writer>(&self, writer: &mut W) -> Result<(), Error> { fn write<W: Writer>(&self, writer: &mut W) -> Result<(), Error> {
let secp = Secp256k1::with_caps(ContextFlag::None); let secp = Secp256k1::with_caps(ContextFlag::None);
writer.write_fixed_bytes(&self.serialize_vec(&secp, true).as_ref())?; writer.write_fixed_bytes(self.serialize_vec(&secp, true))?;
Ok(()) Ok(())
} }
} }
@ -657,7 +644,7 @@ impl Writeable for PublicKey {
impl Readable for PublicKey { impl Readable for PublicKey {
// Read the public key in compressed form // Read the public key in compressed form
fn read(reader: &mut dyn Reader) -> Result<Self, Error> { fn read(reader: &mut dyn Reader) -> Result<Self, Error> {
let buf = reader.read_fixed_bytes(PublicKey::LEN)?; let buf = reader.read_fixed_bytes(COMPRESSED_PUBLIC_KEY_SIZE)?;
let secp = Secp256k1::with_caps(ContextFlag::None); let secp = Secp256k1::with_caps(ContextFlag::None);
let pk = PublicKey::from_slice(&secp, &buf).map_err(|_| Error::CorruptedData)?; let pk = PublicKey::from_slice(&secp, &buf).map_err(|_| Error::CorruptedData)?;
Ok(pk) Ok(pk)
@ -709,9 +696,8 @@ impl<'a> Writer for BinWriter<'a> {
SerializationMode::Full SerializationMode::Full
} }
fn write_fixed_bytes<T: AsFixedBytes>(&mut self, fixed: &T) -> Result<(), Error> { fn write_fixed_bytes<T: AsRef<[u8]>>(&mut self, bytes: T) -> Result<(), Error> {
let bs = fixed.as_ref(); self.sink.write_all(bytes.as_ref())?;
self.sink.write_all(bs)?;
Ok(()) Ok(())
} }
@ -838,20 +824,17 @@ impl Writeable for [u8; 4] {
} }
} }
/// Trait for types that serialize to a known fixed length.
pub trait FixedLength {
/// The length in bytes
const LEN: usize;
}
/// Trait for types that can be added to a PMMR. /// Trait for types that can be added to a PMMR.
pub trait PMMRable: Writeable + Clone + Debug + DefaultHashable { pub trait PMMRable: Writeable + Clone + Debug + DefaultHashable {
/// The type of element actually stored in the MMR data file. /// The type of element actually stored in the MMR data file.
/// This allows us to store Hash elements in the header MMR for variable size BlockHeaders. /// This allows us to store Hash elements in the header MMR for variable size BlockHeaders.
type E: FixedLength + Readable + Writeable + Debug; type E: Readable + Writeable + Debug;
/// Convert the pmmrable into the element to be stored in the MMR data file. /// Convert the pmmrable into the element to be stored in the MMR data file.
fn as_elmt(&self) -> Self::E; fn as_elmt(&self) -> Self::E;
/// Size of each element if "fixed" size. Elements are "variable" size if None.
fn elmt_size() -> Option<u16>;
} }
/// Generic trait to ensure PMMR elements can be hashed with an index /// Generic trait to ensure PMMR elements can be hashed with an index
@ -866,93 +849,6 @@ impl<T: DefaultHashable> PMMRIndexHashable for T {
} }
} }
/// Useful marker trait on types that can be sized byte slices
pub trait AsFixedBytes: Sized + AsRef<[u8]> {
/// The length in bytes
fn len(&self) -> usize;
}
impl<'a> AsFixedBytes for &'a [u8] {
fn len(&self) -> usize {
1
}
}
impl AsFixedBytes for Vec<u8> {
fn len(&self) -> usize {
self.len()
}
}
impl AsFixedBytes for [u8; 1] {
fn len(&self) -> usize {
1
}
}
impl AsFixedBytes for [u8; 2] {
fn len(&self) -> usize {
2
}
}
impl AsFixedBytes for [u8; 4] {
fn len(&self) -> usize {
4
}
}
impl AsFixedBytes for [u8; 6] {
fn len(&self) -> usize {
6
}
}
impl AsFixedBytes for [u8; 8] {
fn len(&self) -> usize {
8
}
}
impl AsFixedBytes for [u8; 20] {
fn len(&self) -> usize {
20
}
}
impl AsFixedBytes for [u8; 32] {
fn len(&self) -> usize {
32
}
}
impl AsFixedBytes for String {
fn len(&self) -> usize {
self.len()
}
}
impl AsFixedBytes for crate::core::hash::Hash {
fn len(&self) -> usize {
32
}
}
impl AsFixedBytes for util::secp::pedersen::RangeProof {
fn len(&self) -> usize {
self.plen
}
}
impl AsFixedBytes for util::secp::Signature {
fn len(&self) -> usize {
64
}
}
impl AsFixedBytes for util::secp::pedersen::Commitment {
fn len(&self) -> usize {
PEDERSEN_COMMITMENT_SIZE
}
}
impl AsFixedBytes for BlindingFactor {
fn len(&self) -> usize {
SECRET_KEY_SIZE
}
}
impl AsFixedBytes for keychain::Identifier {
fn len(&self) -> usize {
IDENTIFIER_SIZE
}
}
// serializer for io::Errorkind, originally auto-generated by serde-derive // serializer for io::Errorkind, originally auto-generated by serde-derive
// slightly modified to handle the #[non_exhaustive] tag on io::ErrorKind // slightly modified to handle the #[non_exhaustive] tag on io::ErrorKind
fn serialize_error_kind<S>( fn serialize_error_kind<S>(

View file

@ -22,7 +22,7 @@ use grin_core::libtx::{
reward, reward,
}; };
use grin_core::pow::Difficulty; use grin_core::pow::Difficulty;
use grin_core::ser::{self, FixedLength, PMMRable, Readable, Reader, Writeable, Writer}; use grin_core::ser::{self, PMMRable, Readable, Reader, Writeable, Writer};
use keychain::{Identifier, Keychain}; use keychain::{Identifier, Keychain};
// utility producing a transaction with 2 inputs and a single outputs // utility producing a transaction with 2 inputs and a single outputs
@ -133,16 +133,16 @@ pub struct TestElem(pub [u32; 4]);
impl DefaultHashable for TestElem {} impl DefaultHashable for TestElem {}
impl FixedLength for TestElem {
const LEN: usize = 16;
}
impl PMMRable for TestElem { impl PMMRable for TestElem {
type E = Self; type E = Self;
fn as_elmt(&self) -> Self::E { fn as_elmt(&self) -> Self::E {
self.clone() self.clone()
} }
fn elmt_size() -> Option<u16> {
Some(16)
}
} }
impl Writeable for TestElem { impl Writeable for TestElem {

View file

@ -21,7 +21,7 @@
//! stream and make sure we get the right number of bytes out. //! stream and make sure we get the right number of bytes out.
use crate::core::ser; use crate::core::ser;
use crate::core::ser::{FixedLength, ProtocolVersion}; use crate::core::ser::ProtocolVersion;
use crate::msg::{ use crate::msg::{
read_body, read_discard, read_header, read_item, write_message, Msg, MsgHeader, read_body, read_discard, read_header, read_item, write_message, Msg, MsgHeader,
MsgHeaderWrapper, MsgHeaderWrapper,

View file

@ -19,7 +19,7 @@ use crate::core::core::hash::Hash;
use crate::core::core::BlockHeader; use crate::core::core::BlockHeader;
use crate::core::pow::Difficulty; use crate::core::pow::Difficulty;
use crate::core::ser::{ use crate::core::ser::{
self, FixedLength, ProtocolVersion, Readable, Reader, StreamingReader, Writeable, Writer, self, ProtocolVersion, Readable, Reader, StreamingReader, Writeable, Writer,
}; };
use crate::core::{consensus, global}; use crate::core::{consensus, global};
use crate::types::{ use crate::types::{
@ -262,6 +262,9 @@ pub struct MsgHeader {
} }
impl MsgHeader { impl MsgHeader {
// 2 magic bytes + 1 type byte + 8 bytes (msg_len)
pub const LEN: usize = 2 + 1 + 8;
/// Creates a new message header. /// Creates a new message header.
pub fn new(msg_type: Type, len: u64) -> MsgHeader { pub fn new(msg_type: Type, len: u64) -> MsgHeader {
MsgHeader { MsgHeader {
@ -272,11 +275,6 @@ impl MsgHeader {
} }
} }
impl FixedLength for MsgHeader {
// 2 magic bytes + 1 type byte + 8 bytes (msg_len)
const LEN: usize = 2 + 1 + 8;
}
impl Writeable for MsgHeader { impl Writeable for MsgHeader {
fn write<W: Writer>(&self, writer: &mut W) -> Result<(), ser::Error> { fn write<W: Writer>(&self, writer: &mut W) -> Result<(), ser::Error> {
ser_multiwrite!( ser_multiwrite!(

View file

@ -19,11 +19,12 @@ use std::{io, time};
use crate::core::core::hash::{Hash, Hashed}; use crate::core::core::hash::{Hash, Hashed};
use crate::core::core::pmmr::{self, family, Backend}; use crate::core::core::pmmr::{self, family, Backend};
use crate::core::core::BlockHeader; use crate::core::core::BlockHeader;
use crate::core::ser::{FixedLength, PMMRable, ProtocolVersion}; use crate::core::ser::{PMMRable, ProtocolVersion};
use crate::leaf_set::LeafSet; use crate::leaf_set::LeafSet;
use crate::prune_list::PruneList; use crate::prune_list::PruneList;
use crate::types::{AppendOnlyFile, DataFile, SizeEntry, SizeInfo}; use crate::types::{AppendOnlyFile, DataFile, SizeEntry, SizeInfo};
use croaring::Bitmap; use croaring::Bitmap;
use std::convert::TryInto;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
const PMMR_HASH_FILE: &str = "pmmr_hash.bin"; const PMMR_HASH_FILE: &str = "pmmr_hash.bin";
@ -229,11 +230,11 @@ impl<T: PMMRable> Backend<T> for PMMRBackend<T> {
impl<T: PMMRable> PMMRBackend<T> { impl<T: PMMRable> PMMRBackend<T> {
/// Instantiates a new PMMR backend. /// Instantiates a new PMMR backend.
/// If optional size is provided then treat as "fixed" size otherwise "variable" size backend.
/// Use the provided dir to store its files. /// Use the provided dir to store its files.
pub fn new<P: AsRef<Path>>( pub fn new<P: AsRef<Path>>(
data_dir: P, data_dir: P,
prunable: bool, prunable: bool,
fixed_size: bool,
version: ProtocolVersion, version: ProtocolVersion,
header: Option<&BlockHeader>, header: Option<&BlockHeader>,
) -> io::Result<PMMRBackend<T>> { ) -> io::Result<PMMRBackend<T>> {
@ -241,8 +242,8 @@ impl<T: PMMRable> PMMRBackend<T> {
// Are we dealing with "fixed size" data elements or "variable size" data elements // Are we dealing with "fixed size" data elements or "variable size" data elements
// maintained in an associated size file? // maintained in an associated size file?
let size_info = if fixed_size { let size_info = if let Some(fixed_size) = T::elmt_size() {
SizeInfo::FixedSize(T::E::LEN as u16) SizeInfo::FixedSize(fixed_size)
} else { } else {
SizeInfo::VariableSize(Box::new(AppendOnlyFile::open( SizeInfo::VariableSize(Box::new(AppendOnlyFile::open(
data_dir.join(PMMR_SIZE_FILE), data_dir.join(PMMR_SIZE_FILE),
@ -252,7 +253,7 @@ impl<T: PMMRable> PMMRBackend<T> {
}; };
// Hash file is always "fixed size" and we use 32 bytes per hash. // Hash file is always "fixed size" and we use 32 bytes per hash.
let hash_size_info = SizeInfo::FixedSize(Hash::LEN as u16); let hash_size_info = SizeInfo::FixedSize(Hash::LEN.try_into().unwrap());
let hash_file = DataFile::open(&data_dir.join(PMMR_HASH_FILE), hash_size_info, version)?; let hash_file = DataFile::open(&data_dir.join(PMMR_HASH_FILE), hash_size_info, version)?;
let data_file = DataFile::open(&data_dir.join(PMMR_DATA_FILE), size_info, version)?; let data_file = DataFile::open(&data_dir.join(PMMR_DATA_FILE), size_info, version)?;

View file

@ -16,8 +16,7 @@ use memmap;
use tempfile::tempfile; use tempfile::tempfile;
use crate::core::ser::{ use crate::core::ser::{
self, BinWriter, FixedLength, ProtocolVersion, Readable, Reader, StreamingReader, Writeable, self, BinWriter, ProtocolVersion, Readable, Reader, StreamingReader, Writeable, Writer,
Writer,
}; };
use std::fmt::Debug; use std::fmt::Debug;
use std::fs::{self, File, OpenOptions}; use std::fs::{self, File, OpenOptions};
@ -39,8 +38,9 @@ pub struct SizeEntry {
pub size: u16, pub size: u16,
} }
impl FixedLength for SizeEntry { impl SizeEntry {
const LEN: usize = 8 + 2; /// Length of a size entry (8 + 2 bytes) for convenience.
pub const LEN: u16 = 8 + 2;
} }
impl Readable for SizeEntry { impl Readable for SizeEntry {

View file

@ -24,22 +24,16 @@ use croaring::Bitmap;
use crate::core::core::hash::DefaultHashable; use crate::core::core::hash::DefaultHashable;
use crate::core::core::pmmr::{Backend, PMMR}; use crate::core::core::pmmr::{Backend, PMMR};
use crate::core::ser::{ use crate::core::ser::{
Error, FixedLength, PMMRIndexHashable, PMMRable, ProtocolVersion, Readable, Reader, Writeable, Error, PMMRIndexHashable, PMMRable, ProtocolVersion, Readable, Reader, Writeable, Writer,
Writer,
}; };
#[test] #[test]
fn pmmr_leaf_idx_iter() { fn pmmr_leaf_idx_iter() {
let (data_dir, elems) = setup("leaf_idx_iter"); let (data_dir, elems) = setup("leaf_idx_iter");
{ {
let mut backend = store::pmmr::PMMRBackend::new( let mut backend =
data_dir.to_string(), store::pmmr::PMMRBackend::new(data_dir.to_string(), true, ProtocolVersion(1), None)
true, .unwrap();
false,
ProtocolVersion(1),
None,
)
.unwrap();
// adding first set of 4 elements and sync // adding first set of 4 elements and sync
let mmr_size = load(0, &elems[0..5], &mut backend); let mmr_size = load(0, &elems[0..5], &mut backend);
@ -62,14 +56,9 @@ fn pmmr_leaf_idx_iter() {
fn pmmr_append() { fn pmmr_append() {
let (data_dir, elems) = setup("append"); let (data_dir, elems) = setup("append");
{ {
let mut backend = store::pmmr::PMMRBackend::new( let mut backend =
data_dir.to_string(), store::pmmr::PMMRBackend::new(data_dir.to_string(), false, ProtocolVersion(1), None)
false, .unwrap();
false,
ProtocolVersion(1),
None,
)
.unwrap();
// adding first set of 4 elements and sync // adding first set of 4 elements and sync
let mut mmr_size = load(0, &elems[0..4], &mut backend); let mut mmr_size = load(0, &elems[0..4], &mut backend);
@ -154,14 +143,9 @@ fn pmmr_compact_leaf_sibling() {
// setup the mmr store with all elements // setup the mmr store with all elements
{ {
let mut backend = store::pmmr::PMMRBackend::new( let mut backend =
data_dir.to_string(), store::pmmr::PMMRBackend::new(data_dir.to_string(), true, ProtocolVersion(1), None)
true, .unwrap();
false,
ProtocolVersion(1),
None,
)
.unwrap();
let mmr_size = load(0, &elems[..], &mut backend); let mmr_size = load(0, &elems[..], &mut backend);
backend.sync().unwrap(); backend.sync().unwrap();
@ -236,14 +220,9 @@ fn pmmr_prune_compact() {
// setup the mmr store with all elements // setup the mmr store with all elements
{ {
let mut backend = store::pmmr::PMMRBackend::new( let mut backend =
data_dir.to_string(), store::pmmr::PMMRBackend::new(data_dir.to_string(), true, ProtocolVersion(1), None)
true, .unwrap();
false,
ProtocolVersion(1),
None,
)
.unwrap();
let mmr_size = load(0, &elems[..], &mut backend); let mmr_size = load(0, &elems[..], &mut backend);
backend.sync().unwrap(); backend.sync().unwrap();
@ -293,14 +272,9 @@ fn pmmr_reload() {
// set everything up with an initial backend // set everything up with an initial backend
{ {
let mut backend = store::pmmr::PMMRBackend::new( let mut backend =
data_dir.to_string(), store::pmmr::PMMRBackend::new(data_dir.to_string(), true, ProtocolVersion(1), None)
true, .unwrap();
false,
ProtocolVersion(1),
None,
)
.unwrap();
let mmr_size = load(0, &elems[..], &mut backend); let mmr_size = load(0, &elems[..], &mut backend);
@ -359,14 +333,9 @@ fn pmmr_reload() {
// create a new backend referencing the data files // create a new backend referencing the data files
// and check everything still works as expected // and check everything still works as expected
{ {
let mut backend = store::pmmr::PMMRBackend::new( let mut backend =
data_dir.to_string(), store::pmmr::PMMRBackend::new(data_dir.to_string(), true, ProtocolVersion(1), None)
true, .unwrap();
false,
ProtocolVersion(1),
None,
)
.unwrap();
assert_eq!(backend.unpruned_size(), mmr_size); assert_eq!(backend.unpruned_size(), mmr_size);
{ {
let pmmr: PMMR<'_, TestElem, _> = PMMR::at(&mut backend, mmr_size); let pmmr: PMMR<'_, TestElem, _> = PMMR::at(&mut backend, mmr_size);
@ -407,7 +376,7 @@ fn pmmr_rewind() {
let (data_dir, elems) = setup("rewind"); let (data_dir, elems) = setup("rewind");
{ {
let mut backend = let mut backend =
store::pmmr::PMMRBackend::new(data_dir.clone(), true, false, ProtocolVersion(1), None) store::pmmr::PMMRBackend::new(data_dir.clone(), true, ProtocolVersion(1), None)
.unwrap(); .unwrap();
// adding elements and keeping the corresponding root // adding elements and keeping the corresponding root
@ -524,7 +493,7 @@ fn pmmr_compact_single_leaves() {
let (data_dir, elems) = setup("compact_single_leaves"); let (data_dir, elems) = setup("compact_single_leaves");
{ {
let mut backend = let mut backend =
store::pmmr::PMMRBackend::new(data_dir.clone(), true, false, ProtocolVersion(1), None) store::pmmr::PMMRBackend::new(data_dir.clone(), true, ProtocolVersion(1), None)
.unwrap(); .unwrap();
let mmr_size = load(0, &elems[0..5], &mut backend); let mmr_size = load(0, &elems[0..5], &mut backend);
backend.sync().unwrap(); backend.sync().unwrap();
@ -560,7 +529,7 @@ fn pmmr_compact_entire_peak() {
let (data_dir, elems) = setup("compact_entire_peak"); let (data_dir, elems) = setup("compact_entire_peak");
{ {
let mut backend = let mut backend =
store::pmmr::PMMRBackend::new(data_dir.clone(), true, false, ProtocolVersion(1), None) store::pmmr::PMMRBackend::new(data_dir.clone(), true, ProtocolVersion(1), None)
.unwrap(); .unwrap();
let mmr_size = load(0, &elems[0..5], &mut backend); let mmr_size = load(0, &elems[0..5], &mut backend);
backend.sync().unwrap(); backend.sync().unwrap();
@ -616,14 +585,9 @@ fn pmmr_compact_horizon() {
let mmr_size; let mmr_size;
{ {
let mut backend = store::pmmr::PMMRBackend::new( let mut backend =
data_dir.clone(), store::pmmr::PMMRBackend::new(data_dir.clone(), true, ProtocolVersion(1), None)
true, .unwrap();
false,
ProtocolVersion(1),
None,
)
.unwrap();
mmr_size = load(0, &elems[..], &mut backend); mmr_size = load(0, &elems[..], &mut backend);
backend.sync().unwrap(); backend.sync().unwrap();
@ -705,7 +669,6 @@ fn pmmr_compact_horizon() {
let backend = store::pmmr::PMMRBackend::<TestElem>::new( let backend = store::pmmr::PMMRBackend::<TestElem>::new(
data_dir.to_string(), data_dir.to_string(),
true, true,
false,
ProtocolVersion(1), ProtocolVersion(1),
None, None,
) )
@ -726,7 +689,6 @@ fn pmmr_compact_horizon() {
let mut backend = store::pmmr::PMMRBackend::<TestElem>::new( let mut backend = store::pmmr::PMMRBackend::<TestElem>::new(
data_dir.to_string(), data_dir.to_string(),
true, true,
false,
ProtocolVersion(1), ProtocolVersion(1),
None, None,
) )
@ -749,7 +711,6 @@ fn pmmr_compact_horizon() {
let backend = store::pmmr::PMMRBackend::<TestElem>::new( let backend = store::pmmr::PMMRBackend::<TestElem>::new(
data_dir.to_string(), data_dir.to_string(),
true, true,
false,
ProtocolVersion(1), ProtocolVersion(1),
None, None,
) )
@ -782,14 +743,9 @@ fn compact_twice() {
// setup the mmr store with all elements // setup the mmr store with all elements
// Scoped to allow Windows to teardown // Scoped to allow Windows to teardown
{ {
let mut backend = store::pmmr::PMMRBackend::new( let mut backend =
data_dir.to_string(), store::pmmr::PMMRBackend::new(data_dir.to_string(), true, ProtocolVersion(1), None)
true, .unwrap();
false,
ProtocolVersion(1),
None,
)
.unwrap();
let mmr_size = load(0, &elems[..], &mut backend); let mmr_size = load(0, &elems[..], &mut backend);
backend.sync().unwrap(); backend.sync().unwrap();
@ -1008,16 +964,16 @@ struct TestElem(u32);
impl DefaultHashable for TestElem {} impl DefaultHashable for TestElem {}
impl FixedLength for TestElem {
const LEN: usize = 4;
}
impl PMMRable for TestElem { impl PMMRable for TestElem {
type E = Self; type E = Self;
fn as_elmt(&self) -> Self::E { fn as_elmt(&self) -> Self::E {
self.clone() self.clone()
} }
fn elmt_size() -> Option<u16> {
Some(4)
}
} }
impl Writeable for TestElem { impl Writeable for TestElem {