Core/grin
0
0
Fork 0
mirror of https://github.com/mimblewimble/grin.git synced 2025-01-20 19:11:08 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

support for v2 txhashset (but only using v1 in mainnent and floonet) (#3051)

Browse source
This commit is contained in:
Antioch Peverell 2019-09-24 09:57:34 +01:00 committed by GitHub
parent 1c072f535c
commit 973a03c826
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 211 additions and 134 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
chain/src/chain.rs
View file

@ -20,7 +20,6 @@ use crate::core::core::merkle_proof::MerkleProof;
use crate::core::core::verifier_cache::VerifierCache;
use crate::core::core::{
Block, BlockHeader, BlockSums, Committed, Output, OutputIdentifier, Transaction, TxKernel,
TxKernelEntry,
};
use crate::core::global;
use crate::core::pow;
@ -176,9 +175,24 @@ impl Chain {
// open the txhashset, creating a new one if necessary
let mut txhashset = txhashset::TxHashSet::open(db_root.clone(), store.clone(), None)?;
let mut header_pmmr =
PMMRHandle::new(&db_root, "header", "header_head", false, true, None)?;
let mut sync_pmmr = PMMRHandle::new(&db_root, "header", "sync_head", false, true, None)?;
let mut header_pmmr = PMMRHandle::new(
&db_root,
"header",
"header_head",
false,
true,
ProtocolVersion(1),
None,
)?;
let mut sync_pmmr = PMMRHandle::new(
&db_root,
"header",
"sync_head",
false,
true,
ProtocolVersion(1),
None,
)?;
setup_head(
&genesis,
@ -661,7 +675,7 @@ impl Chain {
pub fn kernel_data_write(&self, reader: &mut dyn Read) -> Result<(), Error> {
let mut count = 0;
let mut stream = StreamingReader::new(reader, ProtocolVersion::local());
while let Ok(_kernel) = TxKernelEntry::read(&mut stream) {
while let Ok(_kernel) = TxKernel::read(&mut stream) {
count += 1;
}
@ -1146,7 +1160,7 @@ impl Chain {
}
/// as above, for kernels
pub fn get_last_n_kernel(&self, distance: u64) -> Vec<(Hash, TxKernelEntry)> {
pub fn get_last_n_kernel(&self, distance: u64) -> Vec<(Hash, TxKernel)> {
self.txhashset.read().last_n_kernel(distance)
}

111
chain/src/txhashset/txhashset.rs
View file

@ -19,10 +19,8 @@ use crate::core::core::committed::Committed;
use crate::core::core::hash::{Hash, Hashed};
use crate::core::core::merkle_proof::MerkleProof;
use crate::core::core::pmmr::{self, Backend, ReadonlyPMMR, RewindablePMMR, PMMR};
use crate::core::core::{
Block, BlockHeader, Input, Output, OutputIdentifier, TxKernel, TxKernelEntry,
};
use crate::core::ser::{PMMRIndexHashable, PMMRable};
use crate::core::core::{Block, BlockHeader, Input, Output, OutputIdentifier, TxKernel};
use crate::core::ser::{PMMRIndexHashable, PMMRable, ProtocolVersion};
use crate::error::{Error, ErrorKind};
use crate::store::{Batch, ChainStore};
use crate::txhashset::{RewindableKernelView, UTXOView};
@ -62,6 +60,7 @@ impl<T: PMMRable> PMMRHandle<T> {
file_name: &str,
prunable: bool,
fixed_size: bool,
version: ProtocolVersion,
header: Option<&BlockHeader>,
) -> Result<PMMRHandle<T>, Error> {
let path = Path::new(root_dir).join(sub_dir).join(file_name);
@ -69,7 +68,8 @@ impl<T: PMMRable> PMMRHandle<T> {
let path_str = path.to_str().ok_or(Error::from(ErrorKind::Other(
"invalid file path".to_owned(),
)))?;
let backend = PMMRBackend::new(path_str.to_string(), prunable, fixed_size, header)?;
let backend =
PMMRBackend::new(path_str.to_string(), prunable, fixed_size, version, header)?;
let last_pos = backend.unpruned_size();
Ok(PMMRHandle { backend, last_pos })
}
@ -113,33 +113,78 @@ impl TxHashSet {
commit_index: Arc<ChainStore>,
header: Option<&BlockHeader>,
) -> Result<TxHashSet, Error> {
Ok(TxHashSet {
output_pmmr_h: PMMRHandle::new(
&root_dir,
TXHASHSET_SUBDIR,
OUTPUT_SUBDIR,
true,
true,
header,
)?,
rproof_pmmr_h: PMMRHandle::new(
&root_dir,
TXHASHSET_SUBDIR,
RANGE_PROOF_SUBDIR,
true,
true,
header,
)?,
kernel_pmmr_h: PMMRHandle::new(
let output_pmmr_h = PMMRHandle::new(
&root_dir,
TXHASHSET_SUBDIR,
OUTPUT_SUBDIR,
true,
true,
ProtocolVersion(1),
header,
)?;
let rproof_pmmr_h = PMMRHandle::new(
&root_dir,
TXHASHSET_SUBDIR,
RANGE_PROOF_SUBDIR,
true,
true,
ProtocolVersion(1),
header,
)?;
let mut maybe_kernel_handle: Option<PMMRHandle<TxKernel>> = None;
let versions = vec![ProtocolVersion(2), ProtocolVersion(1)];
for version in versions {
let handle = PMMRHandle::new(
&root_dir,
TXHASHSET_SUBDIR,
KERNEL_SUBDIR,
false, // not prunable
false, // variable size kernel data file
version,
None,
)?,
commit_index,
})
)?;
if handle.last_pos == 0 {
debug!(
"attempting to open (empty) kernel PMMR using {:?} - SUCCESS",
version
);
maybe_kernel_handle = Some(handle);
break;
}
let kernel: Option<TxKernel> = ReadonlyPMMR::at(&handle.backend, 1).get_data(1);
if let Some(kernel) = kernel {
if kernel.verify().is_ok() {
debug!(
"attempting to open kernel PMMR using {:?} - SUCCESS",
version
);
maybe_kernel_handle = Some(handle);
break;
} else {
debug!(
"attempting to open kernel PMMR using {:?} - FAIL (verify failed)",
version
);
}
} else {
debug!(
"attempting to open kernel PMMR using {:?} - FAIL (read failed)",
version
);
}
}
if let Some(kernel_pmmr_h) = maybe_kernel_handle {
Ok(TxHashSet {
output_pmmr_h,
rproof_pmmr_h,
kernel_pmmr_h,
commit_index,
})
} else {
Err(ErrorKind::TxHashSetErr(format!("failed to open kernel PMMR")).into())
}
}
/// Close all backend file handles
@ -192,7 +237,7 @@ impl TxHashSet {
}
/// as above, for kernels
pub fn last_n_kernel(&self, distance: u64) -> Vec<(Hash, TxKernelEntry)> {
pub fn last_n_kernel(&self, distance: u64) -> Vec<(Hash, TxKernel)> {
ReadonlyPMMR::at(&self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos)
.get_last_n_insertions(distance)
}
@ -247,9 +292,9 @@ impl TxHashSet {
let mut index = max_index + 1;
while index > min_index {
index -= 1;
if let Some(t) = pmmr.get_data(index) {
if &t.kernel.excess == excess {
return Some((t.kernel, index));
if let Some(kernel) = pmmr.get_data(index) {
if &kernel.excess == excess {
return Some((kernel, index));
}
}
}
@ -258,8 +303,6 @@ impl TxHashSet {
/// Get MMR roots.
pub fn roots(&self) -> TxHashSetRoots {
// let header_pmmr =
// ReadonlyPMMR::at(&self.header_pmmr_h.backend, self.header_pmmr_h.last_pos);
let output_pmmr =
ReadonlyPMMR::at(&self.output_pmmr_h.backend, self.output_pmmr_h.last_pos);
let rproof_pmmr =
@ -268,7 +311,6 @@ impl TxHashSet {
ReadonlyPMMR::at(&self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos);
TxHashSetRoots {
// header_root: header_pmmr.root(),
output_root: output_pmmr.root(),
rproof_root: rproof_pmmr.root(),
kernel_root: kernel_pmmr.root(),
@ -1192,8 +1234,7 @@ impl<'a> Extension<'a> {
.kernel_pmmr
.get_data(n)
.ok_or::<Error>(ErrorKind::TxKernelNotFound.into())?;
tx_kernels.push(kernel.kernel);
tx_kernels.push(kernel);
}
if tx_kernels.len() >= KERNEL_BATCH_SIZE || n >= self.kernel_pmmr.unpruned_size() {

70
core/src/core/transaction.rs
View file

@ -344,15 +344,22 @@ impl Readable for TxKernel {
}
}
/// We store TxKernelEntry in the kernel MMR.
/// We store kernels in the kernel MMR.
/// Note: These are "variable size" to support different kernel featuere variants.
impl PMMRable for TxKernel {
type E = TxKernelEntry;
type E = Self;
fn as_elmt(&self) -> TxKernelEntry {
TxKernelEntry::from_kernel(self)
fn as_elmt(&self) -> Self::E {
self.clone()
}
}
/// Kernels are "variable size" but we need to implement FixedLength for legacy reasons.
/// At some point we will refactor the MMR backend so this is no longer required.
impl FixedLength for TxKernel {
const LEN: usize = 0;
}
impl KernelFeatures {
/// Is this a coinbase kernel?
pub fn is_coinbase(&self) -> bool {
@ -494,61 +501,6 @@ impl TxKernel {
}
}
/// Wrapper around a tx kernel used when maintaining them in the MMR.
/// These will be useful once we implement relative lockheights via relative kernels
/// as a kernel may have an optional rel_kernel but we will not want to store these
/// directly in the kernel MMR.
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct TxKernelEntry {
/// The underlying tx kernel.
pub kernel: TxKernel,
}
impl Writeable for TxKernelEntry {
fn write<W: Writer>(&self, writer: &mut W) -> Result<(), ser::Error> {
self.kernel.write(writer)?;
Ok(())
}
}
impl Readable for TxKernelEntry {
fn read(reader: &mut dyn Reader) -> Result<TxKernelEntry, ser::Error> {
let kernel = TxKernel::read(reader)?;
Ok(TxKernelEntry { kernel })
}
}
impl TxKernelEntry {
/// The excess on the underlying tx kernel.
pub fn excess(&self) -> Commitment {
self.kernel.excess
}
/// Verify the underlying tx kernel.
pub fn verify(&self) -> Result<(), Error> {
self.kernel.verify()
}
/// Build a new tx kernel entry from a kernel.
pub fn from_kernel(kernel: &TxKernel) -> TxKernelEntry {
TxKernelEntry {
kernel: kernel.clone(),
}
}
}
impl From<TxKernel> for TxKernelEntry {
fn from(kernel: TxKernel) -> Self {
TxKernelEntry { kernel }
}
}
impl FixedLength for TxKernelEntry {
const LEN: usize = 17 // features plus fee and lock_height
+ secp::constants::PEDERSEN_COMMITMENT_SIZE
+ secp::constants::AGG_SIGNATURE_SIZE;
}
/// Enum of possible tx weight verification options -
///
/// * As "transaction" checks tx (as block) weight does not exceed max_block_weight.

6
store/src/pmmr.rs
View file

@ -204,13 +204,9 @@ impl<T: PMMRable> PMMRBackend<T> {
data_dir: P,
prunable: bool,
fixed_size: bool,
version: ProtocolVersion,
header: Option<&BlockHeader>,
) -> io::Result<PMMRBackend<T>> {
// Note: Explicit protocol version here.
// Regardless of our "default" protocol version we have existing MMR files
// and we need to be able to support these across upgrades.
let version = ProtocolVersion(1);
let data_dir = data_dir.as_ref();
// Are we dealing with "fixed size" data elements or "variable size" data elements

15
store/src/types.rs
View file

@ -185,6 +185,17 @@ pub struct AppendOnlyFile<T> {
_marker: marker::PhantomData<T>,
}
impl AppendOnlyFile<SizeEntry> {
fn sum_sizes(&self) -> io::Result<u64> {
let mut sum = 0;
for pos in 0..self.buffer_start_pos {
let entry = self.read_as_elmt(pos)?;
sum += entry.size as u64;
}
Ok(sum)
}
}
impl<T> AppendOnlyFile<T>
where
T: Debug + Readable + Writeable,
@ -215,8 +226,9 @@ where
// This will occur during "fast sync" as we do not sync the size_file
// and must build it locally.
// And we can *only* do this after init() the data file (so we know sizes).
let expected_size = aof.size()?;
if let SizeInfo::VariableSize(ref mut size_file) = &mut aof.size_info {
if size_file.size()? == 0 {
if size_file.sum_sizes()? != expected_size {
aof.rebuild_size_file()?;
// (Re)init the entire file as we just rebuilt the size_file
@ -517,6 +529,7 @@ where
if let SizeInfo::VariableSize(ref mut size_file) = &mut self.size_info {
// Note: Reading from data file and writing sizes to the associated (tmp) size_file.
let tmp_path = size_file.path.with_extension("tmp");
debug!("rebuild_size_file: {:?}", tmp_path);
// Scope the reader and writer to within the block so we can safely replace files later on.
{

2
store/tests/lmdb.rs
View file

@ -35,7 +35,7 @@ impl PhatChunkStruct {
}
impl Readable for PhatChunkStruct {
fn read(reader: &mut Reader) -> Result<PhatChunkStruct, ser::Error> {
fn read(reader: &mut dyn Reader) -> Result<PhatChunkStruct, ser::Error> {
let mut retval = PhatChunkStruct::new();
for _ in 0..TEST_ALLOC_SIZE {
retval.phatness = reader.read_u64()?;

115
store/tests/pmmr.rs
View file

@ -24,15 +24,22 @@ use croaring::Bitmap;
use crate::core::core::hash::DefaultHashable;
use crate::core::core::pmmr::{Backend, PMMR};
use crate::core::ser::{
Error, FixedLength, PMMRIndexHashable, PMMRable, Readable, Reader, Writeable, Writer,
Error, FixedLength, PMMRIndexHashable, PMMRable, ProtocolVersion, Readable, Reader, Writeable,
Writer,
};
#[test]
fn pmmr_append() {
let (data_dir, elems) = setup("append");
{
let mut backend =
store::pmmr::PMMRBackend::new(data_dir.to_string(), true, false, None).unwrap();
let mut backend = store::pmmr::PMMRBackend::new(
data_dir.to_string(),
true,
false,
ProtocolVersion(1),
None,
)
.unwrap();
// adding first set of 4 elements and sync
let mut mmr_size = load(0, &elems[0..4], &mut backend);
@ -114,8 +121,14 @@ fn pmmr_compact_leaf_sibling() {
// setup the mmr store with all elements
{
let mut backend =
store::pmmr::PMMRBackend::new(data_dir.to_string(), true, false, None).unwrap();
let mut backend = store::pmmr::PMMRBackend::new(
data_dir.to_string(),
true,
false,
ProtocolVersion(1),
None,
)
.unwrap();
let mmr_size = load(0, &elems[..], &mut backend);
backend.sync().unwrap();
@ -187,8 +200,14 @@ fn pmmr_prune_compact() {
// setup the mmr store with all elements
{
let mut backend =
store::pmmr::PMMRBackend::new(data_dir.to_string(), true, false, None).unwrap();
let mut backend = store::pmmr::PMMRBackend::new(
data_dir.to_string(),
true,
false,
ProtocolVersion(1),
None,
)
.unwrap();
let mmr_size = load(0, &elems[..], &mut backend);
backend.sync().unwrap();
@ -238,8 +257,14 @@ fn pmmr_reload() {
// set everything up with an initial backend
{
let mut backend =
store::pmmr::PMMRBackend::new(data_dir.to_string(), true, false, None).unwrap();
let mut backend = store::pmmr::PMMRBackend::new(
data_dir.to_string(),
true,
false,
ProtocolVersion(1),
None,
)
.unwrap();
let mmr_size = load(0, &elems[..], &mut backend);
@ -298,8 +323,14 @@ fn pmmr_reload() {
// create a new backend referencing the data files
// and check everything still works as expected
{
let mut backend =
store::pmmr::PMMRBackend::new(data_dir.to_string(), true, false, None).unwrap();
let mut backend = store::pmmr::PMMRBackend::new(
data_dir.to_string(),
true,
false,
ProtocolVersion(1),
None,
)
.unwrap();
assert_eq!(backend.unpruned_size(), mmr_size);
{
let pmmr: PMMR<'_, TestElem, _> = PMMR::at(&mut backend, mmr_size);
@ -340,7 +371,8 @@ fn pmmr_rewind() {
let (data_dir, elems) = setup("rewind");
{
let mut backend =
store::pmmr::PMMRBackend::new(data_dir.clone(), true, false, None).unwrap();
store::pmmr::PMMRBackend::new(data_dir.clone(), true, false, ProtocolVersion(1), None)
.unwrap();
// adding elements and keeping the corresponding root
let mut mmr_size = load(0, &elems[0..4], &mut backend);
@ -456,7 +488,8 @@ fn pmmr_compact_single_leaves() {
let (data_dir, elems) = setup("compact_single_leaves");
{
let mut backend =
store::pmmr::PMMRBackend::new(data_dir.clone(), true, false, None).unwrap();
store::pmmr::PMMRBackend::new(data_dir.clone(), true, false, ProtocolVersion(1), None)
.unwrap();
let mmr_size = load(0, &elems[0..5], &mut backend);
backend.sync().unwrap();
@ -491,7 +524,8 @@ fn pmmr_compact_entire_peak() {
let (data_dir, elems) = setup("compact_entire_peak");
{
let mut backend =
store::pmmr::PMMRBackend::new(data_dir.clone(), true, false, None).unwrap();
store::pmmr::PMMRBackend::new(data_dir.clone(), true, false, ProtocolVersion(1), None)
.unwrap();
let mmr_size = load(0, &elems[0..5], &mut backend);
backend.sync().unwrap();
@ -546,8 +580,14 @@ fn pmmr_compact_horizon() {
let mmr_size;
{
let mut backend =
store::pmmr::PMMRBackend::new(data_dir.clone(), true, false, None).unwrap();
let mut backend = store::pmmr::PMMRBackend::new(
data_dir.clone(),
true,
false,
ProtocolVersion(1),
None,
)
.unwrap();
mmr_size = load(0, &elems[..], &mut backend);
backend.sync().unwrap();
@ -626,9 +666,14 @@ fn pmmr_compact_horizon() {
// recheck stored data
{
// recreate backend
let backend =
store::pmmr::PMMRBackend::<TestElem>::new(data_dir.to_string(), true, false, None)
.unwrap();
let backend = store::pmmr::PMMRBackend::<TestElem>::new(
data_dir.to_string(),
true,
false,
ProtocolVersion(1),
None,
)
.unwrap();
assert_eq!(backend.data_size(), 19);
assert_eq!(backend.hash_size(), 35);
@ -642,9 +687,14 @@ fn pmmr_compact_horizon() {
}
{
let mut backend =
store::pmmr::PMMRBackend::<TestElem>::new(data_dir.to_string(), true, false, None)
.unwrap();
let mut backend = store::pmmr::PMMRBackend::<TestElem>::new(
data_dir.to_string(),
true,
false,
ProtocolVersion(1),
None,
)
.unwrap();
{
let mut pmmr: PMMR<'_, TestElem, _> = PMMR::at(&mut backend, mmr_size);
@ -660,9 +710,14 @@ fn pmmr_compact_horizon() {
// recheck stored data
{
// recreate backend
let backend =
store::pmmr::PMMRBackend::<TestElem>::new(data_dir.to_string(), true, false, None)
.unwrap();
let backend = store::pmmr::PMMRBackend::<TestElem>::new(
data_dir.to_string(),
true,
false,
ProtocolVersion(1),
None,
)
.unwrap();
// 0010012001001230
@ -691,8 +746,14 @@ fn compact_twice() {
// setup the mmr store with all elements
// Scoped to allow Windows to teardown
{
let mut backend =
store::pmmr::PMMRBackend::new(data_dir.to_string(), true, false, None).unwrap();
let mut backend = store::pmmr::PMMRBackend::new(
data_dir.to_string(),
true,
false,
ProtocolVersion(1),
None,
)
.unwrap();
let mmr_size = load(0, &elems[..], &mut backend);
backend.sync().unwrap();