PIBD segment p2p messages (#3496)

* Define PIBD segment p2p messages

* Respond to segment requests

* Use specialized (de)ser for output bitmap segments

* Allowed segment height ranges in const
This commit is contained in:
jaspervdm 2020-11-25 21:52:09 +01:00 committed by GitHub
parent 9abb6e3e01
commit 96afc766a1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 454 additions and 3 deletions

View file

@ -153,6 +153,12 @@ pub enum ErrorKind {
/// PIBD segment related error
#[fail(display = "Segment error")]
SegmentError(segment::SegmentError),
/// The segmenter is associated to a different block header
#[fail(display = "Segmenter header mismatch")]
SegmenterHeaderMismatch,
/// Segment height not within allowed range
#[fail(display = "Invalid segment height")]
InvalidSegmentHeight,
}
impl Display for Error {

View file

@ -237,6 +237,14 @@ fn decode_message(
Type::PeerAddrs => Message::PeerAddrs(msg.body()?),
Type::TxHashSetRequest => Message::TxHashSetRequest(msg.body()?),
Type::TxHashSetArchive => Message::TxHashSetArchive(msg.body()?),
Type::GetOutputBitmapSegment => Message::GetOutputBitmapSegment(msg.body()?),
Type::OutputBitmapSegment => Message::OutputBitmapSegment(msg.body()?),
Type::GetOutputSegment => Message::GetOutputSegment(msg.body()?),
Type::OutputSegment => Message::OutputSegment(msg.body()?),
Type::GetRangeProofSegment => Message::GetRangeProofSegment(msg.body()?),
Type::RangeProofSegment => Message::RangeProofSegment(msg.body()?),
Type::GetKernelSegment => Message::GetKernelSegment(msg.body()?),
Type::KernelSegment => Message::KernelSegment(msg.body()?),
Type::Error | Type::Hand | Type::Shake | Type::Headers => {
return Err(Error::UnexpectedMessage)
}

View file

@ -14,10 +14,13 @@
//! Message types that transit over the network and related serialization code.
use crate::chain::txhashset::BitmapSegment;
use crate::conn::Tracker;
use crate::core::core::hash::Hash;
use crate::core::core::transaction::{OutputIdentifier, TxKernel};
use crate::core::core::{
BlockHeader, Transaction, UntrustedBlock, UntrustedBlockHeader, UntrustedCompactBlock,
BlockHeader, Segment, SegmentIdentifier, Transaction, UntrustedBlock, UntrustedBlockHeader,
UntrustedCompactBlock,
};
use crate::core::pow::Difficulty;
use crate::core::ser::{
@ -28,6 +31,7 @@ use crate::types::{
AttachmentMeta, AttachmentUpdate, Capabilities, Error, PeerAddr, ReasonForBan,
MAX_BLOCK_HEADERS, MAX_LOCATORS, MAX_PEER_ADDRS,
};
use crate::util::secp::pedersen::RangeProof;
use bytes::Bytes;
use num::FromPrimitive;
use std::fmt;
@ -70,6 +74,14 @@ enum_from_primitive! {
BanReason = 18,
GetTransaction = 19,
TransactionKernel = 20,
GetOutputBitmapSegment = 21,
OutputBitmapSegment = 22,
GetOutputSegment = 23,
OutputSegment = 24,
GetRangeProofSegment = 25,
RangeProofSegment = 26,
GetKernelSegment = 27,
KernelSegment = 28,
}
}
@ -107,6 +119,14 @@ fn max_msg_size(msg_type: Type) -> u64 {
Type::BanReason => 64,
Type::GetTransaction => 32,
Type::TransactionKernel => 32,
Type::GetOutputBitmapSegment => 41,
Type::OutputBitmapSegment => 2 * max_block_size(),
Type::GetOutputSegment => 41,
Type::OutputSegment => 2 * max_block_size(),
Type::GetRangeProofSegment => 41,
Type::RangeProofSegment => 2 * max_block_size(),
Type::GetKernelSegment => 41,
Type::KernelSegment => 2 * max_block_size(),
}
}
@ -710,6 +730,115 @@ impl Readable for TxHashSetArchive {
}
}
/// Request to get a segment of a (P)MMR at a particular block.
pub struct SegmentRequest {
/// The hash of the block the MMR is associated with
pub block_hash: Hash,
/// The identifier of the requested segment
pub identifier: SegmentIdentifier,
}
impl Readable for SegmentRequest {
fn read<R: Reader>(reader: &mut R) -> Result<Self, ser::Error> {
let block_hash = Readable::read(reader)?;
let identifier = Readable::read(reader)?;
Ok(Self {
block_hash,
identifier,
})
}
}
impl Writeable for SegmentRequest {
fn write<W: Writer>(&self, writer: &mut W) -> Result<(), ser::Error> {
Writeable::write(&self.block_hash, writer)?;
Writeable::write(&self.identifier, writer)
}
}
/// Response to a (P)MMR segment request.
pub struct SegmentResponse<T> {
/// The hash of the block the MMR is associated with
pub block_hash: Hash,
/// The MMR segment
pub segment: Segment<T>,
}
impl<T: Readable> Readable for SegmentResponse<T> {
fn read<R: Reader>(reader: &mut R) -> Result<Self, ser::Error> {
let block_hash = Readable::read(reader)?;
let segment = Readable::read(reader)?;
Ok(Self {
block_hash,
segment,
})
}
}
impl<T: Writeable> Writeable for SegmentResponse<T> {
fn write<W: Writer>(&self, writer: &mut W) -> Result<(), ser::Error> {
Writeable::write(&self.block_hash, writer)?;
Writeable::write(&self.segment, writer)
}
}
/// Response to an output PMMR segment request.
pub struct OutputSegmentResponse {
/// The segment response
pub response: SegmentResponse<OutputIdentifier>,
/// The root hash of the output bitmap MMR
pub output_bitmap_root: Hash,
}
impl Readable for OutputSegmentResponse {
fn read<R: Reader>(reader: &mut R) -> Result<Self, ser::Error> {
let response = Readable::read(reader)?;
let output_bitmap_root = Readable::read(reader)?;
Ok(Self {
response,
output_bitmap_root,
})
}
}
impl Writeable for OutputSegmentResponse {
fn write<W: Writer>(&self, writer: &mut W) -> Result<(), ser::Error> {
Writeable::write(&self.response, writer)?;
Writeable::write(&self.output_bitmap_root, writer)
}
}
/// Response to an output bitmap MMR segment request.
pub struct OutputBitmapSegmentResponse {
/// The hash of the block the MMR is associated with
pub block_hash: Hash,
/// The MMR segment
pub segment: BitmapSegment,
/// The root hash of the output PMMR
pub output_root: Hash,
}
impl Readable for OutputBitmapSegmentResponse {
fn read<R: Reader>(reader: &mut R) -> Result<Self, ser::Error> {
let block_hash = Readable::read(reader)?;
let segment = Readable::read(reader)?;
let output_root = Readable::read(reader)?;
Ok(Self {
block_hash,
segment,
output_root,
})
}
}
impl Writeable for OutputBitmapSegmentResponse {
fn write<W: Writer>(&self, writer: &mut W) -> Result<(), ser::Error> {
Writeable::write(&self.block_hash, writer)?;
Writeable::write(&self.segment, writer)?;
Writeable::write(&self.output_root, writer)
}
}
pub enum Message {
Unknown(u8),
Ping(Ping),
@ -731,6 +860,14 @@ pub enum Message {
TxHashSetRequest(TxHashSetRequest),
TxHashSetArchive(TxHashSetArchive),
Attachment(AttachmentUpdate, Option<Bytes>),
GetOutputBitmapSegment(SegmentRequest),
OutputBitmapSegment(OutputBitmapSegmentResponse),
GetOutputSegment(SegmentRequest),
OutputSegment(OutputSegmentResponse),
GetRangeProofSegment(SegmentRequest),
RangeProofSegment(SegmentResponse<RangeProof>),
GetKernelSegment(SegmentRequest),
KernelSegment(SegmentResponse<TxKernel>),
}
impl fmt::Display for Message {
@ -756,6 +893,14 @@ impl fmt::Display for Message {
Message::TxHashSetRequest(_) => write!(f, "tx hash set request"),
Message::TxHashSetArchive(_) => write!(f, "tx hash set"),
Message::Attachment(_, _) => write!(f, "attachment"),
Message::GetOutputBitmapSegment(_) => write!(f, "get output bitmap segment"),
Message::OutputBitmapSegment(_) => write!(f, "output bitmap segment"),
Message::GetOutputSegment(_) => write!(f, "get output segment"),
Message::OutputSegment(_) => write!(f, "output segment"),
Message::GetRangeProofSegment(_) => write!(f, "get range proof segment"),
Message::RangeProofSegment(_) => write!(f, "range proof segment"),
Message::GetKernelSegment(_) => write!(f, "get kernel segment"),
Message::KernelSegment(_) => write!(f, "kernel segment"),
}
}
}

View file

@ -23,8 +23,10 @@ use std::sync::Arc;
use lru_cache::LruCache;
use crate::chain;
use crate::chain::txhashset::BitmapChunk;
use crate::conn;
use crate::core::core::hash::{Hash, Hashed};
use crate::core::core::{OutputIdentifier, Segment, SegmentIdentifier, TxKernel};
use crate::core::pow::Difficulty;
use crate::core::ser::Writeable;
use crate::core::{core, global};
@ -35,6 +37,7 @@ use crate::types::{
Capabilities, ChainAdapter, Error, NetAdapter, P2PConfig, PeerAddr, PeerInfo, ReasonForBan,
TxHashSetRead,
};
use crate::util::secp::pedersen::RangeProof;
use chrono::prelude::{DateTime, Utc};
const MAX_TRACK_SIZE: usize = 30;
@ -565,6 +568,38 @@ impl ChainAdapter for TrackingAdapter {
fn get_tmpfile_pathname(&self, tmpfile_name: String) -> PathBuf {
self.adapter.get_tmpfile_pathname(tmpfile_name)
}
fn get_kernel_segment(
&self,
hash: Hash,
id: SegmentIdentifier,
) -> Result<Segment<TxKernel>, chain::Error> {
self.adapter.get_kernel_segment(hash, id)
}
fn get_bitmap_segment(
&self,
hash: Hash,
id: SegmentIdentifier,
) -> Result<(Segment<BitmapChunk>, Hash), chain::Error> {
self.adapter.get_bitmap_segment(hash, id)
}
fn get_output_segment(
&self,
hash: Hash,
id: SegmentIdentifier,
) -> Result<(Segment<OutputIdentifier>, Hash), chain::Error> {
self.adapter.get_output_segment(hash, id)
}
fn get_rangeproof_segment(
&self,
hash: Hash,
id: SegmentIdentifier,
) -> Result<Segment<RangeProof>, chain::Error> {
self.adapter.get_rangeproof_segment(hash, id)
}
}
impl NetAdapter for TrackingAdapter {

View file

@ -21,8 +21,10 @@ use std::sync::Arc;
use rand::prelude::*;
use crate::chain;
use crate::chain::txhashset::BitmapChunk;
use crate::core::core;
use crate::core::core::hash::{Hash, Hashed};
use crate::core::core::{OutputIdentifier, Segment, SegmentIdentifier, TxKernel};
use crate::core::global;
use crate::core::pow::Difficulty;
use crate::peer::Peer;
@ -31,6 +33,7 @@ use crate::types::{
Capabilities, ChainAdapter, Error, NetAdapter, P2PConfig, PeerAddr, PeerInfo, ReasonForBan,
TxHashSetRead, MAX_PEER_ADDRS,
};
use crate::util::secp::pedersen::RangeProof;
use chrono::prelude::*;
use chrono::Duration;
@ -625,6 +628,38 @@ impl ChainAdapter for Peers {
fn get_tmpfile_pathname(&self, tmpfile_name: String) -> PathBuf {
self.adapter.get_tmpfile_pathname(tmpfile_name)
}
fn get_kernel_segment(
&self,
hash: Hash,
id: SegmentIdentifier,
) -> Result<Segment<TxKernel>, chain::Error> {
self.adapter.get_kernel_segment(hash, id)
}
fn get_bitmap_segment(
&self,
hash: Hash,
id: SegmentIdentifier,
) -> Result<(Segment<BitmapChunk>, Hash), chain::Error> {
self.adapter.get_bitmap_segment(hash, id)
}
fn get_output_segment(
&self,
hash: Hash,
id: SegmentIdentifier,
) -> Result<(Segment<OutputIdentifier>, Hash), chain::Error> {
self.adapter.get_output_segment(hash, id)
}
fn get_rangeproof_segment(
&self,
hash: Hash,
id: SegmentIdentifier,
) -> Result<Segment<RangeProof>, chain::Error> {
self.adapter.get_rangeproof_segment(hash, id)
}
}
impl NetAdapter for Peers {

View file

@ -16,7 +16,10 @@ use crate::chain;
use crate::conn::MessageHandler;
use crate::core::core::{hash::Hashed, CompactBlock};
use crate::msg::{Consumed, Headers, Message, Msg, PeerAddrs, Pong, TxHashSetArchive, Type};
use crate::msg::{
Consumed, Headers, Message, Msg, OutputBitmapSegmentResponse, OutputSegmentResponse, PeerAddrs,
Pong, SegmentRequest, SegmentResponse, TxHashSetArchive, Type,
};
use crate::types::{AttachmentMeta, Error, NetAdapter, PeerInfo};
use chrono::prelude::Utc;
use rand::{thread_rng, Rng};
@ -287,6 +290,92 @@ impl MessageHandler for Protocol {
Consumed::Attachment(Arc::new(meta), file)
}
Message::GetOutputBitmapSegment(req) => {
let SegmentRequest {
block_hash,
identifier,
} = req;
if let Ok((segment, output_root)) =
self.adapter.get_bitmap_segment(block_hash, identifier)
{
Consumed::Response(Msg::new(
Type::OutputBitmapSegment,
OutputBitmapSegmentResponse {
block_hash,
segment: segment.into(),
output_root,
},
self.peer_info.version,
)?)
} else {
Consumed::None
}
}
Message::GetOutputSegment(req) => {
let SegmentRequest {
block_hash,
identifier,
} = req;
if let Ok((segment, output_bitmap_root)) =
self.adapter.get_output_segment(block_hash, identifier)
{
Consumed::Response(Msg::new(
Type::OutputSegment,
OutputSegmentResponse {
response: SegmentResponse {
block_hash,
segment,
},
output_bitmap_root,
},
self.peer_info.version,
)?)
} else {
Consumed::None
}
}
Message::GetRangeProofSegment(req) => {
let SegmentRequest {
block_hash,
identifier,
} = req;
if let Ok(segment) = self.adapter.get_rangeproof_segment(block_hash, identifier) {
Consumed::Response(Msg::new(
Type::RangeProofSegment,
SegmentResponse {
block_hash,
segment,
},
self.peer_info.version,
)?)
} else {
Consumed::None
}
}
Message::GetKernelSegment(req) => {
let SegmentRequest {
block_hash,
identifier,
} = req;
if let Ok(segment) = self.adapter.get_kernel_segment(block_hash, identifier) {
Consumed::Response(Msg::new(
Type::KernelSegment,
SegmentResponse {
block_hash,
segment,
},
self.peer_info.version,
)?)
} else {
Consumed::None
}
}
Message::OutputBitmapSegment(_)
| Message::OutputSegment(_)
| Message::RangeProofSegment(_)
| Message::KernelSegment(_) => Consumed::None,
Message::Unknown(_) => Consumed::None,
};
Ok(consumed)

View file

@ -21,8 +21,10 @@ use std::thread;
use std::time::Duration;
use crate::chain;
use crate::chain::txhashset::BitmapChunk;
use crate::core::core;
use crate::core::core::hash::Hash;
use crate::core::core::{OutputIdentifier, Segment, SegmentIdentifier, TxKernel};
use crate::core::global;
use crate::core::pow::Difficulty;
use crate::handshake::Handshake;
@ -33,6 +35,7 @@ use crate::types::{
Capabilities, ChainAdapter, Error, NetAdapter, P2PConfig, PeerAddr, PeerInfo, ReasonForBan,
TxHashSetRead,
};
use crate::util::secp::pedersen::RangeProof;
use crate::util::StopState;
use chrono::prelude::{DateTime, Utc};
@ -375,6 +378,38 @@ impl ChainAdapter for DummyAdapter {
fn get_tmpfile_pathname(&self, _tmpfile_name: String) -> PathBuf {
unimplemented!()
}
fn get_kernel_segment(
&self,
_hash: Hash,
_id: SegmentIdentifier,
) -> Result<Segment<TxKernel>, chain::Error> {
unimplemented!()
}
fn get_bitmap_segment(
&self,
_hash: Hash,
_id: SegmentIdentifier,
) -> Result<(Segment<BitmapChunk>, Hash), chain::Error> {
unimplemented!()
}
fn get_output_segment(
&self,
_hash: Hash,
_id: SegmentIdentifier,
) -> Result<(Segment<OutputIdentifier>, Hash), chain::Error> {
unimplemented!()
}
fn get_rangeproof_segment(
&self,
_hash: Hash,
_id: SegmentIdentifier,
) -> Result<Segment<RangeProof>, chain::Error> {
unimplemented!()
}
}
impl NetAdapter for DummyAdapter {

View file

@ -28,12 +28,15 @@ use serde::{Deserialize, Deserializer};
use grin_store;
use crate::chain;
use crate::chain::txhashset::BitmapChunk;
use crate::core::core;
use crate::core::core::hash::Hash;
use crate::core::core::{OutputIdentifier, Segment, SegmentIdentifier, TxKernel};
use crate::core::global;
use crate::core::pow::Difficulty;
use crate::core::ser::{self, ProtocolVersion, Readable, Reader, Writeable, Writer};
use crate::msg::PeerAddrs;
use crate::util::secp::pedersen::RangeProof;
use crate::util::RwLock;
/// Maximum number of block headers a peer should ever send
@ -642,6 +645,30 @@ pub trait ChainAdapter: Sync + Send {
/// Get a tmp file path in above specific tmp dir (create tmp dir if not exist)
/// Delete file if tmp file already exists
fn get_tmpfile_pathname(&self, tmpfile_name: String) -> PathBuf;
fn get_kernel_segment(
&self,
hash: Hash,
id: SegmentIdentifier,
) -> Result<Segment<TxKernel>, chain::Error>;
fn get_bitmap_segment(
&self,
hash: Hash,
id: SegmentIdentifier,
) -> Result<(Segment<BitmapChunk>, Hash), chain::Error>;
fn get_output_segment(
&self,
hash: Hash,
id: SegmentIdentifier,
) -> Result<(Segment<OutputIdentifier>, Hash), chain::Error>;
fn get_rangeproof_segment(
&self,
hash: Hash,
id: SegmentIdentifier,
) -> Result<Segment<RangeProof>, chain::Error>;
}
/// Additional methods required by the protocol that don't need to be

View file

@ -22,6 +22,7 @@ use std::sync::{Arc, Weak};
use std::thread;
use std::time::Instant;
use crate::chain::txhashset::BitmapChunk;
use crate::chain::{
self, BlockStatus, ChainAdapter, Options, SyncState, SyncStatus, TxHashsetDownloadStats,
};
@ -30,17 +31,27 @@ use crate::common::types::{ChainValidationMode, DandelionEpoch, ServerConfig};
use crate::core::core::hash::{Hash, Hashed};
use crate::core::core::transaction::Transaction;
use crate::core::core::verifier_cache::VerifierCache;
use crate::core::core::{BlockHeader, BlockSums, CompactBlock, Inputs, OutputIdentifier};
use crate::core::core::{
BlockHeader, BlockSums, CompactBlock, Inputs, OutputIdentifier, Segment, SegmentIdentifier,
TxKernel,
};
use crate::core::pow::Difficulty;
use crate::core::ser::ProtocolVersion;
use crate::core::{core, global};
use crate::p2p;
use crate::p2p::types::PeerInfo;
use crate::pool::{self, BlockChain, PoolAdapter};
use crate::util::secp::pedersen::RangeProof;
use crate::util::OneTime;
use chrono::prelude::*;
use chrono::Duration;
use rand::prelude::*;
use std::ops::Range;
const KERNEL_SEGMENT_HEIGHT_RANGE: Range<u8> = 9..14;
const BITMAP_SEGMENT_HEIGHT_RANGE: Range<u8> = 9..14;
const OUTPUT_SEGMENT_HEIGHT_RANGE: Range<u8> = 11..16;
const RANGEPROOF_SEGMENT_HEIGHT_RANGE: Range<u8> = 7..12;
/// Implementation of the NetAdapter for the . Gets notified when new
/// blocks and transactions are received and forwards to the chain and pool
@ -477,6 +488,66 @@ where
fn get_tmpfile_pathname(&self, tmpfile_name: String) -> PathBuf {
self.chain().get_tmpfile_pathname(tmpfile_name)
}
fn get_kernel_segment(
&self,
hash: Hash,
id: SegmentIdentifier,
) -> Result<Segment<TxKernel>, chain::Error> {
if !KERNEL_SEGMENT_HEIGHT_RANGE.contains(&id.height) {
return Err(chain::ErrorKind::InvalidSegmentHeight.into());
}
let segmenter = self.chain().segmenter()?;
if segmenter.header().hash() != hash {
return Err(chain::ErrorKind::SegmenterHeaderMismatch.into());
}
segmenter.kernel_segment(id)
}
fn get_bitmap_segment(
&self,
hash: Hash,
id: SegmentIdentifier,
) -> Result<(Segment<BitmapChunk>, Hash), chain::Error> {
if !BITMAP_SEGMENT_HEIGHT_RANGE.contains(&id.height) {
return Err(chain::ErrorKind::InvalidSegmentHeight.into());
}
let segmenter = self.chain().segmenter()?;
if segmenter.header().hash() != hash {
return Err(chain::ErrorKind::SegmenterHeaderMismatch.into());
}
segmenter.bitmap_segment(id)
}
fn get_output_segment(
&self,
hash: Hash,
id: SegmentIdentifier,
) -> Result<(Segment<OutputIdentifier>, Hash), chain::Error> {
if !OUTPUT_SEGMENT_HEIGHT_RANGE.contains(&id.height) {
return Err(chain::ErrorKind::InvalidSegmentHeight.into());
}
let segmenter = self.chain().segmenter()?;
if segmenter.header().hash() != hash {
return Err(chain::ErrorKind::SegmenterHeaderMismatch.into());
}
segmenter.output_segment(id)
}
fn get_rangeproof_segment(
&self,
hash: Hash,
id: SegmentIdentifier,
) -> Result<Segment<RangeProof>, chain::Error> {
if RANGEPROOF_SEGMENT_HEIGHT_RANGE.contains(&id.height) {
return Err(chain::ErrorKind::InvalidSegmentHeight.into());
}
let segmenter = self.chain().segmenter()?;
if segmenter.header().hash() != hash {
return Err(chain::ErrorKind::SegmenterHeaderMismatch.into());
}
segmenter.rangeproof_segment(id)
}
}
impl<B, P, V> NetToChainAdapter<B, P, V>