From 96afc766a19f044607926b40fc0491ef9d7b9570 Mon Sep 17 00:00:00 2001 From: jaspervdm Date: Wed, 25 Nov 2020 21:52:09 +0100 Subject: [PATCH] PIBD segment p2p messages (#3496) * Define PIBD segment p2p messages * Respond to segment requests * Use specialized (de)ser for output bitmap segments * Allowed segment height ranges in const --- chain/src/error.rs | 6 ++ p2p/src/codec.rs | 8 ++ p2p/src/msg.rs | 147 ++++++++++++++++++++++++++++++++- p2p/src/peer.rs | 35 ++++++++ p2p/src/peers.rs | 35 ++++++++ p2p/src/protocol.rs | 91 +++++++++++++++++++- p2p/src/serv.rs | 35 ++++++++ p2p/src/types.rs | 27 ++++++ servers/src/common/adapters.rs | 73 +++++++++++++++- 9 files changed, 454 insertions(+), 3 deletions(-) diff --git a/chain/src/error.rs b/chain/src/error.rs index 561c11df3..6bc87fce5 100644 --- a/chain/src/error.rs +++ b/chain/src/error.rs @@ -153,6 +153,12 @@ pub enum ErrorKind { /// PIBD segment related error #[fail(display = "Segment error")] SegmentError(segment::SegmentError), + /// The segmenter is associated to a different block header + #[fail(display = "Segmenter header mismatch")] + SegmenterHeaderMismatch, + /// Segment height not within allowed range + #[fail(display = "Invalid segment height")] + InvalidSegmentHeight, } impl Display for Error { diff --git a/p2p/src/codec.rs b/p2p/src/codec.rs index 9fd6043c4..293613f47 100644 --- a/p2p/src/codec.rs +++ b/p2p/src/codec.rs @@ -237,6 +237,14 @@ fn decode_message( Type::PeerAddrs => Message::PeerAddrs(msg.body()?), Type::TxHashSetRequest => Message::TxHashSetRequest(msg.body()?), Type::TxHashSetArchive => Message::TxHashSetArchive(msg.body()?), + Type::GetOutputBitmapSegment => Message::GetOutputBitmapSegment(msg.body()?), + Type::OutputBitmapSegment => Message::OutputBitmapSegment(msg.body()?), + Type::GetOutputSegment => Message::GetOutputSegment(msg.body()?), + Type::OutputSegment => Message::OutputSegment(msg.body()?), + Type::GetRangeProofSegment => Message::GetRangeProofSegment(msg.body()?), + Type::RangeProofSegment => Message::RangeProofSegment(msg.body()?), + Type::GetKernelSegment => Message::GetKernelSegment(msg.body()?), + Type::KernelSegment => Message::KernelSegment(msg.body()?), Type::Error | Type::Hand | Type::Shake | Type::Headers => { return Err(Error::UnexpectedMessage) } diff --git a/p2p/src/msg.rs b/p2p/src/msg.rs index 796817ad0..d0ac2e659 100644 --- a/p2p/src/msg.rs +++ b/p2p/src/msg.rs @@ -14,10 +14,13 @@ //! Message types that transit over the network and related serialization code. +use crate::chain::txhashset::BitmapSegment; use crate::conn::Tracker; use crate::core::core::hash::Hash; +use crate::core::core::transaction::{OutputIdentifier, TxKernel}; use crate::core::core::{ - BlockHeader, Transaction, UntrustedBlock, UntrustedBlockHeader, UntrustedCompactBlock, + BlockHeader, Segment, SegmentIdentifier, Transaction, UntrustedBlock, UntrustedBlockHeader, + UntrustedCompactBlock, }; use crate::core::pow::Difficulty; use crate::core::ser::{ @@ -28,6 +31,7 @@ use crate::types::{ AttachmentMeta, AttachmentUpdate, Capabilities, Error, PeerAddr, ReasonForBan, MAX_BLOCK_HEADERS, MAX_LOCATORS, MAX_PEER_ADDRS, }; +use crate::util::secp::pedersen::RangeProof; use bytes::Bytes; use num::FromPrimitive; use std::fmt; @@ -70,6 +74,14 @@ enum_from_primitive! { BanReason = 18, GetTransaction = 19, TransactionKernel = 20, + GetOutputBitmapSegment = 21, + OutputBitmapSegment = 22, + GetOutputSegment = 23, + OutputSegment = 24, + GetRangeProofSegment = 25, + RangeProofSegment = 26, + GetKernelSegment = 27, + KernelSegment = 28, } } @@ -107,6 +119,14 @@ fn max_msg_size(msg_type: Type) -> u64 { Type::BanReason => 64, Type::GetTransaction => 32, Type::TransactionKernel => 32, + Type::GetOutputBitmapSegment => 41, + Type::OutputBitmapSegment => 2 * max_block_size(), + Type::GetOutputSegment => 41, + Type::OutputSegment => 2 * max_block_size(), + Type::GetRangeProofSegment => 41, + Type::RangeProofSegment => 2 * max_block_size(), + Type::GetKernelSegment => 41, + Type::KernelSegment => 2 * max_block_size(), } } @@ -710,6 +730,115 @@ impl Readable for TxHashSetArchive { } } +/// Request to get a segment of a (P)MMR at a particular block. +pub struct SegmentRequest { + /// The hash of the block the MMR is associated with + pub block_hash: Hash, + /// The identifier of the requested segment + pub identifier: SegmentIdentifier, +} + +impl Readable for SegmentRequest { + fn read(reader: &mut R) -> Result { + let block_hash = Readable::read(reader)?; + let identifier = Readable::read(reader)?; + Ok(Self { + block_hash, + identifier, + }) + } +} + +impl Writeable for SegmentRequest { + fn write(&self, writer: &mut W) -> Result<(), ser::Error> { + Writeable::write(&self.block_hash, writer)?; + Writeable::write(&self.identifier, writer) + } +} + +/// Response to a (P)MMR segment request. +pub struct SegmentResponse { + /// The hash of the block the MMR is associated with + pub block_hash: Hash, + /// The MMR segment + pub segment: Segment, +} + +impl Readable for SegmentResponse { + fn read(reader: &mut R) -> Result { + let block_hash = Readable::read(reader)?; + let segment = Readable::read(reader)?; + Ok(Self { + block_hash, + segment, + }) + } +} + +impl Writeable for SegmentResponse { + fn write(&self, writer: &mut W) -> Result<(), ser::Error> { + Writeable::write(&self.block_hash, writer)?; + Writeable::write(&self.segment, writer) + } +} + +/// Response to an output PMMR segment request. +pub struct OutputSegmentResponse { + /// The segment response + pub response: SegmentResponse, + /// The root hash of the output bitmap MMR + pub output_bitmap_root: Hash, +} + +impl Readable for OutputSegmentResponse { + fn read(reader: &mut R) -> Result { + let response = Readable::read(reader)?; + let output_bitmap_root = Readable::read(reader)?; + Ok(Self { + response, + output_bitmap_root, + }) + } +} + +impl Writeable for OutputSegmentResponse { + fn write(&self, writer: &mut W) -> Result<(), ser::Error> { + Writeable::write(&self.response, writer)?; + Writeable::write(&self.output_bitmap_root, writer) + } +} + +/// Response to an output bitmap MMR segment request. +pub struct OutputBitmapSegmentResponse { + /// The hash of the block the MMR is associated with + pub block_hash: Hash, + /// The MMR segment + pub segment: BitmapSegment, + /// The root hash of the output PMMR + pub output_root: Hash, +} + +impl Readable for OutputBitmapSegmentResponse { + fn read(reader: &mut R) -> Result { + let block_hash = Readable::read(reader)?; + let segment = Readable::read(reader)?; + let output_root = Readable::read(reader)?; + Ok(Self { + block_hash, + segment, + output_root, + }) + } +} + +impl Writeable for OutputBitmapSegmentResponse { + fn write(&self, writer: &mut W) -> Result<(), ser::Error> { + Writeable::write(&self.block_hash, writer)?; + Writeable::write(&self.segment, writer)?; + Writeable::write(&self.output_root, writer) + } +} + pub enum Message { Unknown(u8), Ping(Ping), @@ -731,6 +860,14 @@ pub enum Message { TxHashSetRequest(TxHashSetRequest), TxHashSetArchive(TxHashSetArchive), Attachment(AttachmentUpdate, Option), + GetOutputBitmapSegment(SegmentRequest), + OutputBitmapSegment(OutputBitmapSegmentResponse), + GetOutputSegment(SegmentRequest), + OutputSegment(OutputSegmentResponse), + GetRangeProofSegment(SegmentRequest), + RangeProofSegment(SegmentResponse), + GetKernelSegment(SegmentRequest), + KernelSegment(SegmentResponse), } impl fmt::Display for Message { @@ -756,6 +893,14 @@ impl fmt::Display for Message { Message::TxHashSetRequest(_) => write!(f, "tx hash set request"), Message::TxHashSetArchive(_) => write!(f, "tx hash set"), Message::Attachment(_, _) => write!(f, "attachment"), + Message::GetOutputBitmapSegment(_) => write!(f, "get output bitmap segment"), + Message::OutputBitmapSegment(_) => write!(f, "output bitmap segment"), + Message::GetOutputSegment(_) => write!(f, "get output segment"), + Message::OutputSegment(_) => write!(f, "output segment"), + Message::GetRangeProofSegment(_) => write!(f, "get range proof segment"), + Message::RangeProofSegment(_) => write!(f, "range proof segment"), + Message::GetKernelSegment(_) => write!(f, "get kernel segment"), + Message::KernelSegment(_) => write!(f, "kernel segment"), } } } diff --git a/p2p/src/peer.rs b/p2p/src/peer.rs index 6907a8527..617a9172b 100644 --- a/p2p/src/peer.rs +++ b/p2p/src/peer.rs @@ -23,8 +23,10 @@ use std::sync::Arc; use lru_cache::LruCache; use crate::chain; +use crate::chain::txhashset::BitmapChunk; use crate::conn; use crate::core::core::hash::{Hash, Hashed}; +use crate::core::core::{OutputIdentifier, Segment, SegmentIdentifier, TxKernel}; use crate::core::pow::Difficulty; use crate::core::ser::Writeable; use crate::core::{core, global}; @@ -35,6 +37,7 @@ use crate::types::{ Capabilities, ChainAdapter, Error, NetAdapter, P2PConfig, PeerAddr, PeerInfo, ReasonForBan, TxHashSetRead, }; +use crate::util::secp::pedersen::RangeProof; use chrono::prelude::{DateTime, Utc}; const MAX_TRACK_SIZE: usize = 30; @@ -565,6 +568,38 @@ impl ChainAdapter for TrackingAdapter { fn get_tmpfile_pathname(&self, tmpfile_name: String) -> PathBuf { self.adapter.get_tmpfile_pathname(tmpfile_name) } + + fn get_kernel_segment( + &self, + hash: Hash, + id: SegmentIdentifier, + ) -> Result, chain::Error> { + self.adapter.get_kernel_segment(hash, id) + } + + fn get_bitmap_segment( + &self, + hash: Hash, + id: SegmentIdentifier, + ) -> Result<(Segment, Hash), chain::Error> { + self.adapter.get_bitmap_segment(hash, id) + } + + fn get_output_segment( + &self, + hash: Hash, + id: SegmentIdentifier, + ) -> Result<(Segment, Hash), chain::Error> { + self.adapter.get_output_segment(hash, id) + } + + fn get_rangeproof_segment( + &self, + hash: Hash, + id: SegmentIdentifier, + ) -> Result, chain::Error> { + self.adapter.get_rangeproof_segment(hash, id) + } } impl NetAdapter for TrackingAdapter { diff --git a/p2p/src/peers.rs b/p2p/src/peers.rs index 9c6dd2c59..520a7c356 100644 --- a/p2p/src/peers.rs +++ b/p2p/src/peers.rs @@ -21,8 +21,10 @@ use std::sync::Arc; use rand::prelude::*; use crate::chain; +use crate::chain::txhashset::BitmapChunk; use crate::core::core; use crate::core::core::hash::{Hash, Hashed}; +use crate::core::core::{OutputIdentifier, Segment, SegmentIdentifier, TxKernel}; use crate::core::global; use crate::core::pow::Difficulty; use crate::peer::Peer; @@ -31,6 +33,7 @@ use crate::types::{ Capabilities, ChainAdapter, Error, NetAdapter, P2PConfig, PeerAddr, PeerInfo, ReasonForBan, TxHashSetRead, MAX_PEER_ADDRS, }; +use crate::util::secp::pedersen::RangeProof; use chrono::prelude::*; use chrono::Duration; @@ -625,6 +628,38 @@ impl ChainAdapter for Peers { fn get_tmpfile_pathname(&self, tmpfile_name: String) -> PathBuf { self.adapter.get_tmpfile_pathname(tmpfile_name) } + + fn get_kernel_segment( + &self, + hash: Hash, + id: SegmentIdentifier, + ) -> Result, chain::Error> { + self.adapter.get_kernel_segment(hash, id) + } + + fn get_bitmap_segment( + &self, + hash: Hash, + id: SegmentIdentifier, + ) -> Result<(Segment, Hash), chain::Error> { + self.adapter.get_bitmap_segment(hash, id) + } + + fn get_output_segment( + &self, + hash: Hash, + id: SegmentIdentifier, + ) -> Result<(Segment, Hash), chain::Error> { + self.adapter.get_output_segment(hash, id) + } + + fn get_rangeproof_segment( + &self, + hash: Hash, + id: SegmentIdentifier, + ) -> Result, chain::Error> { + self.adapter.get_rangeproof_segment(hash, id) + } } impl NetAdapter for Peers { diff --git a/p2p/src/protocol.rs b/p2p/src/protocol.rs index 15f869497..0b35ec4a6 100644 --- a/p2p/src/protocol.rs +++ b/p2p/src/protocol.rs @@ -16,7 +16,10 @@ use crate::chain; use crate::conn::MessageHandler; use crate::core::core::{hash::Hashed, CompactBlock}; -use crate::msg::{Consumed, Headers, Message, Msg, PeerAddrs, Pong, TxHashSetArchive, Type}; +use crate::msg::{ + Consumed, Headers, Message, Msg, OutputBitmapSegmentResponse, OutputSegmentResponse, PeerAddrs, + Pong, SegmentRequest, SegmentResponse, TxHashSetArchive, Type, +}; use crate::types::{AttachmentMeta, Error, NetAdapter, PeerInfo}; use chrono::prelude::Utc; use rand::{thread_rng, Rng}; @@ -287,6 +290,92 @@ impl MessageHandler for Protocol { Consumed::Attachment(Arc::new(meta), file) } + + Message::GetOutputBitmapSegment(req) => { + let SegmentRequest { + block_hash, + identifier, + } = req; + if let Ok((segment, output_root)) = + self.adapter.get_bitmap_segment(block_hash, identifier) + { + Consumed::Response(Msg::new( + Type::OutputBitmapSegment, + OutputBitmapSegmentResponse { + block_hash, + segment: segment.into(), + output_root, + }, + self.peer_info.version, + )?) + } else { + Consumed::None + } + } + Message::GetOutputSegment(req) => { + let SegmentRequest { + block_hash, + identifier, + } = req; + if let Ok((segment, output_bitmap_root)) = + self.adapter.get_output_segment(block_hash, identifier) + { + Consumed::Response(Msg::new( + Type::OutputSegment, + OutputSegmentResponse { + response: SegmentResponse { + block_hash, + segment, + }, + output_bitmap_root, + }, + self.peer_info.version, + )?) + } else { + Consumed::None + } + } + Message::GetRangeProofSegment(req) => { + let SegmentRequest { + block_hash, + identifier, + } = req; + if let Ok(segment) = self.adapter.get_rangeproof_segment(block_hash, identifier) { + Consumed::Response(Msg::new( + Type::RangeProofSegment, + SegmentResponse { + block_hash, + segment, + }, + self.peer_info.version, + )?) + } else { + Consumed::None + } + } + Message::GetKernelSegment(req) => { + let SegmentRequest { + block_hash, + identifier, + } = req; + if let Ok(segment) = self.adapter.get_kernel_segment(block_hash, identifier) { + Consumed::Response(Msg::new( + Type::KernelSegment, + SegmentResponse { + block_hash, + segment, + }, + self.peer_info.version, + )?) + } else { + Consumed::None + } + } + Message::OutputBitmapSegment(_) + | Message::OutputSegment(_) + | Message::RangeProofSegment(_) + | Message::KernelSegment(_) => Consumed::None, + Message::Unknown(_) => Consumed::None, }; Ok(consumed) diff --git a/p2p/src/serv.rs b/p2p/src/serv.rs index 8760276f3..fd0abf090 100644 --- a/p2p/src/serv.rs +++ b/p2p/src/serv.rs @@ -21,8 +21,10 @@ use std::thread; use std::time::Duration; use crate::chain; +use crate::chain::txhashset::BitmapChunk; use crate::core::core; use crate::core::core::hash::Hash; +use crate::core::core::{OutputIdentifier, Segment, SegmentIdentifier, TxKernel}; use crate::core::global; use crate::core::pow::Difficulty; use crate::handshake::Handshake; @@ -33,6 +35,7 @@ use crate::types::{ Capabilities, ChainAdapter, Error, NetAdapter, P2PConfig, PeerAddr, PeerInfo, ReasonForBan, TxHashSetRead, }; +use crate::util::secp::pedersen::RangeProof; use crate::util::StopState; use chrono::prelude::{DateTime, Utc}; @@ -375,6 +378,38 @@ impl ChainAdapter for DummyAdapter { fn get_tmpfile_pathname(&self, _tmpfile_name: String) -> PathBuf { unimplemented!() } + + fn get_kernel_segment( + &self, + _hash: Hash, + _id: SegmentIdentifier, + ) -> Result, chain::Error> { + unimplemented!() + } + + fn get_bitmap_segment( + &self, + _hash: Hash, + _id: SegmentIdentifier, + ) -> Result<(Segment, Hash), chain::Error> { + unimplemented!() + } + + fn get_output_segment( + &self, + _hash: Hash, + _id: SegmentIdentifier, + ) -> Result<(Segment, Hash), chain::Error> { + unimplemented!() + } + + fn get_rangeproof_segment( + &self, + _hash: Hash, + _id: SegmentIdentifier, + ) -> Result, chain::Error> { + unimplemented!() + } } impl NetAdapter for DummyAdapter { diff --git a/p2p/src/types.rs b/p2p/src/types.rs index faf2d1dfe..476a954e1 100644 --- a/p2p/src/types.rs +++ b/p2p/src/types.rs @@ -28,12 +28,15 @@ use serde::{Deserialize, Deserializer}; use grin_store; use crate::chain; +use crate::chain::txhashset::BitmapChunk; use crate::core::core; use crate::core::core::hash::Hash; +use crate::core::core::{OutputIdentifier, Segment, SegmentIdentifier, TxKernel}; use crate::core::global; use crate::core::pow::Difficulty; use crate::core::ser::{self, ProtocolVersion, Readable, Reader, Writeable, Writer}; use crate::msg::PeerAddrs; +use crate::util::secp::pedersen::RangeProof; use crate::util::RwLock; /// Maximum number of block headers a peer should ever send @@ -642,6 +645,30 @@ pub trait ChainAdapter: Sync + Send { /// Get a tmp file path in above specific tmp dir (create tmp dir if not exist) /// Delete file if tmp file already exists fn get_tmpfile_pathname(&self, tmpfile_name: String) -> PathBuf; + + fn get_kernel_segment( + &self, + hash: Hash, + id: SegmentIdentifier, + ) -> Result, chain::Error>; + + fn get_bitmap_segment( + &self, + hash: Hash, + id: SegmentIdentifier, + ) -> Result<(Segment, Hash), chain::Error>; + + fn get_output_segment( + &self, + hash: Hash, + id: SegmentIdentifier, + ) -> Result<(Segment, Hash), chain::Error>; + + fn get_rangeproof_segment( + &self, + hash: Hash, + id: SegmentIdentifier, + ) -> Result, chain::Error>; } /// Additional methods required by the protocol that don't need to be diff --git a/servers/src/common/adapters.rs b/servers/src/common/adapters.rs index 662c282c0..05daf2858 100644 --- a/servers/src/common/adapters.rs +++ b/servers/src/common/adapters.rs @@ -22,6 +22,7 @@ use std::sync::{Arc, Weak}; use std::thread; use std::time::Instant; +use crate::chain::txhashset::BitmapChunk; use crate::chain::{ self, BlockStatus, ChainAdapter, Options, SyncState, SyncStatus, TxHashsetDownloadStats, }; @@ -30,17 +31,27 @@ use crate::common::types::{ChainValidationMode, DandelionEpoch, ServerConfig}; use crate::core::core::hash::{Hash, Hashed}; use crate::core::core::transaction::Transaction; use crate::core::core::verifier_cache::VerifierCache; -use crate::core::core::{BlockHeader, BlockSums, CompactBlock, Inputs, OutputIdentifier}; +use crate::core::core::{ + BlockHeader, BlockSums, CompactBlock, Inputs, OutputIdentifier, Segment, SegmentIdentifier, + TxKernel, +}; use crate::core::pow::Difficulty; use crate::core::ser::ProtocolVersion; use crate::core::{core, global}; use crate::p2p; use crate::p2p::types::PeerInfo; use crate::pool::{self, BlockChain, PoolAdapter}; +use crate::util::secp::pedersen::RangeProof; use crate::util::OneTime; use chrono::prelude::*; use chrono::Duration; use rand::prelude::*; +use std::ops::Range; + +const KERNEL_SEGMENT_HEIGHT_RANGE: Range = 9..14; +const BITMAP_SEGMENT_HEIGHT_RANGE: Range = 9..14; +const OUTPUT_SEGMENT_HEIGHT_RANGE: Range = 11..16; +const RANGEPROOF_SEGMENT_HEIGHT_RANGE: Range = 7..12; /// Implementation of the NetAdapter for the . Gets notified when new /// blocks and transactions are received and forwards to the chain and pool @@ -477,6 +488,66 @@ where fn get_tmpfile_pathname(&self, tmpfile_name: String) -> PathBuf { self.chain().get_tmpfile_pathname(tmpfile_name) } + + fn get_kernel_segment( + &self, + hash: Hash, + id: SegmentIdentifier, + ) -> Result, chain::Error> { + if !KERNEL_SEGMENT_HEIGHT_RANGE.contains(&id.height) { + return Err(chain::ErrorKind::InvalidSegmentHeight.into()); + } + let segmenter = self.chain().segmenter()?; + if segmenter.header().hash() != hash { + return Err(chain::ErrorKind::SegmenterHeaderMismatch.into()); + } + segmenter.kernel_segment(id) + } + + fn get_bitmap_segment( + &self, + hash: Hash, + id: SegmentIdentifier, + ) -> Result<(Segment, Hash), chain::Error> { + if !BITMAP_SEGMENT_HEIGHT_RANGE.contains(&id.height) { + return Err(chain::ErrorKind::InvalidSegmentHeight.into()); + } + let segmenter = self.chain().segmenter()?; + if segmenter.header().hash() != hash { + return Err(chain::ErrorKind::SegmenterHeaderMismatch.into()); + } + segmenter.bitmap_segment(id) + } + + fn get_output_segment( + &self, + hash: Hash, + id: SegmentIdentifier, + ) -> Result<(Segment, Hash), chain::Error> { + if !OUTPUT_SEGMENT_HEIGHT_RANGE.contains(&id.height) { + return Err(chain::ErrorKind::InvalidSegmentHeight.into()); + } + let segmenter = self.chain().segmenter()?; + if segmenter.header().hash() != hash { + return Err(chain::ErrorKind::SegmenterHeaderMismatch.into()); + } + segmenter.output_segment(id) + } + + fn get_rangeproof_segment( + &self, + hash: Hash, + id: SegmentIdentifier, + ) -> Result, chain::Error> { + if RANGEPROOF_SEGMENT_HEIGHT_RANGE.contains(&id.height) { + return Err(chain::ErrorKind::InvalidSegmentHeight.into()); + } + let segmenter = self.chain().segmenter()?; + if segmenter.header().hash() != hash { + return Err(chain::ErrorKind::SegmenterHeaderMismatch.into()); + } + segmenter.rangeproof_segment(id) + } } impl NetToChainAdapter