Added support for a header chain, in addition to the main chain. Main chain must strictly follow header chain but header chain is allowed to be ahead (for sync).

chain/src/lib.rs

@@ -38,4 +38,4 @@ pub mod types;
 // Re-export the base interface
 
 pub use types::{ChainStore, Tip, ChainAdapter};
-pub use pipe::{NONE, process_block};
+pub use pipe::{SYNC, NONE, process_block, process_block_header, Error};

chain/src/pipe.rs

@@ -14,6 +14,7 @@
 
 //! Implementation of the chain block acceptance (or refusal) pipeline.
 
+use std::convert::From;
 use std::sync::{Arc, Mutex};
 
 use secp;
@@ -24,6 +25,7 @@ use core::core::hash::{Hash, Hashed};
 use core::core::target::Difficulty;
 use core::core::{BlockHeader, Block, Proof};
 use core::pow;
+use core::ser;
 use types;
 use types::{Tip, ChainStore, ChainAdapter, NoopAdapter};
 use store;
@@ -34,6 +36,8 @@ bitflags! {
     const NONE = 0b00000001,
     /// Runs without checking the Proof of Work, mostly to make testing easier.
     const SKIP_POW = 0b00000010,
+    /// Adds block while in syncing mode.
+    const SYNC = 0b00000100,
   }
 }
 
@@ -66,6 +70,18 @@ pub enum Error {
 	InvalidBlockHeight,
 	/// Internal issue when trying to save or load data from store
 	StoreErr(types::Error),
+	SerErr(ser::Error),
+}
+
+impl From<types::Error> for Error {
+	fn from(e: types::Error) -> Error {
+		Error::StoreErr(e)
+	}
+}
+impl From<ser::Error> for Error {
+	fn from(e: ser::Error) -> Error {
+		Error::SerErr(e)
+	}
 }
 
 /// Runs the block processing pipeline, including validation and finding a
@@ -79,7 +95,7 @@ pub fn process_block(b: &Block,
 	// TODO should just take a promise for a block with a full header so we don't
 	// spend resources reading the full block when its header is invalid
 
-	let head = try!(store.head().map_err(&Error::StoreErr));
+	let head = store.head().map_err(&Error::StoreErr)?;
 
 	let mut ctx = BlockContext {
 		opts: opts,
@@ -92,7 +108,11 @@ pub fn process_block(b: &Block,
 	      b.hash(),
 	      b.header.height);
 	try!(check_known(b.hash(), &mut ctx));
-	try!(validate_header(&b, &mut ctx));
+
+	if !ctx.opts.intersects(SYNC) {
+		// in sync mode, the header has already been validated
+		try!(validate_header(&b.header, &mut ctx));
+	}
 	try!(validate_block(b, &mut ctx));
 	info!("Block at {} with hash {} is valid, going to save and append.",
 	      b.header.height,
@@ -102,6 +122,31 @@ pub fn process_block(b: &Block,
 	update_head(b, &mut ctx)
 }
 
+pub fn process_block_header(bh: &BlockHeader,
+                            store: Arc<ChainStore>,
+                            adapter: Arc<ChainAdapter>,
+                            opts: Options)
+                            -> Result<Option<Tip>, Error> {
+
+	let head = store.get_header_head().map_err(&Error::StoreErr)?;
+
+	let mut ctx = BlockContext {
+		opts: opts,
+		store: store,
+		adapter: adapter,
+		head: head,
+	};
+
+	info!("Starting validation pipeline for block header {} at {}.",
+	      bh.hash(),
+	      bh.height);
+	try!(check_known(bh.hash(), &mut ctx));
+	try!(validate_header(&bh, &mut ctx));
+	try!(add_block_header(bh, &mut ctx));
+	// TODO a global lock should be set before that step or even earlier
+	update_header_head(bh, &mut ctx)
+}
+
 /// Quick in-memory check to fast-reject any block we've already handled
 /// recently. Keeps duplicates from the network in check.
 fn check_known(bh: Hash, ctx: &mut BlockContext) -> Result<(), Error> {
@@ -116,8 +161,7 @@ fn check_known(bh: Hash, ctx: &mut BlockContext) -> Result<(), Error> {
 /// to make it as cheap as possible. The different validations are also
 /// arranged by order of cost to have as little DoS surface as possible.
 /// TODO require only the block header (with length information)
-fn validate_header(b: &Block, ctx: &mut BlockContext) -> Result<(), Error> {
+fn validate_header(header: &BlockHeader, ctx: &mut BlockContext) -> Result<(), Error> {
-	let header = &b.header;
 	if header.height > ctx.head.height + 1 {
 		// TODO actually handle orphans and add them to a size-limited set
 		return Err(Error::Unfit("orphan".to_string()));
@@ -157,7 +201,7 @@ fn validate_header(b: &Block, ctx: &mut BlockContext) -> Result<(), Error> {
 		if header.cuckoo_len != cuckoo_sz {
 			return Err(Error::WrongCuckooSize);
 		}
-		if !pow::verify(b) {
+		if !pow::verify(header) {
 			return Err(Error::InvalidPow);
 		}
 	}
@@ -169,7 +213,10 @@ fn validate_header(b: &Block, ctx: &mut BlockContext) -> Result<(), Error> {
 fn validate_block(b: &Block, ctx: &mut BlockContext) -> Result<(), Error> {
 	let curve = secp::Secp256k1::with_caps(secp::ContextFlag::Commit);
 	try!(b.verify(&curve).map_err(&Error::InvalidBlockProof));
-	// TODO check every input exists
+
+	if !ctx.opts.intersects(SYNC) {
+		// TODO check every input exists as a UTXO using the UXTO index
+	}
 	Ok(())
 }
 
@@ -183,15 +230,24 @@ fn add_block(b: &Block, ctx: &mut BlockContext) -> Result<(), Error> {
 	Ok(())
 }
 
+/// Officially adds the block header to our header chain.
+fn add_block_header(bh: &BlockHeader, ctx: &mut BlockContext) -> Result<(), Error> {
+	ctx.store.save_block_header(bh).map_err(&Error::StoreErr);
+
+	Ok(())
+}
+
 /// Directly updates the head if we've just appended a new block to it or handle
 /// the situation where we've just added enough work to have a fork with more
 /// work than the head.
 fn update_head(b: &Block, ctx: &mut BlockContext) -> Result<Option<Tip>, Error> {
 	// if we made a fork with more work than the head (which should also be true
 	// when extending the head), update it
-	let tip = Tip::from_block(b);
+	let tip = Tip::from_block(&b.header);
 	if tip.total_difficulty > ctx.head.total_difficulty {
-		try!(ctx.store.save_head(&tip).map_err(&Error::StoreErr));
+		ctx.store.setup_height(&b.header).map_err(&Error::StoreErr)?;
+		ctx.store.save_head(&tip).map_err(&Error::StoreErr)?;
+
 		ctx.head = tip.clone();
 		info!("Updated head to {} at {}.", b.hash(), b.header.height);
 		Ok(Some(tip))
@@ -199,3 +255,23 @@ fn update_head(b: &Block, ctx: &mut BlockContext) -> Result<Option<Tip>, Error>
 		Ok(None)
 	}
 }
+
+/// Directly updates the head if we've just appended a new block to it or handle
+/// the situation where we've just added enough work to have a fork with more
+/// work than the head.
+fn update_header_head(bh: &BlockHeader, ctx: &mut BlockContext) -> Result<Option<Tip>, Error> {
+	// if we made a fork with more work than the head (which should also be true
+	// when extending the head), update it
+	let tip = Tip::from_block(bh);
+	if tip.total_difficulty > ctx.head.total_difficulty {
+		ctx.store.save_header_head(&tip).map_err(&Error::StoreErr)?;
+
+		ctx.head = tip.clone();
+		info!("Updated block header head to {} at {}.",
+		      bh.hash(),
+		      bh.height);
+		Ok(Some(tip))
+	} else {
+		Ok(None)
+	}
+}

chain/src/store.rs

@@ -17,7 +17,7 @@
 use byteorder::{WriteBytesExt, BigEndian};
 
 use types::*;
-use core::core::hash::Hash;
+use core::core::hash::{Hash, Hashed};
 use core::core::{Block, BlockHeader};
 use grin_store;
 
@@ -28,6 +28,8 @@ const SEP: u8 = ':' as u8;
 const BLOCK_HEADER_PREFIX: u8 = 'h' as u8;
 const BLOCK_PREFIX: u8 = 'b' as u8;
 const HEAD_PREFIX: u8 = 'H' as u8;
+const HEADER_HEAD_PREFIX: u8 = 'I' as u8;
+const HEADER_HEIGHT_PREFIX: u8 = '8' as u8;
 
 /// An implementation of the ChainStore trait backed by a simple key-value
 /// store.
@@ -53,22 +55,78 @@ impl ChainStore for ChainKVStore {
 		self.get_block_header(&head.last_block_h)
 	}
 
-	fn save_block(&self, b: &Block) -> Result<(), Error> {
+	fn save_head(&self, t: &Tip) -> Result<(), Error> {
-		try!(self.db
-			.put_ser(&to_key(BLOCK_PREFIX, &mut b.hash().to_vec())[..], b)
-			.map_err(&to_store_err));
 		self.db
-			.put_ser(&to_key(BLOCK_HEADER_PREFIX, &mut b.hash().to_vec())[..],
+			.batch()
-			         &b.header)
+			.put_ser(&vec![HEAD_PREFIX], t)?
+			.put_ser(&vec![HEADER_HEAD_PREFIX], t)?
+			.write()
 			.map_err(&to_store_err)
 	}
 
+	fn get_header_head(&self) -> Result<Tip, Error> {
+		option_to_not_found(self.db.get_ser(&vec![HEADER_HEAD_PREFIX]))
+	}
+
+	fn save_header_head(&self, t: &Tip) -> Result<(), Error> {
+		self.db.put_ser(&vec![HEADER_HEAD_PREFIX], t).map_err(&to_store_err)
+	}
+
+	fn get_block(&self, h: &Hash) -> Result<Block, Error> {
+		option_to_not_found(self.db.get_ser(&to_key(BLOCK_PREFIX, &mut h.to_vec())))
+	}
+
 	fn get_block_header(&self, h: &Hash) -> Result<BlockHeader, Error> {
 		option_to_not_found(self.db.get_ser(&to_key(BLOCK_HEADER_PREFIX, &mut h.to_vec())))
 	}
 
-	fn save_head(&self, t: &Tip) -> Result<(), Error> {
+	fn save_block(&self, b: &Block) -> Result<(), Error> {
-		self.db.put_ser(&vec![HEAD_PREFIX], t).map_err(&to_store_err)
+		self.db
+			.batch()
+			.put_ser(&to_key(BLOCK_PREFIX, &mut b.hash().to_vec())[..], b)?
+			.put_ser(&to_key(BLOCK_HEADER_PREFIX, &mut b.hash().to_vec())[..],
+			         &b.header)?
+			.write()
+			.map_err(&to_store_err)
+	}
+
+	fn save_block_header(&self, bh: &BlockHeader) -> Result<(), Error> {
+		self.db
+			.put_ser(&to_key(BLOCK_HEADER_PREFIX, &mut bh.hash().to_vec())[..],
+			         bh)
+			.map_err(&to_store_err)
+	}
+
+	fn get_header_by_height(&self, height: u64) -> Result<BlockHeader, Error> {
+		option_to_not_found(self.db.get_ser(&u64_to_key(HEADER_HEIGHT_PREFIX, height)))
+	}
+
+	fn setup_height(&self, bh: &BlockHeader) -> Result<(), Error> {
+		self.db.put_ser(&u64_to_key(HEADER_HEIGHT_PREFIX, bh.height), bh).map_err(&to_store_err)?;
+
+		let mut prev_h = bh.previous;
+		let mut prev_height = bh.height - 1;
+		while prev_height > 0 {
+			let prev = self.get_header_by_height(prev_height)?;
+			if prev.hash() != prev_h {
+				let real_prev = self.get_block_header(&prev_h)?;
+				self.db
+					.put_ser(&u64_to_key(HEADER_HEIGHT_PREFIX, real_prev.height),
+					         &real_prev)
+					.map_err(&to_store_err)?;
+				prev_h = real_prev.previous;
+				prev_height = real_prev.height - 1;
+			} else {
+				break;
+			}
+		}
+		Ok(())
+	}
+}
+
+impl From<grin_store::Error> for Error {
+	fn from(e: grin_store::Error) -> Error {
+		Error::StorageErr(e.to_string())
 	}
 }
 
@@ -78,6 +136,14 @@ fn to_key(prefix: u8, val: &mut Vec<u8>) -> &mut Vec<u8> {
 	val
 }
 
+fn u64_to_key<'a>(prefix: u8, val: u64) -> Vec<u8> {
+	let mut u64_vec = vec![];
+	u64_vec.write_u64::<BigEndian>(val).unwrap();
+	u64_vec.insert(0, SEP);
+	u64_vec.insert(0, prefix);
+	u64_vec
+}
+
 fn to_store_err(e: grin_store::Error) -> Error {
 	Error::StorageErr(format!("{:?}", e))
 }

chain/src/types.rs

@@ -15,7 +15,7 @@
 //! Base types that the block chain pipeline requires.
 
 use core::core::{Block, BlockHeader};
-use core::core::hash::Hash;
+use core::core::hash::{Hash, Hashed};
 use core::core::target::Difficulty;
 use core::ser;
 
@@ -47,12 +47,12 @@ impl Tip {
 	}
 
 	/// Append a new block to this tip, returning a new updated tip.
-	pub fn from_block(b: &Block) -> Tip {
+	pub fn from_block(bh: &BlockHeader) -> Tip {
 		Tip {
-			height: b.header.height,
+			height: bh.height,
-			last_block_h: b.hash(),
+			last_block_h: bh.hash(),
-			prev_block_h: b.header.previous,
+			prev_block_h: bh.previous,
-			total_difficulty: b.header.total_difficulty.clone(),
+			total_difficulty: bh.total_difficulty.clone(),
 		}
 	}
 }
@@ -99,14 +99,35 @@ pub trait ChainStore: Send + Sync {
 	/// Block header for the chain head
 	fn head_header(&self) -> Result<BlockHeader, Error>;
 
+	/// Save the provided tip as the current head of our chain
+	fn save_head(&self, t: &Tip) -> Result<(), Error>;
+
+	/// Gets a block header by hash
+	fn get_block(&self, h: &Hash) -> Result<Block, Error>;
+
 	/// Gets a block header by hash
 	fn get_block_header(&self, h: &Hash) -> Result<BlockHeader, Error>;
 
 	/// Save the provided block in store
 	fn save_block(&self, b: &Block) -> Result<(), Error>;
 
-	/// Save the provided tip as the current head of our chain
+	/// Save the provided block header in store
-	fn save_head(&self, t: &Tip) -> Result<(), Error>;
+	fn save_block_header(&self, bh: &BlockHeader) -> Result<(), Error>;
+
+	/// Get the tip of the header chain
+	fn get_header_head(&self) -> Result<Tip, Error>;
+
+	/// Save the provided tip as the current head of the block header chain
+	fn save_header_head(&self, t: &Tip) -> Result<(), Error>;
+
+	/// Gets the block header at the provided height
+	fn get_header_by_height(&self, height: u64) -> Result<BlockHeader, Error>;
+
+	/// Saves the provided block header at the corresponding height. Also check
+	/// the consistency of the height chain in store by assuring previous
+	/// headers
+	/// are also at their respective heights.
+	fn setup_height(&self, bh: &BlockHeader) -> Result<(), Error>;
 }
 
 /// Bridge between the chain pipeline and the rest of the system. Handles

chain/tests/mine_simple_chain.rs

@@ -40,7 +40,7 @@ fn mine_empty_chain() {
 	let mut gen = grin_core::genesis::genesis();
 	gen.header.cuckoo_len = 16;
 	let diff = gen.header.difficulty.clone();
-	pow::pow(&mut gen, diff).unwrap();
+	pow::pow(&mut gen.header, diff).unwrap();
 	store.save_block(&gen).unwrap();
 
 	// setup a new head tip
@@ -64,7 +64,7 @@ fn mine_empty_chain() {
 		                                             prev.header.cuckoo_len);
 		b.header.difficulty = difficulty.clone();
 
-		pow::pow(&mut b, difficulty).unwrap();
+		pow::pow(&mut b.header, difficulty).unwrap();
 		grin_chain::pipe::process_block(&b,
 		                                arc_store.clone(),
 		                                adapter.clone(),
@@ -93,6 +93,7 @@ fn mine_forks() {
 	// setup a new head tip
 	let tip = Tip::new(gen.hash());
 	store.save_head(&tip).unwrap();
+  println!("WRITTEN HEAD");
 
 	// mine and add a few blocks
 	let mut prev = gen;