Store block header hash by input and output commitment (#104)

* initial pass at indexing block header hashes by commitments
* check block height matches currently indexed block height
when getting block header for an output commit
* add some test coverage
* document get_block_header_by_output_commit

chain/src/chain.rs

@@ -23,7 +23,7 @@ use secp::pedersen::Commitment;
 use core::core::{Block, BlockHeader, Output};
 use core::core::target::Difficulty;
 use core::core::hash::Hash;
-use grin_store;
+use grin_store::Error::NotFoundErr;
 use pipe;
 use store;
 use types::*;
@@ -35,13 +35,13 @@ const MAX_ORPHANS: usize = 20;
 /// Helper macro to transform a Result into an Option with None in case
 /// of error
 macro_rules! none_err {
-  ($trying:expr) => {{
+    ($trying:expr) => {{
-    let tried = $trying;
+        let tried = $trying;
-    if let Err(_) = tried {
+        if let Err(_) = tried {
-      return None;
+            return None;
-    }
+        }
-    tried.unwrap()
+        tried.unwrap()
-  }}
+    }}
 }
 
 /// Facade to the blockchain block processing pipeline and storage. Provides
@@ -71,7 +71,7 @@ impl Chain {
 		let chain_store = store::ChainKVStore::new(db_root).unwrap();
 		match chain_store.head() {
 			Ok(_) => {true},
-			Err(grin_store::Error::NotFoundErr) => false,
+			Err(NotFoundErr) => false,
 			Err(_) => false,
 		}
 	}
@@ -92,7 +92,7 @@ impl Chain {
 		// check if we have a head in store, otherwise the genesis block is it
 		let head = match chain_store.head() {
 			Ok(tip) => tip,
-			Err(grin_store::Error::NotFoundErr) => {
+			Err(NotFoundErr) => {
 				if let None = gen_block {
 					return Err(Error::GenesisBlockRequired);
 				}
@@ -112,6 +112,7 @@ impl Chain {
         // TODO - confirm this was safe to remove based on code above?
 		// let head = chain_store.head()?;
 
+
 		Ok(Chain {
 			store: Arc::new(chain_store),
 			adapter: adapter,
@@ -210,7 +211,7 @@ impl Chain {
 		}
 	}
 
-	/// Gets an unspent output from its commitment. With return None if the
+    /// Gets an unspent output from its commitment. With return None if the
 	/// output
 	/// doesn't exist or has been spent. This querying is done in a way that's
 	/// constistent with the current chain state and more specifically the
@@ -273,6 +274,13 @@ impl Chain {
 		)
 	}
 
+    /// Gets the block header by the provided output commitment
+    pub fn get_block_header_by_output_commit(&self, commit: &Commitment) -> Result<BlockHeader, Error> {
+        self.store.get_block_header_by_output_commit(commit).map_err(
+            &Error::StoreErr,
+        )
+    }
+
 	/// Get the tip of the header chain
 	pub fn get_header_head(&self) -> Result<Tip, Error> {
 		self.store.get_header_head().map_err(&Error::StoreErr)

chain/src/store.rs

@@ -33,6 +33,7 @@ const HEAD_PREFIX: u8 = 'H' as u8;
 const HEADER_HEAD_PREFIX: u8 = 'I' as u8;
 const HEADER_HEIGHT_PREFIX: u8 = '8' as u8;
 const OUTPUT_COMMIT_PREFIX: u8 = 'o' as u8;
+const HEADER_BY_OUTPUT_PREFIX: u8 = 'p' as u8;
 
 /// An implementation of the ChainStore trait backed by a simple key-value
 /// store.
@@ -54,8 +55,7 @@ impl ChainStore for ChainKVStore {
 	}
 
 	fn head_header(&self) -> Result<BlockHeader, Error> {
-		let head: Tip = try!(option_to_not_found(self.db.get_ser(&vec![HEAD_PREFIX])));
+		self.get_block_header(&try!(self.head()).last_block_h)
-		self.get_block_header(&head.last_block_h)
 	}
 
 	fn save_head(&self, t: &Tip) -> Result<(), Error> {
@@ -83,9 +83,7 @@ impl ChainStore for ChainKVStore {
 	}
 
 	fn get_block_header(&self, h: &Hash) -> Result<BlockHeader, Error> {
-		option_to_not_found(self.db.get_ser(
+		option_to_not_found(self.db.get_ser(&to_key(BLOCK_HEADER_PREFIX, &mut h.to_vec())))
-			&to_key(BLOCK_HEADER_PREFIX, &mut h.to_vec()),
-		))
 	}
 
 	fn check_block_exists(&self, h: &Hash) -> Result<bool, Error> {
@@ -97,27 +95,49 @@ impl ChainStore for ChainKVStore {
 		let mut batch = self.db
 			.batch()
 			.put_ser(&to_key(BLOCK_PREFIX, &mut b.hash().to_vec())[..], b)?
-			.put_ser(
+			.put_ser(&to_key(BLOCK_HEADER_PREFIX, &mut b.hash().to_vec())[..], &b.header)?;
-				&to_key(BLOCK_HEADER_PREFIX, &mut b.hash().to_vec())[..],
-				&b.header,
-			)?;
 
 		// saving the full output under its hash, as well as a commitment to hash index
 		for out in &b.outputs {
-			let mut out_bytes = out.commit.as_ref().to_vec();
+			batch = batch
-			batch = batch.put_ser(
+				.put_ser(&to_key(OUTPUT_COMMIT_PREFIX, &mut out.commitment().as_ref().to_vec())[..], out)?
-				&to_key(OUTPUT_COMMIT_PREFIX, &mut out_bytes)[..],
+				.put_ser(&to_key(HEADER_BY_OUTPUT_PREFIX, &mut out.commitment().as_ref().to_vec())[..], &b.hash())?;
-				out,
-			)?;
 		}
 		batch.write()
 	}
 
+	// lookup the block header hash by output commitment
+	// lookup the block header based on this hash
+	// to check the chain is correct compare this block header to
+	// the block header currently indexed at the relevant block height (tbd if actually necessary)
+	//
+	// NOTE: This index is not exhaustive.
+	// This node may not have seen this full block, so may not have populated the index.
+	// Block headers older than some threshold (2 months?) will not necessarily be included
+	// in this index.
+	//
+	fn get_block_header_by_output_commit(&self, commit: &Commitment) -> Result<BlockHeader, Error> {
+		let block_hash = self.db.get_ser(&to_key(
+			HEADER_BY_OUTPUT_PREFIX,
+			&mut commit.as_ref().to_vec(),
+		))?;
+
+		match block_hash {
+			Some(hash) => {
+				let block_header = self.get_block_header(&hash)?;
+				let header_at_height = self.get_header_by_height(block_header.height)?;
+				if block_header.hash() == header_at_height.hash() {
+					Ok(block_header)
+				} else {
+					Err(Error::NotFoundErr)
+				}
+			},
+			None => Err(Error::NotFoundErr)
+		}
+	}
+
 	fn save_block_header(&self, bh: &BlockHeader) -> Result<(), Error> {
-		self.db.put_ser(
+		self.db.put_ser(&to_key(BLOCK_HEADER_PREFIX, &mut bh.hash().to_vec())[..], bh)
-			&to_key(BLOCK_HEADER_PREFIX, &mut bh.hash().to_vec())[..],
-			bh,
-		)
 	}
 
 	fn get_header_by_height(&self, height: u64) -> Result<BlockHeader, Error> {
@@ -131,6 +151,8 @@ impl ChainStore for ChainKVStore {
 		)))
 	}
 
+	// TODO - this looks identical to get_output_by_commit above
+	// TODO - are we sure this returns a hash correctly?
 	fn has_output_commit(&self, commit: &Commitment) -> Result<Hash, Error> {
 		option_to_not_found(self.db.get_ser(&to_key(
 			OUTPUT_COMMIT_PREFIX,
@@ -138,6 +160,9 @@ impl ChainStore for ChainKVStore {
 		)))
 	}
 
+	/// Maintain consistency of the "header_by_height" index by traversing back through the
+	/// current chain and updating "header_by_height" until we reach a block_header
+	/// that is consistent with its height (everything prior to this will be consistent)
 	fn setup_height(&self, bh: &BlockHeader) -> Result<(), Error> {
 		self.db.put_ser(
 			&u64_to_key(HEADER_HEIGHT_PREFIX, bh.height),

chain/src/types.rs

@@ -184,8 +184,11 @@ pub trait ChainStore: Send + Sync {
 	/// Gets an output by its commitment
 	fn get_output_by_commit(&self, commit: &Commitment) -> Result<Output, store::Error>;
 
-	/// Checks whether an output commitment exists and returns the output hash
+    /// Checks whether an output commitment exists and returns the output hash
-	fn has_output_commit(&self, commit: &Commitment) -> Result<Hash, store::Error>;
+    fn has_output_commit(&self, commit: &Commitment) -> Result<Hash, store::Error>;
+
+    /// Gets a block_header for the given input commit
+    fn get_block_header_by_output_commit(&self, commit: &Commitment) -> Result<BlockHeader, store::Error>;
 
 	/// Saves the provided block header at the corresponding height. Also check
 	/// the consistency of the height chain in store by assuring previous

chain/tests/mine_simple_chain.rs

@@ -86,6 +86,26 @@ fn mine_empty_chain() {
 		let head = chain.head().unwrap();
 		assert_eq!(head.height, n);
 		assert_eq!(head.last_block_h, bhash);
+
+        // now check the block_header of the head
+        let header = chain.head_header().unwrap();
+        assert_eq!(header.height, n);
+        assert_eq!(header.hash(), bhash);
+
+        // now check the block itself
+        let block = chain.get_block(&header.hash()).unwrap();
+        assert_eq!(block.header.height, n);
+        assert_eq!(block.hash(), bhash);
+        assert_eq!(block.outputs.len(), 1);
+
+        // now check the block height index
+        let header_by_height = chain.get_header_by_height(n).unwrap();
+        assert_eq!(header_by_height.hash(), bhash);
+
+        // now check the header output index
+        let output = block.outputs[0];
+        let header_by_output_commit = chain.get_block_header_by_output_commit(&output.commitment()).unwrap();
+        assert_eq!(header_by_output_commit.hash(), bhash);
 	}
 }
 

chain/tests/store_indices.rs

@@ -0,0 +1,56 @@
+// Copyright 2017 The Grin Developers
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+extern crate env_logger;
+extern crate grin_chain as chain;
+extern crate grin_core as core;
+extern crate rand;
+extern crate secp256k1zkp as secp;
+
+use std::fs;
+
+use chain::ChainStore;
+use core::core::hash::Hashed;
+use core::core::{Block, BlockHeader};
+use secp::key;
+
+fn clean_output_dir(dir_name: &str) {
+	let _ = fs::remove_dir_all(dir_name);
+}
+
+#[test]
+fn test_various_store_indices() {
+	let _ = env_logger::init();
+	clean_output_dir(".grin");
+
+	let chain_store = &chain::store::ChainKVStore::new(".grin".to_string()).unwrap() as &ChainStore;
+
+	let block = Block::new(&BlockHeader::default(), vec![], key::ONE_KEY).unwrap();
+	let commit = block.outputs[0].commitment();
+	let block_hash = block.hash();
+
+	chain_store.save_block(&block).unwrap();
+	chain_store.setup_height(&block.header).unwrap();
+
+	let block_header = chain_store.get_block_header(&block_hash).unwrap();
+	assert_eq!(block_header.hash(), block_hash);
+
+	let block_header = chain_store.get_header_by_height(1).unwrap();
+	assert_eq!(block_header.hash(), block_hash);
+
+	let block_header = chain_store
+		.get_block_header_by_output_commit(&commit)
+		.unwrap();
+	assert_eq!(block_header.hash(), block_hash);
+}

core/src/ser.rs

@@ -386,6 +386,12 @@ impl Writeable for [u8; 4] {
 	}
 }
 
+impl Writeable for Vec<u8> {
+	fn write<W: Writer>(&self, writer: &mut W) -> Result<(), Error> {
+		writer.write_fixed_bytes(self)
+	}
+}
+
 /// Useful marker trait on types that can be sized byte slices
 pub trait AsFixedBytes: Sized + AsRef<[u8]> {
 	/// The length in bytes