hashed switch commitments - BLAKE2(bJ, r) (#648)

* wip

* derive switch commit hash key from extended key in wallet

* fix failing pool test

core/src/core/block.rs

@@ -775,7 +775,11 @@ impl Block {
 	) -> Result<(Output, TxKernel), keychain::Error> {
 		let commit = keychain.commit(reward(fees), key_id)?;
 		let switch_commit = keychain.switch_commit(key_id)?;
-		let switch_commit_hash = SwitchCommitHash::from_switch_commit(switch_commit);
+		let switch_commit_hash = SwitchCommitHash::from_switch_commit(
+			switch_commit,
+			keychain,
+			key_id,
+		);
 
 		trace!(
 			LOGGER,

core/src/core/build.rs

@@ -96,7 +96,11 @@ pub fn output(value: u64, key_id: Identifier) -> Box<Append> {
 
 		let commit = build.keychain.commit(value, &key_id).unwrap();
 		let switch_commit = build.keychain.switch_commit(&key_id).unwrap();
-		let switch_commit_hash = SwitchCommitHash::from_switch_commit(switch_commit);
+		let switch_commit_hash = SwitchCommitHash::from_switch_commit(
+			switch_commit,
+			build.keychain,
+			&key_id,
+		);
 		trace!(
 			LOGGER,
 			"Builder - Pedersen Commit is: {:?}, Switch Commit is: {:?}",

core/src/core/mod.rs

@@ -264,7 +264,7 @@ mod test {
 		let mut vec = Vec::new();
 		ser::serialize(&mut vec, &tx).expect("serialization failed");
 		println!("{}", vec.len());
-		assert!(vec.len() == 5352);
+		assert!(vec.len() == 5364);
 	}
 
 	#[test]

core/src/core/transaction.rs

@@ -30,11 +30,11 @@ use keychain::{Identifier, Keychain};
 use ser::{self, read_and_verify_sorted, Readable, Reader, Writeable, WriteableSorted, Writer};
 use util;
 
-/// The size to use for the stored blake2 hash of a switch_commitment
-pub const SWITCH_COMMIT_HASH_SIZE: usize = 20;
+/// The size of the blake2 hash of a switch commitment (256 bits)
+pub const SWITCH_COMMIT_HASH_SIZE: usize = 32;
 
-/// The size of the secret key used to generate the switch commitment hash (blake2)
-pub const SWITCH_COMMIT_KEY_SIZE: usize = 20;
+/// The size of the secret key used in to generate blake2 switch commitment hash (256 bits)
+pub const SWITCH_COMMIT_KEY_SIZE: usize = 32;
 
 bitflags! {
 	/// Options for a kernel's structure or use
@@ -486,6 +486,25 @@ impl SwitchCommitHashKey {
 	pub fn zero() -> SwitchCommitHashKey {
 		SwitchCommitHashKey([0; SWITCH_COMMIT_KEY_SIZE])
 	}
+
+	/// Generate a switch commit hash key from the provided keychain and key id.
+	pub fn from_keychain(keychain: &Keychain, key_id: &Identifier) -> SwitchCommitHashKey {
+		SwitchCommitHashKey(
+			keychain.switch_commit_hash_key(key_id)
+				.expect("failed to derive switch commit hash key")
+		)
+	}
+
+	/// Reconstructs a switch commit hash key from a byte slice.
+	pub fn from_bytes(bytes: &[u8]) -> SwitchCommitHashKey {
+		assert!(bytes.len() == 32, "switch_commit_hash_key requires 32 bytes");
+
+		let mut key = [0; SWITCH_COMMIT_KEY_SIZE];
+		for i in 0..min(SWITCH_COMMIT_KEY_SIZE, bytes.len()) {
+			key[i] = bytes[i];
+		}
+		SwitchCommitHashKey(key)
+	}
 }
 
 /// Definition of the switch commitment hash
@@ -529,14 +548,17 @@ impl ::std::fmt::Debug for SwitchCommitHash {
 
 impl SwitchCommitHash {
 	/// Builds a switch commit hash from a switch commit using blake2
-	pub fn from_switch_commit(switch_commit: Commitment) -> SwitchCommitHash {
-		// always use the "zero" key for now
-		let key = SwitchCommitHashKey::zero();
+	pub fn from_switch_commit(
+		switch_commit: Commitment,
+		keychain: &Keychain,
+		key_id: &Identifier,
+	) -> SwitchCommitHash {
+		let key = SwitchCommitHashKey::from_keychain(keychain, key_id);
 		let switch_commit_hash = blake2b(SWITCH_COMMIT_HASH_SIZE, &key.0, &switch_commit.0);
-		let switch_commit_hash = switch_commit_hash.as_bytes();
+		let switch_commit_hash_bytes = switch_commit_hash.as_bytes();
 		let mut h = [0; SWITCH_COMMIT_HASH_SIZE];
 		for i in 0..SWITCH_COMMIT_HASH_SIZE {
-			h[i] = switch_commit_hash[i];
+			h[i] = switch_commit_hash_bytes[i];
 		}
 		SwitchCommitHash(h)
 	}
@@ -939,7 +961,11 @@ mod test {
 		let key_id = keychain.derive_key_id(1).unwrap();
 		let commit = keychain.commit(5, &key_id).unwrap();
 		let switch_commit = keychain.switch_commit(&key_id).unwrap();
-		let switch_commit_hash = SwitchCommitHash::from_switch_commit(switch_commit);
+		let switch_commit_hash = SwitchCommitHash::from_switch_commit(
+			switch_commit,
+			&keychain,
+			&key_id,
+		);
 		let msg = secp::pedersen::ProofMessage::empty();
 		let proof = keychain.range_proof(5, &key_id, commit, msg).unwrap();
 
@@ -966,7 +992,11 @@ mod test {
 
 		let commit = keychain.commit(1003, &key_id).unwrap();
 		let switch_commit = keychain.switch_commit(&key_id).unwrap();
-		let switch_commit_hash = SwitchCommitHash::from_switch_commit(switch_commit);
+		let switch_commit_hash = SwitchCommitHash::from_switch_commit(
+			switch_commit,
+			&keychain,
+			&key_id,
+		);
 		let msg = secp::pedersen::ProofMessage::empty();
 		let proof = keychain.range_proof(1003, &key_id, commit, msg).unwrap();
 
@@ -1027,7 +1057,7 @@ mod test {
 		).unwrap();
 
 		let short_id = input.short_id(&block_hash);
-		assert_eq!(short_id, ShortId::from_hex("ff2c91d85fcd").unwrap());
+		assert_eq!(short_id, ShortId::from_hex("102864956811").unwrap());
 
 		// now generate the short_id for a *very* similar output (single feature flag different)
 		// and check it generates a different short_id
@@ -1042,6 +1072,6 @@ mod test {
 		).unwrap();
 
 		let short_id = input.short_id(&block_hash);
-		assert_eq!(short_id, ShortId::from_hex("b91a8d669bf9").unwrap());
+		assert_eq!(short_id, ShortId::from_hex("b8c189165df1").unwrap());
 	}
 }

keychain/src/extkey.rs

@@ -175,38 +175,17 @@ pub struct ExtendedKey {
 	pub n_child: u32,
 	/// Root key identifier
 	pub root_key_id: Identifier,
-	/// Code of the derivation chain
-	pub chaincode: [u8; 32],
 	/// Actual private key
 	pub key: SecretKey,
+	/// Code of the derivation chain
+	pub chaincode: [u8; 32],
+	/// The bytes of the key used for generating the associated switch_commit_hash
+	pub switch_key: [u8; 32],
+	/// Code of the derivation chain for the switch_commit_hash key
+	pub switch_chaincode: [u8; 32],
 }
 
 impl ExtendedKey {
-	/// Creates a new extended key from a serialized one
-	pub fn from_slice(secp: &Secp256k1, slice: &[u8]) -> Result<ExtendedKey, Error> {
-		// TODO change when ser. ext. size is fixed
-		if slice.len() != 79 {
-			return Err(Error::InvalidSliceSize);
-		}
-		let depth: u8 = slice[0];
-		let root_key_id = Identifier::from_bytes(&slice[1..11]);
-		let n_child = BigEndian::read_u32(&slice[11..15]);
-		let mut chaincode: [u8; 32] = [0; 32];
-		(&mut chaincode).copy_from_slice(&slice[15..47]);
-		let key = match SecretKey::from_slice(secp, &slice[47..79]) {
-			Ok(key) => key,
-			Err(_) => return Err(Error::InvalidExtendedKey),
-		};
-
-		Ok(ExtendedKey {
-			depth,
-			root_key_id,
-			n_child,
-			chaincode,
-			key,
-		})
-	}
-
 	/// Creates a new extended master key from a seed
 	pub fn from_seed(secp: &Secp256k1, seed: &[u8]) -> Result<ExtendedKey, Error> {
 		match seed.len() {
@@ -214,20 +193,36 @@ impl ExtendedKey {
 			_ => return Err(Error::InvalidSeedSize),
 		}
 
-		let derived = blake2b(64, b"Mimble seed", seed);
+		let derived = blake2b(64, b"Grin/MW Seed", seed);
+		let slice = derived.as_bytes();
 
-		let mut chaincode: [u8; 32] = [0; 32];
-		(&mut chaincode).copy_from_slice(&derived.as_bytes()[32..]);
 		// TODO Error handling
-		let secret_key = SecretKey::from_slice(&secp, &derived.as_bytes()[0..32])
+		let key = SecretKey::from_slice(&secp, &slice[0..32])
 			.expect("Error generating from seed");
 
+		let mut chaincode: [u8; 32] = Default::default();
+		(&mut chaincode).copy_from_slice(&slice[32..64]);
+
+		// Now derive the switch_key and switch_chaincode in a similar fashion
+		// but using a different key to ensure there is nothing linking
+		// the secret key and the switch commit hash key for any extended key
+		// we subsequently derive
+		let switch_derived = blake2b(64, b"Grin/MW Switch Seed", seed);
+		let switch_slice = switch_derived.as_bytes();
+
+		let mut switch_key: [u8; 32] = Default::default();
+		(&mut switch_key).copy_from_slice(&switch_slice[0..32]);
+		let mut switch_chaincode: [u8; 32] = Default::default();
+		(&mut switch_chaincode).copy_from_slice(&switch_slice[32..64]);
+
 		let mut ext_key = ExtendedKey {
 			depth: 0,
 			root_key_id: Identifier::zero(),
 			n_child: 0,
-			chaincode: chaincode,
-			key: secret_key,
+			key,
+			chaincode,
+			switch_key,
+			switch_chaincode,
 		};
 
 		ext_key.root_key_id = ext_key.identifier(secp)?;
@@ -247,27 +242,41 @@ impl ExtendedKey {
 	pub fn derive(&self, secp: &Secp256k1, n: u32) -> Result<ExtendedKey, Error> {
 		let mut n_bytes: [u8; 4] = [0; 4];
 		BigEndian::write_u32(&mut n_bytes, n);
+
 		let mut seed = self.key[..].to_vec();
 		seed.extend_from_slice(&n_bytes);
 
 		let derived = blake2b(64, &self.chaincode[..], &seed[..]);
+		let slice = derived.as_bytes();
 
-		let mut secret_key =
-			SecretKey::from_slice(&secp, &derived.as_bytes()[0..32]).expect("Error deriving key");
-		secret_key
-			.add_assign(secp, &self.key)
+		let mut key = SecretKey::from_slice(&secp, &slice[0..32])
+			.expect("Error deriving key");
+		key.add_assign(secp, &self.key)
 			.expect("Error deriving key");
-		// TODO check if key != 0 ?
 
-		let mut chain_code: [u8; 32] = [0; 32];
-		(&mut chain_code).clone_from_slice(&derived.as_bytes()[32..]);
+		let mut chaincode: [u8; 32] = Default::default();
+		(&mut chaincode).copy_from_slice(&slice[32..64]);
+
+		// Now derive the switch_key and switch_chaincode in a similar fashion
+		let mut switch_seed = self.switch_key[..].to_vec();
+		switch_seed.extend_from_slice(&n_bytes);
+		let switch_derived = blake2b(64, &self.switch_chaincode[..], &switch_seed[..]);
+		let switch_slice = switch_derived.as_bytes();
+		let mut switch_key: [u8; 32] = Default::default();
+		(&mut switch_key).copy_from_slice(&switch_slice[0..32]);
+		let mut switch_chaincode: [u8; 32] = Default::default();
+		(&mut switch_chaincode).copy_from_slice(&switch_slice[32..64]);
+
+		// TODO check if key != 0 ?
 
 		Ok(ExtendedKey {
 			depth: self.depth + 1,
 			root_key_id: self.identifier(&secp)?,
 			n_child: n,
-			chaincode: chain_code,
-			key: secret_key,
+			key,
+			chaincode,
+			switch_key,
+			switch_chaincode,
 		})
 	}
 }
@@ -310,11 +319,11 @@ mod test {
 		let s = Secp256k1::new();
 		let seed = from_hex("000102030405060708090a0b0c0d0e0f");
 		let extk = ExtendedKey::from_seed(&s, &seed.as_slice()).unwrap();
-		let sec = from_hex("c3f5ae520f474b390a637de4669c84d0ed9bbc21742577fac930834d3c3083dd");
+		let sec = from_hex("2878a92133b0a7c2fbfb0bd4520ed2e55ea3fa2913200f05c30077d30b193480");
 		let secret_key = SecretKey::from_slice(&s, sec.as_slice()).unwrap();
 		let chaincode =
-			from_hex("e7298e68452b0c6d54837670896e1aee76b118075150d90d4ee416ece106ae72");
-		let identifier = from_hex("83e59c48297b78b34b73");
+			from_hex("3ad40dd836c5ce25dfcbdee5044d92cf6b65bd5475717fa7a56dd4a032cca7c0");
+		let identifier = from_hex("6f7c1a053ca54592e783");
 		let depth = 0;
 		let n_child = 0;
 		assert_eq!(extk.key, secret_key);
@@ -333,17 +342,16 @@ mod test {
 
 	#[test]
 	fn extkey_derivation() {
-		// TODO More test vectors
 		let s = Secp256k1::new();
 		let seed = from_hex("000102030405060708090a0b0c0d0e0f");
 		let extk = ExtendedKey::from_seed(&s, &seed.as_slice()).unwrap();
 		let derived = extk.derive(&s, 0).unwrap();
-		let sec = from_hex("d75f70beb2bd3b56f9b064087934bdedee98e4b5aae6280c58b4eff38847888f");
+		let sec = from_hex("2676a3ab2ded7c79cbd0bd26d448698de5da5af8e809080d3cacfa2ee31a9aa7");
 		let secret_key = SecretKey::from_slice(&s, sec.as_slice()).unwrap();
 		let chaincode =
-			from_hex("243cb881e1549e714db31d23af45540b13ad07941f64a786bbf3313b4de1df52");
-		let root_key_id = from_hex("83e59c48297b78b34b73");
-		let identifier = from_hex("0185adb4d8b730099c93");
+			from_hex("9bc90b148f4c9478205d6ca72c58bbda2902be1e5082de05d56339a74a5314a3");
+		let root_key_id = from_hex("6f7c1a053ca54592e783");
+		let identifier = from_hex("5f2ec8ee00e8bca002fa");
 		let depth = 1;
 		let n_child = 0;
 		assert_eq!(derived.key, secret_key);

keychain/src/keychain.rs

@@ -112,14 +112,19 @@ impl Keychain {
 	}
 
 	fn derived_key(&self, key_id: &Identifier) -> Result<SecretKey, Error> {
-		trace!(LOGGER, "Derived Key by key_id: {}", key_id);
-
 		// first check our overrides and just return the key if we have one in there
 		if let Some(key) = self.key_overrides.get(key_id) {
 			trace!(LOGGER, "... Derived Key (using override) key_id: {}", key_id);
 			return Ok(*key);
 		}
 
+		let extkey = self.derived_extended_key(key_id)?;
+		Ok(extkey.key)
+	}
+
+	fn derived_extended_key(&self, key_id: &Identifier) -> Result<extkey::ExtendedKey, Error> {
+		trace!(LOGGER, "Derived Key by key_id: {}", key_id);
+
 		// then check the derivation cache to see if we have previously derived this key
 		// if so use the derivation from the cache to derive the key
 		{
@@ -146,7 +151,7 @@ impl Keychain {
 				}
 
 				if extkey_id == *key_id {
-					return Ok(extkey.key);
+					return Ok(extkey);
 				}
 			}
 		}
@@ -160,10 +165,10 @@ impl Keychain {
 	fn derived_key_from_index(
 		&self,
 		derivation: u32,
-	) -> Result<SecretKey, Error> {
+	) -> Result<extkey::ExtendedKey, Error> {
 		trace!(LOGGER, "Derived Key (fast) by derivation: {}", derivation);
 		let extkey = self.extkey.derive(&self.secp, derivation)?;
-		return Ok(extkey.key)
+		return Ok(extkey)
 	}
 
 	pub fn commit(&self, amount: u64, key_id: &Identifier) -> Result<Commitment, Error> {
@@ -177,8 +182,8 @@ impl Keychain {
 		amount: u64,
 		derivation: u32,
 	) -> Result<Commitment, Error> {
-		let skey = self.derived_key_from_index(derivation)?;
-		let commit = self.secp.commit(amount, skey)?;
+		let extkey = self.derived_key_from_index(derivation)?;
+		let commit = self.secp.commit(amount, extkey.key)?;
 		Ok(commit)
 	}
 
@@ -189,13 +194,26 @@ impl Keychain {
 	}
 
 	pub fn switch_commit_from_index(&self, index:u32) -> Result<Commitment, Error> {
-		//just do this directly, because cache seems really slow for wallet reconstruct
+		// just do this directly, because cache seems really slow for wallet reconstruct
 		let skey = self.extkey.derive(&self.secp, index)?;
 		let skey = skey.key;
 		let commit = self.secp.switch_commit(skey)?;
 		Ok(commit)
 	}
 
+	pub fn switch_commit_hash_key(&self, key_id: &Identifier) -> Result<[u8; 32], Error> {
+		// first check our overrides and just return zero key if we have an override
+		// we allow keys to be overridden for testing
+		// and do not care about switch_commit_hash_keys in this case
+		if let Some(_) = self.key_overrides.get(key_id) {
+			let key: [u8; 32] = Default::default();
+			return Ok(key);
+		}
+
+		let extkey = self.derived_extended_key(key_id)?;
+		Ok(extkey.switch_key)
+	}
+
 	pub fn range_proof(
 		&self,
 		amount: u64,

pool/src/graph.rs

@@ -309,7 +309,11 @@ mod tests {
 
 		let output_commit = keychain.commit(70, &key_id1).unwrap();
 		let switch_commit = keychain.switch_commit(&key_id1).unwrap();
-		let switch_commit_hash = SwitchCommitHash::from_switch_commit(switch_commit);
+		let switch_commit_hash = SwitchCommitHash::from_switch_commit(
+			switch_commit,
+			&keychain,
+			&key_id1,
+		);
 
 		let inputs = vec![
 			core::transaction::Input::new(

pool/src/pool.rs

@@ -1312,7 +1312,11 @@ mod tests {
 		let key_id = keychain.derive_key_id(value as u32).unwrap();
 		let commit = keychain.commit(value, &key_id).unwrap();
 		let switch_commit = keychain.switch_commit(&key_id).unwrap();
-		let switch_commit_hash = SwitchCommitHash::from_switch_commit(switch_commit);
+		let switch_commit_hash = SwitchCommitHash::from_switch_commit(
+			switch_commit,
+			&keychain,
+			&key_id,
+		);
 		let msg = secp::pedersen::ProofMessage::empty();
 		let proof = keychain.range_proof(value, &key_id, commit, msg).unwrap();
 
@@ -1330,7 +1334,11 @@ mod tests {
 		let key_id = keychain.derive_key_id(value as u32).unwrap();
 		let commit = keychain.commit(value, &key_id).unwrap();
 		let switch_commit = keychain.switch_commit(&key_id).unwrap();
-		let switch_commit_hash = SwitchCommitHash::from_switch_commit(switch_commit);
+		let switch_commit_hash = SwitchCommitHash::from_switch_commit(
+			switch_commit,
+			&keychain,
+			&key_id,
+		);
 		let msg = secp::pedersen::ProofMessage::empty();
 		let proof = keychain.range_proof(value, &key_id, commit, msg).unwrap();
 

wallet/src/restore.rs

@@ -156,10 +156,16 @@ fn find_utxos_with_key(
 	);
 
 	for output in block_outputs.outputs.iter().filter(|x| !x.spent) {
-		for i in 0..*key_iterations {
-			let expected_hash = SwitchCommitHash::from_switch_commit(switch_commit_cache[i as usize]);
+		for i in 1..*key_iterations {
+			let key_id = &keychain.derive_key_id(i as u32).unwrap();
 
 			if let Ok(x) = output.switch_commit_hash() {
+				let expected_hash = SwitchCommitHash::from_switch_commit(
+					switch_commit_cache[i as usize],
+					&keychain,
+					&key_id,
+				);
+
 				if x == expected_hash {
 					info!(
 						LOGGER,
@@ -170,7 +176,6 @@ fn find_utxos_with_key(
 
 					// add it to result set here
 					let commit_id = from_hex(output.commit.clone()).unwrap();
-					let key_id = keychain.derive_key_id(i as u32).unwrap();
 
 					let res = retrieve_amount_and_coinbase_status(
 						config,