Utility to automatically generate mainnet genesis block (#2122)

* Get last bitcon block hash, setup genesis header without PoW (for now)

* More a few properties to mainnet genesis. Don't get too excited, several are placeholders.

* Mine a valid Cuckaroo solution for genesis block

* Use miner as library to get a solution for genesis. Replace final values in genesis.rs before committing it.

* Complete genesis replacement

* Fixed various replacements to obtain a compilable, well-formed genesis

* Check plugin errors, uncomment PoW validation

* Fixes to nonce handling in genesis mining

* Also produce full block hashes

* Fix genesis hash test

Cargo.toml

@@ -13,6 +13,7 @@ edition = "2018"
 
 [workspace]
 members = ["api", "chain", "config", "core", "keychain", "p2p", "servers", "store", "util", "pool", "wallet"]
+exclude = ["etc/gen_gen"]
 
 [[bin]]
 name = "grin"

core/src/core/block.rs

@@ -535,8 +535,8 @@ impl Block {
 	/// Consumes this block and returns a new block with the coinbase output
 	/// and kernels added
 	pub fn with_reward(mut self, reward_out: Output, reward_kern: TxKernel) -> Block {
-		self.body.outputs.push(reward_out);
-		self.body.kernels.push(reward_kern);
+		self.body.outputs = vec![reward_out];
+		self.body.kernels = vec![reward_kern];
 		self
 	}
 

core/src/genesis.rs

@@ -14,12 +14,21 @@
 
 //! Definition of the genesis block. Placeholder for now.
 
+// required for genesis replacement
+//! #![allow(unused_imports)]
+
 use chrono::prelude::{TimeZone, Utc};
 
-use crate::consensus;
 use crate::core;
 use crate::global;
 use crate::pow::{Difficulty, Proof, ProofOfWork};
+use crate::util;
+use crate::util::secp::constants::SINGLE_BULLET_PROOF_SIZE;
+use crate::util::secp::pedersen::{Commitment, RangeProof};
+use crate::util::secp::Signature;
+
+use crate::core::hash::Hash;
+use crate::keychain::BlindingFactor;
 
 /// Genesis block definition for development networks. The proof of work size
 /// is small enough to mine it on the fly, so it does not contain its own
@@ -110,7 +119,6 @@ pub fn genesis_testnet3() -> core::Block {
 pub fn genesis_testnet4() -> core::Block {
 	core::Block::with_header(core::BlockHeader {
 		height: 0,
-		// previous: core::hash::Hash([0xff; 32]),
 		timestamp: Utc.ymd(2018, 10, 17).and_hms(20, 0, 0),
 		pow: ProofOfWork {
 			total_difficulty: Difficulty::from_num(global::initial_block_difficulty()),
@@ -132,19 +140,62 @@ pub fn genesis_testnet4() -> core::Block {
 		..Default::default()
 	})
 }
+
 /// Placeholder for mainnet genesis block, will definitely change before
 /// release so no use trying to pre-mine it.
 pub fn genesis_main() -> core::Block {
-	core::Block::with_header(core::BlockHeader {
+	let gen = core::Block::with_header(core::BlockHeader {
 		height: 0,
-		// previous: core::hash::Hash([0xff; 32]),
-		timestamp: Utc.ymd(2018, 8, 14).and_hms(0, 0, 0),
+		timestamp: Utc.ymd(2019, 1, 15).and_hms(12, 0, 0), // REPLACE
+		prev_root: Hash::default(),                        // REPLACE
+		output_root: Hash::default(),                      // REPLACE
+		range_proof_root: Hash::default(),                 // REPLACE
+		kernel_root: Hash::default(),                      // REPLACE
+		total_kernel_offset: BlindingFactor::zero(),       // REPLACE
+		output_mmr_size: 1,
+		kernel_mmr_size: 1,
 		pow: ProofOfWork {
-			total_difficulty: Difficulty::from_num(global::initial_block_difficulty()),
-			secondary_scaling: 1,
-			nonce: global::get_genesis_nonce(),
-			proof: Proof::zero(consensus::PROOFSIZE),
+			total_difficulty: Difficulty::from_num(10_u64.pow(8)),
+			secondary_scaling: 1856,
+			nonce: 1, // REPLACE
+			proof: Proof {
+				nonces: vec![0; 42], // REPLACE
+				edge_bits: 29,
+			},
 		},
 		..Default::default()
-	})
+	});
+	let kernel = core::TxKernel {
+		features: core::KernelFeatures::COINBASE_KERNEL,
+		fee: 0,
+		lock_height: 0,
+		excess: Commitment::from_vec(vec![]), // REPLACE
+		excess_sig: Signature::from_raw_data(&[0; 64]).unwrap(), //REPLACE
+	};
+	let output = core::Output {
+		features: core::OutputFeatures::COINBASE_OUTPUT,
+		commit: Commitment::from_vec(vec![]), // REPLACE
+		proof: RangeProof {
+			plen: SINGLE_BULLET_PROOF_SIZE,
+			proof: [0; SINGLE_BULLET_PROOF_SIZE], // REPLACE
+		},
+	};
+	gen.with_reward(output, kernel)
+}
+
+#[cfg(test)]
+mod test {
+	use super::*;
+	use crate::ser;
+	use crate::core::hash::Hashed;
+
+	// TODO hardcode the hashes once genesis is set
+	#[test]
+	fn mainnet_genesis_hash() {
+		let gen_hash = genesis_main().hash();
+		println!("mainnet genesis hash: {}", gen_hash.to_hex());
+		let gen_bin = ser::ser_vec(&genesis_main()).unwrap();
+		println!("mainnet genesis full hash: {}\n", gen_bin.hash().to_hex());
+		//assert_eq!(gene_hash.to_hex, "");
+	}
 }

core/src/global.rs

@@ -315,6 +315,8 @@ pub fn get_genesis_nonce() -> u64 {
 		ChainTypes::AutomatedTesting => 0,
 		// Magic nonce for current genesis block at cuckatoo15
 		ChainTypes::UserTesting => 27944,
+		// Placeholder, obviously not the right value
+		ChainTypes::Mainnet => 0,
 		// Magic nonce for genesis block for testnet2 (cuckatoo29)
 		_ => panic!("Pre-set"),
 	}

etc/gen_gen/Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+name = "grin_gen_gen"
+version = "0.0.1"
+edition = "2018"
+authors = ["Grin Developers <mimblewimble@lists.launchpad.net>"]
+description = "Utility to automate the generation of Grin's genesis block"
+license = "Apache-2.0"
+repository = "https://github.com/mimblewimble/grin"
+keywords = [ "crypto", "grin", "mimblewimble" ]
+readme = "README.md"
+
+[[bin]]
+name = "gen_gen"
+path = "src/bin/gen_gen.rs"
+
+[dependencies]
+chrono = "0.4.4"
+cuckoo_miner = "0.4.2"
+curl = "0.4.19"
+grin_core = "0.4.2"
+grin_chain = "0.4.2"
+grin_keychain = "0.4.2"
+grin_miner_plugin = "0.4.2"
+grin_store = "0.4.2"
+grin_util = "0.4.2"
+serde_json = "1"
+
+[patch.crates-io]
+grin_core = { path = "../../core" }
+grin_keychain = { path = "../../keychain" }

etc/gen_gen/README.md

@@ -0,0 +1,18 @@
+# Genesis Genesis
+
+This crate isn't strictly part of grin but allows the generation and release of a new Grin Genesis in an automated fashion. The process is the following:
+
+* Prepare a multisig output and kernel to use as coinbase. In the case of Grin mainnet, this is done and owned by the council treasurers. This can be down a few days prior.
+* Grab the latest bitcoin block hash from publicly available APIs.
+* Build a genesis block with the prepared coinbase and the bitcoin block hash as `prev_root`. The timestamp of the block is set to 30 min ahead to leave enough time to run a build and start a node.
+* Mine the block so we have at least a valid Cuckatoo Cycle. We don't require a given difficulty for that solution.
+* Finalize the block with the proof-of-work, setting a high-enough difficulty (to be agreed upon separately).
+* Commit the block information to github.
+* Tag version 1.0.0 (scary).
+
+N.B. This was written while listening to Genesis. Unfortunately, I'm not rich enough to do it while driving a Genesis. And that'd be dangerous.
+
+# Usage
+
+1. Build this crate.
+2. From its root run `./target/release/gen-gen --coinbase <file> --difficulty <u64> --tag <version>`

etc/gen_gen/src/bin/gen_gen.rs

@@ -0,0 +1,324 @@
+// Copyright 2018 The Grin Developers
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Main for building the genesis generation utility.
+
+use std::io::{BufRead, Write};
+use std::sync::Arc;
+use std::{fs, io, path, process};
+
+use chrono::prelude::Utc;
+use chrono::{Datelike, Duration, Timelike};
+use curl;
+use serde_json;
+
+use cuckoo_miner as cuckoo;
+use grin_chain as chain;
+use grin_core as core;
+use grin_miner_plugin as plugin;
+use grin_store as store;
+use grin_util as util;
+
+use grin_core::core::hash::Hashed;
+use grin_core::core::verifier_cache::LruVerifierCache;
+use grin_keychain::{BlindingFactor, ExtKeychain, Keychain};
+
+static BCHAIN_INFO_URL: &str = "https://blockchain.info/latestblock";
+static BCYPHER_URL: &str = "https://api.blockcypher.com/v1/btc/main";
+static BCHAIR_URL: &str = "https://api.blockchair.com/bitcoin/blocks?limit=2";
+
+static GENESIS_RS_PATH: &str = "../../core/src/genesis.rs";
+static PLUGIN_PATH: &str = "./cuckaroo_mean_cuda_29.cuckooplugin";
+
+fn main() {
+	if !path::Path::new(GENESIS_RS_PATH).exists() {
+		panic!(
+			"File {} not found, make sure you're running this from the gen_gen directory",
+			GENESIS_RS_PATH
+		);
+	}
+	if !path::Path::new(PLUGIN_PATH).exists() {
+		panic!(
+			"File {} not found, make sure you're running this from the gen_gen directory",
+			PLUGIN_PATH
+		);
+	}
+
+	// get the latest bitcoin hash
+	let h1 = get_bchain_head();
+	let h2 = get_bcypher_head();
+	let h3 = get_bchair_head();
+	if h1 != h2 || h1 != h3 {
+		panic!(
+			"Bitcoin chain head is inconsistent, please retry ({}, {}, {}).",
+			h1, h2, h3
+		);
+	}
+	println!("Using bitcoin block hash {}", h1);
+
+	// build the basic parts of the genesis block header
+	let mut gen = core::genesis::genesis_main();
+	gen.header.timestamp = Utc::now() + Duration::minutes(30);
+	gen.header.prev_root = core::core::hash::Hash::from_hex(&h1).unwrap();
+
+	// TODO get the proper keychain and/or raw coinbase
+	let keychain = ExtKeychain::from_random_seed().unwrap();
+	let key_id = ExtKeychain::derive_key_id(0, 1, 0, 0, 0);
+	let reward = core::libtx::reward::output(&keychain, &key_id, 0, 0).unwrap();
+	gen = gen.with_reward(reward.0, reward.1);
+
+	{
+		// setup a tmp chain to set block header roots
+		core::global::set_mining_mode(core::global::ChainTypes::AutomatedTesting);
+		let tmp_chain = setup_chain(".grin.tmp", core::pow::mine_genesis_block().unwrap());
+		tmp_chain.set_txhashset_roots(&mut gen).unwrap();
+	}
+
+	// mine a Cuckaroo29 block
+	core::global::set_mining_mode(core::global::ChainTypes::Mainnet);
+	let plugin_lib = cuckoo::PluginLibrary::new(PLUGIN_PATH).unwrap();
+	let mut params = plugin_lib.get_default_params();
+	params.mutate_nonce = false;
+	let solver_ctx = plugin_lib.create_solver_ctx(&mut params);
+
+	let mut solver_sols = plugin::SolverSolutions::default();
+	let mut solver_stats = plugin::SolverStats::default();
+	let mut nonce = 0;
+	while solver_sols.num_sols == 0 {
+		solver_sols = plugin::SolverSolutions::default();
+		gen.header.pow.nonce = nonce;
+		let _ = plugin_lib.run_solver(
+			solver_ctx,
+			gen.header.pre_pow(),
+			nonce,
+			1,
+			&mut solver_sols,
+			&mut solver_stats,
+		);
+		if solver_stats.has_errored {
+			println!(
+				"Plugin {} has errored, device: {}. Reason: {}",
+				solver_stats.get_plugin_name(),
+				solver_stats.get_device_name(),
+				solver_stats.get_error_reason(),
+			);
+			return;
+		}
+
+		nonce += 1;
+	}
+
+	// Set the PoW solution and make sure the block is mostly valid
+	gen.header.pow.proof.nonces = solver_sols.sols[0].to_u64s();
+	assert!(gen.header.pow.is_secondary(), "Not a secondary header");
+	println!("Built genesis:\n{:?}", gen);
+	core::pow::verify_size(&gen.header).unwrap();
+	gen.validate(
+		&BlindingFactor::zero(),
+		Arc::new(util::RwLock::new(LruVerifierCache::new())),
+	)
+	.unwrap();
+
+	println!("\nFinal genesis cyclehash: {}", gen.hash().to_hex());
+	let gen_bin = core::ser::ser_vec(&gen).unwrap();
+	println!("Final genesis full hash: {}\n", gen_bin.hash().to_hex());
+
+	update_genesis_rs(&gen);
+	println!("genesis.rs has been updated, check it and run mainnet_genesis_hash test");
+	println!("also check bitcoin block {} hasn't been orphaned.", h1);
+	println!("press c+enter to proceed.");
+	let mut input = String::new();
+	io::stdin().read_line(&mut input).unwrap();
+	if input != "c\n" {
+		return;
+	}
+
+	// Commit genesis block info in git and tag
+	process::Command::new("git")
+		.args(&["commit", "-am", "Minor: finalized genesis block"])
+		.status()
+		.expect("git commit failed");
+	process::Command::new("git")
+		.args(&["tag", "-a", "v1.0", "-m", "Mainnet release"])
+		.status()
+		.expect("git tag failed");
+	process::Command::new("git")
+		.args(&["push", "origin", "v1.0"])
+		.status()
+		.expect("git tag push failed");
+	println!("All done!");
+}
+
+fn update_genesis_rs(gen: &core::core::Block) {
+	// set the replacement patterns
+	let mut replacements = vec![];
+	replacements.push((
+		"timestamp".to_string(),
+		format!(
+			"Utc.ymd({}, {}, {}).and_hms({}, {}, {})",
+			gen.header.timestamp.date().year(),
+			gen.header.timestamp.date().month(),
+			gen.header.timestamp.date().day(),
+			gen.header.timestamp.time().hour(),
+			gen.header.timestamp.time().minute(),
+			gen.header.timestamp.time().second(),
+		),
+	));
+	replacements.push((
+		"prev_root".to_string(),
+		format!(
+			"Hash::from_hex(\"{}\").unwrap()",
+			gen.header.prev_root.to_hex()
+		),
+	));
+	replacements.push((
+		"output_root".to_string(),
+		format!(
+			"Hash::from_hex(\"{}\").unwrap()",
+			gen.header.output_root.to_hex()
+		),
+	));
+	replacements.push((
+		"range_proof_root".to_string(),
+		format!(
+			"Hash::from_hex(\"{}\").unwrap()",
+			gen.header.range_proof_root.to_hex()
+		),
+	));
+	replacements.push((
+		"kernel_root".to_string(),
+		format!(
+			"Hash::from_hex(\"{}\").unwrap()",
+			gen.header.kernel_root.to_hex()
+		),
+	));
+	replacements.push((
+		"total_kernel_offset".to_string(),
+		format!(
+			"BlindingFactor::from_hex(\"{}\").unwrap()",
+			gen.header.total_kernel_offset.to_hex()
+		),
+	));
+	replacements.push(("nonce".to_string(), format!("{}", gen.header.pow.nonce)));
+	replacements.push((
+		"nonces".to_string(),
+		format!("vec!{:?}", gen.header.pow.proof.nonces),
+	));
+	replacements.push((
+		"excess".to_string(),
+		format!(
+			"Commitment::from_vec(util::from_hex({:x?}.to_string()).unwrap())",
+			util::to_hex(gen.kernels()[0].excess.0.to_vec())
+		),
+	));
+	replacements.push((
+		"excess_sig".to_string(),
+		format!(
+			"Signature::from_raw_data(&{:?}).unwrap()",
+			gen.kernels()[0].excess_sig.to_raw_data().to_vec(),
+		),
+	));
+	replacements.push((
+		"commit".to_string(),
+		format!(
+			"Commitment::from_vec(util::from_hex({:x?}.to_string()).unwrap())",
+			util::to_hex(gen.outputs()[0].commitment().0.to_vec())
+		),
+	));
+	replacements.push((
+		"proof".to_string(),
+		format!("{:?}", gen.outputs()[0].proof.bytes().to_vec()),
+	));
+
+	// check each possible replacement in the file, remove the replacement from
+	// the list when found to avoid double replacements
+	let mut replaced = String::new();
+	{
+		let genesis_rs = fs::File::open(GENESIS_RS_PATH).unwrap();
+		let reader = io::BufReader::new(&genesis_rs);
+		for rline in reader.lines() {
+			let line = rline.unwrap();
+			let mut has_replaced = false;
+			if line.contains("REPLACE") {
+				for (pos, replacement) in replacements.iter().enumerate() {
+					if line.contains(&replacement.0) {
+						replaced.push_str(&format!("{}: {},\n", replacement.0, replacement.1));
+						replacements.remove(pos);
+						has_replaced = true;
+						break;
+					}
+				}
+			}
+			if !has_replaced {
+				replaced.push_str(&format!("{}\n", line));
+			}
+		}
+	}
+	let mut genesis_rs = fs::File::create(GENESIS_RS_PATH).unwrap();
+	genesis_rs.write_all(replaced.as_bytes()).unwrap();
+}
+
+fn setup_chain(dir_name: &str, genesis: core::core::Block) -> chain::Chain {
+	util::init_test_logger();
+	let _ = fs::remove_dir_all(dir_name);
+	let verifier_cache = Arc::new(util::RwLock::new(
+		core::core::verifier_cache::LruVerifierCache::new(),
+	));
+	let db_env = Arc::new(store::new_env(dir_name.to_string()));
+	chain::Chain::init(
+		dir_name.to_string(),
+		db_env,
+		Arc::new(chain::types::NoopAdapter {}),
+		genesis,
+		core::pow::verify_size,
+		verifier_cache,
+		false,
+	)
+	.unwrap()
+}
+
+fn get_bchain_head() -> String {
+	get_json(BCHAIN_INFO_URL)["hash"]
+		.as_str()
+		.unwrap()
+		.to_string()
+}
+
+fn get_bcypher_head() -> String {
+	get_json(BCYPHER_URL)["hash"].as_str().unwrap().to_string()
+}
+
+fn get_bchair_head() -> String {
+	get_json(BCHAIR_URL)["data"][0]["hash"]
+		.as_str()
+		.unwrap()
+		.to_string()
+}
+
+fn get_json(url: &str) -> serde_json::Value {
+	let mut body = Vec::new();
+	let mut easy = curl::easy::Easy::new();
+	easy.url(url).unwrap();
+	{
+		let mut transfer = easy.transfer();
+		transfer
+			.write_function(|data| {
+				body.extend_from_slice(data);
+				Ok(data.len())
+			})
+			.unwrap();
+		transfer.perform().unwrap();
+	}
+	serde_json::from_slice(&body).unwrap()
+}