Get back the lost code of #2575 (#2771)

chain/src/chain.rs

@@ -680,7 +680,7 @@ impl Chain {
 		}
 
 		// prepares the zip and return the corresponding Read
-		let txhashset_reader = txhashset::zip_read(self.db_root.clone(), &header, None)?;
+		let txhashset_reader = txhashset::zip_read(self.db_root.clone(), &header)?;
 		Ok((
 			header.output_mmr_size,
 			header.kernel_mmr_size,

chain/src/txhashset/txhashset.rs

@@ -32,12 +32,12 @@ use crate::util::secp::pedersen::{Commitment, RangeProof};
 use crate::util::{file, secp_static, zip};
 use croaring::Bitmap;
 use grin_store;
-use grin_store::pmmr::{PMMRBackend, PMMR_FILES};
+use grin_store::pmmr::{clean_files_by_prefix, PMMRBackend, PMMR_FILES};
 use std::collections::HashSet;
 use std::fs::{self, File};
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
-use std::time::{Instant, SystemTime, UNIX_EPOCH};
+use std::time::Instant;
 
 const HEADERHASHSET_SUBDIR: &'static str = "header";
 const TXHASHSET_SUBDIR: &'static str = "txhashset";
@@ -1417,22 +1417,38 @@ impl<'a> Extension<'a> {
 
 /// Packages the txhashset data files into a zip and returns a Read to the
 /// resulting file
-pub fn zip_read(root_dir: String, header: &BlockHeader, rand: Option<u32>) -> Result<File, Error> {
-	let ts = if let None = rand {
-		let now = SystemTime::now();
-		now.duration_since(UNIX_EPOCH).unwrap().subsec_micros()
-	} else {
-		rand.unwrap()
-	};
-	let txhashset_zip = format!("{}_{}.zip", TXHASHSET_ZIP, ts);
+pub fn zip_read(root_dir: String, header: &BlockHeader) -> Result<File, Error> {
+	let txhashset_zip = format!("{}_{}.zip", TXHASHSET_ZIP, header.hash().to_string());
 
 	let txhashset_path = Path::new(&root_dir).join(TXHASHSET_SUBDIR);
 	let zip_path = Path::new(&root_dir).join(txhashset_zip);
-	// create the zip archive
-	{
+
+	// if file exist, just re-use it
+	let zip_file = File::open(zip_path.clone());
+	if let Ok(zip) = zip_file {
+		return Ok(zip);
+	} else {
+		// clean up old zips.
+		// Theoretically, we only need clean-up those zip files older than STATE_SYNC_THRESHOLD.
+		// But practically, these zip files are not small ones, we just keep the zips in last one hour
+		let data_dir = Path::new(&root_dir);
+		let pattern = format!("{}_", TXHASHSET_ZIP);
+		if let Ok(n) = clean_files_by_prefix(data_dir.clone(), &pattern, 60 * 60) {
+			debug!(
+				"{} zip files have been clean up in folder: {:?}",
+				n, data_dir
+			);
+		}
+	}
+
+	// otherwise, create the zip archive
+	let path_to_be_cleanup = {
 		// Temp txhashset directory
-		let temp_txhashset_path =
-			Path::new(&root_dir).join(format!("{}_zip_{}", TXHASHSET_SUBDIR, ts));
+		let temp_txhashset_path = Path::new(&root_dir).join(format!(
+			"{}_zip_{}",
+			TXHASHSET_SUBDIR,
+			header.hash().to_string()
+		));
 		// Remove temp dir if it exist
 		if temp_txhashset_path.exists() {
 			fs::remove_dir_all(&temp_txhashset_path)?;
@@ -1444,10 +1460,21 @@ pub fn zip_read(root_dir: String, header: &BlockHeader, rand: Option<u32>) -> Re
 		// Compress zip
 		zip::compress(&temp_txhashset_path, &File::create(zip_path.clone())?)
 			.map_err(|ze| ErrorKind::Other(ze.to_string()))?;
-	}
+
+		temp_txhashset_path
+	};
 
 	// open it again to read it back
-	let zip_file = File::open(zip_path)?;
+	let zip_file = File::open(zip_path.clone())?;
+
+	// clean-up temp txhashset directory.
+	if let Err(e) = fs::remove_dir_all(&path_to_be_cleanup) {
+		warn!(
+			"txhashset zip file: {:?} fail to remove, err: {}",
+			zip_path.to_str(),
+			e
+		);
+	}
 	Ok(zip_file)
 }
 

chain/tests/test_txhashset.rs

@@ -22,12 +22,12 @@ use std::fs::{self, File, OpenOptions};
 use std::iter::FromIterator;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
-use std::time::{SystemTime, UNIX_EPOCH};
 
 use crate::chain::store::ChainStore;
 use crate::chain::txhashset;
 use crate::core::core::BlockHeader;
 use crate::util::file;
+use grin_core::core::hash::Hashed;
 
 fn clean_output_dir(dir_name: &str) {
 	let _ = fs::remove_dir_all(dir_name);
@@ -35,52 +35,48 @@ fn clean_output_dir(dir_name: &str) {
 
 #[test]
 fn test_unexpected_zip() {
-	let now = SystemTime::now();
-	let rand = now.duration_since(UNIX_EPOCH).unwrap().subsec_micros();
-
 	let db_root = format!(".grin_txhashset_zip");
 	clean_output_dir(&db_root);
 	{
 		let chain_store = ChainStore::new(&db_root).unwrap();
 		let store = Arc::new(chain_store);
 		txhashset::TxHashSet::open(db_root.clone(), store.clone(), None).unwrap();
+		let head = BlockHeader::default();
 		// First check if everything works out of the box
-		assert!(txhashset::zip_read(db_root.clone(), &BlockHeader::default(), Some(rand)).is_ok());
-		let zip_path = Path::new(&db_root).join(format!("txhashset_snapshot_{}.zip", rand));
+		assert!(txhashset::zip_read(db_root.clone(), &head).is_ok());
+		let zip_path = Path::new(&db_root).join(format!(
+			"txhashset_snapshot_{}.zip",
+			head.hash().to_string()
+		));
 		let zip_file = File::open(&zip_path).unwrap();
-		assert!(txhashset::zip_write(
-			PathBuf::from(db_root.clone()),
-			zip_file,
-			&BlockHeader::default()
-		)
-		.is_ok());
+		assert!(txhashset::zip_write(PathBuf::from(db_root.clone()), zip_file, &head).is_ok());
 		// Remove temp txhashset dir
-		fs::remove_dir_all(Path::new(&db_root).join(format!("txhashset_zip_{}", rand))).unwrap();
+		fs::remove_dir_all(
+			Path::new(&db_root).join(format!("txhashset_zip_{}", head.hash().to_string())),
+		);
 		// Then add strange files in the original txhashset folder
 		write_file(db_root.clone());
-		assert!(txhashset::zip_read(db_root.clone(), &BlockHeader::default(), Some(rand)).is_ok());
+		assert!(txhashset::zip_read(db_root.clone(), &head).is_ok());
 		// Check that the temp dir dos not contains the strange files
-		let txhashset_zip_path = Path::new(&db_root).join(format!("txhashset_zip_{}", rand));
+		let txhashset_zip_path =
+			Path::new(&db_root).join(format!("txhashset_zip_{}", head.hash().to_string()));
 		assert!(txhashset_contains_expected_files(
-			format!("txhashset_zip_{}", rand),
+			format!("txhashset_zip_{}", head.hash().to_string()),
 			txhashset_zip_path.clone()
 		));
-		fs::remove_dir_all(Path::new(&db_root).join(format!("txhashset_zip_{}", rand))).unwrap();
+		fs::remove_dir_all(
+			Path::new(&db_root).join(format!("txhashset_zip_{}", head.hash().to_string())),
+		);
 
 		let zip_file = File::open(zip_path).unwrap();
-		assert!(txhashset::zip_write(
-			PathBuf::from(db_root.clone()),
-			zip_file,
-			&BlockHeader::default()
-		)
-		.is_ok());
+		assert!(txhashset::zip_write(PathBuf::from(db_root.clone()), zip_file, &head).is_ok());
 		// Check that the txhashset dir dos not contains the strange files
 		let txhashset_path = Path::new(&db_root).join("txhashset");
 		assert!(txhashset_contains_expected_files(
 			"txhashset".to_string(),
 			txhashset_path.clone()
 		));
-		fs::remove_dir_all(Path::new(&db_root).join("txhashset")).unwrap();
+		fs::remove_dir_all(Path::new(&db_root).join("txhashset"));
 	}
 	// Cleanup chain directory
 	clean_output_dir(&db_root);