Core/grin
0
0
Fork 0
mirror of https://github.com/mimblewimble/grin.git synced 2025-01-21 03:21:08 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

Use smaller LMDB Allocation Size in non-production modes (#3264)

Browse source 
* add option to override lmdb allocation size in testing

* simplify to use global::is_production_mode
This commit is contained in:
Yeastplume 2020-03-10 09:30:12 +00:00 committed by GitHub
parent 31bd2d923a
commit ee5fe1ac63
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 21 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
store/src/lmdb.rs
View file

@ -22,11 +22,14 @@ use lmdb_zero as lmdb;
use lmdb_zero::traits::CreateCursor; use lmdb_zero::traits::CreateCursor;
use lmdb_zero::LmdbResultExt; use lmdb_zero::LmdbResultExt;
use crate::core::global;
use crate::core::ser::{self, ProtocolVersion}; use crate::core::ser::{self, ProtocolVersion};
use crate::util::{RwLock, RwLockReadGuard}; use crate::util::{RwLock, RwLockReadGuard};
/// number of bytes to grow the database by when needed /// number of bytes to grow the database by when needed
pub const ALLOC_CHUNK_SIZE: usize = 134_217_728; //128 MB pub const ALLOC_CHUNK_SIZE_DEFAULT: usize = 134_217_728; //128 MB
/// And for test mode, to avoid too much disk allocation on windows
pub const ALLOC_CHUNK_SIZE_DEFAULT_TEST: usize = 1_048_576; //1 MB
const RESIZE_PERCENT: f32 = 0.9; const RESIZE_PERCENT: f32 = 0.9;
/// Want to ensure that each resize gives us at least this % /// Want to ensure that each resize gives us at least this %
/// of total space free /// of total space free
@ -73,6 +76,7 @@ pub struct Store {
db: Arc<RwLock<Option<Arc<lmdb::Database<'static>>>>>, db: Arc<RwLock<Option<Arc<lmdb::Database<'static>>>>>,
name: String, name: String,
version: ProtocolVersion, version: ProtocolVersion,
alloc_chunk_size: usize,
} }
impl Store { impl Store {
@ -105,6 +109,11 @@ impl Store {
env_builder.set_maxreaders(max_readers)?; env_builder.set_maxreaders(max_readers)?;
} }
let alloc_chunk_size = match global::is_production_mode() {
true => ALLOC_CHUNK_SIZE_DEFAULT,
false => ALLOC_CHUNK_SIZE_DEFAULT_TEST,
};
let env = unsafe { env_builder.open(&full_path, lmdb::open::NOTLS, 0o600)? }; let env = unsafe { env_builder.open(&full_path, lmdb::open::NOTLS, 0o600)? };
debug!( debug!(
@ -117,6 +126,7 @@ impl Store {
db: Arc::new(RwLock::new(None)), db: Arc::new(RwLock::new(None)),
name: db_name, name: db_name,
version: DEFAULT_DB_VERSION, version: DEFAULT_DB_VERSION,
alloc_chunk_size,
}; };
{ {
@ -133,11 +143,16 @@ impl Store {
/// Construct a new store using a specific protocol version. /// Construct a new store using a specific protocol version.
/// Permits access to the db with legacy protocol versions for db migrations. /// Permits access to the db with legacy protocol versions for db migrations.
pub fn with_version(&self, version: ProtocolVersion) -> Store { pub fn with_version(&self, version: ProtocolVersion) -> Store {
let alloc_chunk_size = match global::is_production_mode() {
true => ALLOC_CHUNK_SIZE_DEFAULT,
false => ALLOC_CHUNK_SIZE_DEFAULT_TEST,
};
Store { Store {
env: self.env.clone(), env: self.env.clone(),
db: self.db.clone(), db: self.db.clone(),
name: self.name.clone(), name: self.name.clone(),
version: version, version: version,
alloc_chunk_size,
} }
} }
@ -171,7 +186,7 @@ impl Store {
); );
if size_used as f32 / env_info.mapsize as f32 > resize_percent if size_used as f32 / env_info.mapsize as f32 > resize_percent
|| env_info.mapsize < ALLOC_CHUNK_SIZE || env_info.mapsize < self.alloc_chunk_size
{ {
trace!("Resize threshold met (percent-based)"); trace!("Resize threshold met (percent-based)");
Ok(true) Ok(true)
@ -188,12 +203,12 @@ impl Store {
let stat = self.env.stat()?; let stat = self.env.stat()?;
let size_used = stat.psize as usize * env_info.last_pgno; let size_used = stat.psize as usize * env_info.last_pgno;
let new_mapsize = if env_info.mapsize < ALLOC_CHUNK_SIZE { let new_mapsize = if env_info.mapsize < self.alloc_chunk_size {
ALLOC_CHUNK_SIZE self.alloc_chunk_size
} else { } else {
let mut tot = env_info.mapsize; let mut tot = env_info.mapsize;
while size_used as f32 / tot as f32 > RESIZE_MIN_TARGET_PERCENT { while size_used as f32 / tot as f32 > RESIZE_MIN_TARGET_PERCENT {
tot += ALLOC_CHUNK_SIZE; tot += self.alloc_chunk_size;
} }
tot tot
}; };

2
store/tests/lmdb.rs
View file

@ -20,7 +20,7 @@ use grin_core::ser::{self, Readable, Reader, Writeable, Writer};
use std::fs; use std::fs;
const WRITE_CHUNK_SIZE: usize = 20; const WRITE_CHUNK_SIZE: usize = 20;
const TEST_ALLOC_SIZE: usize = store::lmdb::ALLOC_CHUNK_SIZE / 8 / WRITE_CHUNK_SIZE; const TEST_ALLOC_SIZE: usize = store::lmdb::ALLOC_CHUNK_SIZE_DEFAULT / 8 / WRITE_CHUNK_SIZE;
#[derive(Clone)] #[derive(Clone)]
struct PhatChunkStruct { struct PhatChunkStruct {