make sure we give up waiting on lock eventually

This commit is contained in:
Cyryl Płotnicki 2020-12-27 22:17:22 +00:00
parent b1f8e8a709
commit 43fb1d86fd
3 changed files with 63 additions and 18 deletions

13
Cargo.lock generated
View file

@ -48,6 +48,7 @@ dependencies = [
"sha2", "sha2",
"tempfile", "tempfile",
"thiserror", "thiserror",
"two-rusty-forks",
"uuid", "uuid",
"vfs", "vfs",
"walkdir", "walkdir",
@ -813,6 +814,18 @@ dependencies = [
"winapi", "winapi",
] ]
[[package]]
name = "two-rusty-forks"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21ad7c05d9d393a945f0320da5bc22906e58b90f1f5ac0163f92778a4f4dbbe6"
dependencies = [
"fnv",
"quick-error",
"tempfile",
"wait-timeout",
]
[[package]] [[package]]
name = "typenum" name = "typenum"
version = "1.12.0" version = "1.12.0"

View file

@ -31,6 +31,7 @@ walkdir = "2.3"
[dev-dependencies] [dev-dependencies]
proptest = "0.10" proptest = "0.10"
two-rusty-forks = "0.4.0"
[dev-dependencies.cargo-husky] [dev-dependencies.cargo-husky]
version = "1" version = "1"

View file

@ -1,8 +1,7 @@
use anyhow::Result; use anyhow::Result;
#[cfg(feature = "failpoints")]
use anyhow::*; use anyhow::*;
use fail::fail_point; use fail::fail_point;
use std::io::Write; use std::{io::Write, time::Instant};
use uuid::Uuid; use uuid::Uuid;
use vfs::VfsPath; use vfs::VfsPath;
@ -13,12 +12,18 @@ pub struct Lock {
path: VfsPath, path: VfsPath,
} }
const MAX_TIMEOUT_MILLIS: u16 = 8192;
impl Lock { impl Lock {
pub fn lock(index_directory: &VfsPath) -> Result<Self> { pub fn lock(index_directory: &VfsPath) -> Result<Self> {
Lock::lock_with_timeout(index_directory, MAX_TIMEOUT_MILLIS)
}
pub fn lock_with_timeout(index_directory: &VfsPath, max_timeout_millis: u16) -> Result<Self> {
let mut buffer = [0u8; 16]; let mut buffer = [0u8; 16];
OsRng.fill_bytes(&mut buffer); OsRng.fill_bytes(&mut buffer);
let id = Uuid::from_bytes(buffer); let id = Uuid::from_bytes(buffer);
Lock::wait_to_have_sole_lock(id, index_directory)?; Lock::wait_to_have_sole_lock(id, index_directory, max_timeout_millis)?;
let path = Lock::lock_file_path(index_directory, id)?; let path = Lock::lock_file_path(index_directory, id)?;
Ok(Lock { path }) Ok(Lock { path })
} }
@ -35,14 +40,20 @@ impl Lock {
Ok(()) Ok(())
} }
fn wait_to_have_sole_lock(lock_id: Uuid, index_directory: &VfsPath) -> Result<()> { fn wait_to_have_sole_lock(lock_id: Uuid, index_directory: &VfsPath, max_timeout_millis: u16) -> Result<()> {
let start_time = Instant::now();
let _ = Lock::create_lock_file(lock_id, index_directory); let _ = Lock::create_lock_file(lock_id, index_directory);
while !Lock::sole_lock(lock_id, index_directory)? { while !Lock::sole_lock(lock_id, index_directory)? {
let path = Lock::lock_file_path(index_directory, lock_id)?; let path = Lock::lock_file_path(index_directory, lock_id)?;
if path.exists() {
path.remove_file()?; path.remove_file()?;
let sleep_duration = time::Duration::from_millis((OsRng.next_u32() % 256).into()); }
let sleep_duration = time::Duration::from_millis((OsRng.next_u32() % 64).into());
thread::sleep(sleep_duration); thread::sleep(sleep_duration);
let _ = Lock::create_lock_file(lock_id, index_directory); let _ = Lock::create_lock_file(lock_id, index_directory);
if start_time.elapsed().as_millis() > max_timeout_millis.into() {
return Err(anyhow!("timed out waiting on lock"));
}
} }
Ok(()) Ok(())
} }
@ -51,12 +62,15 @@ impl Lock {
let my_lock_file_path = Lock::lock_file_path(index_directory, lock_id)?; let my_lock_file_path = Lock::lock_file_path(index_directory, lock_id)?;
let locks = Lock::all_locks(index_directory)?; let locks = Lock::all_locks(index_directory)?;
let mut only_mine = true; let mut only_mine = true;
for path in locks { for path in &locks {
if path != my_lock_file_path { if path != &my_lock_file_path {
only_mine = false; only_mine = false;
break; break;
} }
} }
if locks.iter().count() == 0 {
return Ok(false);
}
Ok(only_mine) Ok(only_mine)
} }
@ -98,6 +112,8 @@ mod must {
use anyhow::*; use anyhow::*;
#[cfg(feature = "failpoints")] #[cfg(feature = "failpoints")]
use fail::FailScenario; use fail::FailScenario;
#[cfg(feature = "failpoints")]
use two_rusty_forks::rusty_fork_test;
use vfs::{MemoryFS, VfsPath}; use vfs::{MemoryFS, VfsPath};
#[test] #[test]
@ -112,19 +128,34 @@ mod must {
Ok(()) Ok(())
} }
#[test]
#[cfg(feature = "failpoints")] #[cfg(feature = "failpoints")]
fn be_able_to_lock_under_high_contention() -> Result<()> { rusty_fork_test! {
#[test]
fn be_able_to_lock_when_creating_lock_file_fails_sometimes() {
let scenario = FailScenario::setup(); let scenario = FailScenario::setup();
fail::cfg("create-lock-file", "90%10*return(some lock file creation error)->off").map_err(|e| anyhow!(e))?; fail::cfg("create-lock-file", "90%10*return(some lock file creation error)->off").map_err(|e| anyhow!(e)).unwrap();
{ {
let path = MemoryFS::new().into(); let path = MemoryFS::new().into();
let lock = Lock::lock(&path)?; let lock = Lock::lock(&path).unwrap();
lock.release()?; lock.release().unwrap();
} }
scenario.teardown(); scenario.teardown();
Ok(()) }
}
#[cfg(feature = "failpoints")]
rusty_fork_test! {
#[test]
fn know_to_give_up_when_creating_lock_file_always_fails() {
let scenario = FailScenario::setup();
fail::cfg("create-lock-file", "return(persistent lock file creation error)").map_err(|e| anyhow!(e)).unwrap();
let path = MemoryFS::new().into();
assert!(Lock::lock_with_timeout(&path, 1).is_err());
scenario.teardown();
}
} }
} }