test under lock contention

Cargo.lock

@@ -31,6 +31,7 @@ dependencies = [
  "atomicwrites",
  "base64",
  "cargo-husky",
+ "fail",
  "femme",
  "fs2",
  "glob",
@@ -198,6 +199,17 @@ version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
 
+[[package]]
+name = "fail"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3be3c61c59fdc91f5dbc3ea31ee8623122ce80057058be560654c5d410d181a6"
+dependencies = [
+ "lazy_static",
+ "log",
+ "rand 0.7.3",
+]
+
 [[package]]
 name = "femme"
 version = "2.1.1"

Cargo.toml

@@ -9,6 +9,7 @@ license = "AGPL-3.0"
 anyhow = "1.0"
 atomicwrites = "0.2"
 base64 = "0.13"
+fail = "0.4"
 femme = "2.1"
 fs2 = "0.4"
 glob = "0.3"
@@ -38,3 +39,6 @@ features = ["run-for-all", "prepush-hook", "run-cargo-check", "run-cargo-test",
 
 [profile.release]
 debug = 1
+
+[features]
+failpoints = [ "fail/failpoints" ]

scripts/test.sh

@@ -8,5 +8,5 @@ fi
 cargo fmt -- --check
 cargo clippy --all-targets --all-features -- -D warnings
 cargo check --frozen
-cargo test
-cargo test -- --ignored
+cargo test --all-targets --all-features
+cargo test --all-targets --all-features -- --ignored

src/index/lock.rs

@@ -1,4 +1,7 @@
 use anyhow::Result;
+#[cfg(feature = "failpoints")]
+use anyhow::*;
+use fail::fail_point;
 use std::io::Write;
 use uuid::Uuid;
 use vfs::VfsPath;
@@ -33,13 +36,13 @@ impl Lock {
     }
 
     fn wait_to_have_sole_lock(lock_id: Uuid, index_directory: &VfsPath) -> Result<()> {
-        Lock::create_lock_file(lock_id, index_directory)?;
+        let _ = Lock::create_lock_file(lock_id, index_directory);
         while !Lock::sole_lock(lock_id, index_directory)? {
             let path = Lock::lock_file_path(index_directory, lock_id)?;
             path.remove_file()?;
             let sleep_duration = time::Duration::from_millis((OsRng.next_u32() % 256).into());
             thread::sleep(sleep_duration);
-            Lock::create_lock_file(lock_id, index_directory)?;
+            let _ = Lock::create_lock_file(lock_id, index_directory);
         }
         Ok(())
     }
@@ -67,6 +70,7 @@ impl Lock {
 
     fn create_lock_file(lock_id: Uuid, index_directory: &VfsPath) -> Result<()> {
         let lock_file_path = Lock::lock_file_path(index_directory, lock_id)?;
+        fail_point!("create-lock-file", |e: Option<String>| Err(anyhow!(e.unwrap())));
         let mut file = lock_file_path.create_file()?;
         let lock_id_text = lock_id.to_hyphenated().to_string();
         let lock_id_bytes = lock_id_text.as_bytes();
@@ -89,6 +93,11 @@ impl Drop for Lock {
 mod must {
     use super::Lock;
     use anyhow::Result;
+
+    #[cfg(feature = "failpoints")]
+    use anyhow::*;
+    #[cfg(feature = "failpoints")]
+    use fail::FailScenario;
     use vfs::{MemoryFS, VfsPath};
 
     #[test]
@@ -102,4 +111,20 @@ mod must {
         assert_eq!(entries, 0);
         Ok(())
     }
+
+    #[test]
+    #[cfg(feature = "failpoints")]
+    fn be_able_to_lock_under_high_contention() -> Result<()> {
+        let scenario = FailScenario::setup();
+        fail::cfg("create-lock-file", "90%10*return(some lock file creation error)->off").map_err(|e| anyhow!(e))?;
+
+        {
+            let path = MemoryFS::new().into();
+            let lock = Lock::lock(&path)?;
+            lock.release()?;
+        }
+
+        scenario.teardown();
+        Ok(())
+    }
 }