Move encoder out

src/index/io.rs

@@ -5,6 +5,7 @@ use uuid::Uuid;
 
 use crate::index::item::IndexItem;
 use crate::index::{lock, Index};
+use crate::io::error_correcting_encoder;
 use crate::repository::ItemId;
 use anyhow::Result;
 use anyhow::*;
@@ -12,8 +13,6 @@ use lock::Lock;
 use nix::unistd::getpid;
 use std::{cmp::max, io::Write};
 
-const DATA_SHARD_SIZE: u16 = 4096;
-
 impl Index {
     pub fn load(repository_path: &VfsPath) -> Result<Self> {
         if !repository_path.exists() {
@@ -71,7 +70,7 @@ impl Index {
         let serialised = serde_json::to_string(&self)?;
 
         let bytes = serialised.as_bytes();
-        let encoded = Index::encode(bytes)?;
+        let encoded = error_correcting_encoder::encode(bytes)?;
 
         {
             let mut file = index_file_path.create_file()?;
@@ -79,9 +78,13 @@ impl Index {
             file.flush()?;
         }
 
-        let mut file = index_file_path.open_file()?;
+        let readback = {
-        let mut readback = vec![];
+            let mut file = index_file_path.open_file()?;
-        file.read_to_end(&mut readback)?;
+            let mut readback = vec![];
+            file.read_to_end(&mut readback)?;
+            readback
+        };
+
         if readback != encoded {
             Err(anyhow!("index readback incorrect"))
         } else {
@@ -89,10 +92,6 @@ impl Index {
         }
     }
 
-    fn encode(bytes: &[u8]) -> Result<&[u8]> {
-        Ok(bytes)
-    }
-
     fn load_from_file(index_file_path: &VfsPath) -> Result<Self> {
         let index_text = index_file_path
             .read_to_string()
@@ -159,39 +158,4 @@ mod must {
 
         Ok(())
     }
-
-    #[test]
-    fn survive_file_corruption() -> Result<()> {
-        let repository_path: VfsPath = MemoryFS::new().into();
-        let mut original = Index::new()?;
-
-        original.save(&repository_path)?;
-        corrupt(&repository_path)?;
-        let loaded = Index::load(&repository_path)?;
-
-        assert_eq!(original, loaded);
-
-        Ok(())
-    }
-
-    fn corrupt(repository_path: &VfsPath) -> Result<()> {
-        let index_file_path = &Index::index_file_path_for_repository_path(&repository_path)?;
-
-        let size = dbg!(index_file_path.metadata()?.len as usize);
-        let corrupt_byte_index = rand::thread_rng().gen_range::<usize, _>(0..size);
-
-        let corrupted = {
-            let mut file = index_file_path.open_file()?;
-            let mut buffer = vec![];
-            file.read_to_end(&mut buffer)?;
-            buffer[corrupt_byte_index] = rand::thread_rng().gen::<u8>();
-            buffer
-        };
-        {
-            let mut file = index_file_path.create_file()?;
-            file.write_all(&corrupted)?;
-        }
-
-        Ok(())
-    }
 }

src/io/error_correcting_encoder.rs

@@ -0,0 +1,47 @@
+use anyhow::Result;
+
+pub fn encode(bytes: &[u8]) -> Result<&[u8]> {
+    // find number of shards and parity blocks
+    // encode checksum with each block
+    // when decoding, remove blocks with invalid checksum
+
+    Ok(bytes)
+}
+
+pub fn decode(bytes: &[u8]) -> Result<&[u8]> {
+    // find number of shards and parity blocks
+    // encode checksum with each block
+    // when decoding, remove blocks with invalid checksum
+
+    Ok(bytes)
+}
+
+mod must {
+
+    use anyhow::Result;
+    use rand::{thread_rng, Rng, RngCore};
+    use vfs::{MemoryFS, VfsPath};
+
+    use super::{decode, encode};
+
+    #[test]
+    fn survive_data_corruption() -> Result<()> {
+        let mut original: [u8; 32] = [0; 32];
+        thread_rng().fill_bytes(&mut original);
+
+        let encoded = encode(&original)?;
+
+        let size = dbg!(encoded.len());
+        let corrupt_byte_index = rand::thread_rng().gen_range::<usize, _>(0..size);
+
+        let mut corrupted: [u8; 32] = [0; 32];
+        corrupted.copy_from_slice(encoded);
+        corrupted[corrupt_byte_index] = rand::thread_rng().gen::<u8>();
+
+        let decoded = decode(&corrupted)?;
+
+        assert_eq!(decoded, original);
+
+        Ok(())
+    }
+}

src/io/mod.rs

@@ -0,0 +1 @@
+pub mod error_correcting_encoder;

src/lib.rs

@@ -4,4 +4,5 @@ pub mod restore;
 pub mod test;
 
 mod index;
+mod io;
 mod version;