diff --git a/Cargo.lock b/Cargo.lock index a1b68f5..c6270e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15,6 +15,76 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" +[[package]] +name = "aho-corasick" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf8dcb5b4bbaa28653b647d8c77bd4ed40183b48882e130c1f1ffb73de069fd7" + +[[package]] +name = "async-log" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5ba33304f3165922dd93565a36dda8ac96c28b1a926ad535ec904e48ecad59b" +dependencies = [ + "async-log-attributes", + "backtrace", + "log", +] + +[[package]] +name = "async-log" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac39aabd463ad2b1fadb34c22f48e5d160fa60b7cd69a426362e71bde49ebb6d" +dependencies = [ + "async-log-attributes", + "backtrace", + "log", +] + +[[package]] +name = "async-log-attributes" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da97f8e61b19a72f67d8932de8b0905f7d41a1d7b9501b9938c7755f96f6362d" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "syn 0.15.44", +] + +[[package]] +name = "atomicwrites" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a2baf2feb820299c53c7ad1cc4f5914a220a1cb76d7ce321d2522a94b54651f" +dependencies = [ + "nix 0.14.1", + "tempdir", + "winapi", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.0.1" @@ -39,17 +109,42 @@ dependencies = [ name = "bakare" version = "0.1.0" dependencies = [ + "anyhow", + "async-log 2.0.0", + "atomicwrites", + "base64", "cargo-husky", - "failure", + "femme", + "fs2", + "glob", "hex", + "log", + "nix 0.19.0", + "rand 0.7.3", + "rayon", "rust-crypto", "serde", "serde_cbor", + "serde_json", "sha2", "tempfile", + "thiserror", + "uuid", "walkdir", ] +[[package]] +name = "base64" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + [[package]] name = "block-buffer" version = "0.9.0" @@ -59,12 +154,24 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bumpalo" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" + [[package]] name = "cargo-husky" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b02b629252fe8ef6460461409564e2c21d0c8e77e0944f3d189ff06c4e932ad" +[[package]] +name = "cc" +version = "1.0.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1770ced377336a88a67c473594ccc14eca6f4559217c34f64aac8f83d641b40" + [[package]] name = "cfg-if" version = "0.1.10" @@ -77,12 +184,104 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "clicolors-control" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90082ee5dcdd64dc4e9e0d37fbf3ee325419e39c0092191e0393df65518f741e" +dependencies = [ + "atty", + "lazy_static", + "libc", + "winapi", +] + +[[package]] +name = "cloudabi" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467" +dependencies = [ + "bitflags", +] + +[[package]] +name = "console" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ca57c2c14b8a2bf3105bc9d15574aad80babf6a9c44b1058034cdf8bd169628" +dependencies = [ + "atty", + "clicolors-control", + "encode_unicode", + "lazy_static", + "libc", + "parking_lot", + "regex", + "termios", + "unicode-width", + "winapi", +] + +[[package]] +name = "const_fn" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c478836e029dcef17fb47c89023448c64f781a046e0300e257ad8225ae59afab" + [[package]] name = "cpuid-bool" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" +[[package]] +name = "crossbeam-channel" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0f606a85340376eef0d6d8fec399e6d4a544d648386c6645eb6d0653b27d9f" +dependencies = [ + "cfg-if 1.0.0", + "const_fn", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec91540d98355f690a86367e566ecad2e9e579f230230eb7c21398372be73ea5" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "const_fn", + "lazy_static", +] + [[package]] name = "digest" version = "0.9.0" @@ -93,25 +292,43 @@ dependencies = [ ] [[package]] -name = "failure" -version = "0.1.8" +name = "either" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + +[[package]] +name = "femme" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d9ac2da60a2a97a0643d082f1cb25901a90e6f2f69f03baf9763721aa31ee6a" dependencies = [ - "backtrace", - "failure_derive", + "async-log 1.1.0", + "cfg-if 0.1.10", + "console", + "js-sys", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-bindgen", + "web-sys", ] [[package]] -name = "failure_derive" -version = "0.1.8" +name = "fs2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", + "libc", + "winapi", ] [[package]] @@ -153,24 +370,102 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce" +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + [[package]] name = "half" version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d36fab90f82edc3c747f9d438e06cf0a491055896f2a279638bb5beed6c40177" +[[package]] +name = "hermit-abi" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8" +dependencies = [ + "libc", +] + [[package]] name = "hex" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" +[[package]] +name = "instant" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb1fc4429a33e1f80d41dc9fea4d108a88bec1de8053878898ae448a0b52f613" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "itoa" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" + +[[package]] +name = "js-sys" +version = "0.3.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca059e81d9486668f12d455a4ea6daa600bd408134cd17e3d3fb5a32d1f016f8" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + [[package]] name = "libc" version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614" +[[package]] +name = "lock_api" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28247cc5a5be2f05fbcd76dd0cf2c7d3b5400cb978a28042abcd4fa0b3f8261c" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +dependencies = [ + "cfg-if 0.1.10", +] + +[[package]] +name = "memchr" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" + +[[package]] +name = "memoffset" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" +dependencies = [ + "autocfg", +] + [[package]] name = "miniz_oxide" version = "0.4.3" @@ -181,6 +476,41 @@ dependencies = [ "autocfg", ] +[[package]] +name = "nix" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c722bee1037d430d0f8e687bbdbf222f27cc6e4e68d5caf630857bb2b6dbdce" +dependencies = [ + "bitflags", + "cc", + "cfg-if 0.1.10", + "libc", + "void", +] + +[[package]] +name = "nix" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85db2feff6bf70ebc3a4793191517d5f0331100a2f10f9bf93b5e5214f32b7b7" +dependencies = [ + "bitflags", + "cc", + "cfg-if 0.1.10", + "libc", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "object" version = "0.22.0" @@ -193,19 +523,63 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "parking_lot" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4893845fa2ca272e647da5d0e46660a314ead9c2fdd9a883aabc32e481a8733" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b" +dependencies = [ + "cfg-if 0.1.10", + "cloudabi", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi", +] + [[package]] name = "ppv-lite86" version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" +[[package]] +name = "proc-macro2" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" +dependencies = [ + "unicode-xid 0.1.0", +] + [[package]] name = "proc-macro2" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" dependencies = [ - "unicode-xid", + "unicode-xid 0.2.1", +] + +[[package]] +name = "quote" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" +dependencies = [ + "proc-macro2 0.4.30", ] [[package]] @@ -214,7 +588,7 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" dependencies = [ - "proc-macro2", + "proc-macro2 1.0.24", ] [[package]] @@ -296,6 +670,31 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rayon" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "lazy_static", + "num_cpus", +] + [[package]] name = "rdrand" version = "0.4.0" @@ -311,6 +710,24 @@ version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" +[[package]] +name = "regex" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38cf2c13ed4745de91a5eb834e11c00bcc3709e773173b2ce4c56c9fbde04b9c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", +] + +[[package]] +name = "regex-syntax" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b181ba2dcf07aaccad5448e8ead58db5b742cf85dfe035e2227f137a539a189" + [[package]] name = "remove_dir_all" version = "0.5.3" @@ -345,6 +762,12 @@ version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + [[package]] name = "same-file" version = "1.0.6" @@ -354,6 +777,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + [[package]] name = "serde" version = "1.0.117" @@ -379,9 +808,20 @@ version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cbd1ae72adb44aab48f325a02444a5fc079349a8d804c1fc922aed3f7454c74e" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", +] + +[[package]] +name = "serde_json" +version = "1.0.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcac07dbffa1c65e7f816ab9eba78eb142c6d44410f4eeba1e26e4f5dfa56b95" +dependencies = [ + "itoa", + "ryu", + "serde", ] [[package]] @@ -397,27 +837,42 @@ dependencies = [ "opaque-debug", ] +[[package]] +name = "smallvec" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252" + +[[package]] +name = "syn" +version = "0.15.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "unicode-xid 0.1.0", +] + [[package]] name = "syn" version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac" dependencies = [ - "proc-macro2", - "quote", - "unicode-xid", + "proc-macro2 1.0.24", + "quote 1.0.7", + "unicode-xid 0.2.1", ] [[package]] -name = "synstructure" -version = "0.12.4" +name = "tempdir" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" +checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" dependencies = [ - "proc-macro2", - "quote", - "syn", - "unicode-xid", + "rand 0.4.6", + "remove_dir_all", ] [[package]] @@ -434,6 +889,44 @@ dependencies = [ "winapi", ] +[[package]] +name = "termios" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411c5bf740737c7918b8b1fe232dca4dc9f8e754b8ad5e20966814001ed0ac6b" +dependencies = [ + "libc", +] + +[[package]] +name = "thiserror" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9ae34b84616eedaaf1e9dd6026dbe00dcafa92aa0c8077cb69df1fcfe5e53e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ba20f23e85b10754cd195504aebf6a27e2e6cbe28c17778a0c930724628dd56" +dependencies = [ + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", +] + +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +dependencies = [ + "lazy_static", +] + [[package]] name = "time" version = "0.1.44" @@ -451,18 +944,45 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" + [[package]] name = "unicode-xid" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +[[package]] +name = "uuid" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fde2f6a4bea1d6e007c4ad38c6839fa71cbb63b6dbf5b595aa38dc9b1093c11" +dependencies = [ + "rand 0.7.3", +] + [[package]] name = "version_check" version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + [[package]] name = "walkdir" version = "2.3.1" @@ -486,6 +1006,72 @@ version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +[[package]] +name = "wasm-bindgen" +version = "0.2.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac64ead5ea5f05873d7c12b545865ca2b8d28adfc50a49b84770a3a97265d42" +dependencies = [ + "cfg-if 0.1.10", + "serde", + "serde_json", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f22b422e2a757c35a73774860af8e112bff612ce6cb604224e8e47641a9e4f68" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b13312a745c08c469f0b292dd2fcd6411dba5f7160f593da6ef69b64e407038" +dependencies = [ + "quote 1.0.7", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f249f06ef7ee334cc3b8ff031bfc11ec99d00f34d86da7498396dc1e3b1498fe" +dependencies = [ + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d649a3145108d7d3fbcde896a468d1bd636791823c9921135218ad89be08307" + +[[package]] +name = "web-sys" +version = "0.3.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bf6ef87ad7ae8008e15a355ce696bed26012b7caa21605188cfd8214ab51e2d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 9db949c..18a7a18 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,17 +6,30 @@ edition = "2018" license = "AGPL-3.0" [dependencies] -failure = "0.1" +anyhow = "1.0" +async-log = "2.0" +atomicwrites = "0.2" +base64 = "0.11" +femme = "1.3" +fs2 = "0.4" +glob = "0.3" +hex = "0.4" +log = "0.4" +nix = "0.19" +rand = "0.7" +rayon = "1.1" rust-crypto = "0.2" serde = { version = "1.0", features = ["derive"] } serde_cbor = "0.11" -tempfile = "3.1" -walkdir = "2.3" +serde_json = "1.0" sha2 = "0.9" -hex = "0.4" +tempfile = "3.1" +thiserror = "1.0" +uuid = { version = "0.8", features = ["v4"] } +walkdir = "2.3" [dev-dependencies.cargo-husky] version = "1" default-features = false -features = ["run-for-all", "prepush-hook", "run-cargo-test", "run-cargo-clippy", "run-cargo-fmt"] +features = ["run-for-all", "prepush-hook", "run-cargo-clippy", "run-cargo-fmt"] diff --git a/README.md b/README.md index a53361e..ae4d191 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,10 @@ Motivation: All the backup systems are either slow or crashing or both on my backup. Tried duply: +
- Works but is very slow - +Works but is very slow + ``` --------------[ Backup Statistics ]-------------- StartTime 1547198362.85 (Fri Jan 11 09:19:22 2019) @@ -29,32 +30,37 @@ Errors 0
- Tried restic: -* crashes with OOM -* corrupts repo if you suspend one backup process and launch another from a different computer -Goals for bakare: -* fast -* using max bandwidth -* use max cpu -* use max disk I/O -* memory usage limit -* encryption by default - asymmetric, creates a keypair for you -* deduplication of file data -* fuzzy find by file name in stored files -* failure to process one file should not affect any other files -* intermittent network failures should not make the whole process fail (test with random packet drop) -* system suspend/resume should not make the repo become corrupted, even in the presence of other active backup processes running on other computers, targeting same repo - this is where `restic` fails +- crashes with OOM +- corrupts repo if you suspend one backup process and launch another from a different computer -Nice to have: -* daemon that listens for file events and updates a list of files to be backed up on the next backup run - or a `continous backup` mode - the daemon uploads the file whenever it sees the change -* peer2peer mode - people storing encrypted backups for each other -* relay mode, where daemon works on one or more central points with local storage (e.g. NAS) and various computers sync with that central location. Then though the central locaiton uploads everything to the other location, typically the cloud. + Goals for bakare: -Implementation: -* test with randomly created dirs and files, with property based tests and fuzzer -* see if we can use `salsa` for recomputaiton -* index corruption tests - mutate random byte and see if everything is readable -* network packet drop tests +- fast +- using max bandwidth +- use max cpu +- use max disk I/O +- memory usage limit +- encryption by default - asymmetric, creates a keypair for you +- deduplication of file data +- fuzzy find by file name in stored files +- failure to process one file should not affect any other files +- intermittent network failures should not make the whole process fail (test with random packet drop) +- system suspend/resume should not make the repo become corrupted, even in the presence of other active backup processes running on other computers, targeting same repo - this is where `restic` fails + Nice to have: + +- daemon that listens for file events and updates a list of files to be backed up on the next backup run - or a `continous backup` mode - the daemon uploads the file whenever it sees the change +- peer2peer mode - people storing encrypted backups for each other +- relay mode, where daemon works on one or more central points with local storage (e.g. NAS) and various computers sync with that central location. Then though the central locaiton uploads everything to the other location, typically the cloud. + + Implementation: + +- automatic node discovery - two roles: data publisher and data persister - should be able to figure out which node is which automatically +- test with randomly created dirs and files, with property based tests and fuzzer +- see if we can use `salsa` for recomputaiton +- index corruption tests - mutate random byte and see if everything is readable +- network packet drop tests +- use bevy for decoupling ? +- remove all `unwraps` diff --git a/shell.nix b/shell.nix index d59fdb0..fe04ba8 100644 --- a/shell.nix +++ b/shell.nix @@ -5,10 +5,14 @@ let in with nixpkgs; stdenv.mkDerivation { - name = "genpass_shell"; + name = "bakare_shell"; buildInputs = [ channel.rust cacert openssl openssh zlib - llvm pkgconfig git + pkgconfig clang llvm + git ]; + shellHook = '' + export RUST_SRC_PATH="${channel.rust-src}/lib/rustlib/src/rust/src" + ''; } diff --git a/src/backup.rs b/src/backup.rs index 0fa974e..5aa07b8 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -1,8 +1,9 @@ +use anyhow::Result; +use anyhow::*; use std::path::Path; use walkdir::WalkDir; -use crate::error::BakareError; use crate::repository::Repository; pub struct Engine<'a> { @@ -11,21 +12,29 @@ pub struct Engine<'a> { } impl<'a> Engine<'a> { - pub fn new(source_path: &'a Path, repository: &'a mut Repository<'a>) -> Result { + pub fn new(source_path: &'a Path, repository: &'a mut Repository<'a>) -> Result { if source_path.ancestors().any(|a| a == repository.path()) { - return Err(BakareError::SourceSameAsRepository); + return Err(anyhow!("source same as repository")); } Ok(Engine { source_path, repository }) } - pub fn backup(&mut self) -> Result<(), BakareError> { + pub fn backup(&mut self) -> Result<()> { let walker = WalkDir::new(self.source_path); + let save_every = 16; + let mut save_counter = 0; for maybe_entry in walker { let entry = maybe_entry?; if entry.path() != self.source_path { self.repository.store(entry.path())?; } + save_counter += 1; + if save_counter == save_every { + save_counter = 0; + self.repository.save_index()?; + } } + self.repository.save_index()?; Ok(()) } } diff --git a/src/error.rs b/src/error.rs deleted file mode 100644 index a7eb556..0000000 --- a/src/error.rs +++ /dev/null @@ -1,46 +0,0 @@ -use std::io; - -use failure::Fail; -use std::path::StripPrefixError; - -#[derive(Debug, Fail)] -pub enum BakareError { - #[fail(display = "io error")] - IOError(Option), - #[fail(display = "backup source same as repository")] - SourceSameAsRepository, - #[fail(display = "repository path is not absolute")] - RepositoryPathNotAbsolute, - #[fail(display = "path to store is not absolute")] - PathToStoreNotAbsolute, - #[fail(display = "directory used in place of a file")] - DirectoryNotFile, - #[fail(display = "corrupted repository - cannot find file")] - CorruptedRepoNoFile, - #[fail(display = "index loading error")] - IndexLoadingError(Option), -} - -impl From for BakareError { - fn from(e: io::Error) -> Self { - BakareError::IOError(Some(e)) - } -} - -impl From for BakareError { - fn from(e: walkdir::Error) -> Self { - BakareError::IOError(e.into_io_error()) - } -} - -impl From for BakareError { - fn from(_: StripPrefixError) -> Self { - BakareError::IOError(None) - } -} - -impl From for BakareError { - fn from(e: serde_cbor::Error) -> Self { - BakareError::IndexLoadingError(Some(e)) - } -} diff --git a/src/index/io.rs b/src/index/io.rs new file mode 100644 index 0000000..a4052d8 --- /dev/null +++ b/src/index/io.rs @@ -0,0 +1,105 @@ +use atomicwrites::{AllowOverwrite, AtomicFile}; +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; + +use uuid::Uuid; + +use crate::index::item::IndexItem; +use crate::index::{lock, Index}; +use crate::repository::ItemId; +use anyhow::Result; +use anyhow::*; +use lock::Lock; +use nix::unistd::getpid; +use std::{cmp::max, io::Write}; + +impl Index { + pub fn load(repository_path: &Path) -> Result { + if !repository_path.exists() { + let mut index = Index::new(repository_path); + index.save()?; + } + let lock = Lock::new(repository_path)?; + let index = Index::load_from_file(&Index::index_file_path_for_repository_path(repository_path))?; + lock.release()?; + log::debug!( + "[{}] loaded index from {}, version: {}", + getpid(), + repository_path.to_string_lossy(), + index.version + ); + Ok(index) + } + + pub fn save(&mut self) -> Result<()> { + let lock_id = Uuid::new_v4(); + let lock = Lock::new(&self.index_directory())?; + if self.index_file_path().exists() { + let index = Index::load_from_file(&Index::index_file_path_for_repository_path(&self.index_directory()))?; + self.merge_items_by_file_id(index.items_by_file_id); + self.merge_newest_items(index.newest_items_by_source_path); + self.version = max(self.version.clone(), index.version); + } + self.version = self.version.next(); + self.write_index_to_file(self.index_file_path())?; + lock.release()?; + log::debug!("[{}] saved index version {} with lock id {}", getpid(), self.version, lock_id,); + Ok(()) + } + + fn write_index_to_file(&mut self, path: T) -> Result<()> + where + T: AsRef, + { + fs::create_dir_all(path.as_ref().parent().unwrap()).context("create index directory")?; + + let file = AtomicFile::new(&path, AllowOverwrite); + + file.write(|f| { + let contents = serde_json::to_string(&self)?; + f.write_all(contents.as_bytes()) + }) + .context("writing index to disk")?; + + Ok(()) + } + + fn index_file_path(&self) -> PathBuf { + Path::new(&self.index_path).to_path_buf() + } + + fn load_from_file>(index_file_path: T) -> Result { + let path_text = format!("{}", index_file_path.as_ref().to_string_lossy()); + let index_text = + fs::read_to_string(path_text.clone()).context(format!("reading index file contents from {}", path_text))?; + + let mut index: Index = serde_json::from_str(&index_text).context(format!("cannot read index from: {}", index_text))?; + index.index_path = path_text; + Ok(index) + } + + fn merge_newest_items(&mut self, old_newest_items: HashMap) { + for (source_path, old_newest_item) in old_newest_items { + if let Some(new_newest_item) = self.newest_items_by_source_path.get(&source_path) { + if old_newest_item.version() > new_newest_item.version() { + self.newest_items_by_source_path.insert(source_path, old_newest_item); + } + } else { + self.newest_items_by_source_path.insert(source_path, old_newest_item); + } + } + } + + fn merge_items_by_file_id(&mut self, old_items_by_file_id: HashMap) { + self.items_by_file_id.extend(old_items_by_file_id); + } + + fn index_file_path_for_repository_path(path: &Path) -> PathBuf { + path.join("index") + } + + fn index_directory(&self) -> PathBuf { + self.index_file_path().parent().unwrap().to_path_buf() + } +} diff --git a/src/index/item.rs b/src/index/item.rs new file mode 100644 index 0000000..605ac81 --- /dev/null +++ b/src/index/item.rs @@ -0,0 +1,59 @@ +use serde::{Deserialize, Serialize}; + +use crate::repository::ItemId; +use crate::{repository_item::RepositoryItem, version::Version}; + +#[derive(Clone, Debug, PartialOrd, PartialEq, Ord, Eq, Serialize, Deserialize)] +pub struct IndexItem { + relative_path: String, + original_source_path: String, + id: ItemId, + version: Version, +} + +impl IndexItem { + pub fn from(original_source_path: String, relative_path: String, id: ItemId, version: Version) -> IndexItem { + IndexItem { + relative_path, + original_source_path, + id, + version, + } + } + + pub fn next_version(&self, id: ItemId, relative_path: String) -> IndexItem { + IndexItem { + original_source_path: self.original_source_path.clone(), + version: self.version.next(), + relative_path, + id, + } + } + + pub fn version(&self) -> Version { + self.version.clone() + } + + pub fn id(&self) -> ItemId { + self.id.clone() + } + + pub fn relative_path(&self) -> &str { + &self.relative_path + } + + pub fn original_source_path(&self) -> &str { + &self.original_source_path + } +} + +impl From for IndexItem { + fn from(i: RepositoryItem) -> Self { + IndexItem { + relative_path: i.relative_path().to_string_lossy().to_string(), + original_source_path: i.original_source_path().to_string_lossy().to_string(), + id: i.id().clone(), + version: i.version().clone(), + } + } +} diff --git a/src/index/lock.rs b/src/index/lock.rs new file mode 100644 index 0000000..f6c7a21 --- /dev/null +++ b/src/index/lock.rs @@ -0,0 +1,83 @@ +use anyhow::Result; +use anyhow::*; +use atomicwrites::{AtomicFile, DisallowOverwrite}; +use glob::{glob, Paths}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use uuid::Uuid; + +use rand::{rngs::OsRng, RngCore}; +use std::{thread, time}; + +pub struct Lock { + path: PathBuf, +} + +impl Lock { + pub fn new(index_directory: &Path) -> Result { + let mut buffer = [0u8; 16]; + OsRng.fill_bytes(&mut buffer); + let id = Uuid::from_bytes(buffer); + Lock::wait_to_have_sole_lock(id, index_directory)?; + let path = Lock::lock_file_path(index_directory, id); + Ok(Lock { path }) + } + + pub fn release(self) -> Result<()> { + if self.path.exists() { + std::fs::remove_file(self.path)?; + } + Ok(()) + } + + fn wait_to_have_sole_lock(lock_id: Uuid, index_directory: &Path) -> Result<()> { + Lock::create_lock_file(lock_id, index_directory)?; + while !Lock::sole_lock(lock_id, index_directory)? { + let path = Lock::lock_file_path(index_directory, lock_id); + std::fs::remove_file(path)?; + let sleep_duration = time::Duration::from_millis((OsRng.next_u32() % 256).into()); + thread::sleep(sleep_duration); + Lock::create_lock_file(lock_id, index_directory)?; + } + Ok(()) + } + + fn sole_lock(lock_id: Uuid, index_directory: &Path) -> Result { + let my_lock_file_path = Lock::lock_file_path(index_directory, lock_id); + let locks = Lock::all_locks(index_directory)?; + let mut only_mine = true; + for path in locks { + let path = path?; + if path.to_string_lossy() != my_lock_file_path.to_string_lossy() { + only_mine = false; + break; + } + } + Ok(only_mine) + } + + fn all_locks(index_directory: &Path) -> Result { + let locks_glob = Lock::locks_glob(index_directory); + Ok(glob(&locks_glob)?) + } + + fn create_lock_file(lock_id: Uuid, index_directory: &Path) -> Result<()> { + let lock_file_path = Lock::lock_file_path(index_directory, lock_id); + let file = AtomicFile::new(lock_file_path, DisallowOverwrite); + match file.write(|f| f.write_all(lock_id.to_hyphenated().to_string().as_bytes())) { + Ok(_) => Ok(()), + Err(e) => Err(anyhow!("error acquiring lock: {}", e)), + } + } + + fn lock_file_path(path: &Path, lock_id: Uuid) -> PathBuf { + let path_text = &format!("{}/{}.lock", path.to_string_lossy(), lock_id); + Path::new(path_text).to_path_buf() + } + + fn locks_glob(path: &Path) -> String { + format!("{}/*.lock", path.to_string_lossy()) + } +} + +//TODO: add drop implementation diff --git a/src/index.rs b/src/index/mod.rs similarity index 55% rename from src/index.rs rename to src/index/mod.rs index 726d805..0a44ec2 100644 --- a/src/index.rs +++ b/src/index/mod.rs @@ -1,21 +1,18 @@ use std::collections::hash_map::Iter; use std::collections::HashMap; -use std::fs::File; use std::path::Path; use serde::{Deserialize, Serialize}; -use crate::error::BakareError; -use crate::repository::{ItemId, Version}; -use crate::repository_item::RepositoryItem; +use crate::index::item::IndexItem; +use crate::repository::ItemId; +use crate::{repository_item::RepositoryItem, version::Version}; +use anyhow::Result; +use anyhow::*; -#[derive(Clone, Debug, PartialOrd, PartialEq, Ord, Eq, Serialize, Deserialize)] -pub struct IndexItem { - relative_path: String, - original_source_path: String, - id: ItemId, - version: Version, -} +mod io; +mod item; +mod lock; #[derive(Serialize, Deserialize)] pub struct Index { @@ -23,38 +20,7 @@ pub struct Index { items_by_file_id: HashMap, index_path: String, repository_path: String, -} - -pub struct IndexItemIterator<'a> { - iterator: Iter<'a, String, IndexItem>, -} - -impl<'a> Iterator for IndexItemIterator<'a> { - type Item = IndexItem; - - fn next(&mut self) -> Option { - self.iterator.next().map(|i| i.1.clone()) - } -} - -impl IndexItem { - fn from(original_source_path: String, relative_path: String, id: ItemId, version: Version) -> IndexItem { - IndexItem { - relative_path, - original_source_path, - id, - version, - } - } - - fn next_version(&self, id: ItemId, relative_path: String) -> IndexItem { - IndexItem { - original_source_path: self.original_source_path.clone(), - version: self.version.next(), - relative_path, - id, - } - } + version: Version, } impl Index { @@ -64,26 +30,10 @@ impl Index { items_by_file_id: Default::default(), index_path: repository_path.join("index").to_string_lossy().to_string(), repository_path: repository_path.to_string_lossy().to_string(), + version: Version::default(), } } - pub fn load(path: &Path) -> Result { - let index_file_path = path.join("index"); - let index_file = File::open(index_file_path)?; - let index: Index = serde_cbor::from_reader(index_file)?; - Ok(index) - } - - pub fn save(&self) -> Result<(), BakareError> { - let index_file = File::create(self.index_file_path())?; - serde_cbor::to_writer(index_file, &self)?; - Ok(()) - } - - pub fn index_file_path(&self) -> &Path { - Path::new(&self.index_path) - } - pub fn remember(&mut self, original_source_path: &Path, relative_path: &Path, id: ItemId) { let item = if let Some(old) = self .newest_items_by_source_path @@ -99,30 +49,30 @@ impl Index { ) }; - self.items_by_file_id.insert(item.id.clone(), item.clone()); + self.items_by_file_id.insert(item.id(), item.clone()); self.newest_items_by_source_path .insert(original_source_path.to_string_lossy().to_string(), item); } pub fn repository_item(&self, i: &IndexItem) -> RepositoryItem { let index_item = i.clone(); - let relative_path = Path::new(&index_item.relative_path); + let relative_path = Path::new(index_item.relative_path()); let repository_path = Path::new(&self.repository_path); - let original_source_path = Path::new(&index_item.original_source_path); + let original_source_path = Path::new(index_item.original_source_path()); let absolute_path = repository_path.join(relative_path); let absolute_path = absolute_path.as_path(); RepositoryItem::from( original_source_path, absolute_path, relative_path, - index_item.id, - index_item.version, + index_item.id(), + index_item.version(), ) } - pub fn newest_item_by_source_path(&self, path: &Path) -> Result, BakareError> { + pub fn newest_item_by_source_path(&self, path: &Path) -> Result> { if !path.is_absolute() { - return Err(BakareError::RepositoryPathNotAbsolute); + return Err(anyhow!("repository path not absolute")); } Ok(self .newest_items_by_source_path @@ -130,7 +80,7 @@ impl Index { .cloned()) } - pub fn item_by_id(&self, id: &ItemId) -> Result, BakareError> { + pub fn item_by_id(&self, id: &ItemId) -> Result> { Ok(self.items_by_file_id.get(id).cloned()) } @@ -141,13 +91,14 @@ impl Index { } } -impl From for IndexItem { - fn from(i: RepositoryItem) -> Self { - IndexItem { - relative_path: i.relative_path().to_string_lossy().to_string(), - original_source_path: i.original_source_path().to_string_lossy().to_string(), - id: i.id().clone(), - version: i.version().clone(), - } +pub struct IndexItemIterator<'a> { + iterator: Iter<'a, String, IndexItem>, +} + +impl<'a> Iterator for IndexItemIterator<'a> { + type Item = IndexItem; + + fn next(&mut self) -> Option { + self.iterator.next().map(|i| i.1.clone()) } } diff --git a/src/lib.rs b/src/lib.rs index fd6768b..f88bb6d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,4 @@ pub mod backup; -pub mod error; pub mod repository; pub mod restore; pub mod source; @@ -8,3 +7,4 @@ pub mod test; mod index; mod repository_item; +mod version; diff --git a/src/repository.rs b/src/repository.rs index e8b8e19..02a4291 100644 --- a/src/repository.rs +++ b/src/repository.rs @@ -1,15 +1,16 @@ -use std::path::{Path, PathBuf}; -use std::{fmt, fs, io}; - -use crate::error::BakareError; -use crate::index::{Index, IndexItemIterator}; -use crate::repository_item::RepositoryItem; -use serde::{Deserialize, Serialize}; -use sha2::Digest; -use sha2::Sha512; use std::fmt::Formatter; use std::fs::File; use std::io::BufReader; +use std::path::{Path, PathBuf}; +use std::{fmt, fs, io}; + +use crate::index::{Index, IndexItemIterator}; +use crate::repository_item::RepositoryItem; +use anyhow::Result; +use anyhow::*; +use serde::{Deserialize, Serialize}; +use sha2::Digest; +use sha2::Sha512; use walkdir::WalkDir; /// represents a place where backup is stored an can be restored from. @@ -25,16 +26,33 @@ pub struct Repository<'a> { const DATA_DIR_NAME: &str = "data"; #[derive(Clone, Debug, PartialOrd, PartialEq, Ord, Eq, Serialize, Deserialize, Hash)] -pub struct ItemId(Box<[u8]>); - -#[derive(Clone, Debug, PartialOrd, PartialEq, Ord, Eq, Serialize, Deserialize, Hash)] -pub struct Version(u128); +pub struct ItemId(#[serde(with = "base64")] Vec); pub struct RepositoryItemIterator<'a> { iterator: IndexItemIterator<'a>, index: &'a Index, } +mod base64 { + use ::base64; + use serde::{de, Deserialize, Deserializer, Serializer}; + + pub fn serialize(bytes: &[u8], serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&base64::encode(bytes)) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let s = <&str>::deserialize(deserializer)?; + base64::decode(s).map_err(de::Error::custom) + } +} + impl<'a> Iterator for RepositoryItemIterator<'a> { type Item = RepositoryItem; @@ -43,18 +61,6 @@ impl<'a> Iterator for RepositoryItemIterator<'a> { } } -impl Version { - pub fn next(&self) -> Self { - Version(self.0 + 1) - } -} - -impl Default for Version { - fn default() -> Self { - Version(1) - } -} - impl AsRef<[u8]> for ItemId { fn as_ref(&self) -> &[u8] { &self.0 @@ -63,7 +69,7 @@ impl AsRef<[u8]> for ItemId { impl From<&[u8]> for ItemId { fn from(a: &[u8]) -> Self { - ItemId(Box::from(a)) + ItemId(a.into()) } } @@ -74,15 +80,15 @@ impl fmt::Display for ItemId { } impl<'a> Repository<'a> { - pub fn init(path: &Path) -> Result<(), BakareError> { - let index = Index::new(path); + pub fn init(path: &Path) -> Result<()> { + let mut index = Index::new(path); index.save()?; Ok(()) } - pub fn open(path: &Path) -> Result { + pub fn open(path: &Path) -> Result { if !path.is_absolute() { - return Err(BakareError::RepositoryPathNotAbsolute); + return Err(anyhow!("path to repository not absolute")); } let index = Index::load(path)?; @@ -93,9 +99,13 @@ impl<'a> Repository<'a> { &self.path } - pub fn store(&mut self, source_path: &Path) -> Result<(), BakareError> { + pub fn save_index(&mut self) -> Result<()> { + self.index.save() + } + + pub fn store(&mut self, source_path: &Path) -> Result<()> { if !source_path.is_absolute() { - return Err(BakareError::PathToStoreNotAbsolute); + return Err(anyhow!("path to store not absolute")); } let id = Repository::calculate_id(source_path)?; let destination_path = self.data_dir(); @@ -103,23 +113,23 @@ impl<'a> Repository<'a> { let destination_path = Path::new(&destination_path); if source_path.is_file() { - fs::create_dir_all(destination_path.parent().unwrap())?; + let parent = destination_path.parent().unwrap(); + fs::create_dir_all(parent)?; fs::copy(source_path, destination_path)?; let relative_path = destination_path.strip_prefix(self.path)?; self.index.remember(source_path, relative_path, id); - self.index.save()?; } Ok(()) } - pub fn newest_item_by_source_path(&self, path: &Path) -> Result, BakareError> { + pub fn newest_item_by_source_path(&self, path: &Path) -> Result> { Ok(self .index .newest_item_by_source_path(path)? .map(|i| self.index.repository_item(&i))) } - pub fn item_by_id(&self, id: &ItemId) -> Result, BakareError> { + pub fn item_by_id(&self, id: &ItemId) -> Result> { self.index.item_by_id(id).map(|i| i.map(|i| self.index.repository_item(&i))) } @@ -130,7 +140,7 @@ impl<'a> Repository<'a> { } } - pub fn data_weight(&self) -> Result { + pub fn data_weight(&self) -> Result { let total_size = WalkDir::new(self.data_dir()) .into_iter() .filter_map(|entry| entry.ok()) @@ -144,7 +154,7 @@ impl<'a> Repository<'a> { self.path().join(DATA_DIR_NAME) } - fn calculate_id(source_path: &Path) -> Result { + fn calculate_id(source_path: &Path) -> Result { let source_file = File::open(source_path)?; let mut reader = BufReader::new(source_file); let mut hasher = Sha512::new(); diff --git a/src/repository_item.rs b/src/repository_item.rs index 2ba1ea2..23f9d83 100644 --- a/src/repository_item.rs +++ b/src/repository_item.rs @@ -1,5 +1,6 @@ -use crate::error::BakareError; -use crate::repository::{ItemId, Version}; +use crate::{repository::ItemId, version::Version}; +use anyhow::Result; +use anyhow::*; use std::fmt::{Display, Formatter}; use std::path::Path; use std::{fmt, fs}; @@ -24,23 +25,22 @@ impl RepositoryItem { } } - pub fn save(&self, save_to: &Path) -> Result<(), BakareError> { + pub fn save(&self, save_to: &Path) -> Result<()> { if !save_to.is_absolute() { - return Err(BakareError::PathToStoreNotAbsolute); + return Err(anyhow!("path to store not absolute")); } let target_path = save_to.join(&self.original_source_path.strip_prefix("/")?); if !target_path.is_absolute() { - return Err(BakareError::PathToStoreNotAbsolute); + return Err(anyhow!("path to store not absolute")); } let parent = target_path.parent().unwrap(); if !parent.exists() { fs::create_dir_all(parent)?; } if !self.absolute_path.exists() { - return Err(BakareError::CorruptedRepoNoFile); + return Err(anyhow!("corrupted repository")); } - println!("restoring {} to {}", &self.absolute_path.display(), &target_path.display()); fs::copy(&self.absolute_path, &target_path)?; Ok(()) diff --git a/src/restore.rs b/src/restore.rs index ae56b2e..aac7551 100644 --- a/src/restore.rs +++ b/src/restore.rs @@ -1,30 +1,32 @@ use std::path::Path; -use crate::error::BakareError; use crate::repository::Repository; use crate::repository_item::RepositoryItem; +use anyhow::Result; +use anyhow::*; pub struct Engine<'a> { - repository: &'a Repository<'a>, + repository: &'a mut Repository<'a>, target_path: &'a Path, } impl<'a> Engine<'a> { - pub fn new(repository: &'a Repository, target_path: &'a Path) -> Result { + pub fn new(repository: &'a mut Repository<'a>, target_path: &'a Path) -> Result { if !target_path.is_absolute() { - return Err(BakareError::PathToStoreNotAbsolute); + return Err(anyhow!("path to store not absolute")); } Ok(Engine { repository, target_path }) } - pub fn restore_all(&self) -> Result<(), BakareError> { + pub fn restore_all(&mut self) -> Result<()> { for item in self.repository.newest_items() { self.restore(&item)?; } + self.repository.save_index()?; Ok(()) } - pub fn restore(&self, item: &RepositoryItem) -> Result<(), BakareError> { + pub fn restore(&self, item: &RepositoryItem) -> Result<()> { item.save(self.target_path)?; Ok(()) } diff --git a/src/test/assertions.rs b/src/test/assertions.rs index d62d896..27fbdf4 100644 --- a/src/test/assertions.rs +++ b/src/test/assertions.rs @@ -5,15 +5,15 @@ use std::path::Path; use tempfile::tempdir; use walkdir::WalkDir; -use crate::error::BakareError; use crate::repository::{ItemId, Repository}; use crate::repository_item::RepositoryItem; use crate::source::TempSource; use crate::{backup, restore}; +use anyhow::Result; -pub fn assert_same_after_restore(source_path: &Path) -> Result<(), BakareError> { - let repository_path = tempdir()?.into_path(); - let restore_target = tempdir()?.into_path(); +pub fn assert_same_after_restore(source_path: &Path) -> Result<()> { + let repository_path = tempdir().unwrap().into_path(); + let restore_target = tempdir().unwrap().into_path(); assert_ne!(source_path, repository_path); assert_ne!(repository_path, restore_target); @@ -25,8 +25,8 @@ pub fn assert_same_after_restore(source_path: &Path) -> Result<(), BakareError> backup_engine.backup()?; } { - let restore_repository = Repository::open(repository_path.as_path())?; - let restore_engine = restore::Engine::new(&restore_repository, &restore_target)?; + let mut restore_repository = Repository::open(repository_path.as_path())?; + let mut restore_engine = restore::Engine::new(&mut restore_repository, &restore_target)?; restore_engine.restore_all()?; } @@ -34,15 +34,12 @@ pub fn assert_same_after_restore(source_path: &Path) -> Result<(), BakareError> Ok(()) } -pub fn assert_restored_file_contents( - repository_path: &Path, - source_file_full_path: &Path, - contents: &str, -) -> Result<(), BakareError> { - let restore_repository = Repository::open(repository_path)?; - let restore_target = tempdir()?; - let restore_engine = restore::Engine::new(&restore_repository, &restore_target.path())?; +pub fn assert_restored_file_contents(repository_path: &Path, source_file_full_path: &Path, contents: &str) -> Result<()> { + let mut restore_repository = Repository::open(repository_path)?; let item = restore_repository.newest_item_by_source_path(&source_file_full_path)?; + let restore_target = tempdir().unwrap(); + let restore_engine = restore::Engine::new(&mut restore_repository, &restore_target.path())?; + restore_engine.restore(&item.unwrap())?; let restored_file_path = restore_target.path().join(source_file_full_path.strip_prefix("/")?); assert_target_file_contents(&restored_file_path, contents) @@ -53,17 +50,17 @@ pub fn assert_restored_from_version_has_contents( source_file_full_path: &Path, old_contents: &str, old_id: &ItemId, -) -> Result<(), BakareError> { - let restore_repository = Repository::open(repository_path)?; - let restore_target = tempdir()?; - let restore_engine = restore::Engine::new(&restore_repository, &restore_target.path())?; +) -> Result<()> { + let mut restore_repository = Repository::open(repository_path)?; let old_item = restore_repository.item_by_id(&old_id)?; + let restore_target = tempdir().unwrap(); + let restore_engine = restore::Engine::new(&mut restore_repository, &restore_target.path())?; restore_engine.restore(&old_item.unwrap())?; let restored_file_path = restore_target.path().join(source_file_full_path.strip_prefix("/")?); assert_target_file_contents(&restored_file_path, old_contents) } -pub fn newest_item(repository_path: &Path, source_file_full_path: &Path) -> Result { +pub fn newest_item(repository_path: &Path, source_file_full_path: &Path) -> Result { let item = { let reading_repository = Repository::open(repository_path)?; let item = reading_repository.newest_item_by_source_path(&source_file_full_path)?; @@ -73,10 +70,10 @@ pub fn newest_item(repository_path: &Path, source_file_full_path: &Path) -> Resu Ok(item) } -pub fn restore_all_from_reloaded_repository(repository_path: &Path, restore_target: &Path) -> Result<(), BakareError> { +pub fn restore_all_from_reloaded_repository(repository_path: &Path, restore_target: &Path) -> Result<()> { { - let restore_repository = Repository::open(repository_path)?; - let restore_engine = restore::Engine::new(&restore_repository, &restore_target)?; + let mut restore_repository = Repository::open(repository_path)?; + let mut restore_engine = restore::Engine::new(&mut restore_repository, &restore_target)?; restore_engine.restore_all()?; Ok(()) } @@ -87,49 +84,49 @@ pub fn backup_file_with_contents( repository_path: &Path, source_file_relative_path: &str, contents: &str, -) -> Result<(), BakareError> { +) -> Result<()> { { let mut backup_repository = Repository::open(repository_path)?; let mut backup_engine = backup::Engine::new(source.path(), &mut backup_repository)?; - source.write_text_to_file(source_file_relative_path, contents)?; + source.write_text_to_file(source_file_relative_path, contents).unwrap(); backup_engine.backup()?; Ok(()) } } -pub fn data_weight(repository_path: &Path) -> Result { +pub fn data_weight(repository_path: &Path) -> Result { { let repository = Repository::open(repository_path)?; Ok(repository.data_weight()?) } } -fn assert_directory_trees_have_same_contents(left: &Path, right: &Path) -> Result<(), BakareError> { +fn assert_directory_trees_have_same_contents(left: &Path, right: &Path) -> Result<()> { let left_files = get_sorted_files_recursively(left)?; let right_files = get_sorted_files_recursively(right)?; let pairs = left_files.iter().zip(right_files); for (l, r) in pairs { assert_eq!(l.file_name(), r.file_name()); - let mut fl = File::open(l)?; - let mut fr = File::open(r)?; + let mut fl = File::open(l).unwrap(); + let mut fr = File::open(r).unwrap(); let mut bl = vec![]; let mut br = vec![]; - fl.read_to_end(&mut bl)?; - fr.read_to_end(&mut br)?; + fl.read_to_end(&mut bl).unwrap(); + fr.read_to_end(&mut br).unwrap(); assert_eq!(bl, br); } Ok(()) } -fn get_sorted_files_recursively(path: &Path) -> Result>, BakareError> { - let walker = WalkDir::new(path).sort_by(|a, b| a.file_name().cmp(b.file_name())); +pub fn get_sorted_files_recursively>(path: T) -> Result>> { + let walker = WalkDir::new(path.as_ref()).sort_by(|a, b| a.file_name().cmp(b.file_name())); let mut result = vec![]; for maybe_entry in walker { let entry = maybe_entry?; - if entry.path() == path { + if entry.path() == path.as_ref() { continue; } if entry.path().is_file() { @@ -140,10 +137,13 @@ fn get_sorted_files_recursively(path: &Path) -> Result>, BakareErr Ok(result) } -fn assert_target_file_contents(restored_path: &Path, expected_contents: &str) -> Result<(), BakareError> { +fn assert_target_file_contents(restored_path: &Path, expected_contents: &str) -> Result<()> { let mut actual_contents = String::new(); assert!(restored_path.exists(), "Expected '{}' to be there", restored_path.display()); - File::open(restored_path)?.read_to_string(&mut actual_contents)?; + File::open(restored_path) + .unwrap() + .read_to_string(&mut actual_contents) + .unwrap(); assert_eq!(expected_contents, actual_contents); Ok(()) } diff --git a/src/version.rs b/src/version.rs new file mode 100644 index 0000000..15d3699 --- /dev/null +++ b/src/version.rs @@ -0,0 +1,23 @@ +use serde::{Deserialize, Serialize}; +use std::fmt::Display; + +#[derive(Clone, Debug, PartialOrd, PartialEq, Ord, Eq, Serialize, Deserialize, Hash)] +pub struct Version(u128); + +impl Version { + pub fn next(&self) -> Self { + Version(self.0 + 1) + } +} + +impl Default for Version { + fn default() -> Self { + Version(1) + } +} + +impl Display for Version { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} diff --git a/tests/concurrency_tests.rs b/tests/concurrency_tests.rs new file mode 100644 index 0000000..7ff6bde --- /dev/null +++ b/tests/concurrency_tests.rs @@ -0,0 +1,115 @@ +use std::fs; +use std::path::Path; + +use anyhow::Result; +use async_log::span; +use bakare::repository::Repository; +use bakare::source::TempSource; +use bakare::test::assertions::*; +use bakare::{backup, restore}; +use nix::sys::wait::{waitpid, WaitStatus}; +use nix::unistd::{fork, getpid, ForkResult}; +use tempfile::tempdir; + +#[test] +fn handle_concurrent_backups() -> Result<()> { + setup_logger(); + let repository_path = &tempdir().unwrap().into_path(); + Repository::init(repository_path)?; + + let parallel_backups_number = 16; + let files_per_backup_number = 16; + let total_number_of_files = parallel_backups_number * files_per_backup_number; + let finished_backup_runs = backup_in_parallel(repository_path, parallel_backups_number, files_per_backup_number)?; + assert_eq!(finished_backup_runs.len(), parallel_backups_number); + + let all_restored_files = restore_all(repository_path)?; + assert_eq!(all_restored_files.len(), total_number_of_files); + + for i in 0..parallel_backups_number { + for j in 0..files_per_backup_number { + let id = file_id(i, j); + let file = all_restored_files.iter().find(|f| f.ends_with(id.clone())); + assert!(file.unwrap().exists(), "file {:?} does not exist", file); + let contents = fs::read_to_string(file.unwrap()).unwrap(); + assert_eq!(id.to_string(), contents.to_owned()); + } + } + Ok(()) +} + +fn backup_in_parallel( + repository_path: T, + parallel_backups_number: usize, + files_per_backup_number: usize, +) -> Result> +where + T: AsRef + Sync, +{ + let task_numbers = (0..parallel_backups_number).collect::>(); + let mut child_pids = vec![]; + span!("[{}] acquiring children for parent", getpid(), { + for task_number in &task_numbers { + match unsafe { fork() } { + Ok(ForkResult::Parent { child }) => { + child_pids.push(child); + } + Ok(ForkResult::Child) => { + backup_process(*task_number, &repository_path, files_per_backup_number)?; + std::process::exit(0); + } + + Err(_) => panic!("fork failed"), + } + } + }); + span!("[{}] waiting for {} children", getpid(), child_pids.len(), { + for pid in child_pids { + log::debug!("[{}] waiting for a child {} to exit", getpid(), pid); + let status = waitpid(Some(pid), None)?; + match status { + WaitStatus::Exited(pid, code) => { + assert!(code == 0, "failed the wait for {} with code {}", pid, code); + } + WaitStatus::Signaled(pid, _, _) => assert!(false, "failed with signal for {}", pid), + _ => panic!("unknown state"), + } + } + }); + Ok(task_numbers) +} + +fn backup_process(task_number: usize, repository_path: T, files_per_backup_number: usize) -> Result<()> +where + T: AsRef + Sync, +{ + let mut repository = Repository::open(repository_path.as_ref())?; + let source = TempSource::new().unwrap(); + let mut backup_engine = backup::Engine::new(source.path(), &mut repository)?; + for i in 0..files_per_backup_number { + let id = file_id(task_number, i); + source.write_text_to_file(&id, &id).unwrap(); + } + backup_engine.backup()?; + Ok(()) +} + +fn restore_all>(repository_path: T) -> Result>> { + let restore_target = tempdir().unwrap().into_path(); + let mut restore_repository = Repository::open(repository_path.as_ref())?; + let mut restore_engine = restore::Engine::new(&mut restore_repository, restore_target.as_ref())?; + restore_engine.restore_all()?; + get_sorted_files_recursively(&restore_target) +} + +fn setup_logger() { + let logger = femme::pretty::Logger::new(); + async_log::Logger::wrap(logger, rand::random::) + .start(log::LevelFilter::Trace) + .unwrap(); +} + +fn file_id(i: usize, j: usize) -> String { + format!("{}-{}", i, j) +} +// TODO handle stale leftover locks diff --git a/tests/system_tests.rs b/tests/system_tests.rs index 97e9405..fc9a2a3 100644 --- a/tests/system_tests.rs +++ b/tests/system_tests.rs @@ -1,27 +1,27 @@ use tempfile::tempdir; +use anyhow::Result; use bakare::backup; -use bakare::error::BakareError; use bakare::repository::Repository; use bakare::source::TempSource; use bakare::test::assertions::*; #[test] -fn restore_multiple_files() -> Result<(), BakareError> { - let source = TempSource::new()?; +fn restore_multiple_files() -> Result<()> { + let source = TempSource::new().unwrap(); - source.write_text_to_file("first", "some contents")?; - source.write_text_to_file("second", "some contents")?; - source.write_text_to_file("third", "some other contents")?; + source.write_text_to_file("first", "some contents").unwrap(); + source.write_text_to_file("second", "some contents").unwrap(); + source.write_text_to_file("third", "some other contents").unwrap(); assert_same_after_restore(source.path()) } #[test] -fn restore_files_after_reopening_repository() -> Result<(), BakareError> { - let source = TempSource::new()?; - let repository_path = &tempdir()?.into_path(); - let restore_target = tempdir()?.into_path(); +fn restore_files_after_reopening_repository() -> Result<()> { + let source = TempSource::new().unwrap(); + let repository_path = &tempdir().unwrap().into_path(); + let restore_target = tempdir().unwrap().into_path(); Repository::init(repository_path)?; let source_file_relative_path = "some file path"; @@ -36,9 +36,9 @@ fn restore_files_after_reopening_repository() -> Result<(), BakareError> { } #[test] -fn restore_older_version_of_file() -> Result<(), BakareError> { - let source = TempSource::new()?; - let repository_path = tempdir()?.into_path(); +fn restore_older_version_of_file() -> Result<()> { + let source = TempSource::new().unwrap(); + let repository_path = tempdir().unwrap().into_path(); Repository::init(repository_path.as_path())?; let source_file_relative_path = "some path"; @@ -57,9 +57,9 @@ fn restore_older_version_of_file() -> Result<(), BakareError> { } #[test] -fn newer_version_should_be_greater_than_earlier_version() -> Result<(), BakareError> { - let source = TempSource::new()?; - let repository_path = tempdir()?.into_path(); +fn newer_version_should_be_greater_than_earlier_version() -> Result<()> { + let source = TempSource::new().unwrap(); + let repository_path = tempdir().unwrap().into_path(); Repository::init(repository_path.as_path())?; let source_file_relative_path = "some path"; @@ -81,9 +81,9 @@ fn newer_version_should_be_greater_than_earlier_version() -> Result<(), BakareEr } #[test] -fn store_duplicated_files_just_once() -> Result<(), BakareError> { - let source = TempSource::new()?; - let repository_path = &tempdir()?.into_path(); +fn store_duplicated_files_just_once() -> Result<()> { + let source = TempSource::new().unwrap(); + let repository_path = &tempdir().unwrap().into_path(); Repository::init(repository_path)?; assert_eq!(data_weight(&repository_path)?, 0); @@ -103,9 +103,9 @@ fn store_duplicated_files_just_once() -> Result<(), BakareError> { } #[test] -fn restore_latest_version_by_default() -> Result<(), BakareError> { - let source = TempSource::new()?; - let repository_path = &tempdir()?.into_path(); +fn restore_latest_version_by_default() -> Result<()> { + let source = TempSource::new().unwrap(); + let repository_path = &tempdir().unwrap().into_path(); Repository::init(repository_path)?; let source_file_relative_path = "some path"; @@ -118,16 +118,14 @@ fn restore_latest_version_by_default() -> Result<(), BakareError> { } #[test] -fn forbid_backup_of_paths_within_repository() -> Result<(), BakareError> { - let repository_path = &tempdir()?.into_path(); +fn forbid_backup_of_paths_within_repository() -> Result<()> { + let repository_path = &tempdir().unwrap().into_path(); Repository::init(repository_path)?; let mut repository = Repository::open(repository_path)?; - let error = backup::Engine::new(repository_path, &mut repository).err().unwrap(); - let correct_error = matches!(error, BakareError::SourceSameAsRepository); - assert!(correct_error); + let error = backup::Engine::new(repository_path, &mut repository); + assert!(error.is_err()); Ok(()) } - -// TODO: test concurrent writes // TODO: index corruption // TODO: encryption +// TODO: resume from sleep while backup in progress