cyplo/bakare
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add basic safety around running multiple backup processes concurrently.

Browse source
This commit is contained in:
Cyryl Płotnicki 2020-11-08 14:27:26 +00:00
parent 502fc153a3
commit 90a86ef95f
18 changed files with 1221 additions and 303 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

642
Cargo.lock generated
View file

@ -15,6 +15,76 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"
[[package]]
name = "aho-corasick"
version = "0.7.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5"
dependencies = [
"memchr",
]
[[package]]
name = "anyhow"
version = "1.0.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf8dcb5b4bbaa28653b647d8c77bd4ed40183b48882e130c1f1ffb73de069fd7"
[[package]]
name = "async-log"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f5ba33304f3165922dd93565a36dda8ac96c28b1a926ad535ec904e48ecad59b"
dependencies = [
"async-log-attributes",
"backtrace",
"log",
]
[[package]]
name = "async-log"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac39aabd463ad2b1fadb34c22f48e5d160fa60b7cd69a426362e71bde49ebb6d"
dependencies = [
"async-log-attributes",
"backtrace",
"log",
]
[[package]]
name = "async-log-attributes"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da97f8e61b19a72f67d8932de8b0905f7d41a1d7b9501b9938c7755f96f6362d"
dependencies = [
"proc-macro2 0.4.30",
"quote 0.6.13",
"syn 0.15.44",
]
[[package]]
name = "atomicwrites"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a2baf2feb820299c53c7ad1cc4f5914a220a1cb76d7ce321d2522a94b54651f"
dependencies = [
"nix 0.14.1",
"tempdir",
"winapi",
]
[[package]]
name = "atty"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
dependencies = [
"hermit-abi",
"libc",
"winapi",
]
[[package]] [[package]]
name = "autocfg" name = "autocfg"
version = "1.0.1" version = "1.0.1"
@ -39,17 +109,42 @@ dependencies = [
name = "bakare" name = "bakare"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"anyhow",
"async-log 2.0.0",
"atomicwrites",
"base64",
"cargo-husky", "cargo-husky",
"failure", "femme",
"fs2",
"glob",
"hex", "hex",
"log",
"nix 0.19.0",
"rand 0.7.3",
"rayon",
"rust-crypto", "rust-crypto",
"serde", "serde",
"serde_cbor", "serde_cbor",
"serde_json",
"sha2", "sha2",
"tempfile", "tempfile",
"thiserror",
"uuid",
"walkdir", "walkdir",
] ]
[[package]]
name = "base64"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
[[package]]
name = "bitflags"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
[[package]] [[package]]
name = "block-buffer" name = "block-buffer"
version = "0.9.0" version = "0.9.0"
@ -59,12 +154,24 @@ dependencies = [
"generic-array", "generic-array",
] ]
[[package]]
name = "bumpalo"
version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820"
[[package]] [[package]]
name = "cargo-husky" name = "cargo-husky"
version = "1.5.0" version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b02b629252fe8ef6460461409564e2c21d0c8e77e0944f3d189ff06c4e932ad" checksum = "7b02b629252fe8ef6460461409564e2c21d0c8e77e0944f3d189ff06c4e932ad"
[[package]]
name = "cc"
version = "1.0.62"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1770ced377336a88a67c473594ccc14eca6f4559217c34f64aac8f83d641b40"
[[package]] [[package]]
name = "cfg-if" name = "cfg-if"
version = "0.1.10" version = "0.1.10"
@ -77,12 +184,104 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "clicolors-control"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90082ee5dcdd64dc4e9e0d37fbf3ee325419e39c0092191e0393df65518f741e"
dependencies = [
"atty",
"lazy_static",
"libc",
"winapi",
]
[[package]]
name = "cloudabi"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467"
dependencies = [
"bitflags",
]
[[package]]
name = "console"
version = "0.7.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ca57c2c14b8a2bf3105bc9d15574aad80babf6a9c44b1058034cdf8bd169628"
dependencies = [
"atty",
"clicolors-control",
"encode_unicode",
"lazy_static",
"libc",
"parking_lot",
"regex",
"termios",
"unicode-width",
"winapi",
]
[[package]]
name = "const_fn"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c478836e029dcef17fb47c89023448c64f781a046e0300e257ad8225ae59afab"
[[package]] [[package]]
name = "cpuid-bool" name = "cpuid-bool"
version = "0.1.2" version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634"
[[package]]
name = "crossbeam-channel"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0f606a85340376eef0d6d8fec399e6d4a544d648386c6645eb6d0653b27d9f"
dependencies = [
"cfg-if 1.0.0",
"const_fn",
"crossbeam-utils",
"lazy_static",
"memoffset",
"scopeguard",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec91540d98355f690a86367e566ecad2e9e579f230230eb7c21398372be73ea5"
dependencies = [
"autocfg",
"cfg-if 1.0.0",
"const_fn",
"lazy_static",
]
[[package]] [[package]]
name = "digest" name = "digest"
version = "0.9.0" version = "0.9.0"
@ -93,25 +292,43 @@ dependencies = [
] ]
[[package]] [[package]]
name = "failure" name = "either"
version = "0.1.8" version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
[[package]]
name = "encode_unicode"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
[[package]]
name = "femme"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d9ac2da60a2a97a0643d082f1cb25901a90e6f2f69f03baf9763721aa31ee6a"
dependencies = [ dependencies = [
"backtrace", "async-log 1.1.0",
"failure_derive", "cfg-if 0.1.10",
"console",
"js-sys",
"log",
"serde",
"serde_derive",
"serde_json",
"wasm-bindgen",
"web-sys",
] ]
[[package]] [[package]]
name = "failure_derive" name = "fs2"
version = "0.1.8" version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
dependencies = [ dependencies = [
"proc-macro2", "libc",
"quote", "winapi",
"syn",
"synstructure",
] ]
[[package]] [[package]]
@ -153,24 +370,102 @@ version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce" checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce"
[[package]]
name = "glob"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
[[package]] [[package]]
name = "half" name = "half"
version = "1.6.0" version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d36fab90f82edc3c747f9d438e06cf0a491055896f2a279638bb5beed6c40177" checksum = "d36fab90f82edc3c747f9d438e06cf0a491055896f2a279638bb5beed6c40177"
[[package]]
name = "hermit-abi"
version = "0.1.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8"
dependencies = [
"libc",
]
[[package]] [[package]]
name = "hex" name = "hex"
version = "0.4.2" version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35"
[[package]]
name = "instant"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb1fc4429a33e1f80d41dc9fea4d108a88bec1de8053878898ae448a0b52f613"
dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "itoa"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6"
[[package]]
name = "js-sys"
version = "0.3.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca059e81d9486668f12d455a4ea6daa600bd408134cd17e3d3fb5a32d1f016f8"
dependencies = [
"wasm-bindgen",
]
[[package]]
name = "lazy_static"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]] [[package]]
name = "libc" name = "libc"
version = "0.2.80" version = "0.2.80"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614" checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614"
[[package]]
name = "lock_api"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28247cc5a5be2f05fbcd76dd0cf2c7d3b5400cb978a28042abcd4fa0b3f8261c"
dependencies = [
"scopeguard",
]
[[package]]
name = "log"
version = "0.4.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b"
dependencies = [
"cfg-if 0.1.10",
]
[[package]]
name = "memchr"
version = "2.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
[[package]]
name = "memoffset"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa"
dependencies = [
"autocfg",
]
[[package]] [[package]]
name = "miniz_oxide" name = "miniz_oxide"
version = "0.4.3" version = "0.4.3"
@ -181,6 +476,41 @@ dependencies = [
"autocfg", "autocfg",
] ]
[[package]]
name = "nix"
version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c722bee1037d430d0f8e687bbdbf222f27cc6e4e68d5caf630857bb2b6dbdce"
dependencies = [
"bitflags",
"cc",
"cfg-if 0.1.10",
"libc",
"void",
]
[[package]]
name = "nix"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85db2feff6bf70ebc3a4793191517d5f0331100a2f10f9bf93b5e5214f32b7b7"
dependencies = [
"bitflags",
"cc",
"cfg-if 0.1.10",
"libc",
]
[[package]]
name = "num_cpus"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
dependencies = [
"hermit-abi",
"libc",
]
[[package]] [[package]]
name = "object" name = "object"
version = "0.22.0" version = "0.22.0"
@ -193,19 +523,63 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
[[package]]
name = "parking_lot"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4893845fa2ca272e647da5d0e46660a314ead9c2fdd9a883aabc32e481a8733"
dependencies = [
"instant",
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b"
dependencies = [
"cfg-if 0.1.10",
"cloudabi",
"instant",
"libc",
"redox_syscall",
"smallvec",
"winapi",
]
[[package]] [[package]]
name = "ppv-lite86" name = "ppv-lite86"
version = "0.2.10" version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
[[package]]
name = "proc-macro2"
version = "0.4.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759"
dependencies = [
"unicode-xid 0.1.0",
]
[[package]] [[package]]
name = "proc-macro2" name = "proc-macro2"
version = "1.0.24" version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71"
dependencies = [ dependencies = [
"unicode-xid", "unicode-xid 0.2.1",
]
[[package]]
name = "quote"
version = "0.6.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1"
dependencies = [
"proc-macro2 0.4.30",
] ]
[[package]] [[package]]
@ -214,7 +588,7 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2 1.0.24",
] ]
[[package]] [[package]]
@ -296,6 +670,31 @@ dependencies = [
"rand_core 0.5.1", "rand_core 0.5.1",
] ]
[[package]]
name = "rayon"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674"
dependencies = [
"autocfg",
"crossbeam-deque",
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
"crossbeam-utils",
"lazy_static",
"num_cpus",
]
[[package]] [[package]]
name = "rdrand" name = "rdrand"
version = "0.4.0" version = "0.4.0"
@ -311,6 +710,24 @@ version = "0.1.57"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce"
[[package]]
name = "regex"
version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38cf2c13ed4745de91a5eb834e11c00bcc3709e773173b2ce4c56c9fbde04b9c"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
"thread_local",
]
[[package]]
name = "regex-syntax"
version = "0.6.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b181ba2dcf07aaccad5448e8ead58db5b742cf85dfe035e2227f137a539a189"
[[package]] [[package]]
name = "remove_dir_all" name = "remove_dir_all"
version = "0.5.3" version = "0.5.3"
@ -345,6 +762,12 @@ version = "0.3.24"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" checksum = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda"
[[package]]
name = "ryu"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
[[package]] [[package]]
name = "same-file" name = "same-file"
version = "1.0.6" version = "1.0.6"
@ -354,6 +777,12 @@ dependencies = [
"winapi-util", "winapi-util",
] ]
[[package]]
name = "scopeguard"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]] [[package]]
name = "serde" name = "serde"
version = "1.0.117" version = "1.0.117"
@ -379,9 +808,20 @@ version = "1.0.117"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cbd1ae72adb44aab48f325a02444a5fc079349a8d804c1fc922aed3f7454c74e" checksum = "cbd1ae72adb44aab48f325a02444a5fc079349a8d804c1fc922aed3f7454c74e"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2 1.0.24",
"quote", "quote 1.0.7",
"syn", "syn 1.0.48",
]
[[package]]
name = "serde_json"
version = "1.0.59"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcac07dbffa1c65e7f816ab9eba78eb142c6d44410f4eeba1e26e4f5dfa56b95"
dependencies = [
"itoa",
"ryu",
"serde",
] ]
[[package]] [[package]]
@ -397,27 +837,42 @@ dependencies = [
"opaque-debug", "opaque-debug",
] ]
[[package]]
name = "smallvec"
version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252"
[[package]]
name = "syn"
version = "0.15.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5"
dependencies = [
"proc-macro2 0.4.30",
"quote 0.6.13",
"unicode-xid 0.1.0",
]
[[package]] [[package]]
name = "syn" name = "syn"
version = "1.0.48" version = "1.0.48"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac" checksum = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2 1.0.24",
"quote", "quote 1.0.7",
"unicode-xid", "unicode-xid 0.2.1",
] ]
[[package]] [[package]]
name = "synstructure" name = "tempdir"
version = "0.12.4" version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8"
dependencies = [ dependencies = [
"proc-macro2", "rand 0.4.6",
"quote", "remove_dir_all",
"syn",
"unicode-xid",
] ]
[[package]] [[package]]
@ -434,6 +889,44 @@ dependencies = [
"winapi", "winapi",
] ]
[[package]]
name = "termios"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "411c5bf740737c7918b8b1fe232dca4dc9f8e754b8ad5e20966814001ed0ac6b"
dependencies = [
"libc",
]
[[package]]
name = "thiserror"
version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e9ae34b84616eedaaf1e9dd6026dbe00dcafa92aa0c8077cb69df1fcfe5e53e"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ba20f23e85b10754cd195504aebf6a27e2e6cbe28c17778a0c930724628dd56"
dependencies = [
"proc-macro2 1.0.24",
"quote 1.0.7",
"syn 1.0.48",
]
[[package]]
name = "thread_local"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14"
dependencies = [
"lazy_static",
]
[[package]] [[package]]
name = "time" name = "time"
version = "0.1.44" version = "0.1.44"
@ -451,18 +944,45 @@ version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33"
[[package]]
name = "unicode-width"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3"
[[package]]
name = "unicode-xid"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"
[[package]] [[package]]
name = "unicode-xid" name = "unicode-xid"
version = "0.2.1" version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"
[[package]]
name = "uuid"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9fde2f6a4bea1d6e007c4ad38c6839fa71cbb63b6dbf5b595aa38dc9b1093c11"
dependencies = [
"rand 0.7.3",
]
[[package]] [[package]]
name = "version_check" name = "version_check"
version = "0.9.2" version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed"
[[package]]
name = "void"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
[[package]] [[package]]
name = "walkdir" name = "walkdir"
version = "2.3.1" version = "2.3.1"
@ -486,6 +1006,72 @@ version = "0.10.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
[[package]]
name = "wasm-bindgen"
version = "0.2.68"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ac64ead5ea5f05873d7c12b545865ca2b8d28adfc50a49b84770a3a97265d42"
dependencies = [
"cfg-if 0.1.10",
"serde",
"serde_json",
"wasm-bindgen-macro",
]
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.68"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f22b422e2a757c35a73774860af8e112bff612ce6cb604224e8e47641a9e4f68"
dependencies = [
"bumpalo",
"lazy_static",
"log",
"proc-macro2 1.0.24",
"quote 1.0.7",
"syn 1.0.48",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.68"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b13312a745c08c469f0b292dd2fcd6411dba5f7160f593da6ef69b64e407038"
dependencies = [
"quote 1.0.7",
"wasm-bindgen-macro-support",
]
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.68"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f249f06ef7ee334cc3b8ff031bfc11ec99d00f34d86da7498396dc1e3b1498fe"
dependencies = [
"proc-macro2 1.0.24",
"quote 1.0.7",
"syn 1.0.48",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.68"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d649a3145108d7d3fbcde896a468d1bd636791823c9921135218ad89be08307"
[[package]]
name = "web-sys"
version = "0.3.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bf6ef87ad7ae8008e15a355ce696bed26012b7caa21605188cfd8214ab51e2d"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]] [[package]]
name = "winapi" name = "winapi"
version = "0.3.9" version = "0.3.9"

23
Cargo.toml
View file

@ -6,17 +6,30 @@ edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
[dependencies] [dependencies]
failure = "0.1" anyhow = "1.0"
async-log = "2.0"
atomicwrites = "0.2"
base64 = "0.11"
femme = "1.3"
fs2 = "0.4"
glob = "0.3"
hex = "0.4"
log = "0.4"
nix = "0.19"
rand = "0.7"
rayon = "1.1"
rust-crypto = "0.2" rust-crypto = "0.2"
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
serde_cbor = "0.11" serde_cbor = "0.11"
tempfile = "3.1" serde_json = "1.0"
walkdir = "2.3"
sha2 = "0.9" sha2 = "0.9"
hex = "0.4" tempfile = "3.1"
thiserror = "1.0"
uuid = { version = "0.8", features = ["v4"] }
walkdir = "2.3"
[dev-dependencies.cargo-husky] [dev-dependencies.cargo-husky]
version = "1" version = "1"
default-features = false default-features = false
features = ["run-for-all", "prepush-hook", "run-cargo-test", "run-cargo-clippy", "run-cargo-fmt"] features = ["run-for-all", "prepush-hook", "run-cargo-clippy", "run-cargo-fmt"]

56
README.md
View file

@ -2,8 +2,9 @@ Motivation:
All the backup systems are either slow or crashing or both on my backup. All the backup systems are either slow or crashing or both on my backup.
Tried duply: Tried duply:
<details> <details>
<summary>Works but is very slow</summary> <summary>Works but is very slow</summary>
``` ```
--------------[ Backup Statistics ]-------------- --------------[ Backup Statistics ]--------------
@ -29,32 +30,37 @@ Errors 0
</details> </details>
Tried restic: Tried restic:
* crashes with OOM
* corrupts repo if you suspend one backup process and launch another from a different computer
Goals for bakare: - crashes with OOM
* fast - corrupts repo if you suspend one backup process and launch another from a different computer
* using max bandwidth
* use max cpu
* use max disk I/O
* memory usage limit
* encryption by default - asymmetric, creates a keypair for you
* deduplication of file data
* fuzzy find by file name in stored files
* failure to process one file should not affect any other files
* intermittent network failures should not make the whole process fail (test with random packet drop)
* system suspend/resume should not make the repo become corrupted, even in the presence of other active backup processes running on other computers, targeting same repo - this is where `restic` fails
Nice to have: Goals for bakare:
* daemon that listens for file events and updates a list of files to be backed up on the next backup run - or a `continous backup` mode - the daemon uploads the file whenever it sees the change
* peer2peer mode - people storing encrypted backups for each other
* relay mode, where daemon works on one or more central points with local storage (e.g. NAS) and various computers sync with that central location. Then though the central locaiton uploads everything to the other location, typically the cloud.
Implementation: - fast
* test with randomly created dirs and files, with property based tests and fuzzer - using max bandwidth
* see if we can use `salsa` for recomputaiton - use max cpu
* index corruption tests - mutate random byte and see if everything is readable - use max disk I/O
* network packet drop tests - memory usage limit
- encryption by default - asymmetric, creates a keypair for you
- deduplication of file data
- fuzzy find by file name in stored files
- failure to process one file should not affect any other files
- intermittent network failures should not make the whole process fail (test with random packet drop)
- system suspend/resume should not make the repo become corrupted, even in the presence of other active backup processes running on other computers, targeting same repo - this is where `restic` fails
Nice to have:
- daemon that listens for file events and updates a list of files to be backed up on the next backup run - or a `continous backup` mode - the daemon uploads the file whenever it sees the change
- peer2peer mode - people storing encrypted backups for each other
- relay mode, where daemon works on one or more central points with local storage (e.g. NAS) and various computers sync with that central location. Then though the central locaiton uploads everything to the other location, typically the cloud.
Implementation:
- automatic node discovery - two roles: data publisher and data persister - should be able to figure out which node is which automatically
- test with randomly created dirs and files, with property based tests and fuzzer
- see if we can use `salsa` for recomputaiton
- index corruption tests - mutate random byte and see if everything is readable
- network packet drop tests
- use bevy for decoupling ?
- remove all `unwraps`

8
shell.nix
View file

@ -5,10 +5,14 @@ let
in in
with nixpkgs; with nixpkgs;
stdenv.mkDerivation { stdenv.mkDerivation {
name = "genpass_shell"; name = "bakare_shell";
buildInputs = [ buildInputs = [
channel.rust channel.rust
cacert openssl openssh zlib cacert openssl openssh zlib
llvm pkgconfig git pkgconfig clang llvm
git
]; ];
shellHook = ''
export RUST_SRC_PATH="${channel.rust-src}/lib/rustlib/src/rust/src"
'';
} }

17
src/backup.rs
View file

@ -1,8 +1,9 @@
use anyhow::Result;
use anyhow::*;
use std::path::Path; use std::path::Path;
use walkdir::WalkDir; use walkdir::WalkDir;
use crate::error::BakareError;
use crate::repository::Repository; use crate::repository::Repository;
pub struct Engine<'a> { pub struct Engine<'a> {
@ -11,21 +12,29 @@ pub struct Engine<'a> {
} }
impl<'a> Engine<'a> { impl<'a> Engine<'a> {
pub fn new(source_path: &'a Path, repository: &'a mut Repository<'a>) -> Result<Self, BakareError> { pub fn new(source_path: &'a Path, repository: &'a mut Repository<'a>) -> Result<Self> {
if source_path.ancestors().any(|a| a == repository.path()) { if source_path.ancestors().any(|a| a == repository.path()) {
return Err(BakareError::SourceSameAsRepository); return Err(anyhow!("source same as repository"));
} }
Ok(Engine { source_path, repository }) Ok(Engine { source_path, repository })
} }
pub fn backup(&mut self) -> Result<(), BakareError> { pub fn backup(&mut self) -> Result<()> {
let walker = WalkDir::new(self.source_path); let walker = WalkDir::new(self.source_path);
let save_every = 16;
let mut save_counter = 0;
for maybe_entry in walker { for maybe_entry in walker {
let entry = maybe_entry?; let entry = maybe_entry?;
if entry.path() != self.source_path { if entry.path() != self.source_path {
self.repository.store(entry.path())?; self.repository.store(entry.path())?;
} }
save_counter += 1;
if save_counter == save_every {
save_counter = 0;
self.repository.save_index()?;
} }
}
self.repository.save_index()?;
Ok(()) Ok(())
} }
} }

46
src/error.rs
View file

@ -1,46 +0,0 @@
use std::io;
use failure::Fail;
use std::path::StripPrefixError;
#[derive(Debug, Fail)]
pub enum BakareError {
#[fail(display = "io error")]
IOError(Option<io::Error>),
#[fail(display = "backup source same as repository")]
SourceSameAsRepository,
#[fail(display = "repository path is not absolute")]
RepositoryPathNotAbsolute,
#[fail(display = "path to store is not absolute")]
PathToStoreNotAbsolute,
#[fail(display = "directory used in place of a file")]
DirectoryNotFile,
#[fail(display = "corrupted repository - cannot find file")]
CorruptedRepoNoFile,
#[fail(display = "index loading error")]
IndexLoadingError(Option<serde_cbor::Error>),
}
impl From<io::Error> for BakareError {
fn from(e: io::Error) -> Self {
BakareError::IOError(Some(e))
}
}
impl From<walkdir::Error> for BakareError {
fn from(e: walkdir::Error) -> Self {
BakareError::IOError(e.into_io_error())
}
}
impl From<StripPrefixError> for BakareError {
fn from(_: StripPrefixError) -> Self {
BakareError::IOError(None)
}
}
impl From<serde_cbor::Error> for BakareError {
fn from(e: serde_cbor::Error) -> Self {
BakareError::IndexLoadingError(Some(e))
}
}

105
src/index/io.rs Normal file
View file

@ -0,0 +1,105 @@
use atomicwrites::{AllowOverwrite, AtomicFile};
use std::collections::HashMap;
use std::fs;
use std::path::{Path, PathBuf};
use uuid::Uuid;
use crate::index::item::IndexItem;
use crate::index::{lock, Index};
use crate::repository::ItemId;
use anyhow::Result;
use anyhow::*;
use lock::Lock;
use nix::unistd::getpid;
use std::{cmp::max, io::Write};
impl Index {
pub fn load(repository_path: &Path) -> Result<Self> {
if !repository_path.exists() {
let mut index = Index::new(repository_path);
index.save()?;
}
let lock = Lock::new(repository_path)?;
let index = Index::load_from_file(&Index::index_file_path_for_repository_path(repository_path))?;
lock.release()?;
log::debug!(
"[{}] loaded index from {}, version: {}",
getpid(),
repository_path.to_string_lossy(),
index.version
);
Ok(index)
}
pub fn save(&mut self) -> Result<()> {
let lock_id = Uuid::new_v4();
let lock = Lock::new(&self.index_directory())?;
if self.index_file_path().exists() {
let index = Index::load_from_file(&Index::index_file_path_for_repository_path(&self.index_directory()))?;
self.merge_items_by_file_id(index.items_by_file_id);
self.merge_newest_items(index.newest_items_by_source_path);
self.version = max(self.version.clone(), index.version);
}
self.version = self.version.next();
self.write_index_to_file(self.index_file_path())?;
lock.release()?;
log::debug!("[{}] saved index version {} with lock id {}", getpid(), self.version, lock_id,);
Ok(())
}
fn write_index_to_file<T>(&mut self, path: T) -> Result<()>
where
T: AsRef<Path>,
{
fs::create_dir_all(path.as_ref().parent().unwrap()).context("create index directory")?;
let file = AtomicFile::new(&path, AllowOverwrite);
file.write(|f| {
let contents = serde_json::to_string(&self)?;
f.write_all(contents.as_bytes())
})
.context("writing index to disk")?;
Ok(())
}
fn index_file_path(&self) -> PathBuf {
Path::new(&self.index_path).to_path_buf()
}
fn load_from_file<T: AsRef<Path>>(index_file_path: T) -> Result<Self> {
let path_text = format!("{}", index_file_path.as_ref().to_string_lossy());
let index_text =
fs::read_to_string(path_text.clone()).context(format!("reading index file contents from {}", path_text))?;
let mut index: Index = serde_json::from_str(&index_text).context(format!("cannot read index from: {}", index_text))?;
index.index_path = path_text;
Ok(index)
}
fn merge_newest_items(&mut self, old_newest_items: HashMap<String, IndexItem>) {
for (source_path, old_newest_item) in old_newest_items {
if let Some(new_newest_item) = self.newest_items_by_source_path.get(&source_path) {
if old_newest_item.version() > new_newest_item.version() {
self.newest_items_by_source_path.insert(source_path, old_newest_item);
}
} else {
self.newest_items_by_source_path.insert(source_path, old_newest_item);
}
}
}
fn merge_items_by_file_id(&mut self, old_items_by_file_id: HashMap<ItemId, IndexItem>) {
self.items_by_file_id.extend(old_items_by_file_id);
}
fn index_file_path_for_repository_path(path: &Path) -> PathBuf {
path.join("index")
}
fn index_directory(&self) -> PathBuf {
self.index_file_path().parent().unwrap().to_path_buf()
}
}

59
src/index/item.rs Normal file
View file

@ -0,0 +1,59 @@
use serde::{Deserialize, Serialize};
use crate::repository::ItemId;
use crate::{repository_item::RepositoryItem, version::Version};
#[derive(Clone, Debug, PartialOrd, PartialEq, Ord, Eq, Serialize, Deserialize)]
pub struct IndexItem {
relative_path: String,
original_source_path: String,
id: ItemId,
version: Version,
}
impl IndexItem {
pub fn from(original_source_path: String, relative_path: String, id: ItemId, version: Version) -> IndexItem {
IndexItem {
relative_path,
original_source_path,
id,
version,
}
}
pub fn next_version(&self, id: ItemId, relative_path: String) -> IndexItem {
IndexItem {
original_source_path: self.original_source_path.clone(),
version: self.version.next(),
relative_path,
id,
}
}
pub fn version(&self) -> Version {
self.version.clone()
}
pub fn id(&self) -> ItemId {
self.id.clone()
}
pub fn relative_path(&self) -> &str {
&self.relative_path
}
pub fn original_source_path(&self) -> &str {
&self.original_source_path
}
}
impl From<RepositoryItem> for IndexItem {
fn from(i: RepositoryItem) -> Self {
IndexItem {
relative_path: i.relative_path().to_string_lossy().to_string(),
original_source_path: i.original_source_path().to_string_lossy().to_string(),
id: i.id().clone(),
version: i.version().clone(),
}
}
}

83
src/index/lock.rs Normal file
View file

@ -0,0 +1,83 @@
use anyhow::Result;
use anyhow::*;
use atomicwrites::{AtomicFile, DisallowOverwrite};
use glob::{glob, Paths};
use std::io::Write;
use std::path::{Path, PathBuf};
use uuid::Uuid;
use rand::{rngs::OsRng, RngCore};
use std::{thread, time};
pub struct Lock {
path: PathBuf,
}
impl Lock {
pub fn new(index_directory: &Path) -> Result<Self> {
let mut buffer = [0u8; 16];
OsRng.fill_bytes(&mut buffer);
let id = Uuid::from_bytes(buffer);
Lock::wait_to_have_sole_lock(id, index_directory)?;
let path = Lock::lock_file_path(index_directory, id);
Ok(Lock { path })
}
pub fn release(self) -> Result<()> {
if self.path.exists() {
std::fs::remove_file(self.path)?;
}
Ok(())
}
fn wait_to_have_sole_lock(lock_id: Uuid, index_directory: &Path) -> Result<()> {
Lock::create_lock_file(lock_id, index_directory)?;
while !Lock::sole_lock(lock_id, index_directory)? {
let path = Lock::lock_file_path(index_directory, lock_id);
std::fs::remove_file(path)?;
let sleep_duration = time::Duration::from_millis((OsRng.next_u32() % 256).into());
thread::sleep(sleep_duration);
Lock::create_lock_file(lock_id, index_directory)?;
}
Ok(())
}
fn sole_lock(lock_id: Uuid, index_directory: &Path) -> Result<bool> {
let my_lock_file_path = Lock::lock_file_path(index_directory, lock_id);
let locks = Lock::all_locks(index_directory)?;
let mut only_mine = true;
for path in locks {
let path = path?;
if path.to_string_lossy() != my_lock_file_path.to_string_lossy() {
only_mine = false;
break;
}
}
Ok(only_mine)
}
fn all_locks(index_directory: &Path) -> Result<Paths> {
let locks_glob = Lock::locks_glob(index_directory);
Ok(glob(&locks_glob)?)
}
fn create_lock_file(lock_id: Uuid, index_directory: &Path) -> Result<()> {
let lock_file_path = Lock::lock_file_path(index_directory, lock_id);
let file = AtomicFile::new(lock_file_path, DisallowOverwrite);
match file.write(|f| f.write_all(lock_id.to_hyphenated().to_string().as_bytes())) {
Ok(_) => Ok(()),
Err(e) => Err(anyhow!("error acquiring lock: {}", e)),
}
}
fn lock_file_path(path: &Path, lock_id: Uuid) -> PathBuf {
let path_text = &format!("{}/{}.lock", path.to_string_lossy(), lock_id);
Path::new(path_text).to_path_buf()
}
fn locks_glob(path: &Path) -> String {
format!("{}/*.lock", path.to_string_lossy())
}
}
//TODO: add drop implementation

103
src/index.rs → src/index/mod.rs
View file

@ -1,21 +1,18 @@
use std::collections::hash_map::Iter; use std::collections::hash_map::Iter;
use std::collections::HashMap; use std::collections::HashMap;
use std::fs::File;
use std::path::Path; use std::path::Path;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use crate::error::BakareError; use crate::index::item::IndexItem;
use crate::repository::{ItemId, Version}; use crate::repository::ItemId;
use crate::repository_item::RepositoryItem; use crate::{repository_item::RepositoryItem, version::Version};
use anyhow::Result;
use anyhow::*;
#[derive(Clone, Debug, PartialOrd, PartialEq, Ord, Eq, Serialize, Deserialize)] mod io;
pub struct IndexItem { mod item;
relative_path: String, mod lock;
original_source_path: String,
id: ItemId,
version: Version,
}
#[derive(Serialize, Deserialize)] #[derive(Serialize, Deserialize)]
pub struct Index { pub struct Index {
@ -23,38 +20,7 @@ pub struct Index {
items_by_file_id: HashMap<ItemId, IndexItem>, items_by_file_id: HashMap<ItemId, IndexItem>,
index_path: String, index_path: String,
repository_path: String, repository_path: String,
} version: Version,
pub struct IndexItemIterator<'a> {
iterator: Iter<'a, String, IndexItem>,
}
impl<'a> Iterator for IndexItemIterator<'a> {
type Item = IndexItem;
fn next(&mut self) -> Option<Self::Item> {
self.iterator.next().map(|i| i.1.clone())
}
}
impl IndexItem {
fn from(original_source_path: String, relative_path: String, id: ItemId, version: Version) -> IndexItem {
IndexItem {
relative_path,
original_source_path,
id,
version,
}
}
fn next_version(&self, id: ItemId, relative_path: String) -> IndexItem {
IndexItem {
original_source_path: self.original_source_path.clone(),
version: self.version.next(),
relative_path,
id,
}
}
} }
impl Index { impl Index {
@ -64,26 +30,10 @@ impl Index {
items_by_file_id: Default::default(), items_by_file_id: Default::default(),
index_path: repository_path.join("index").to_string_lossy().to_string(), index_path: repository_path.join("index").to_string_lossy().to_string(),
repository_path: repository_path.to_string_lossy().to_string(), repository_path: repository_path.to_string_lossy().to_string(),
version: Version::default(),
} }
} }
pub fn load(path: &Path) -> Result<Self, BakareError> {
let index_file_path = path.join("index");
let index_file = File::open(index_file_path)?;
let index: Index = serde_cbor::from_reader(index_file)?;
Ok(index)
}
pub fn save(&self) -> Result<(), BakareError> {
let index_file = File::create(self.index_file_path())?;
serde_cbor::to_writer(index_file, &self)?;
Ok(())
}
pub fn index_file_path(&self) -> &Path {
Path::new(&self.index_path)
}
pub fn remember(&mut self, original_source_path: &Path, relative_path: &Path, id: ItemId) { pub fn remember(&mut self, original_source_path: &Path, relative_path: &Path, id: ItemId) {
let item = if let Some(old) = self let item = if let Some(old) = self
.newest_items_by_source_path .newest_items_by_source_path
@ -99,30 +49,30 @@ impl Index {
) )
}; };
self.items_by_file_id.insert(item.id.clone(), item.clone()); self.items_by_file_id.insert(item.id(), item.clone());
self.newest_items_by_source_path self.newest_items_by_source_path
.insert(original_source_path.to_string_lossy().to_string(), item); .insert(original_source_path.to_string_lossy().to_string(), item);
} }
pub fn repository_item(&self, i: &IndexItem) -> RepositoryItem { pub fn repository_item(&self, i: &IndexItem) -> RepositoryItem {
let index_item = i.clone(); let index_item = i.clone();
let relative_path = Path::new(&index_item.relative_path); let relative_path = Path::new(index_item.relative_path());
let repository_path = Path::new(&self.repository_path); let repository_path = Path::new(&self.repository_path);
let original_source_path = Path::new(&index_item.original_source_path); let original_source_path = Path::new(index_item.original_source_path());
let absolute_path = repository_path.join(relative_path); let absolute_path = repository_path.join(relative_path);
let absolute_path = absolute_path.as_path(); let absolute_path = absolute_path.as_path();
RepositoryItem::from( RepositoryItem::from(
original_source_path, original_source_path,
absolute_path, absolute_path,
relative_path, relative_path,
index_item.id, index_item.id(),
index_item.version, index_item.version(),
) )
} }
pub fn newest_item_by_source_path(&self, path: &Path) -> Result<Option<IndexItem>, BakareError> { pub fn newest_item_by_source_path(&self, path: &Path) -> Result<Option<IndexItem>> {
if !path.is_absolute() { if !path.is_absolute() {
return Err(BakareError::RepositoryPathNotAbsolute); return Err(anyhow!("repository path not absolute"));
} }
Ok(self Ok(self
.newest_items_by_source_path .newest_items_by_source_path
@ -130,7 +80,7 @@ impl Index {
.cloned()) .cloned())
} }
pub fn item_by_id(&self, id: &ItemId) -> Result<Option<IndexItem>, BakareError> { pub fn item_by_id(&self, id: &ItemId) -> Result<Option<IndexItem>> {
Ok(self.items_by_file_id.get(id).cloned()) Ok(self.items_by_file_id.get(id).cloned())
} }
@ -141,13 +91,14 @@ impl Index {
} }
} }
impl From<RepositoryItem> for IndexItem { pub struct IndexItemIterator<'a> {
fn from(i: RepositoryItem) -> Self { iterator: Iter<'a, String, IndexItem>,
IndexItem { }
relative_path: i.relative_path().to_string_lossy().to_string(),
original_source_path: i.original_source_path().to_string_lossy().to_string(), impl<'a> Iterator for IndexItemIterator<'a> {
id: i.id().clone(), type Item = IndexItem;
version: i.version().clone(),
} fn next(&mut self) -> Option<Self::Item> {
self.iterator.next().map(|i| i.1.clone())
} }
} }

2
src/lib.rs
View file

@ -1,5 +1,4 @@
pub mod backup; pub mod backup;
pub mod error;
pub mod repository; pub mod repository;
pub mod restore; pub mod restore;
pub mod source; pub mod source;
@ -8,3 +7,4 @@ pub mod test;
mod index; mod index;
mod repository_item; mod repository_item;
mod version;

86
src/repository.rs
View file

@ -1,15 +1,16 @@
use std::path::{Path, PathBuf};
use std::{fmt, fs, io};
use crate::error::BakareError;
use crate::index::{Index, IndexItemIterator};
use crate::repository_item::RepositoryItem;
use serde::{Deserialize, Serialize};
use sha2::Digest;
use sha2::Sha512;
use std::fmt::Formatter; use std::fmt::Formatter;
use std::fs::File; use std::fs::File;
use std::io::BufReader; use std::io::BufReader;
use std::path::{Path, PathBuf};
use std::{fmt, fs, io};
use crate::index::{Index, IndexItemIterator};
use crate::repository_item::RepositoryItem;
use anyhow::Result;
use anyhow::*;
use serde::{Deserialize, Serialize};
use sha2::Digest;
use sha2::Sha512;
use walkdir::WalkDir; use walkdir::WalkDir;
/// represents a place where backup is stored an can be restored from. /// represents a place where backup is stored an can be restored from.
@ -25,16 +26,33 @@ pub struct Repository<'a> {
const DATA_DIR_NAME: &str = "data"; const DATA_DIR_NAME: &str = "data";
#[derive(Clone, Debug, PartialOrd, PartialEq, Ord, Eq, Serialize, Deserialize, Hash)] #[derive(Clone, Debug, PartialOrd, PartialEq, Ord, Eq, Serialize, Deserialize, Hash)]
pub struct ItemId(Box<[u8]>); pub struct ItemId(#[serde(with = "base64")] Vec<u8>);
#[derive(Clone, Debug, PartialOrd, PartialEq, Ord, Eq, Serialize, Deserialize, Hash)]
pub struct Version(u128);
pub struct RepositoryItemIterator<'a> { pub struct RepositoryItemIterator<'a> {
iterator: IndexItemIterator<'a>, iterator: IndexItemIterator<'a>,
index: &'a Index, index: &'a Index,
} }
mod base64 {
use ::base64;
use serde::{de, Deserialize, Deserializer, Serializer};
pub fn serialize<S>(bytes: &[u8], serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str(&base64::encode(bytes))
}
pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
where
D: Deserializer<'de>,
{
let s = <&str>::deserialize(deserializer)?;
base64::decode(s).map_err(de::Error::custom)
}
}
impl<'a> Iterator for RepositoryItemIterator<'a> { impl<'a> Iterator for RepositoryItemIterator<'a> {
type Item = RepositoryItem; type Item = RepositoryItem;
@ -43,18 +61,6 @@ impl<'a> Iterator for RepositoryItemIterator<'a> {
} }
} }
impl Version {
pub fn next(&self) -> Self {
Version(self.0 + 1)
}
}
impl Default for Version {
fn default() -> Self {
Version(1)
}
}
impl AsRef<[u8]> for ItemId { impl AsRef<[u8]> for ItemId {
fn as_ref(&self) -> &[u8] { fn as_ref(&self) -> &[u8] {
&self.0 &self.0
@ -63,7 +69,7 @@ impl AsRef<[u8]> for ItemId {
impl From<&[u8]> for ItemId { impl From<&[u8]> for ItemId {
fn from(a: &[u8]) -> Self { fn from(a: &[u8]) -> Self {
ItemId(Box::from(a)) ItemId(a.into())
} }
} }
@ -74,15 +80,15 @@ impl fmt::Display for ItemId {
} }
impl<'a> Repository<'a> { impl<'a> Repository<'a> {
pub fn init(path: &Path) -> Result<(), BakareError> { pub fn init(path: &Path) -> Result<()> {
let index = Index::new(path); let mut index = Index::new(path);
index.save()?; index.save()?;
Ok(()) Ok(())
} }
pub fn open(path: &Path) -> Result<Repository, BakareError> { pub fn open(path: &Path) -> Result<Repository> {
if !path.is_absolute() { if !path.is_absolute() {
return Err(BakareError::RepositoryPathNotAbsolute); return Err(anyhow!("path to repository not absolute"));
} }
let index = Index::load(path)?; let index = Index::load(path)?;
@ -93,9 +99,13 @@ impl<'a> Repository<'a> {
&self.path &self.path
} }
pub fn store(&mut self, source_path: &Path) -> Result<(), BakareError> { pub fn save_index(&mut self) -> Result<()> {
self.index.save()
}
pub fn store(&mut self, source_path: &Path) -> Result<()> {
if !source_path.is_absolute() { if !source_path.is_absolute() {
return Err(BakareError::PathToStoreNotAbsolute); return Err(anyhow!("path to store not absolute"));
} }
let id = Repository::calculate_id(source_path)?; let id = Repository::calculate_id(source_path)?;
let destination_path = self.data_dir(); let destination_path = self.data_dir();
@ -103,23 +113,23 @@ impl<'a> Repository<'a> {
let destination_path = Path::new(&destination_path); let destination_path = Path::new(&destination_path);
if source_path.is_file() { if source_path.is_file() {
fs::create_dir_all(destination_path.parent().unwrap())?; let parent = destination_path.parent().unwrap();
fs::create_dir_all(parent)?;
fs::copy(source_path, destination_path)?; fs::copy(source_path, destination_path)?;
let relative_path = destination_path.strip_prefix(self.path)?; let relative_path = destination_path.strip_prefix(self.path)?;
self.index.remember(source_path, relative_path, id); self.index.remember(source_path, relative_path, id);
self.index.save()?;
} }
Ok(()) Ok(())
} }
pub fn newest_item_by_source_path(&self, path: &Path) -> Result<Option<RepositoryItem>, BakareError> { pub fn newest_item_by_source_path(&self, path: &Path) -> Result<Option<RepositoryItem>> {
Ok(self Ok(self
.index .index
.newest_item_by_source_path(path)? .newest_item_by_source_path(path)?
.map(|i| self.index.repository_item(&i))) .map(|i| self.index.repository_item(&i)))
} }
pub fn item_by_id(&self, id: &ItemId) -> Result<Option<RepositoryItem>, BakareError> { pub fn item_by_id(&self, id: &ItemId) -> Result<Option<RepositoryItem>> {
self.index.item_by_id(id).map(|i| i.map(|i| self.index.repository_item(&i))) self.index.item_by_id(id).map(|i| i.map(|i| self.index.repository_item(&i)))
} }
@ -130,7 +140,7 @@ impl<'a> Repository<'a> {
} }
} }
pub fn data_weight(&self) -> Result<u64, BakareError> { pub fn data_weight(&self) -> Result<u64> {
let total_size = WalkDir::new(self.data_dir()) let total_size = WalkDir::new(self.data_dir())
.into_iter() .into_iter()
.filter_map(|entry| entry.ok()) .filter_map(|entry| entry.ok())
@ -144,7 +154,7 @@ impl<'a> Repository<'a> {
self.path().join(DATA_DIR_NAME) self.path().join(DATA_DIR_NAME)
} }
fn calculate_id(source_path: &Path) -> Result<ItemId, BakareError> { fn calculate_id(source_path: &Path) -> Result<ItemId> {
let source_file = File::open(source_path)?; let source_file = File::open(source_path)?;
let mut reader = BufReader::new(source_file); let mut reader = BufReader::new(source_file);
let mut hasher = Sha512::new(); let mut hasher = Sha512::new();

14
src/repository_item.rs
View file

@ -1,5 +1,6 @@
use crate::error::BakareError; use crate::{repository::ItemId, version::Version};
use crate::repository::{ItemId, Version}; use anyhow::Result;
use anyhow::*;
use std::fmt::{Display, Formatter}; use std::fmt::{Display, Formatter};
use std::path::Path; use std::path::Path;
use std::{fmt, fs}; use std::{fmt, fs};
@ -24,23 +25,22 @@ impl RepositoryItem {
} }
} }
pub fn save(&self, save_to: &Path) -> Result<(), BakareError> { pub fn save(&self, save_to: &Path) -> Result<()> {
if !save_to.is_absolute() { if !save_to.is_absolute() {
return Err(BakareError::PathToStoreNotAbsolute); return Err(anyhow!("path to store not absolute"));
} }
let target_path = save_to.join(&self.original_source_path.strip_prefix("/")?); let target_path = save_to.join(&self.original_source_path.strip_prefix("/")?);
if !target_path.is_absolute() { if !target_path.is_absolute() {
return Err(BakareError::PathToStoreNotAbsolute); return Err(anyhow!("path to store not absolute"));
} }
let parent = target_path.parent().unwrap(); let parent = target_path.parent().unwrap();
if !parent.exists() { if !parent.exists() {
fs::create_dir_all(parent)?; fs::create_dir_all(parent)?;
} }
if !self.absolute_path.exists() { if !self.absolute_path.exists() {
return Err(BakareError::CorruptedRepoNoFile); return Err(anyhow!("corrupted repository"));
} }
println!("restoring {} to {}", &self.absolute_path.display(), &target_path.display());
fs::copy(&self.absolute_path, &target_path)?; fs::copy(&self.absolute_path, &target_path)?;
Ok(()) Ok(())

14
src/restore.rs
View file

@ -1,30 +1,32 @@
use std::path::Path; use std::path::Path;
use crate::error::BakareError;
use crate::repository::Repository; use crate::repository::Repository;
use crate::repository_item::RepositoryItem; use crate::repository_item::RepositoryItem;
use anyhow::Result;
use anyhow::*;
pub struct Engine<'a> { pub struct Engine<'a> {
repository: &'a Repository<'a>, repository: &'a mut Repository<'a>,
target_path: &'a Path, target_path: &'a Path,
} }
impl<'a> Engine<'a> { impl<'a> Engine<'a> {
pub fn new(repository: &'a Repository, target_path: &'a Path) -> Result<Self, BakareError> { pub fn new(repository: &'a mut Repository<'a>, target_path: &'a Path) -> Result<Self> {
if !target_path.is_absolute() { if !target_path.is_absolute() {
return Err(BakareError::PathToStoreNotAbsolute); return Err(anyhow!("path to store not absolute"));
} }
Ok(Engine { repository, target_path }) Ok(Engine { repository, target_path })
} }
pub fn restore_all(&self) -> Result<(), BakareError> { pub fn restore_all(&mut self) -> Result<()> {
for item in self.repository.newest_items() { for item in self.repository.newest_items() {
self.restore(&item)?; self.restore(&item)?;
} }
self.repository.save_index()?;
Ok(()) Ok(())
} }
pub fn restore(&self, item: &RepositoryItem) -> Result<(), BakareError> { pub fn restore(&self, item: &RepositoryItem) -> Result<()> {
item.save(self.target_path)?; item.save(self.target_path)?;
Ok(()) Ok(())
} }

70
src/test/assertions.rs
View file

@ -5,15 +5,15 @@ use std::path::Path;
use tempfile::tempdir; use tempfile::tempdir;
use walkdir::WalkDir; use walkdir::WalkDir;
use crate::error::BakareError;
use crate::repository::{ItemId, Repository}; use crate::repository::{ItemId, Repository};
use crate::repository_item::RepositoryItem; use crate::repository_item::RepositoryItem;
use crate::source::TempSource; use crate::source::TempSource;
use crate::{backup, restore}; use crate::{backup, restore};
use anyhow::Result;
pub fn assert_same_after_restore(source_path: &Path) -> Result<(), BakareError> { pub fn assert_same_after_restore(source_path: &Path) -> Result<()> {
let repository_path = tempdir()?.into_path(); let repository_path = tempdir().unwrap().into_path();
let restore_target = tempdir()?.into_path(); let restore_target = tempdir().unwrap().into_path();
assert_ne!(source_path, repository_path); assert_ne!(source_path, repository_path);
assert_ne!(repository_path, restore_target); assert_ne!(repository_path, restore_target);
@ -25,8 +25,8 @@ pub fn assert_same_after_restore(source_path: &Path) -> Result<(), BakareError>
backup_engine.backup()?; backup_engine.backup()?;
} }
{ {
let restore_repository = Repository::open(repository_path.as_path())?; let mut restore_repository = Repository::open(repository_path.as_path())?;
let restore_engine = restore::Engine::new(&restore_repository, &restore_target)?; let mut restore_engine = restore::Engine::new(&mut restore_repository, &restore_target)?;
restore_engine.restore_all()?; restore_engine.restore_all()?;
} }
@ -34,15 +34,12 @@ pub fn assert_same_after_restore(source_path: &Path) -> Result<(), BakareError>
Ok(()) Ok(())
} }
pub fn assert_restored_file_contents( pub fn assert_restored_file_contents(repository_path: &Path, source_file_full_path: &Path, contents: &str) -> Result<()> {
repository_path: &Path, let mut restore_repository = Repository::open(repository_path)?;
source_file_full_path: &Path,
contents: &str,
) -> Result<(), BakareError> {
let restore_repository = Repository::open(repository_path)?;
let restore_target = tempdir()?;
let restore_engine = restore::Engine::new(&restore_repository, &restore_target.path())?;
let item = restore_repository.newest_item_by_source_path(&source_file_full_path)?; let item = restore_repository.newest_item_by_source_path(&source_file_full_path)?;
let restore_target = tempdir().unwrap();
let restore_engine = restore::Engine::new(&mut restore_repository, &restore_target.path())?;
restore_engine.restore(&item.unwrap())?; restore_engine.restore(&item.unwrap())?;
let restored_file_path = restore_target.path().join(source_file_full_path.strip_prefix("/")?); let restored_file_path = restore_target.path().join(source_file_full_path.strip_prefix("/")?);
assert_target_file_contents(&restored_file_path, contents) assert_target_file_contents(&restored_file_path, contents)
@ -53,17 +50,17 @@ pub fn assert_restored_from_version_has_contents(
source_file_full_path: &Path, source_file_full_path: &Path,
old_contents: &str, old_contents: &str,
old_id: &ItemId, old_id: &ItemId,
) -> Result<(), BakareError> { ) -> Result<()> {
let restore_repository = Repository::open(repository_path)?; let mut restore_repository = Repository::open(repository_path)?;
let restore_target = tempdir()?;
let restore_engine = restore::Engine::new(&restore_repository, &restore_target.path())?;
let old_item = restore_repository.item_by_id(&old_id)?; let old_item = restore_repository.item_by_id(&old_id)?;
let restore_target = tempdir().unwrap();
let restore_engine = restore::Engine::new(&mut restore_repository, &restore_target.path())?;
restore_engine.restore(&old_item.unwrap())?; restore_engine.restore(&old_item.unwrap())?;
let restored_file_path = restore_target.path().join(source_file_full_path.strip_prefix("/")?); let restored_file_path = restore_target.path().join(source_file_full_path.strip_prefix("/")?);
assert_target_file_contents(&restored_file_path, old_contents) assert_target_file_contents(&restored_file_path, old_contents)
} }
pub fn newest_item(repository_path: &Path, source_file_full_path: &Path) -> Result<RepositoryItem, BakareError> { pub fn newest_item(repository_path: &Path, source_file_full_path: &Path) -> Result<RepositoryItem> {
let item = { let item = {
let reading_repository = Repository::open(repository_path)?; let reading_repository = Repository::open(repository_path)?;
let item = reading_repository.newest_item_by_source_path(&source_file_full_path)?; let item = reading_repository.newest_item_by_source_path(&source_file_full_path)?;
@ -73,10 +70,10 @@ pub fn newest_item(repository_path: &Path, source_file_full_path: &Path) -> Resu
Ok(item) Ok(item)
} }
pub fn restore_all_from_reloaded_repository(repository_path: &Path, restore_target: &Path) -> Result<(), BakareError> { pub fn restore_all_from_reloaded_repository(repository_path: &Path, restore_target: &Path) -> Result<()> {
{ {
let restore_repository = Repository::open(repository_path)?; let mut restore_repository = Repository::open(repository_path)?;
let restore_engine = restore::Engine::new(&restore_repository, &restore_target)?; let mut restore_engine = restore::Engine::new(&mut restore_repository, &restore_target)?;
restore_engine.restore_all()?; restore_engine.restore_all()?;
Ok(()) Ok(())
} }
@ -87,49 +84,49 @@ pub fn backup_file_with_contents(
repository_path: &Path, repository_path: &Path,
source_file_relative_path: &str, source_file_relative_path: &str,
contents: &str, contents: &str,
) -> Result<(), BakareError> { ) -> Result<()> {
{ {
let mut backup_repository = Repository::open(repository_path)?; let mut backup_repository = Repository::open(repository_path)?;
let mut backup_engine = backup::Engine::new(source.path(), &mut backup_repository)?; let mut backup_engine = backup::Engine::new(source.path(), &mut backup_repository)?;
source.write_text_to_file(source_file_relative_path, contents)?; source.write_text_to_file(source_file_relative_path, contents).unwrap();
backup_engine.backup()?; backup_engine.backup()?;
Ok(()) Ok(())
} }
} }
pub fn data_weight(repository_path: &Path) -> Result<u64, BakareError> { pub fn data_weight(repository_path: &Path) -> Result<u64> {
{ {
let repository = Repository::open(repository_path)?; let repository = Repository::open(repository_path)?;
Ok(repository.data_weight()?) Ok(repository.data_weight()?)
} }
} }
fn assert_directory_trees_have_same_contents(left: &Path, right: &Path) -> Result<(), BakareError> { fn assert_directory_trees_have_same_contents(left: &Path, right: &Path) -> Result<()> {
let left_files = get_sorted_files_recursively(left)?; let left_files = get_sorted_files_recursively(left)?;
let right_files = get_sorted_files_recursively(right)?; let right_files = get_sorted_files_recursively(right)?;
let pairs = left_files.iter().zip(right_files); let pairs = left_files.iter().zip(right_files);
for (l, r) in pairs { for (l, r) in pairs {
assert_eq!(l.file_name(), r.file_name()); assert_eq!(l.file_name(), r.file_name());
let mut fl = File::open(l)?; let mut fl = File::open(l).unwrap();
let mut fr = File::open(r)?; let mut fr = File::open(r).unwrap();
let mut bl = vec![]; let mut bl = vec![];
let mut br = vec![]; let mut br = vec![];
fl.read_to_end(&mut bl)?; fl.read_to_end(&mut bl).unwrap();
fr.read_to_end(&mut br)?; fr.read_to_end(&mut br).unwrap();
assert_eq!(bl, br); assert_eq!(bl, br);
} }
Ok(()) Ok(())
} }
fn get_sorted_files_recursively(path: &Path) -> Result<Vec<Box<Path>>, BakareError> { pub fn get_sorted_files_recursively<T: AsRef<Path>>(path: T) -> Result<Vec<Box<Path>>> {
let walker = WalkDir::new(path).sort_by(|a, b| a.file_name().cmp(b.file_name())); let walker = WalkDir::new(path.as_ref()).sort_by(|a, b| a.file_name().cmp(b.file_name()));
let mut result = vec![]; let mut result = vec![];
for maybe_entry in walker { for maybe_entry in walker {
let entry = maybe_entry?; let entry = maybe_entry?;
if entry.path() == path { if entry.path() == path.as_ref() {
continue; continue;
} }
if entry.path().is_file() { if entry.path().is_file() {
@ -140,10 +137,13 @@ fn get_sorted_files_recursively(path: &Path) -> Result<Vec<Box<Path>>, BakareErr
Ok(result) Ok(result)
} }
fn assert_target_file_contents(restored_path: &Path, expected_contents: &str) -> Result<(), BakareError> { fn assert_target_file_contents(restored_path: &Path, expected_contents: &str) -> Result<()> {
let mut actual_contents = String::new(); let mut actual_contents = String::new();
assert!(restored_path.exists(), "Expected '{}' to be there", restored_path.display()); assert!(restored_path.exists(), "Expected '{}' to be there", restored_path.display());
File::open(restored_path)?.read_to_string(&mut actual_contents)?; File::open(restored_path)
.unwrap()
.read_to_string(&mut actual_contents)
.unwrap();
assert_eq!(expected_contents, actual_contents); assert_eq!(expected_contents, actual_contents);
Ok(()) Ok(())
} }

23
src/version.rs Normal file
View file

@ -0,0 +1,23 @@
use serde::{Deserialize, Serialize};
use std::fmt::Display;
#[derive(Clone, Debug, PartialOrd, PartialEq, Ord, Eq, Serialize, Deserialize, Hash)]
pub struct Version(u128);
impl Version {
pub fn next(&self) -> Self {
Version(self.0 + 1)
}
}
impl Default for Version {
fn default() -> Self {
Version(1)
}
}
impl Display for Version {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.0)
}
}

115
tests/concurrency_tests.rs Normal file
View file

@ -0,0 +1,115 @@
use std::fs;
use std::path::Path;
use anyhow::Result;
use async_log::span;
use bakare::repository::Repository;
use bakare::source::TempSource;
use bakare::test::assertions::*;
use bakare::{backup, restore};
use nix::sys::wait::{waitpid, WaitStatus};
use nix::unistd::{fork, getpid, ForkResult};
use tempfile::tempdir;
#[test]
fn handle_concurrent_backups() -> Result<()> {
setup_logger();
let repository_path = &tempdir().unwrap().into_path();
Repository::init(repository_path)?;
let parallel_backups_number = 16;
let files_per_backup_number = 16;
let total_number_of_files = parallel_backups_number * files_per_backup_number;
let finished_backup_runs = backup_in_parallel(repository_path, parallel_backups_number, files_per_backup_number)?;
assert_eq!(finished_backup_runs.len(), parallel_backups_number);
let all_restored_files = restore_all(repository_path)?;
assert_eq!(all_restored_files.len(), total_number_of_files);
for i in 0..parallel_backups_number {
for j in 0..files_per_backup_number {
let id = file_id(i, j);
let file = all_restored_files.iter().find(|f| f.ends_with(id.clone()));
assert!(file.unwrap().exists(), "file {:?} does not exist", file);
let contents = fs::read_to_string(file.unwrap()).unwrap();
assert_eq!(id.to_string(), contents.to_owned());
}
}
Ok(())
}
fn backup_in_parallel<T>(
repository_path: T,
parallel_backups_number: usize,
files_per_backup_number: usize,
) -> Result<Vec<usize>>
where
T: AsRef<Path> + Sync,
{
let task_numbers = (0..parallel_backups_number).collect::<Vec<_>>();
let mut child_pids = vec![];
span!("[{}] acquiring children for parent", getpid(), {
for task_number in &task_numbers {
match unsafe { fork() } {
Ok(ForkResult::Parent { child }) => {
child_pids.push(child);
}
Ok(ForkResult::Child) => {
backup_process(*task_number, &repository_path, files_per_backup_number)?;
std::process::exit(0);
}
Err(_) => panic!("fork failed"),
}
}
});
span!("[{}] waiting for {} children", getpid(), child_pids.len(), {
for pid in child_pids {
log::debug!("[{}] waiting for a child {} to exit", getpid(), pid);
let status = waitpid(Some(pid), None)?;
match status {
WaitStatus::Exited(pid, code) => {
assert!(code == 0, "failed the wait for {} with code {}", pid, code);
}
WaitStatus::Signaled(pid, _, _) => assert!(false, "failed with signal for {}", pid),
_ => panic!("unknown state"),
}
}
});
Ok(task_numbers)
}
fn backup_process<T>(task_number: usize, repository_path: T, files_per_backup_number: usize) -> Result<()>
where
T: AsRef<Path> + Sync,
{
let mut repository = Repository::open(repository_path.as_ref())?;
let source = TempSource::new().unwrap();
let mut backup_engine = backup::Engine::new(source.path(), &mut repository)?;
for i in 0..files_per_backup_number {
let id = file_id(task_number, i);
source.write_text_to_file(&id, &id).unwrap();
}
backup_engine.backup()?;
Ok(())
}
fn restore_all<T: AsRef<Path>>(repository_path: T) -> Result<Vec<Box<Path>>> {
let restore_target = tempdir().unwrap().into_path();
let mut restore_repository = Repository::open(repository_path.as_ref())?;
let mut restore_engine = restore::Engine::new(&mut restore_repository, restore_target.as_ref())?;
restore_engine.restore_all()?;
get_sorted_files_recursively(&restore_target)
}
fn setup_logger() {
let logger = femme::pretty::Logger::new();
async_log::Logger::wrap(logger, rand::random::<u64>)
.start(log::LevelFilter::Trace)
.unwrap();
}
fn file_id(i: usize, j: usize) -> String {
format!("{}-{}", i, j)
}
// TODO handle stale leftover locks

56
tests/system_tests.rs
View file

@ -1,27 +1,27 @@
use tempfile::tempdir; use tempfile::tempdir;
use anyhow::Result;
use bakare::backup; use bakare::backup;
use bakare::error::BakareError;
use bakare::repository::Repository; use bakare::repository::Repository;
use bakare::source::TempSource; use bakare::source::TempSource;
use bakare::test::assertions::*; use bakare::test::assertions::*;
#[test] #[test]
fn restore_multiple_files() -> Result<(), BakareError> { fn restore_multiple_files() -> Result<()> {
let source = TempSource::new()?; let source = TempSource::new().unwrap();
source.write_text_to_file("first", "some contents")?; source.write_text_to_file("first", "some contents").unwrap();
source.write_text_to_file("second", "some contents")?; source.write_text_to_file("second", "some contents").unwrap();
source.write_text_to_file("third", "some other contents")?; source.write_text_to_file("third", "some other contents").unwrap();
assert_same_after_restore(source.path()) assert_same_after_restore(source.path())
} }
#[test] #[test]
fn restore_files_after_reopening_repository() -> Result<(), BakareError> { fn restore_files_after_reopening_repository() -> Result<()> {
let source = TempSource::new()?; let source = TempSource::new().unwrap();
let repository_path = &tempdir()?.into_path(); let repository_path = &tempdir().unwrap().into_path();
let restore_target = tempdir()?.into_path(); let restore_target = tempdir().unwrap().into_path();
Repository::init(repository_path)?; Repository::init(repository_path)?;
let source_file_relative_path = "some file path"; let source_file_relative_path = "some file path";
@ -36,9 +36,9 @@ fn restore_files_after_reopening_repository() -> Result<(), BakareError> {
} }
#[test] #[test]
fn restore_older_version_of_file() -> Result<(), BakareError> { fn restore_older_version_of_file() -> Result<()> {
let source = TempSource::new()?; let source = TempSource::new().unwrap();
let repository_path = tempdir()?.into_path(); let repository_path = tempdir().unwrap().into_path();
Repository::init(repository_path.as_path())?; Repository::init(repository_path.as_path())?;
let source_file_relative_path = "some path"; let source_file_relative_path = "some path";
@ -57,9 +57,9 @@ fn restore_older_version_of_file() -> Result<(), BakareError> {
} }
#[test] #[test]
fn newer_version_should_be_greater_than_earlier_version() -> Result<(), BakareError> { fn newer_version_should_be_greater_than_earlier_version() -> Result<()> {
let source = TempSource::new()?; let source = TempSource::new().unwrap();
let repository_path = tempdir()?.into_path(); let repository_path = tempdir().unwrap().into_path();
Repository::init(repository_path.as_path())?; Repository::init(repository_path.as_path())?;
let source_file_relative_path = "some path"; let source_file_relative_path = "some path";
@ -81,9 +81,9 @@ fn newer_version_should_be_greater_than_earlier_version() -> Result<(), BakareEr
} }
#[test] #[test]
fn store_duplicated_files_just_once() -> Result<(), BakareError> { fn store_duplicated_files_just_once() -> Result<()> {
let source = TempSource::new()?; let source = TempSource::new().unwrap();
let repository_path = &tempdir()?.into_path(); let repository_path = &tempdir().unwrap().into_path();
Repository::init(repository_path)?; Repository::init(repository_path)?;
assert_eq!(data_weight(&repository_path)?, 0); assert_eq!(data_weight(&repository_path)?, 0);
@ -103,9 +103,9 @@ fn store_duplicated_files_just_once() -> Result<(), BakareError> {
} }
#[test] #[test]
fn restore_latest_version_by_default() -> Result<(), BakareError> { fn restore_latest_version_by_default() -> Result<()> {
let source = TempSource::new()?; let source = TempSource::new().unwrap();
let repository_path = &tempdir()?.into_path(); let repository_path = &tempdir().unwrap().into_path();
Repository::init(repository_path)?; Repository::init(repository_path)?;
let source_file_relative_path = "some path"; let source_file_relative_path = "some path";
@ -118,16 +118,14 @@ fn restore_latest_version_by_default() -> Result<(), BakareError> {
} }
#[test] #[test]
fn forbid_backup_of_paths_within_repository() -> Result<(), BakareError> { fn forbid_backup_of_paths_within_repository() -> Result<()> {
let repository_path = &tempdir()?.into_path(); let repository_path = &tempdir().unwrap().into_path();
Repository::init(repository_path)?; Repository::init(repository_path)?;
let mut repository = Repository::open(repository_path)?; let mut repository = Repository::open(repository_path)?;
let error = backup::Engine::new(repository_path, &mut repository).err().unwrap(); let error = backup::Engine::new(repository_path, &mut repository);
let correct_error = matches!(error, BakareError::SourceSameAsRepository); assert!(error.is_err());
assert!(correct_error);
Ok(()) Ok(())
} }
// TODO: test concurrent writes
// TODO: index corruption // TODO: index corruption
// TODO: encryption // TODO: encryption
// TODO: resume from sleep while backup in progress