Add basic safety around running multiple backup processes concurrently.
This commit is contained in:
Cyryl Płotnicki
parent
502fc153a3
commit
90a86ef95f
18 changed files with 1221 additions and 303 deletions
642
Cargo.lock
generated
642
Cargo.lock
generated
|
|
@ -15,6 +15,76 @@ version = "0.2.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "0.7.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bf8dcb5b4bbaa28653b647d8c77bd4ed40183b48882e130c1f1ffb73de069fd7"
|
||||
|
||||
[[package]]
|
||||
name = "async-log"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f5ba33304f3165922dd93565a36dda8ac96c28b1a926ad535ec904e48ecad59b"
|
||||
dependencies = [
|
||||
"async-log-attributes",
|
||||
"backtrace",
|
||||
"log",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-log"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac39aabd463ad2b1fadb34c22f48e5d160fa60b7cd69a426362e71bde49ebb6d"
|
||||
dependencies = [
|
||||
"async-log-attributes",
|
||||
"backtrace",
|
||||
"log",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-log-attributes"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "da97f8e61b19a72f67d8932de8b0905f7d41a1d7b9501b9938c7755f96f6362d"
|
||||
dependencies = [
|
||||
"proc-macro2 0.4.30",
|
||||
"quote 0.6.13",
|
||||
"syn 0.15.44",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "atomicwrites"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a2baf2feb820299c53c7ad1cc4f5914a220a1cb76d7ce321d2522a94b54651f"
|
||||
dependencies = [
|
||||
"nix 0.14.1",
|
||||
"tempdir",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "atty"
|
||||
version = "0.2.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.0.1"
|
||||
|
|
@ -39,17 +109,42 @@ dependencies = [
|
|||
name = "bakare"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-log 2.0.0",
|
||||
"atomicwrites",
|
||||
"base64",
|
||||
"cargo-husky",
|
||||
"failure",
|
||||
"femme",
|
||||
"fs2",
|
||||
"glob",
|
||||
"hex",
|
||||
"log",
|
||||
"nix 0.19.0",
|
||||
"rand 0.7.3",
|
||||
"rayon",
|
||||
"rust-crypto",
|
||||
"serde",
|
||||
"serde_cbor",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
"uuid",
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.9.0"
|
||||
|
|
@ -59,12 +154,24 @@ dependencies = [
|
|||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820"
|
||||
|
||||
[[package]]
|
||||
name = "cargo-husky"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7b02b629252fe8ef6460461409564e2c21d0c8e77e0944f3d189ff06c4e932ad"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.0.62"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1770ced377336a88a67c473594ccc14eca6f4559217c34f64aac8f83d641b40"
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "0.1.10"
|
||||
|
|
@ -77,12 +184,104 @@ version = "1.0.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||
|
||||
[[package]]
|
||||
name = "clicolors-control"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "90082ee5dcdd64dc4e9e0d37fbf3ee325419e39c0092191e0393df65518f741e"
|
||||
dependencies = [
|
||||
"atty",
|
||||
"lazy_static",
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cloudabi"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "console"
|
||||
version = "0.7.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ca57c2c14b8a2bf3105bc9d15574aad80babf6a9c44b1058034cdf8bd169628"
|
||||
dependencies = [
|
||||
"atty",
|
||||
"clicolors-control",
|
||||
"encode_unicode",
|
||||
"lazy_static",
|
||||
"libc",
|
||||
"parking_lot",
|
||||
"regex",
|
||||
"termios",
|
||||
"unicode-width",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "const_fn"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c478836e029dcef17fb47c89023448c64f781a046e0300e257ad8225ae59afab"
|
||||
|
||||
[[package]]
|
||||
name = "cpuid-bool"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634"
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-channel"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-deque"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec0f606a85340376eef0d6d8fec399e6d4a544d648386c6645eb6d0653b27d9f"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"const_fn",
|
||||
"crossbeam-utils",
|
||||
"lazy_static",
|
||||
"memoffset",
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec91540d98355f690a86367e566ecad2e9e579f230230eb7c21398372be73ea5"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"cfg-if 1.0.0",
|
||||
"const_fn",
|
||||
"lazy_static",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.9.0"
|
||||
|
|
@ -93,25 +292,43 @@ dependencies = [
|
|||
]
|
||||
|
||||
[[package]]
|
||||
name = "failure"
|
||||
version = "0.1.8"
|
||||
name = "either"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86"
|
||||
checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
|
||||
|
||||
[[package]]
|
||||
name = "encode_unicode"
|
||||
version = "0.3.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
|
||||
|
||||
[[package]]
|
||||
name = "femme"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4d9ac2da60a2a97a0643d082f1cb25901a90e6f2f69f03baf9763721aa31ee6a"
|
||||
dependencies = [
|
||||
"backtrace",
|
||||
"failure_derive",
|
||||
"async-log 1.1.0",
|
||||
"cfg-if 0.1.10",
|
||||
"console",
|
||||
"js-sys",
|
||||
"log",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"serde_json",
|
||||
"wasm-bindgen",
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "failure_derive"
|
||||
version = "0.1.8"
|
||||
name = "fs2"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4"
|
||||
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"synstructure",
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -153,24 +370,102 @@ version = "0.23.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce"
|
||||
|
||||
[[package]]
|
||||
name = "glob"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
|
||||
|
||||
[[package]]
|
||||
name = "half"
|
||||
version = "1.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d36fab90f82edc3c747f9d438e06cf0a491055896f2a279638bb5beed6c40177"
|
||||
|
||||
[[package]]
|
||||
name = "hermit-abi"
|
||||
version = "0.1.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hex"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35"
|
||||
|
||||
[[package]]
|
||||
name = "instant"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cb1fc4429a33e1f80d41dc9fea4d108a88bec1de8053878898ae448a0b52f613"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "0.4.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6"
|
||||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ca059e81d9486668f12d455a4ea6daa600bd408134cd17e3d3fb5a32d1f016f8"
|
||||
dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lazy_static"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.80"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614"
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "28247cc5a5be2f05fbcd76dd0cf2c7d3b5400cb978a28042abcd4fa0b3f8261c"
|
||||
dependencies = [
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b"
|
||||
dependencies = [
|
||||
"cfg-if 0.1.10",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
version = "0.5.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.4.3"
|
||||
|
|
@ -181,6 +476,41 @@ dependencies = [
|
|||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nix"
|
||||
version = "0.14.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c722bee1037d430d0f8e687bbdbf222f27cc6e4e68d5caf630857bb2b6dbdce"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"cc",
|
||||
"cfg-if 0.1.10",
|
||||
"libc",
|
||||
"void",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nix"
|
||||
version = "0.19.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85db2feff6bf70ebc3a4793191517d5f0331100a2f10f9bf93b5e5214f32b7b7"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"cc",
|
||||
"cfg-if 0.1.10",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num_cpus"
|
||||
version = "1.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "object"
|
||||
version = "0.22.0"
|
||||
|
|
@ -193,19 +523,63 @@ version = "0.3.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a4893845fa2ca272e647da5d0e46660a314ead9c2fdd9a883aabc32e481a8733"
|
||||
dependencies = [
|
||||
"instant",
|
||||
"lock_api",
|
||||
"parking_lot_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot_core"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b"
|
||||
dependencies = [
|
||||
"cfg-if 0.1.10",
|
||||
"cloudabi",
|
||||
"instant",
|
||||
"libc",
|
||||
"redox_syscall",
|
||||
"smallvec",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "0.4.30"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759"
|
||||
dependencies = [
|
||||
"unicode-xid 0.1.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71"
|
||||
dependencies = [
|
||||
"unicode-xid",
|
||||
"unicode-xid 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "0.6.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1"
|
||||
dependencies = [
|
||||
"proc-macro2 0.4.30",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -214,7 +588,7 @@ version = "1.0.7"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"proc-macro2 1.0.24",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -296,6 +670,31 @@ dependencies = [
|
|||
"rand_core 0.5.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"crossbeam-deque",
|
||||
"either",
|
||||
"rayon-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon-core"
|
||||
version = "1.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a"
|
||||
dependencies = [
|
||||
"crossbeam-channel",
|
||||
"crossbeam-deque",
|
||||
"crossbeam-utils",
|
||||
"lazy_static",
|
||||
"num_cpus",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rdrand"
|
||||
version = "0.4.0"
|
||||
|
|
@ -311,6 +710,24 @@ version = "0.1.57"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce"
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "38cf2c13ed4745de91a5eb834e11c00bcc3709e773173b2ce4c56c9fbde04b9c"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-syntax",
|
||||
"thread_local",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.6.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3b181ba2dcf07aaccad5448e8ead58db5b742cf85dfe035e2227f137a539a189"
|
||||
|
||||
[[package]]
|
||||
name = "remove_dir_all"
|
||||
version = "0.5.3"
|
||||
|
|
@ -345,6 +762,12 @@ version = "0.3.24"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda"
|
||||
|
||||
[[package]]
|
||||
name = "ryu"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
|
||||
|
||||
[[package]]
|
||||
name = "same-file"
|
||||
version = "1.0.6"
|
||||
|
|
@ -354,6 +777,12 @@ dependencies = [
|
|||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.117"
|
||||
|
|
@ -379,9 +808,20 @@ version = "1.0.117"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cbd1ae72adb44aab48f325a02444a5fc079349a8d804c1fc922aed3f7454c74e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.7",
|
||||
"syn 1.0.48",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.59"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dcac07dbffa1c65e7f816ab9eba78eb142c6d44410f4eeba1e26e4f5dfa56b95"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"ryu",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -397,27 +837,42 @@ dependencies = [
|
|||
"opaque-debug",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "0.15.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5"
|
||||
dependencies = [
|
||||
"proc-macro2 0.4.30",
|
||||
"quote 0.6.13",
|
||||
"unicode-xid 0.1.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.48"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-xid",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.7",
|
||||
"unicode-xid 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "synstructure"
|
||||
version = "0.12.4"
|
||||
name = "tempdir"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701"
|
||||
checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"unicode-xid",
|
||||
"rand 0.4.6",
|
||||
"remove_dir_all",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -434,6 +889,44 @@ dependencies = [
|
|||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "termios"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "411c5bf740737c7918b8b1fe232dca4dc9f8e754b8ad5e20966814001ed0ac6b"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "1.0.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0e9ae34b84616eedaaf1e9dd6026dbe00dcafa92aa0c8077cb69df1fcfe5e53e"
|
||||
dependencies = [
|
||||
"thiserror-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "1.0.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ba20f23e85b10754cd195504aebf6a27e2e6cbe28c17778a0c930724628dd56"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.7",
|
||||
"syn 1.0.48",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thread_local"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "time"
|
||||
version = "0.1.44"
|
||||
|
|
@ -451,18 +944,45 @@ version = "1.12.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-width"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-xid"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-xid"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9fde2f6a4bea1d6e007c4ad38c6839fa71cbb63b6dbf5b595aa38dc9b1093c11"
|
||||
dependencies = [
|
||||
"rand 0.7.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed"
|
||||
|
||||
[[package]]
|
||||
name = "void"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
|
||||
|
||||
[[package]]
|
||||
name = "walkdir"
|
||||
version = "2.3.1"
|
||||
|
|
@ -486,6 +1006,72 @@ version = "0.10.0+wasi-snapshot-preview1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.68"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1ac64ead5ea5f05873d7c12b545865ca2b8d28adfc50a49b84770a3a97265d42"
|
||||
dependencies = [
|
||||
"cfg-if 0.1.10",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"wasm-bindgen-macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-backend"
|
||||
version = "0.2.68"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f22b422e2a757c35a73774860af8e112bff612ce6cb604224e8e47641a9e4f68"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.7",
|
||||
"syn 1.0.48",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.68"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6b13312a745c08c469f0b292dd2fcd6411dba5f7160f593da6ef69b64e407038"
|
||||
dependencies = [
|
||||
"quote 1.0.7",
|
||||
"wasm-bindgen-macro-support",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro-support"
|
||||
version = "0.2.68"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f249f06ef7ee334cc3b8ff031bfc11ec99d00f34d86da7498396dc1e3b1498fe"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.24",
|
||||
"quote 1.0.7",
|
||||
"syn 1.0.48",
|
||||
"wasm-bindgen-backend",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-shared"
|
||||
version = "0.2.68"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d649a3145108d7d3fbcde896a468d1bd636791823c9921135218ad89be08307"
|
||||
|
||||
[[package]]
|
||||
name = "web-sys"
|
||||
version = "0.3.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4bf6ef87ad7ae8008e15a355ce696bed26012b7caa21605188cfd8214ab51e2d"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
|
|
|
|||
23
Cargo.toml
23
Cargo.toml
|
|
@ -6,17 +6,30 @@ edition = "2018"
|
|||
license = "AGPL-3.0"
|
||||
|
||||
[dependencies]
|
||||
failure = "0.1"
|
||||
anyhow = "1.0"
|
||||
async-log = "2.0"
|
||||
atomicwrites = "0.2"
|
||||
base64 = "0.11"
|
||||
femme = "1.3"
|
||||
fs2 = "0.4"
|
||||
glob = "0.3"
|
||||
hex = "0.4"
|
||||
log = "0.4"
|
||||
nix = "0.19"
|
||||
rand = "0.7"
|
||||
rayon = "1.1"
|
||||
rust-crypto = "0.2"
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_cbor = "0.11"
|
||||
tempfile = "3.1"
|
||||
walkdir = "2.3"
|
||||
serde_json = "1.0"
|
||||
sha2 = "0.9"
|
||||
hex = "0.4"
|
||||
tempfile = "3.1"
|
||||
thiserror = "1.0"
|
||||
uuid = { version = "0.8", features = ["v4"] }
|
||||
walkdir = "2.3"
|
||||
|
||||
[dev-dependencies.cargo-husky]
|
||||
version = "1"
|
||||
default-features = false
|
||||
features = ["run-for-all", "prepush-hook", "run-cargo-test", "run-cargo-clippy", "run-cargo-fmt"]
|
||||
features = ["run-for-all", "prepush-hook", "run-cargo-clippy", "run-cargo-fmt"]
|
||||
|
||||
|
|
|
|||
48
README.md
48
README.md
|
|
@ -2,6 +2,7 @@ Motivation:
|
|||
All the backup systems are either slow or crashing or both on my backup.
|
||||
|
||||
Tried duply:
|
||||
|
||||
<details>
|
||||
<summary>Works but is very slow</summary>
|
||||
|
||||
|
|
@ -29,32 +30,37 @@ Errors 0
|
|||
|
||||
</details>
|
||||
|
||||
|
||||
Tried restic:
|
||||
* crashes with OOM
|
||||
* corrupts repo if you suspend one backup process and launch another from a different computer
|
||||
|
||||
- crashes with OOM
|
||||
- corrupts repo if you suspend one backup process and launch another from a different computer
|
||||
|
||||
Goals for bakare:
|
||||
* fast
|
||||
* using max bandwidth
|
||||
* use max cpu
|
||||
* use max disk I/O
|
||||
* memory usage limit
|
||||
* encryption by default - asymmetric, creates a keypair for you
|
||||
* deduplication of file data
|
||||
* fuzzy find by file name in stored files
|
||||
* failure to process one file should not affect any other files
|
||||
* intermittent network failures should not make the whole process fail (test with random packet drop)
|
||||
* system suspend/resume should not make the repo become corrupted, even in the presence of other active backup processes running on other computers, targeting same repo - this is where `restic` fails
|
||||
|
||||
- fast
|
||||
- using max bandwidth
|
||||
- use max cpu
|
||||
- use max disk I/O
|
||||
- memory usage limit
|
||||
- encryption by default - asymmetric, creates a keypair for you
|
||||
- deduplication of file data
|
||||
- fuzzy find by file name in stored files
|
||||
- failure to process one file should not affect any other files
|
||||
- intermittent network failures should not make the whole process fail (test with random packet drop)
|
||||
- system suspend/resume should not make the repo become corrupted, even in the presence of other active backup processes running on other computers, targeting same repo - this is where `restic` fails
|
||||
|
||||
Nice to have:
|
||||
* daemon that listens for file events and updates a list of files to be backed up on the next backup run - or a `continous backup` mode - the daemon uploads the file whenever it sees the change
|
||||
* peer2peer mode - people storing encrypted backups for each other
|
||||
* relay mode, where daemon works on one or more central points with local storage (e.g. NAS) and various computers sync with that central location. Then though the central locaiton uploads everything to the other location, typically the cloud.
|
||||
|
||||
- daemon that listens for file events and updates a list of files to be backed up on the next backup run - or a `continous backup` mode - the daemon uploads the file whenever it sees the change
|
||||
- peer2peer mode - people storing encrypted backups for each other
|
||||
- relay mode, where daemon works on one or more central points with local storage (e.g. NAS) and various computers sync with that central location. Then though the central locaiton uploads everything to the other location, typically the cloud.
|
||||
|
||||
Implementation:
|
||||
* test with randomly created dirs and files, with property based tests and fuzzer
|
||||
* see if we can use `salsa` for recomputaiton
|
||||
* index corruption tests - mutate random byte and see if everything is readable
|
||||
* network packet drop tests
|
||||
|
||||
- automatic node discovery - two roles: data publisher and data persister - should be able to figure out which node is which automatically
|
||||
- test with randomly created dirs and files, with property based tests and fuzzer
|
||||
- see if we can use `salsa` for recomputaiton
|
||||
- index corruption tests - mutate random byte and see if everything is readable
|
||||
- network packet drop tests
|
||||
- use bevy for decoupling ?
|
||||
- remove all `unwraps`
|
||||
|
|
|
|||
|
|
@ -5,10 +5,14 @@ let
|
|||
in
|
||||
with nixpkgs;
|
||||
stdenv.mkDerivation {
|
||||
name = "genpass_shell";
|
||||
name = "bakare_shell";
|
||||
buildInputs = [
|
||||
channel.rust
|
||||
cacert openssl openssh zlib
|
||||
llvm pkgconfig git
|
||||
pkgconfig clang llvm
|
||||
git
|
||||
];
|
||||
shellHook = ''
|
||||
export RUST_SRC_PATH="${channel.rust-src}/lib/rustlib/src/rust/src"
|
||||
'';
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
use anyhow::Result;
|
||||
use anyhow::*;
|
||||
use std::path::Path;
|
||||
|
||||
use walkdir::WalkDir;
|
||||
|
||||
use crate::error::BakareError;
|
||||
use crate::repository::Repository;
|
||||
|
||||
pub struct Engine<'a> {
|
||||
|
|
@ -11,21 +12,29 @@ pub struct Engine<'a> {
|
|||
}
|
||||
|
||||
impl<'a> Engine<'a> {
|
||||
pub fn new(source_path: &'a Path, repository: &'a mut Repository<'a>) -> Result<Self, BakareError> {
|
||||
pub fn new(source_path: &'a Path, repository: &'a mut Repository<'a>) -> Result<Self> {
|
||||
if source_path.ancestors().any(|a| a == repository.path()) {
|
||||
return Err(BakareError::SourceSameAsRepository);
|
||||
return Err(anyhow!("source same as repository"));
|
||||
}
|
||||
Ok(Engine { source_path, repository })
|
||||
}
|
||||
|
||||
pub fn backup(&mut self) -> Result<(), BakareError> {
|
||||
pub fn backup(&mut self) -> Result<()> {
|
||||
let walker = WalkDir::new(self.source_path);
|
||||
let save_every = 16;
|
||||
let mut save_counter = 0;
|
||||
for maybe_entry in walker {
|
||||
let entry = maybe_entry?;
|
||||
if entry.path() != self.source_path {
|
||||
self.repository.store(entry.path())?;
|
||||
}
|
||||
save_counter += 1;
|
||||
if save_counter == save_every {
|
||||
save_counter = 0;
|
||||
self.repository.save_index()?;
|
||||
}
|
||||
}
|
||||
self.repository.save_index()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
|
|||
46
src/error.rs
46
src/error.rs
|
|
@ -1,46 +0,0 @@
|
|||
use std::io;
|
||||
|
||||
use failure::Fail;
|
||||
use std::path::StripPrefixError;
|
||||
|
||||
#[derive(Debug, Fail)]
|
||||
pub enum BakareError {
|
||||
#[fail(display = "io error")]
|
||||
IOError(Option<io::Error>),
|
||||
#[fail(display = "backup source same as repository")]
|
||||
SourceSameAsRepository,
|
||||
#[fail(display = "repository path is not absolute")]
|
||||
RepositoryPathNotAbsolute,
|
||||
#[fail(display = "path to store is not absolute")]
|
||||
PathToStoreNotAbsolute,
|
||||
#[fail(display = "directory used in place of a file")]
|
||||
DirectoryNotFile,
|
||||
#[fail(display = "corrupted repository - cannot find file")]
|
||||
CorruptedRepoNoFile,
|
||||
#[fail(display = "index loading error")]
|
||||
IndexLoadingError(Option<serde_cbor::Error>),
|
||||
}
|
||||
|
||||
impl From<io::Error> for BakareError {
|
||||
fn from(e: io::Error) -> Self {
|
||||
BakareError::IOError(Some(e))
|
||||
}
|
||||
}
|
||||
|
||||
impl From<walkdir::Error> for BakareError {
|
||||
fn from(e: walkdir::Error) -> Self {
|
||||
BakareError::IOError(e.into_io_error())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<StripPrefixError> for BakareError {
|
||||
fn from(_: StripPrefixError) -> Self {
|
||||
BakareError::IOError(None)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<serde_cbor::Error> for BakareError {
|
||||
fn from(e: serde_cbor::Error) -> Self {
|
||||
BakareError::IndexLoadingError(Some(e))
|
||||
}
|
||||
}
|
||||
105
src/index/io.rs
Normal file
105
src/index/io.rs
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
use atomicwrites::{AllowOverwrite, AtomicFile};
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::index::item::IndexItem;
|
||||
use crate::index::{lock, Index};
|
||||
use crate::repository::ItemId;
|
||||
use anyhow::Result;
|
||||
use anyhow::*;
|
||||
use lock::Lock;
|
||||
use nix::unistd::getpid;
|
||||
use std::{cmp::max, io::Write};
|
||||
|
||||
impl Index {
|
||||
pub fn load(repository_path: &Path) -> Result<Self> {
|
||||
if !repository_path.exists() {
|
||||
let mut index = Index::new(repository_path);
|
||||
index.save()?;
|
||||
}
|
||||
let lock = Lock::new(repository_path)?;
|
||||
let index = Index::load_from_file(&Index::index_file_path_for_repository_path(repository_path))?;
|
||||
lock.release()?;
|
||||
log::debug!(
|
||||
"[{}] loaded index from {}, version: {}",
|
||||
getpid(),
|
||||
repository_path.to_string_lossy(),
|
||||
index.version
|
||||
);
|
||||
Ok(index)
|
||||
}
|
||||
|
||||
pub fn save(&mut self) -> Result<()> {
|
||||
let lock_id = Uuid::new_v4();
|
||||
let lock = Lock::new(&self.index_directory())?;
|
||||
if self.index_file_path().exists() {
|
||||
let index = Index::load_from_file(&Index::index_file_path_for_repository_path(&self.index_directory()))?;
|
||||
self.merge_items_by_file_id(index.items_by_file_id);
|
||||
self.merge_newest_items(index.newest_items_by_source_path);
|
||||
self.version = max(self.version.clone(), index.version);
|
||||
}
|
||||
self.version = self.version.next();
|
||||
self.write_index_to_file(self.index_file_path())?;
|
||||
lock.release()?;
|
||||
log::debug!("[{}] saved index version {} with lock id {}", getpid(), self.version, lock_id,);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_index_to_file<T>(&mut self, path: T) -> Result<()>
|
||||
where
|
||||
T: AsRef<Path>,
|
||||
{
|
||||
fs::create_dir_all(path.as_ref().parent().unwrap()).context("create index directory")?;
|
||||
|
||||
let file = AtomicFile::new(&path, AllowOverwrite);
|
||||
|
||||
file.write(|f| {
|
||||
let contents = serde_json::to_string(&self)?;
|
||||
f.write_all(contents.as_bytes())
|
||||
})
|
||||
.context("writing index to disk")?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn index_file_path(&self) -> PathBuf {
|
||||
Path::new(&self.index_path).to_path_buf()
|
||||
}
|
||||
|
||||
fn load_from_file<T: AsRef<Path>>(index_file_path: T) -> Result<Self> {
|
||||
let path_text = format!("{}", index_file_path.as_ref().to_string_lossy());
|
||||
let index_text =
|
||||
fs::read_to_string(path_text.clone()).context(format!("reading index file contents from {}", path_text))?;
|
||||
|
||||
let mut index: Index = serde_json::from_str(&index_text).context(format!("cannot read index from: {}", index_text))?;
|
||||
index.index_path = path_text;
|
||||
Ok(index)
|
||||
}
|
||||
|
||||
fn merge_newest_items(&mut self, old_newest_items: HashMap<String, IndexItem>) {
|
||||
for (source_path, old_newest_item) in old_newest_items {
|
||||
if let Some(new_newest_item) = self.newest_items_by_source_path.get(&source_path) {
|
||||
if old_newest_item.version() > new_newest_item.version() {
|
||||
self.newest_items_by_source_path.insert(source_path, old_newest_item);
|
||||
}
|
||||
} else {
|
||||
self.newest_items_by_source_path.insert(source_path, old_newest_item);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn merge_items_by_file_id(&mut self, old_items_by_file_id: HashMap<ItemId, IndexItem>) {
|
||||
self.items_by_file_id.extend(old_items_by_file_id);
|
||||
}
|
||||
|
||||
fn index_file_path_for_repository_path(path: &Path) -> PathBuf {
|
||||
path.join("index")
|
||||
}
|
||||
|
||||
fn index_directory(&self) -> PathBuf {
|
||||
self.index_file_path().parent().unwrap().to_path_buf()
|
||||
}
|
||||
}
|
||||
59
src/index/item.rs
Normal file
59
src/index/item.rs
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::repository::ItemId;
|
||||
use crate::{repository_item::RepositoryItem, version::Version};
|
||||
|
||||
#[derive(Clone, Debug, PartialOrd, PartialEq, Ord, Eq, Serialize, Deserialize)]
|
||||
pub struct IndexItem {
|
||||
relative_path: String,
|
||||
original_source_path: String,
|
||||
id: ItemId,
|
||||
version: Version,
|
||||
}
|
||||
|
||||
impl IndexItem {
|
||||
pub fn from(original_source_path: String, relative_path: String, id: ItemId, version: Version) -> IndexItem {
|
||||
IndexItem {
|
||||
relative_path,
|
||||
original_source_path,
|
||||
id,
|
||||
version,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn next_version(&self, id: ItemId, relative_path: String) -> IndexItem {
|
||||
IndexItem {
|
||||
original_source_path: self.original_source_path.clone(),
|
||||
version: self.version.next(),
|
||||
relative_path,
|
||||
id,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn version(&self) -> Version {
|
||||
self.version.clone()
|
||||
}
|
||||
|
||||
pub fn id(&self) -> ItemId {
|
||||
self.id.clone()
|
||||
}
|
||||
|
||||
pub fn relative_path(&self) -> &str {
|
||||
&self.relative_path
|
||||
}
|
||||
|
||||
pub fn original_source_path(&self) -> &str {
|
||||
&self.original_source_path
|
||||
}
|
||||
}
|
||||
|
||||
impl From<RepositoryItem> for IndexItem {
|
||||
fn from(i: RepositoryItem) -> Self {
|
||||
IndexItem {
|
||||
relative_path: i.relative_path().to_string_lossy().to_string(),
|
||||
original_source_path: i.original_source_path().to_string_lossy().to_string(),
|
||||
id: i.id().clone(),
|
||||
version: i.version().clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
83
src/index/lock.rs
Normal file
83
src/index/lock.rs
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
use anyhow::Result;
|
||||
use anyhow::*;
|
||||
use atomicwrites::{AtomicFile, DisallowOverwrite};
|
||||
use glob::{glob, Paths};
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use uuid::Uuid;
|
||||
|
||||
use rand::{rngs::OsRng, RngCore};
|
||||
use std::{thread, time};
|
||||
|
||||
pub struct Lock {
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl Lock {
|
||||
pub fn new(index_directory: &Path) -> Result<Self> {
|
||||
let mut buffer = [0u8; 16];
|
||||
OsRng.fill_bytes(&mut buffer);
|
||||
let id = Uuid::from_bytes(buffer);
|
||||
Lock::wait_to_have_sole_lock(id, index_directory)?;
|
||||
let path = Lock::lock_file_path(index_directory, id);
|
||||
Ok(Lock { path })
|
||||
}
|
||||
|
||||
pub fn release(self) -> Result<()> {
|
||||
if self.path.exists() {
|
||||
std::fs::remove_file(self.path)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn wait_to_have_sole_lock(lock_id: Uuid, index_directory: &Path) -> Result<()> {
|
||||
Lock::create_lock_file(lock_id, index_directory)?;
|
||||
while !Lock::sole_lock(lock_id, index_directory)? {
|
||||
let path = Lock::lock_file_path(index_directory, lock_id);
|
||||
std::fs::remove_file(path)?;
|
||||
let sleep_duration = time::Duration::from_millis((OsRng.next_u32() % 256).into());
|
||||
thread::sleep(sleep_duration);
|
||||
Lock::create_lock_file(lock_id, index_directory)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn sole_lock(lock_id: Uuid, index_directory: &Path) -> Result<bool> {
|
||||
let my_lock_file_path = Lock::lock_file_path(index_directory, lock_id);
|
||||
let locks = Lock::all_locks(index_directory)?;
|
||||
let mut only_mine = true;
|
||||
for path in locks {
|
||||
let path = path?;
|
||||
if path.to_string_lossy() != my_lock_file_path.to_string_lossy() {
|
||||
only_mine = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(only_mine)
|
||||
}
|
||||
|
||||
fn all_locks(index_directory: &Path) -> Result<Paths> {
|
||||
let locks_glob = Lock::locks_glob(index_directory);
|
||||
Ok(glob(&locks_glob)?)
|
||||
}
|
||||
|
||||
fn create_lock_file(lock_id: Uuid, index_directory: &Path) -> Result<()> {
|
||||
let lock_file_path = Lock::lock_file_path(index_directory, lock_id);
|
||||
let file = AtomicFile::new(lock_file_path, DisallowOverwrite);
|
||||
match file.write(|f| f.write_all(lock_id.to_hyphenated().to_string().as_bytes())) {
|
||||
Ok(_) => Ok(()),
|
||||
Err(e) => Err(anyhow!("error acquiring lock: {}", e)),
|
||||
}
|
||||
}
|
||||
|
||||
fn lock_file_path(path: &Path, lock_id: Uuid) -> PathBuf {
|
||||
let path_text = &format!("{}/{}.lock", path.to_string_lossy(), lock_id);
|
||||
Path::new(path_text).to_path_buf()
|
||||
}
|
||||
|
||||
fn locks_glob(path: &Path) -> String {
|
||||
format!("{}/*.lock", path.to_string_lossy())
|
||||
}
|
||||
}
|
||||
|
||||
//TODO: add drop implementation
|
||||
|
|
@ -1,21 +1,18 @@
|
|||
use std::collections::hash_map::Iter;
|
||||
use std::collections::HashMap;
|
||||
use std::fs::File;
|
||||
use std::path::Path;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::error::BakareError;
|
||||
use crate::repository::{ItemId, Version};
|
||||
use crate::repository_item::RepositoryItem;
|
||||
use crate::index::item::IndexItem;
|
||||
use crate::repository::ItemId;
|
||||
use crate::{repository_item::RepositoryItem, version::Version};
|
||||
use anyhow::Result;
|
||||
use anyhow::*;
|
||||
|
||||
#[derive(Clone, Debug, PartialOrd, PartialEq, Ord, Eq, Serialize, Deserialize)]
|
||||
pub struct IndexItem {
|
||||
relative_path: String,
|
||||
original_source_path: String,
|
||||
id: ItemId,
|
||||
version: Version,
|
||||
}
|
||||
mod io;
|
||||
mod item;
|
||||
mod lock;
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct Index {
|
||||
|
|
@ -23,38 +20,7 @@ pub struct Index {
|
|||
items_by_file_id: HashMap<ItemId, IndexItem>,
|
||||
index_path: String,
|
||||
repository_path: String,
|
||||
}
|
||||
|
||||
pub struct IndexItemIterator<'a> {
|
||||
iterator: Iter<'a, String, IndexItem>,
|
||||
}
|
||||
|
||||
impl<'a> Iterator for IndexItemIterator<'a> {
|
||||
type Item = IndexItem;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
self.iterator.next().map(|i| i.1.clone())
|
||||
}
|
||||
}
|
||||
|
||||
impl IndexItem {
|
||||
fn from(original_source_path: String, relative_path: String, id: ItemId, version: Version) -> IndexItem {
|
||||
IndexItem {
|
||||
relative_path,
|
||||
original_source_path,
|
||||
id,
|
||||
version,
|
||||
}
|
||||
}
|
||||
|
||||
fn next_version(&self, id: ItemId, relative_path: String) -> IndexItem {
|
||||
IndexItem {
|
||||
original_source_path: self.original_source_path.clone(),
|
||||
version: self.version.next(),
|
||||
relative_path,
|
||||
id,
|
||||
}
|
||||
}
|
||||
version: Version,
|
||||
}
|
||||
|
||||
impl Index {
|
||||
|
|
@ -64,26 +30,10 @@ impl Index {
|
|||
items_by_file_id: Default::default(),
|
||||
index_path: repository_path.join("index").to_string_lossy().to_string(),
|
||||
repository_path: repository_path.to_string_lossy().to_string(),
|
||||
version: Version::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn load(path: &Path) -> Result<Self, BakareError> {
|
||||
let index_file_path = path.join("index");
|
||||
let index_file = File::open(index_file_path)?;
|
||||
let index: Index = serde_cbor::from_reader(index_file)?;
|
||||
Ok(index)
|
||||
}
|
||||
|
||||
pub fn save(&self) -> Result<(), BakareError> {
|
||||
let index_file = File::create(self.index_file_path())?;
|
||||
serde_cbor::to_writer(index_file, &self)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn index_file_path(&self) -> &Path {
|
||||
Path::new(&self.index_path)
|
||||
}
|
||||
|
||||
pub fn remember(&mut self, original_source_path: &Path, relative_path: &Path, id: ItemId) {
|
||||
let item = if let Some(old) = self
|
||||
.newest_items_by_source_path
|
||||
|
|
@ -99,30 +49,30 @@ impl Index {
|
|||
)
|
||||
};
|
||||
|
||||
self.items_by_file_id.insert(item.id.clone(), item.clone());
|
||||
self.items_by_file_id.insert(item.id(), item.clone());
|
||||
self.newest_items_by_source_path
|
||||
.insert(original_source_path.to_string_lossy().to_string(), item);
|
||||
}
|
||||
|
||||
pub fn repository_item(&self, i: &IndexItem) -> RepositoryItem {
|
||||
let index_item = i.clone();
|
||||
let relative_path = Path::new(&index_item.relative_path);
|
||||
let relative_path = Path::new(index_item.relative_path());
|
||||
let repository_path = Path::new(&self.repository_path);
|
||||
let original_source_path = Path::new(&index_item.original_source_path);
|
||||
let original_source_path = Path::new(index_item.original_source_path());
|
||||
let absolute_path = repository_path.join(relative_path);
|
||||
let absolute_path = absolute_path.as_path();
|
||||
RepositoryItem::from(
|
||||
original_source_path,
|
||||
absolute_path,
|
||||
relative_path,
|
||||
index_item.id,
|
||||
index_item.version,
|
||||
index_item.id(),
|
||||
index_item.version(),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn newest_item_by_source_path(&self, path: &Path) -> Result<Option<IndexItem>, BakareError> {
|
||||
pub fn newest_item_by_source_path(&self, path: &Path) -> Result<Option<IndexItem>> {
|
||||
if !path.is_absolute() {
|
||||
return Err(BakareError::RepositoryPathNotAbsolute);
|
||||
return Err(anyhow!("repository path not absolute"));
|
||||
}
|
||||
Ok(self
|
||||
.newest_items_by_source_path
|
||||
|
|
@ -130,7 +80,7 @@ impl Index {
|
|||
.cloned())
|
||||
}
|
||||
|
||||
pub fn item_by_id(&self, id: &ItemId) -> Result<Option<IndexItem>, BakareError> {
|
||||
pub fn item_by_id(&self, id: &ItemId) -> Result<Option<IndexItem>> {
|
||||
Ok(self.items_by_file_id.get(id).cloned())
|
||||
}
|
||||
|
||||
|
|
@ -141,13 +91,14 @@ impl Index {
|
|||
}
|
||||
}
|
||||
|
||||
impl From<RepositoryItem> for IndexItem {
|
||||
fn from(i: RepositoryItem) -> Self {
|
||||
IndexItem {
|
||||
relative_path: i.relative_path().to_string_lossy().to_string(),
|
||||
original_source_path: i.original_source_path().to_string_lossy().to_string(),
|
||||
id: i.id().clone(),
|
||||
version: i.version().clone(),
|
||||
}
|
||||
pub struct IndexItemIterator<'a> {
|
||||
iterator: Iter<'a, String, IndexItem>,
|
||||
}
|
||||
|
||||
impl<'a> Iterator for IndexItemIterator<'a> {
|
||||
type Item = IndexItem;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
self.iterator.next().map(|i| i.1.clone())
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,4 @@
|
|||
pub mod backup;
|
||||
pub mod error;
|
||||
pub mod repository;
|
||||
pub mod restore;
|
||||
pub mod source;
|
||||
|
|
@ -8,3 +7,4 @@ pub mod test;
|
|||
|
||||
mod index;
|
||||
mod repository_item;
|
||||
mod version;
|
||||
|
|
|
|||
|
|
@ -1,15 +1,16 @@
|
|||
use std::path::{Path, PathBuf};
|
||||
use std::{fmt, fs, io};
|
||||
|
||||
use crate::error::BakareError;
|
||||
use crate::index::{Index, IndexItemIterator};
|
||||
use crate::repository_item::RepositoryItem;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::Digest;
|
||||
use sha2::Sha512;
|
||||
use std::fmt::Formatter;
|
||||
use std::fs::File;
|
||||
use std::io::BufReader;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::{fmt, fs, io};
|
||||
|
||||
use crate::index::{Index, IndexItemIterator};
|
||||
use crate::repository_item::RepositoryItem;
|
||||
use anyhow::Result;
|
||||
use anyhow::*;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::Digest;
|
||||
use sha2::Sha512;
|
||||
use walkdir::WalkDir;
|
||||
|
||||
/// represents a place where backup is stored an can be restored from.
|
||||
|
|
@ -25,16 +26,33 @@ pub struct Repository<'a> {
|
|||
const DATA_DIR_NAME: &str = "data";
|
||||
|
||||
#[derive(Clone, Debug, PartialOrd, PartialEq, Ord, Eq, Serialize, Deserialize, Hash)]
|
||||
pub struct ItemId(Box<[u8]>);
|
||||
|
||||
#[derive(Clone, Debug, PartialOrd, PartialEq, Ord, Eq, Serialize, Deserialize, Hash)]
|
||||
pub struct Version(u128);
|
||||
pub struct ItemId(#[serde(with = "base64")] Vec<u8>);
|
||||
|
||||
pub struct RepositoryItemIterator<'a> {
|
||||
iterator: IndexItemIterator<'a>,
|
||||
index: &'a Index,
|
||||
}
|
||||
|
||||
mod base64 {
|
||||
use ::base64;
|
||||
use serde::{de, Deserialize, Deserializer, Serializer};
|
||||
|
||||
pub fn serialize<S>(bytes: &[u8], serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
serializer.serialize_str(&base64::encode(bytes))
|
||||
}
|
||||
|
||||
pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
let s = <&str>::deserialize(deserializer)?;
|
||||
base64::decode(s).map_err(de::Error::custom)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Iterator for RepositoryItemIterator<'a> {
|
||||
type Item = RepositoryItem;
|
||||
|
||||
|
|
@ -43,18 +61,6 @@ impl<'a> Iterator for RepositoryItemIterator<'a> {
|
|||
}
|
||||
}
|
||||
|
||||
impl Version {
|
||||
pub fn next(&self) -> Self {
|
||||
Version(self.0 + 1)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Version {
|
||||
fn default() -> Self {
|
||||
Version(1)
|
||||
}
|
||||
}
|
||||
|
||||
impl AsRef<[u8]> for ItemId {
|
||||
fn as_ref(&self) -> &[u8] {
|
||||
&self.0
|
||||
|
|
@ -63,7 +69,7 @@ impl AsRef<[u8]> for ItemId {
|
|||
|
||||
impl From<&[u8]> for ItemId {
|
||||
fn from(a: &[u8]) -> Self {
|
||||
ItemId(Box::from(a))
|
||||
ItemId(a.into())
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -74,15 +80,15 @@ impl fmt::Display for ItemId {
|
|||
}
|
||||
|
||||
impl<'a> Repository<'a> {
|
||||
pub fn init(path: &Path) -> Result<(), BakareError> {
|
||||
let index = Index::new(path);
|
||||
pub fn init(path: &Path) -> Result<()> {
|
||||
let mut index = Index::new(path);
|
||||
index.save()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn open(path: &Path) -> Result<Repository, BakareError> {
|
||||
pub fn open(path: &Path) -> Result<Repository> {
|
||||
if !path.is_absolute() {
|
||||
return Err(BakareError::RepositoryPathNotAbsolute);
|
||||
return Err(anyhow!("path to repository not absolute"));
|
||||
}
|
||||
|
||||
let index = Index::load(path)?;
|
||||
|
|
@ -93,9 +99,13 @@ impl<'a> Repository<'a> {
|
|||
&self.path
|
||||
}
|
||||
|
||||
pub fn store(&mut self, source_path: &Path) -> Result<(), BakareError> {
|
||||
pub fn save_index(&mut self) -> Result<()> {
|
||||
self.index.save()
|
||||
}
|
||||
|
||||
pub fn store(&mut self, source_path: &Path) -> Result<()> {
|
||||
if !source_path.is_absolute() {
|
||||
return Err(BakareError::PathToStoreNotAbsolute);
|
||||
return Err(anyhow!("path to store not absolute"));
|
||||
}
|
||||
let id = Repository::calculate_id(source_path)?;
|
||||
let destination_path = self.data_dir();
|
||||
|
|
@ -103,23 +113,23 @@ impl<'a> Repository<'a> {
|
|||
let destination_path = Path::new(&destination_path);
|
||||
|
||||
if source_path.is_file() {
|
||||
fs::create_dir_all(destination_path.parent().unwrap())?;
|
||||
let parent = destination_path.parent().unwrap();
|
||||
fs::create_dir_all(parent)?;
|
||||
fs::copy(source_path, destination_path)?;
|
||||
let relative_path = destination_path.strip_prefix(self.path)?;
|
||||
self.index.remember(source_path, relative_path, id);
|
||||
self.index.save()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn newest_item_by_source_path(&self, path: &Path) -> Result<Option<RepositoryItem>, BakareError> {
|
||||
pub fn newest_item_by_source_path(&self, path: &Path) -> Result<Option<RepositoryItem>> {
|
||||
Ok(self
|
||||
.index
|
||||
.newest_item_by_source_path(path)?
|
||||
.map(|i| self.index.repository_item(&i)))
|
||||
}
|
||||
|
||||
pub fn item_by_id(&self, id: &ItemId) -> Result<Option<RepositoryItem>, BakareError> {
|
||||
pub fn item_by_id(&self, id: &ItemId) -> Result<Option<RepositoryItem>> {
|
||||
self.index.item_by_id(id).map(|i| i.map(|i| self.index.repository_item(&i)))
|
||||
}
|
||||
|
||||
|
|
@ -130,7 +140,7 @@ impl<'a> Repository<'a> {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn data_weight(&self) -> Result<u64, BakareError> {
|
||||
pub fn data_weight(&self) -> Result<u64> {
|
||||
let total_size = WalkDir::new(self.data_dir())
|
||||
.into_iter()
|
||||
.filter_map(|entry| entry.ok())
|
||||
|
|
@ -144,7 +154,7 @@ impl<'a> Repository<'a> {
|
|||
self.path().join(DATA_DIR_NAME)
|
||||
}
|
||||
|
||||
fn calculate_id(source_path: &Path) -> Result<ItemId, BakareError> {
|
||||
fn calculate_id(source_path: &Path) -> Result<ItemId> {
|
||||
let source_file = File::open(source_path)?;
|
||||
let mut reader = BufReader::new(source_file);
|
||||
let mut hasher = Sha512::new();
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
use crate::error::BakareError;
|
||||
use crate::repository::{ItemId, Version};
|
||||
use crate::{repository::ItemId, version::Version};
|
||||
use anyhow::Result;
|
||||
use anyhow::*;
|
||||
use std::fmt::{Display, Formatter};
|
||||
use std::path::Path;
|
||||
use std::{fmt, fs};
|
||||
|
|
@ -24,23 +25,22 @@ impl RepositoryItem {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn save(&self, save_to: &Path) -> Result<(), BakareError> {
|
||||
pub fn save(&self, save_to: &Path) -> Result<()> {
|
||||
if !save_to.is_absolute() {
|
||||
return Err(BakareError::PathToStoreNotAbsolute);
|
||||
return Err(anyhow!("path to store not absolute"));
|
||||
}
|
||||
|
||||
let target_path = save_to.join(&self.original_source_path.strip_prefix("/")?);
|
||||
if !target_path.is_absolute() {
|
||||
return Err(BakareError::PathToStoreNotAbsolute);
|
||||
return Err(anyhow!("path to store not absolute"));
|
||||
}
|
||||
let parent = target_path.parent().unwrap();
|
||||
if !parent.exists() {
|
||||
fs::create_dir_all(parent)?;
|
||||
}
|
||||
if !self.absolute_path.exists() {
|
||||
return Err(BakareError::CorruptedRepoNoFile);
|
||||
return Err(anyhow!("corrupted repository"));
|
||||
}
|
||||
println!("restoring {} to {}", &self.absolute_path.display(), &target_path.display());
|
||||
fs::copy(&self.absolute_path, &target_path)?;
|
||||
|
||||
Ok(())
|
||||
|
|
|
|||
|
|
@ -1,30 +1,32 @@
|
|||
use std::path::Path;
|
||||
|
||||
use crate::error::BakareError;
|
||||
use crate::repository::Repository;
|
||||
use crate::repository_item::RepositoryItem;
|
||||
use anyhow::Result;
|
||||
use anyhow::*;
|
||||
|
||||
pub struct Engine<'a> {
|
||||
repository: &'a Repository<'a>,
|
||||
repository: &'a mut Repository<'a>,
|
||||
target_path: &'a Path,
|
||||
}
|
||||
|
||||
impl<'a> Engine<'a> {
|
||||
pub fn new(repository: &'a Repository, target_path: &'a Path) -> Result<Self, BakareError> {
|
||||
pub fn new(repository: &'a mut Repository<'a>, target_path: &'a Path) -> Result<Self> {
|
||||
if !target_path.is_absolute() {
|
||||
return Err(BakareError::PathToStoreNotAbsolute);
|
||||
return Err(anyhow!("path to store not absolute"));
|
||||
}
|
||||
Ok(Engine { repository, target_path })
|
||||
}
|
||||
|
||||
pub fn restore_all(&self) -> Result<(), BakareError> {
|
||||
pub fn restore_all(&mut self) -> Result<()> {
|
||||
for item in self.repository.newest_items() {
|
||||
self.restore(&item)?;
|
||||
}
|
||||
self.repository.save_index()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn restore(&self, item: &RepositoryItem) -> Result<(), BakareError> {
|
||||
pub fn restore(&self, item: &RepositoryItem) -> Result<()> {
|
||||
item.save(self.target_path)?;
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,15 +5,15 @@ use std::path::Path;
|
|||
use tempfile::tempdir;
|
||||
use walkdir::WalkDir;
|
||||
|
||||
use crate::error::BakareError;
|
||||
use crate::repository::{ItemId, Repository};
|
||||
use crate::repository_item::RepositoryItem;
|
||||
use crate::source::TempSource;
|
||||
use crate::{backup, restore};
|
||||
use anyhow::Result;
|
||||
|
||||
pub fn assert_same_after_restore(source_path: &Path) -> Result<(), BakareError> {
|
||||
let repository_path = tempdir()?.into_path();
|
||||
let restore_target = tempdir()?.into_path();
|
||||
pub fn assert_same_after_restore(source_path: &Path) -> Result<()> {
|
||||
let repository_path = tempdir().unwrap().into_path();
|
||||
let restore_target = tempdir().unwrap().into_path();
|
||||
|
||||
assert_ne!(source_path, repository_path);
|
||||
assert_ne!(repository_path, restore_target);
|
||||
|
|
@ -25,8 +25,8 @@ pub fn assert_same_after_restore(source_path: &Path) -> Result<(), BakareError>
|
|||
backup_engine.backup()?;
|
||||
}
|
||||
{
|
||||
let restore_repository = Repository::open(repository_path.as_path())?;
|
||||
let restore_engine = restore::Engine::new(&restore_repository, &restore_target)?;
|
||||
let mut restore_repository = Repository::open(repository_path.as_path())?;
|
||||
let mut restore_engine = restore::Engine::new(&mut restore_repository, &restore_target)?;
|
||||
restore_engine.restore_all()?;
|
||||
}
|
||||
|
||||
|
|
@ -34,15 +34,12 @@ pub fn assert_same_after_restore(source_path: &Path) -> Result<(), BakareError>
|
|||
Ok(())
|
||||
}
|
||||
|
||||
pub fn assert_restored_file_contents(
|
||||
repository_path: &Path,
|
||||
source_file_full_path: &Path,
|
||||
contents: &str,
|
||||
) -> Result<(), BakareError> {
|
||||
let restore_repository = Repository::open(repository_path)?;
|
||||
let restore_target = tempdir()?;
|
||||
let restore_engine = restore::Engine::new(&restore_repository, &restore_target.path())?;
|
||||
pub fn assert_restored_file_contents(repository_path: &Path, source_file_full_path: &Path, contents: &str) -> Result<()> {
|
||||
let mut restore_repository = Repository::open(repository_path)?;
|
||||
let item = restore_repository.newest_item_by_source_path(&source_file_full_path)?;
|
||||
let restore_target = tempdir().unwrap();
|
||||
let restore_engine = restore::Engine::new(&mut restore_repository, &restore_target.path())?;
|
||||
|
||||
restore_engine.restore(&item.unwrap())?;
|
||||
let restored_file_path = restore_target.path().join(source_file_full_path.strip_prefix("/")?);
|
||||
assert_target_file_contents(&restored_file_path, contents)
|
||||
|
|
@ -53,17 +50,17 @@ pub fn assert_restored_from_version_has_contents(
|
|||
source_file_full_path: &Path,
|
||||
old_contents: &str,
|
||||
old_id: &ItemId,
|
||||
) -> Result<(), BakareError> {
|
||||
let restore_repository = Repository::open(repository_path)?;
|
||||
let restore_target = tempdir()?;
|
||||
let restore_engine = restore::Engine::new(&restore_repository, &restore_target.path())?;
|
||||
) -> Result<()> {
|
||||
let mut restore_repository = Repository::open(repository_path)?;
|
||||
let old_item = restore_repository.item_by_id(&old_id)?;
|
||||
let restore_target = tempdir().unwrap();
|
||||
let restore_engine = restore::Engine::new(&mut restore_repository, &restore_target.path())?;
|
||||
restore_engine.restore(&old_item.unwrap())?;
|
||||
let restored_file_path = restore_target.path().join(source_file_full_path.strip_prefix("/")?);
|
||||
assert_target_file_contents(&restored_file_path, old_contents)
|
||||
}
|
||||
|
||||
pub fn newest_item(repository_path: &Path, source_file_full_path: &Path) -> Result<RepositoryItem, BakareError> {
|
||||
pub fn newest_item(repository_path: &Path, source_file_full_path: &Path) -> Result<RepositoryItem> {
|
||||
let item = {
|
||||
let reading_repository = Repository::open(repository_path)?;
|
||||
let item = reading_repository.newest_item_by_source_path(&source_file_full_path)?;
|
||||
|
|
@ -73,10 +70,10 @@ pub fn newest_item(repository_path: &Path, source_file_full_path: &Path) -> Resu
|
|||
Ok(item)
|
||||
}
|
||||
|
||||
pub fn restore_all_from_reloaded_repository(repository_path: &Path, restore_target: &Path) -> Result<(), BakareError> {
|
||||
pub fn restore_all_from_reloaded_repository(repository_path: &Path, restore_target: &Path) -> Result<()> {
|
||||
{
|
||||
let restore_repository = Repository::open(repository_path)?;
|
||||
let restore_engine = restore::Engine::new(&restore_repository, &restore_target)?;
|
||||
let mut restore_repository = Repository::open(repository_path)?;
|
||||
let mut restore_engine = restore::Engine::new(&mut restore_repository, &restore_target)?;
|
||||
restore_engine.restore_all()?;
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -87,49 +84,49 @@ pub fn backup_file_with_contents(
|
|||
repository_path: &Path,
|
||||
source_file_relative_path: &str,
|
||||
contents: &str,
|
||||
) -> Result<(), BakareError> {
|
||||
) -> Result<()> {
|
||||
{
|
||||
let mut backup_repository = Repository::open(repository_path)?;
|
||||
let mut backup_engine = backup::Engine::new(source.path(), &mut backup_repository)?;
|
||||
source.write_text_to_file(source_file_relative_path, contents)?;
|
||||
source.write_text_to_file(source_file_relative_path, contents).unwrap();
|
||||
backup_engine.backup()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn data_weight(repository_path: &Path) -> Result<u64, BakareError> {
|
||||
pub fn data_weight(repository_path: &Path) -> Result<u64> {
|
||||
{
|
||||
let repository = Repository::open(repository_path)?;
|
||||
Ok(repository.data_weight()?)
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_directory_trees_have_same_contents(left: &Path, right: &Path) -> Result<(), BakareError> {
|
||||
fn assert_directory_trees_have_same_contents(left: &Path, right: &Path) -> Result<()> {
|
||||
let left_files = get_sorted_files_recursively(left)?;
|
||||
let right_files = get_sorted_files_recursively(right)?;
|
||||
|
||||
let pairs = left_files.iter().zip(right_files);
|
||||
for (l, r) in pairs {
|
||||
assert_eq!(l.file_name(), r.file_name());
|
||||
let mut fl = File::open(l)?;
|
||||
let mut fr = File::open(r)?;
|
||||
let mut fl = File::open(l).unwrap();
|
||||
let mut fr = File::open(r).unwrap();
|
||||
let mut bl = vec![];
|
||||
let mut br = vec![];
|
||||
fl.read_to_end(&mut bl)?;
|
||||
fr.read_to_end(&mut br)?;
|
||||
fl.read_to_end(&mut bl).unwrap();
|
||||
fr.read_to_end(&mut br).unwrap();
|
||||
assert_eq!(bl, br);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn get_sorted_files_recursively(path: &Path) -> Result<Vec<Box<Path>>, BakareError> {
|
||||
let walker = WalkDir::new(path).sort_by(|a, b| a.file_name().cmp(b.file_name()));
|
||||
pub fn get_sorted_files_recursively<T: AsRef<Path>>(path: T) -> Result<Vec<Box<Path>>> {
|
||||
let walker = WalkDir::new(path.as_ref()).sort_by(|a, b| a.file_name().cmp(b.file_name()));
|
||||
|
||||
let mut result = vec![];
|
||||
|
||||
for maybe_entry in walker {
|
||||
let entry = maybe_entry?;
|
||||
if entry.path() == path {
|
||||
if entry.path() == path.as_ref() {
|
||||
continue;
|
||||
}
|
||||
if entry.path().is_file() {
|
||||
|
|
@ -140,10 +137,13 @@ fn get_sorted_files_recursively(path: &Path) -> Result<Vec<Box<Path>>, BakareErr
|
|||
Ok(result)
|
||||
}
|
||||
|
||||
fn assert_target_file_contents(restored_path: &Path, expected_contents: &str) -> Result<(), BakareError> {
|
||||
fn assert_target_file_contents(restored_path: &Path, expected_contents: &str) -> Result<()> {
|
||||
let mut actual_contents = String::new();
|
||||
assert!(restored_path.exists(), "Expected '{}' to be there", restored_path.display());
|
||||
File::open(restored_path)?.read_to_string(&mut actual_contents)?;
|
||||
File::open(restored_path)
|
||||
.unwrap()
|
||||
.read_to_string(&mut actual_contents)
|
||||
.unwrap();
|
||||
assert_eq!(expected_contents, actual_contents);
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
23
src/version.rs
Normal file
23
src/version.rs
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
use serde::{Deserialize, Serialize};
|
||||
use std::fmt::Display;
|
||||
|
||||
#[derive(Clone, Debug, PartialOrd, PartialEq, Ord, Eq, Serialize, Deserialize, Hash)]
|
||||
pub struct Version(u128);
|
||||
|
||||
impl Version {
|
||||
pub fn next(&self) -> Self {
|
||||
Version(self.0 + 1)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Version {
|
||||
fn default() -> Self {
|
||||
Version(1)
|
||||
}
|
||||
}
|
||||
|
||||
impl Display for Version {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{}", self.0)
|
||||
}
|
||||
}
|
||||
115
tests/concurrency_tests.rs
Normal file
115
tests/concurrency_tests.rs
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::Result;
|
||||
use async_log::span;
|
||||
use bakare::repository::Repository;
|
||||
use bakare::source::TempSource;
|
||||
use bakare::test::assertions::*;
|
||||
use bakare::{backup, restore};
|
||||
use nix::sys::wait::{waitpid, WaitStatus};
|
||||
use nix::unistd::{fork, getpid, ForkResult};
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[test]
|
||||
fn handle_concurrent_backups() -> Result<()> {
|
||||
setup_logger();
|
||||
let repository_path = &tempdir().unwrap().into_path();
|
||||
Repository::init(repository_path)?;
|
||||
|
||||
let parallel_backups_number = 16;
|
||||
let files_per_backup_number = 16;
|
||||
let total_number_of_files = parallel_backups_number * files_per_backup_number;
|
||||
let finished_backup_runs = backup_in_parallel(repository_path, parallel_backups_number, files_per_backup_number)?;
|
||||
assert_eq!(finished_backup_runs.len(), parallel_backups_number);
|
||||
|
||||
let all_restored_files = restore_all(repository_path)?;
|
||||
assert_eq!(all_restored_files.len(), total_number_of_files);
|
||||
|
||||
for i in 0..parallel_backups_number {
|
||||
for j in 0..files_per_backup_number {
|
||||
let id = file_id(i, j);
|
||||
let file = all_restored_files.iter().find(|f| f.ends_with(id.clone()));
|
||||
assert!(file.unwrap().exists(), "file {:?} does not exist", file);
|
||||
let contents = fs::read_to_string(file.unwrap()).unwrap();
|
||||
assert_eq!(id.to_string(), contents.to_owned());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn backup_in_parallel<T>(
|
||||
repository_path: T,
|
||||
parallel_backups_number: usize,
|
||||
files_per_backup_number: usize,
|
||||
) -> Result<Vec<usize>>
|
||||
where
|
||||
T: AsRef<Path> + Sync,
|
||||
{
|
||||
let task_numbers = (0..parallel_backups_number).collect::<Vec<_>>();
|
||||
let mut child_pids = vec![];
|
||||
span!("[{}] acquiring children for parent", getpid(), {
|
||||
for task_number in &task_numbers {
|
||||
match unsafe { fork() } {
|
||||
Ok(ForkResult::Parent { child }) => {
|
||||
child_pids.push(child);
|
||||
}
|
||||
Ok(ForkResult::Child) => {
|
||||
backup_process(*task_number, &repository_path, files_per_backup_number)?;
|
||||
std::process::exit(0);
|
||||
}
|
||||
|
||||
Err(_) => panic!("fork failed"),
|
||||
}
|
||||
}
|
||||
});
|
||||
span!("[{}] waiting for {} children", getpid(), child_pids.len(), {
|
||||
for pid in child_pids {
|
||||
log::debug!("[{}] waiting for a child {} to exit", getpid(), pid);
|
||||
let status = waitpid(Some(pid), None)?;
|
||||
match status {
|
||||
WaitStatus::Exited(pid, code) => {
|
||||
assert!(code == 0, "failed the wait for {} with code {}", pid, code);
|
||||
}
|
||||
WaitStatus::Signaled(pid, _, _) => assert!(false, "failed with signal for {}", pid),
|
||||
_ => panic!("unknown state"),
|
||||
}
|
||||
}
|
||||
});
|
||||
Ok(task_numbers)
|
||||
}
|
||||
|
||||
fn backup_process<T>(task_number: usize, repository_path: T, files_per_backup_number: usize) -> Result<()>
|
||||
where
|
||||
T: AsRef<Path> + Sync,
|
||||
{
|
||||
let mut repository = Repository::open(repository_path.as_ref())?;
|
||||
let source = TempSource::new().unwrap();
|
||||
let mut backup_engine = backup::Engine::new(source.path(), &mut repository)?;
|
||||
for i in 0..files_per_backup_number {
|
||||
let id = file_id(task_number, i);
|
||||
source.write_text_to_file(&id, &id).unwrap();
|
||||
}
|
||||
backup_engine.backup()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn restore_all<T: AsRef<Path>>(repository_path: T) -> Result<Vec<Box<Path>>> {
|
||||
let restore_target = tempdir().unwrap().into_path();
|
||||
let mut restore_repository = Repository::open(repository_path.as_ref())?;
|
||||
let mut restore_engine = restore::Engine::new(&mut restore_repository, restore_target.as_ref())?;
|
||||
restore_engine.restore_all()?;
|
||||
get_sorted_files_recursively(&restore_target)
|
||||
}
|
||||
|
||||
fn setup_logger() {
|
||||
let logger = femme::pretty::Logger::new();
|
||||
async_log::Logger::wrap(logger, rand::random::<u64>)
|
||||
.start(log::LevelFilter::Trace)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn file_id(i: usize, j: usize) -> String {
|
||||
format!("{}-{}", i, j)
|
||||
}
|
||||
// TODO handle stale leftover locks
|
||||
|
|
@ -1,27 +1,27 @@
|
|||
use tempfile::tempdir;
|
||||
|
||||
use anyhow::Result;
|
||||
use bakare::backup;
|
||||
use bakare::error::BakareError;
|
||||
use bakare::repository::Repository;
|
||||
use bakare::source::TempSource;
|
||||
use bakare::test::assertions::*;
|
||||
|
||||
#[test]
|
||||
fn restore_multiple_files() -> Result<(), BakareError> {
|
||||
let source = TempSource::new()?;
|
||||
fn restore_multiple_files() -> Result<()> {
|
||||
let source = TempSource::new().unwrap();
|
||||
|
||||
source.write_text_to_file("first", "some contents")?;
|
||||
source.write_text_to_file("second", "some contents")?;
|
||||
source.write_text_to_file("third", "some other contents")?;
|
||||
source.write_text_to_file("first", "some contents").unwrap();
|
||||
source.write_text_to_file("second", "some contents").unwrap();
|
||||
source.write_text_to_file("third", "some other contents").unwrap();
|
||||
|
||||
assert_same_after_restore(source.path())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restore_files_after_reopening_repository() -> Result<(), BakareError> {
|
||||
let source = TempSource::new()?;
|
||||
let repository_path = &tempdir()?.into_path();
|
||||
let restore_target = tempdir()?.into_path();
|
||||
fn restore_files_after_reopening_repository() -> Result<()> {
|
||||
let source = TempSource::new().unwrap();
|
||||
let repository_path = &tempdir().unwrap().into_path();
|
||||
let restore_target = tempdir().unwrap().into_path();
|
||||
Repository::init(repository_path)?;
|
||||
|
||||
let source_file_relative_path = "some file path";
|
||||
|
|
@ -36,9 +36,9 @@ fn restore_files_after_reopening_repository() -> Result<(), BakareError> {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn restore_older_version_of_file() -> Result<(), BakareError> {
|
||||
let source = TempSource::new()?;
|
||||
let repository_path = tempdir()?.into_path();
|
||||
fn restore_older_version_of_file() -> Result<()> {
|
||||
let source = TempSource::new().unwrap();
|
||||
let repository_path = tempdir().unwrap().into_path();
|
||||
Repository::init(repository_path.as_path())?;
|
||||
|
||||
let source_file_relative_path = "some path";
|
||||
|
|
@ -57,9 +57,9 @@ fn restore_older_version_of_file() -> Result<(), BakareError> {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn newer_version_should_be_greater_than_earlier_version() -> Result<(), BakareError> {
|
||||
let source = TempSource::new()?;
|
||||
let repository_path = tempdir()?.into_path();
|
||||
fn newer_version_should_be_greater_than_earlier_version() -> Result<()> {
|
||||
let source = TempSource::new().unwrap();
|
||||
let repository_path = tempdir().unwrap().into_path();
|
||||
Repository::init(repository_path.as_path())?;
|
||||
|
||||
let source_file_relative_path = "some path";
|
||||
|
|
@ -81,9 +81,9 @@ fn newer_version_should_be_greater_than_earlier_version() -> Result<(), BakareEr
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn store_duplicated_files_just_once() -> Result<(), BakareError> {
|
||||
let source = TempSource::new()?;
|
||||
let repository_path = &tempdir()?.into_path();
|
||||
fn store_duplicated_files_just_once() -> Result<()> {
|
||||
let source = TempSource::new().unwrap();
|
||||
let repository_path = &tempdir().unwrap().into_path();
|
||||
Repository::init(repository_path)?;
|
||||
assert_eq!(data_weight(&repository_path)?, 0);
|
||||
|
||||
|
|
@ -103,9 +103,9 @@ fn store_duplicated_files_just_once() -> Result<(), BakareError> {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn restore_latest_version_by_default() -> Result<(), BakareError> {
|
||||
let source = TempSource::new()?;
|
||||
let repository_path = &tempdir()?.into_path();
|
||||
fn restore_latest_version_by_default() -> Result<()> {
|
||||
let source = TempSource::new().unwrap();
|
||||
let repository_path = &tempdir().unwrap().into_path();
|
||||
Repository::init(repository_path)?;
|
||||
|
||||
let source_file_relative_path = "some path";
|
||||
|
|
@ -118,16 +118,14 @@ fn restore_latest_version_by_default() -> Result<(), BakareError> {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn forbid_backup_of_paths_within_repository() -> Result<(), BakareError> {
|
||||
let repository_path = &tempdir()?.into_path();
|
||||
fn forbid_backup_of_paths_within_repository() -> Result<()> {
|
||||
let repository_path = &tempdir().unwrap().into_path();
|
||||
Repository::init(repository_path)?;
|
||||
let mut repository = Repository::open(repository_path)?;
|
||||
let error = backup::Engine::new(repository_path, &mut repository).err().unwrap();
|
||||
let correct_error = matches!(error, BakareError::SourceSameAsRepository);
|
||||
assert!(correct_error);
|
||||
let error = backup::Engine::new(repository_path, &mut repository);
|
||||
assert!(error.is_err());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// TODO: test concurrent writes
|
||||
// TODO: index corruption
|
||||
// TODO: encryption
|
||||
// TODO: resume from sleep while backup in progress
|
||||
|
|
|
|||
Loading…
Reference in a new issue