basic uploader 1st take

main
Cyryl Płotnicki 2023-02-25 10:04:23 +00:00
parent dc2e6061c3
commit 8b1490bada
3 changed files with 125 additions and 13 deletions

36
Cargo.lock generated
View File

@ -835,6 +835,16 @@ version = "0.3.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
[[package]]
name = "mime_guess"
version = "2.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef"
dependencies = [
"mime",
"unicase",
]
[[package]]
name = "miniz_oxide"
version = "0.6.2"
@ -886,6 +896,7 @@ dependencies = [
"indicatif",
"jwalk",
"log",
"mime_guess",
"rayon",
"reqwest",
"serde",
@ -894,6 +905,7 @@ dependencies = [
"simple_logger",
"structopt",
"tokio",
"tokio-util",
]
[[package]]
@ -1147,6 +1159,7 @@ dependencies = [
"js-sys",
"log",
"mime",
"mime_guess",
"native-tls",
"once_cell",
"percent-encoding",
@ -1161,6 +1174,7 @@ dependencies = [
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"wasm-streams",
"web-sys",
"winreg",
]
@ -1537,6 +1551,15 @@ version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
[[package]]
name = "unicase"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
dependencies = [
"version_check",
]
[[package]]
name = "unicode-bidi"
version = "0.3.10"
@ -1687,6 +1710,19 @@ version = "0.2.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f"
[[package]]
name = "wasm-streams"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078"
dependencies = [
"futures-util",
"js-sys",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
]
[[package]]
name = "web-sys"
version = "0.3.60"

View File

@ -12,11 +12,13 @@ hex ="0.4"
indicatif = "0.17"
jwalk = "0.8"
log = "0.4"
mime_guess = "2"
rayon = "1"
reqwest = { version = "0.11", features = ["gzip", "brotli", "json"] }
reqwest = { version = "0.11", features = ["gzip", "brotli", "json", "multipart", "stream", "blocking"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
sha1 = "0.10"
simple_logger="4"
structopt = "0.3"
tokio = { version = "1", features = ["full"] }
tokio = { version = "1", features = ["full"] }
tokio-util = { version ="0.7", features = ["codec"]}

View File

@ -2,15 +2,17 @@ use std::str::FromStr;
use std::{collections::HashMap, fs, io};
use anyhow::{anyhow, Context, Result};
use camino::{Utf8Path, Utf8PathBuf};
use chrono::prelude::*;
use camino::Utf8PathBuf;
use futures::future::join_all;
use futures::{Future, FutureExt};
use indicatif::ProgressBar;
use jwalk::WalkDir;
use rayon::prelude::*;
use reqwest::{multipart, Body, Response};
use sha1::{Digest, Sha1};
use simple_logger::SimpleLogger;
use structopt::StructOpt;
use tokio_util::codec::{BytesCodec, FramedRead};
struct FileEntry {
path: Utf8PathBuf,
@ -30,21 +32,93 @@ const BASE_PATH: &str = "https://neocities.org/api";
#[tokio::main]
async fn main() -> Result<()> {
let options = Options::from_args();
SimpleLogger::new().init()?;
SimpleLogger::new()
.with_level(log::LevelFilter::Debug)
.init()?;
let http_client = reqwest::Client::new();
let server_entries = fetch_server_list(&http_client, &options.api_key).await?;
let local_entries = hash(&options.directory_to_upload).await?;
let (to_upload, to_delete) = calculate_diff(&server_entries, &local_entries).await;
let upload_progress = ProgressBar::new(to_upload.len() as u64);
println!("to upload: {:#?}", to_upload);
upload(to_upload, &options.directory_to_upload, &http_client, &options.api_key).await?;
delete(to_delete, &http_client, &options.api_key).await?;
log::info!("All done!");
Ok(())
}
async fn upload(
relative_paths_to_upload: Vec<&str>,
root_directory: &Utf8PathBuf,
http_client: &reqwest::Client,
api_key: &str,
) -> Result<()> {
let upload_progress = ProgressBar::new(relative_paths_to_upload.len() as u64);
let all_upload_requests: Vec<_> = relative_paths_to_upload
.iter()
.map(|path| {
upload_file(path, root_directory, http_client, api_key).then(|f| async {
upload_progress.inc(1);
match f {
Ok(response_f) => {
let response = response_f.await?;
if !response.status().is_success() {
Err(anyhow!(format!(
"{}: {}",
response.url(),
response.status().canonical_reason().unwrap_or_default()
)))
} else {
Ok(response)
}
}
Err(err) => Err(err).context(path.to_string()),
}
})
})
.collect();
let all_responses = join_all(all_upload_requests).await;
let all_ok = all_responses.iter().all(|result| result.is_ok());
if !all_ok {
log::warn!("there were some problems deleting files on the server");
log::warn!("{:#?}", all_responses);
} else {
let count = all_responses.len();
if count > 0 {
log::info!("{} files deleted on the server", count);
}
}
upload_progress.finish_and_clear();
Ok(())
}
async fn upload_file(
path: &str,
root_directory: &Utf8PathBuf,
http_client: &reqwest::Client,
api_key: &str,
) -> Result<impl Future<Output = Result<Response, reqwest::Error>>> {
let path = root_directory.join(path);
let file = tokio::fs::File::open(&path).await?;
let stream = FramedRead::new(file, BytesCodec::new());
let file_body = Body::wrap_stream(stream);
let file_name = path.file_name().unwrap_or_default().to_string();
let mime_type = mime_guess::from_path(&path).first_raw().unwrap_or_default();
let file_part = multipart::Part::stream(file_body)
.file_name(file_name)
.mime_str(mime_type)?;
let form = multipart::Form::new().part("file", file_part);
Ok(http_client
.post(format!("{BASE_PATH}/upload"))
.bearer_auth(api_key)
.multipart(form)
.send())
}
async fn delete(to_delete: Vec<&str>, http_client: &reqwest::Client, api_key: &str) -> Result<()> {
let all_deletion_requests: Vec<_> = to_delete
.iter()
@ -71,7 +145,7 @@ async fn delete(to_delete: Vec<&str>, http_client: &reqwest::Client, api_key: &s
Ok(response)
}
}
Err(err) => Err(anyhow!("error issuing request")),
Err(err) => Err(anyhow!(format!("{err}"))),
});
let mut all_responses = all_responses;
let all_ok = all_responses.all(|result| result.is_ok());
@ -130,7 +204,7 @@ async fn fetch_server_list(
progress_bar.set_message("fetching file list from server....");
let response: serde_json::Value = http_client
.get(format!("{BASE_PATH}/list"))
.bearer_auth(&api_key)
.bearer_auth(api_key)
.send()
.await?
.json()
@ -155,10 +229,10 @@ async fn hash(directory: &Utf8PathBuf) -> Result<HashMap<String, FileEntry>> {
if !entry.path.is_dir() {
let file = fs::File::open(&entry.path)?;
entry.sha1_hash = Some(hash_file(file)?);
progress_bar.inc(1);
}
let path = entry.path.strip_prefix(directory)?;
Ok((path.to_string(), entry))
progress_bar.inc(1);
let relative_path = entry.path.strip_prefix(directory)?;
Ok((relative_path.to_string(), entry))
})
.collect();
progress_bar.finish_and_clear();