Skip to content
Snippets Groups Projects
Commit 6c4f6b2f authored by Simon Sapin's avatar Simon Sapin
Browse files

Upload to takobasu with rsync

parent c6e49a0f
No related branches found
No related tags found
1 merge request!1Initial implementation
......@@ -204,6 +204,7 @@
"indicatif",
"num_cpus",
"sha2",
"shell-escape",
"structopt",
"tar",
"time",
......@@ -318,6 +319,12 @@
]
[[package]]
name = "shell-escape"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "45bb67a18fa91266cc7807181f62f9178a6873bfad7dc788c42e6430db40184f"
[[package]]
name = "strsim"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
......
......@@ -17,6 +17,7 @@
num_cpus = "1.13"
sha2 = "0.9"
shell-escape = "0.1"
structopt = "0.3"
tar = "0.4"
tempfile = "3.2"
......
......@@ -10,6 +10,9 @@
use std::str;
use structopt::StructOpt;
const UPLOAD_SSH_HOST: &str = "takobasu";
const UPLOAD_REMOTE_DIRECTORY: &str = "/data/static/static-asv-repos/static/megafine";
#[derive(StructOpt)]
enum Command {
NewDataset(NewDataset),
......@@ -27,7 +30,7 @@
/// The directory to archive.
/// The dataset name is taken from the last component of this path.
directory: PathBuf,
source_directory: PathBuf,
}
struct Megafine {
......@@ -40,5 +43,5 @@
fn run(&self, megafine: &Megafine) -> Result<()> {
println!(
"New dataset from directory {}",
style(self.directory.display()).bold()
style(self.source_directory.display()).bold()
);
......@@ -44,6 +47,6 @@
);
if !fs::metadata(&self.directory)?.is_dir() {
if !fs::metadata(&self.source_directory)?.is_dir() {
Err("Not a directory")?
}
let base_name = self
......@@ -46,8 +49,8 @@
Err("Not a directory")?
}
let base_name = self
.directory
.source_directory
.file_name()
.ok_or("No base name")?
.to_str()
......@@ -58,7 +61,7 @@
.arg("id")
.arg("--id")
.arg("-R")
.arg(&self.directory),
.arg(&self.source_directory),
)? {
format!("_{}", stdout)
} else {
......@@ -68,8 +71,36 @@
let archive_name = format!("{}_{}{}.tar.zst", base_name, today, id);
let bold = style(&archive_name).bold();
let archive_path = megafine.tmp_dir.join(&archive_name);
if archive_path.exists() {
let remote_path = format!("{}/{}", UPLOAD_REMOTE_DIRECTORY, archive_name);
let escaped_remote_path = shell_escape::unix::escape(remote_path.into());
let remote_exists = if let Some(stdout) = subprocess_output(
process::Command::new("ssh")
.arg(UPLOAD_SSH_HOST)
.arg(format!(
"if [ -e {} ]; then echo exists; fi",
escaped_remote_path
)),
)? {
stdout == "exists"
} else {
return Err(format!(
"Consider adding a {} section in {}",
style(format!("Host {}", UPLOAD_SSH_HOST)).bold(),
style("~/.ssh/config").bold()
)
.into());
};
if remote_exists {
if self.overwrite {
println!("Overwriting remote {}", bold)
} else {
Err(format!("Remote {} already exists", bold))?
}
}
let tmp_path = megafine.tmp_dir.join(&archive_name);
let local_exists = tmp_path.exists();
if local_exists {
if self.overwrite {
println!("Overwriting tmp/{}", bold)
} else {
......@@ -97,10 +128,10 @@
zstd.multithread(num_cpus::get() as _)?;
let mut tar = tar::Builder::new(zstd);
tar.follow_symlinks(false);
tar.append_dir_all(base_name, &self.directory)?;
tar.append_dir_all(base_name, &self.source_directory)?;
// Peel the onion
let zstd = tar.into_inner()?;
let Tee(mut tmp_file, sha) = zstd.finish()?;
let size = tmp_file.stream_position()?;
tmp_file.as_file_mut().sync_all()?;
if self.overwrite {
......@@ -101,8 +132,8 @@
// Peel the onion
let zstd = tar.into_inner()?;
let Tee(mut tmp_file, sha) = zstd.finish()?;
let size = tmp_file.stream_position()?;
tmp_file.as_file_mut().sync_all()?;
if self.overwrite {
tmp_file.persist(&archive_path)?;
tmp_file.persist(&tmp_path)?;
} else {
......@@ -108,8 +139,8 @@
} else {
tmp_file.persist_noclobber(&archive_path)?;
tmp_file.persist_noclobber(&tmp_path)?;
}
let hash = hex::encode(sha.finalize());
println!("Compressed to {}", style(BinaryBytes(size)).bold());
println!("SHA-256: {}", style(hash).bold());
......@@ -110,9 +141,19 @@
}
let hash = hex::encode(sha.finalize());
println!("Compressed to {}", style(BinaryBytes(size)).bold());
println!("SHA-256: {}", style(hash).bold());
let rsync_status = process::Command::new("rsync")
.arg("--info=progress2")
.arg("-h")
.arg(tmp_path)
.arg(format!("{}:{}", UPLOAD_SSH_HOST, escaped_remote_path))
.status()?;
if !rsync_status.success() {
Err("")?
}
Ok(())
}
}
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment