# HG changeset patch # User Simon Sapin <simon.sapin@octobus.net> # Date 1636126527 -3600 # Fri Nov 05 16:35:27 2021 +0100 # Node ID 41f4eac6224c8565da474e699a65cfb2ee6fc316 # Parent 53d4e9e0cf0f6c1dd6b220672f2e1bceaa748c5c Write details about new datasets to a TOML file diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -1,3 +1,5 @@ ^target/ ^tmp/ ^cache/ +^datasets/new.toml + diff --git a/Cargo.lock b/Cargo.lock --- a/Cargo.lock +++ b/Cargo.lock @@ -139,6 +139,17 @@ ] [[package]] +name = "getrandom" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] name = "heck" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -203,11 +214,14 @@ "hex", "indicatif", "num_cpus", + "serde", "sha2", "shell-escape", "structopt", "tar", + "tempfile", "time", + "toml", "zstd", ] @@ -240,6 +254,12 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] +name = "ppv-lite86" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed0cfbc8191465bed66e1718596ee0b0b35d5ee1f41c5df2189d0fe8bde535ba" + +[[package]] name = "proc-macro-error" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -282,6 +302,46 @@ ] [[package]] +name = "rand" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" +dependencies = [ + "rand_core", +] + +[[package]] name = "redox_syscall" version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -306,6 +366,35 @@ checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" [[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "serde" +version = "1.0.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7bc1a1ab1961464eae040d96713baa5a724a8152c1222492465b54322ec508b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] name = "sha2" version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -377,6 +466,20 @@ ] [[package]] +name = "tempfile" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +dependencies = [ + "cfg-if", + "libc", + "rand", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] name = "terminal_size" version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -405,6 +508,15 @@ ] [[package]] +name = "toml" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +dependencies = [ + "serde", +] + +[[package]] name = "typenum" version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -441,67 +553,6 @@ checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" [[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "xattr" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" -dependencies = [ - "libc", -] - -[[package]] -name = "zstd" -version = "0.9.0+zstd.1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07749a5dc2cb6b36661290245e350f15ec3bbb304e493db54a1d354480522ccd" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "4.1.1+zstd.1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c91c90f2c593b003603e5e0493c837088df4469da25aafff8bce42ba48caf079" -dependencies = [ - "libc", - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "1.6.1+zstd.1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "615120c7a2431d16cf1cf979e7fc31ba7a5b5e5707b29c8a99e5dbf8a8392a33" -dependencies = [ - "cc", - "libc", -] -ecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" - -[[package]] name = "wasi" version = "0.10.2+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -566,11 +617,3 @@ "cc", "libc", ] -name = "zstd-sys" -version = "1.6.1+zstd.1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "615120c7a2431d16cf1cf979e7fc31ba7a5b5e5707b29c8a99e5dbf8a8392a33" -dependencies = [ - "cc", - "libc", -] diff --git a/Cargo.toml b/Cargo.toml --- a/Cargo.toml +++ b/Cargo.toml @@ -16,10 +16,12 @@ #Â https://github.com/rust-lang/rust/issues/74479 num_cpus = "1.13" +serde = { version = "1.0", features = ["derive"] } sha2 = "0.9" shell-escape = "0.1" structopt = "0.3" tar = "0.4" tempfile = "3.2" time = "0.3" +toml = "0.5" zstd = { version = "0.9", features = ["zstdmt"] } diff --git a/src/main.rs b/src/main.rs --- a/src/main.rs +++ b/src/main.rs @@ -2,9 +2,11 @@ use dialoguer::Confirm; use indicatif::BinaryBytes; use sha2::{Digest, Sha256}; +use std::env; use std::fs; use std::io; use std::io::Seek; +use std::io::Write; use std::path::PathBuf; use std::process; use std::str; @@ -12,14 +14,22 @@ const UPLOAD_SSH_HOST: &str = "takobasu"; const UPLOAD_REMOTE_DIRECTORY: &str = "/data/static/static-asv-repos/static/megafine"; +const DOWNLOAD_BASE_URL: &str = "https://static.octobus.net/megafine/"; + +#[derive(serde::Serialize)] +struct Dataset { + name: String, + url: String, + sha256: String, +} #[derive(StructOpt)] enum Command { - NewDataset(NewDataset), + NewDataset(NewDatasetCommand), } #[derive(StructOpt)] -struct NewDataset { +struct NewDatasetCommand { /// Skip confirmation prompts #[structopt(long)] no_confirm: bool, @@ -36,11 +46,12 @@ struct Megafine { tmp_dir: PathBuf, cache_dir: PathBuf, + datasets: PathBuf, } type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>; -impl NewDataset { +impl NewDatasetCommand { fn run(&self, megafine: &Megafine) -> Result<()> { println!( "New dataset from directory {}", @@ -70,7 +81,8 @@ println!("Not a Mercurial repository?"); format!("") }; - let archive_name = format!("{}_{}{}.tar.zst", base_name, today, id); + let dataset_name = format!("{}_{}{}", base_name, today, id); + let archive_name = format!("{}.tar.zst", dataset_name); let bold = style(&archive_name).bold(); let remote_path = format!("{}/{}", UPLOAD_REMOTE_DIRECTORY, archive_name); @@ -144,7 +156,7 @@ let hash = hex::encode(sha.finalize()); println!("Compressed to {}", style(BinaryBytes(size)).bold()); - println!("SHA-256: {}", style(hash).bold()); + println!("SHA-256: {}", style(&hash).bold()); let rsync_status = process::Command::new("rsync") .arg("--info=progress2") @@ -156,13 +168,37 @@ Err("")? } + #[derive(serde::Serialize)] + struct NewDatasetToml { + datasets: [Dataset; 1], + } + + let to_toml = NewDatasetToml { + datasets: [Dataset { + url: format!("{}{}", DOWNLOAD_BASE_URL, archive_name), + name: dataset_name, + sha256: hash, + }], + }; + let toml_path = megafine.datasets.join("new.toml"); + let mut toml_file = fs::OpenOptions::new() + .append(true) + .create(true) + .open(&toml_path)?; + if toml_file.metadata()?.len() == 0 { + toml_file.write_all(b"# Generated by ./run new-dataset\n")?; + } + toml_file.write_all(b"\n")?; + toml_file.write_all(&toml::to_vec(&to_toml)?)?; + println!("Appended to {}", style(toml_path.display()).bold()); + Ok(()) } } impl Megafine { fn new() -> Result<Self> { - let exe = fs::canonicalize(std::env::current_exe()?)?; + let exe = fs::canonicalize(env::current_exe()?)?; // TODO: use a `try` block when available // https://github.com/rust-lang/rust/issues/31436 @@ -182,9 +218,11 @@ exe.display() ) })?; + let repo = repo.strip_prefix(env::current_dir()?).unwrap_or(repo); Ok(Self { tmp_dir: repo.join("tmp"), cache_dir: repo.join("cache"), + datasets: repo.join("datasets"), }) }