diff --git a/Cargo.lock b/Cargo.lock index 5a7ddc2122..f13603f54a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -623,6 +623,27 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" +[[package]] +name = "bzip2" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42b7c3cbf0fa9c1b82308d57191728ca0256cb821220f4e2fd410a72ade26e3b" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.9+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad3b39a260062fca31f7b0b12f207e8f2590a67d32ec7d59c20484b07ea7285e" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "c_linked_list" version = "1.1.1" @@ -1579,13 +1600,11 @@ dependencies = [ "enr", "eth2_config", "eth2_ssz", - "handlebars", - "reqwest", "serde", - "serde_json", "serde_yaml", "tempdir", "types", + "zip", ] [[package]] @@ -1997,9 +2016,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" +checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" dependencies = [ "cfg-if", "libc", @@ -2092,20 +2111,6 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d36fab90f82edc3c747f9d438e06cf0a491055896f2a279638bb5beed6c40177" -[[package]] -name = "handlebars" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5deefd4816fb852b1ff3cb48f6c41da67be2d0e1d20b26a7a3b076da11f064b1" -dependencies = [ - "log 0.4.11", - "pest", - "pest_derive", - "quick-error 2.0.0", - "serde", - "serde_json", -] - [[package]] name = "hashbrown" version = "0.6.3" @@ -2263,7 +2268,7 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" dependencies = [ - "quick-error 1.2.3", + "quick-error", ] [[package]] @@ -2441,9 +2446,9 @@ dependencies = [ [[package]] name = "integer-sqrt" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d3038ef3da477fd70e6161006ef37f919c24bfe7244062f67a69defe72c02aa" +checksum = "276ec31bcb4a9ee45f58bec6f9ec700ae4cf4f4f8f2fa7e06cb406bd5ffdd770" dependencies = [ "num-traits", ] @@ -2601,9 +2606,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.76" +version = "0.2.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "755456fae044e6fa1ebbbd1b3e902ae19e73097ed4ed87bb79934a867c007bc3" +checksum = "f2f96b10ec2560088a8e76961b00d47107b3a625fecb76dedb29ee7ccbf98235" [[package]] name = "libflate" @@ -3676,49 +3681,6 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" -[[package]] -name = "pest" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" -dependencies = [ - "ucd-trie", -] - -[[package]] -name = "pest_derive" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "833d1ae558dc601e9a60366421196a8d94bc0ac980476d0b67e1d0988d72b2d0" -dependencies = [ - "pest", - "pest_generator", -] - -[[package]] -name = "pest_generator" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55" -dependencies = [ - "pest", - "pest_meta", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "pest_meta" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54be6e404f5317079812fc8f9f5279de376d8856929e21c184ecf6bbd692a11d" -dependencies = [ - "maplit", - "pest", - "sha-1 0.8.2", -] - [[package]] name = "petgraph" version = "0.5.1" @@ -3785,6 +3747,12 @@ dependencies = [ "web-sys", ] +[[package]] +name = "podio" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b18befed8bc2b61abc79a457295e7e838417326da1586050b919414073977f19" + [[package]] name = "poly1305" version = "0.6.0" @@ -3837,9 +3805,9 @@ checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" [[package]] name = "proc-macro2" -version = "1.0.20" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "175c513d55719db99da20232b06cda8bab6b83ec2d04e3283edf0213c37c1a29" +checksum = "36e28516df94f3dd551a587da5357459d9b36d945a7c37c3557928c1c2ff2a2c" dependencies = [ "unicode-xid", ] @@ -3964,12 +3932,6 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" -[[package]] -name = "quick-error" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ac73b1112776fc109b2e61909bc46c7e1bf0d7f690ffb1676553acce16d5cda" - [[package]] name = "quickcheck" version = "0.9.2" @@ -4997,9 +4959,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.3.12" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03088793f677dce356f3ccc2edb1b314ad191ab702a5de3faf49304f7e104918" +checksum = "b1fa70dc5c8104ec096f4fe7ede7a221d35ae13dcd19ba1ad9a81d2cab9a1c44" dependencies = [ "cfg-if", "libc", @@ -5905,12 +5867,6 @@ dependencies = [ "tree_hash_derive", ] -[[package]] -name = "ucd-trie" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" - [[package]] name = "uhttp_sse" version = "0.5.1" @@ -6542,3 +6498,16 @@ dependencies = [ "syn", "synstructure", ] + +[[package]] +name = "zip" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58287c28d78507f5f91f2a4cf1e8310e2c76fd4c6932f93ac60fd1ceb402db7d" +dependencies = [ + "bzip2", + "crc32fast", + "flate2", + "podio", + "time 0.1.44", +] diff --git a/common/eth2_config/src/lib.rs b/common/eth2_config/src/lib.rs index 4a393c14b1..ee3898ecff 100644 --- a/common/eth2_config/src/lib.rs +++ b/common/eth2_config/src/lib.rs @@ -48,22 +48,29 @@ impl Eth2Config { /// Used by the `eth2_testnet_config` crate to initialize testnet directories during build and /// access them at runtime. #[derive(Copy, Clone, Debug, PartialEq)] -pub struct Eth2NetDirectory<'a> { +pub struct Eth2NetArchiveAndDirectory<'a> { pub name: &'a str, pub unique_id: &'a str, + pub archive_name: &'a str, pub commit: &'a str, - pub url_template: &'a str, pub genesis_is_known: bool, } -impl<'a> Eth2NetDirectory<'a> { +impl<'a> Eth2NetArchiveAndDirectory<'a> { /// The directory that should be used to store files downloaded for this net. - pub fn dir(&self) -> PathBuf { + fn pwd(&self) -> PathBuf { env::var("CARGO_MANIFEST_DIR") .expect("should know manifest dir") .parse::() .expect("should parse manifest dir as path") - .join(self.unique_id) + } + + pub fn dir(&self) -> PathBuf { + self.pwd().join(self.unique_id) + } + + pub fn archive_fullpath(&self) -> PathBuf { + self.pwd().join(self.archive_name) } } @@ -72,19 +79,23 @@ macro_rules! unique_id { ($name: tt, $commit: tt, $genesis_is_known: tt) => { concat!("testnet_", $name, "_", $commit, "_", $genesis_is_known); }; + + ($name: tt, $commit: tt) => { + concat!("testnet_", $name, "_", $commit, ".zip"); + }; } macro_rules! define_net { - ($title: ident, $macro_title: tt, $name: tt, $commit: tt, $url_template: tt, $genesis_is_known: tt) => { + ($title: ident, $macro_title: tt, $name: tt, $commit: tt, $genesis_is_known: tt) => { #[macro_use] pub mod $title { use super::*; - pub const ETH2_NET_DIR: Eth2NetDirectory = Eth2NetDirectory { + pub const ETH2_NET_DIR: Eth2NetArchiveAndDirectory = Eth2NetArchiveAndDirectory { name: $name, unique_id: unique_id!($name, $commit, $genesis_is_known), + archive_name: unique_id!($name, $commit), commit: $commit, - url_template: $url_template, genesis_is_known: $genesis_is_known, }; @@ -110,7 +121,6 @@ define_net!( include_altona_file, "altona", "a94e00c1a03df851f960fcf44a79f2a6b1d29af1", - "https://raw.githubusercontent.com/sigp/witti/{{ commit }}/altona/lighthouse/{{ file }}", true ); @@ -119,7 +129,6 @@ define_net!( include_medalla_file, "medalla", "09bbf2c9d108944ac934f94ec6a1d0684ca062a5", - "https://raw.githubusercontent.com/sigp/witti/{{ commit }}/medalla/{{ file }}", true ); diff --git a/common/eth2_testnet_config/.gitignore b/common/eth2_testnet_config/.gitignore index e3b2b30d11..eddff80821 100644 --- a/common/eth2_testnet_config/.gitignore +++ b/common/eth2_testnet_config/.gitignore @@ -1,4 +1,4 @@ -testnet* +testnet*/* schlesi-* witti-* altona* diff --git a/common/eth2_testnet_config/Cargo.toml b/common/eth2_testnet_config/Cargo.toml index f8cd65380d..e8b7bd842b 100644 --- a/common/eth2_testnet_config/Cargo.toml +++ b/common/eth2_testnet_config/Cargo.toml @@ -7,10 +7,8 @@ edition = "2018" build = "build.rs" [build-dependencies] -reqwest = { version = "0.10.4", features = ["blocking", "native-tls-vendored"] } +zip = "0.5" eth2_config = { path = "../eth2_config"} -handlebars = "3.3.0" -serde_json = "1.0.56" [dev-dependencies] tempdir = "0.3.7" @@ -21,4 +19,4 @@ serde_yaml = "0.8.11" types = { path = "../../consensus/types"} enr = { version = "0.1.0", features = ["libsecp256k1", "ed25519"] } eth2_ssz = "0.1.2" -eth2_config = { path = "../eth2_config"} +eth2_config = { path = "../eth2_config"} \ No newline at end of file diff --git a/common/eth2_testnet_config/build.rs b/common/eth2_testnet_config/build.rs index 9eb110197f..eeda318073 100644 --- a/common/eth2_testnet_config/build.rs +++ b/common/eth2_testnet_config/build.rs @@ -1,80 +1,68 @@ //! Downloads a testnet configuration from Github. -use eth2_config::{altona, medalla, Eth2NetDirectory}; -use handlebars::Handlebars; -use serde_json::json; +use eth2_config::{altona, medalla, Eth2NetArchiveAndDirectory}; +use std::fs; use std::fs::File; -use std::io::Write; +use std::io; +use zip::ZipArchive; -const ETH2_NET_DIRS: &[Eth2NetDirectory<'static>] = &[altona::ETH2_NET_DIR, medalla::ETH2_NET_DIR]; +const ETH2_NET_DIRS: &[Eth2NetArchiveAndDirectory<'static>] = + &[altona::ETH2_NET_DIR, medalla::ETH2_NET_DIR]; fn main() { for testnet in ETH2_NET_DIRS { let testnet_dir = testnet.dir(); - + let archive_fullpath = testnet.archive_fullpath(); + //no need to do anything if archives have already been uncompressed before if !testnet_dir.exists() { - std::fs::create_dir_all(&testnet_dir) - .unwrap_or_else(|_| panic!("Unable to create {:?}", testnet_dir)); + if archive_fullpath.exists() { + //uncompress archive and continue + let archive_file = match File::open(&archive_fullpath) { + Ok(f) => f, + Err(e) => panic!("Problem opening archive file: {}", e), + }; - match get_all_files(testnet) { - Ok(()) => (), - Err(e) => { - std::fs::remove_dir_all(&testnet_dir).unwrap_or_else(|_| panic!( - "{}. Failed to remove {:?}, please remove the directory manually because it may contains incomplete testnet data.", - e, - testnet_dir, - )); - panic!(e); - } + match uncompress(archive_file) { + Ok(_) => (), + Err(e) => panic!(e), + }; + } else { + panic!( + "Couldn't find testnet archive at this location: {:?}", + archive_fullpath + ); } } } } -fn get_all_files(testnet: &Eth2NetDirectory<'static>) -> Result<(), String> { - get_file(testnet, "boot_enr.yaml")?; - get_file(testnet, "config.yaml")?; - get_file(testnet, "deploy_block.txt")?; - get_file(testnet, "deposit_contract.txt")?; +fn uncompress(archive_file: File) -> Result<(), String> { + let mut archive = + ZipArchive::new(archive_file).map_err(|e| format!("Error with zip file: {}", e))?; + for i in 0..archive.len() { + let mut file = archive + .by_index(i) + .map_err(|e| format!("Error retrieving file {} inside zip: {}", i, e))?; - if testnet.genesis_is_known { - get_file(testnet, "genesis.ssz")?; - } else { - File::create(testnet.dir().join("genesis.ssz")).unwrap(); + let outpath = file.sanitized_name(); + + if file.name().ends_with('/') { + fs::create_dir_all(&outpath) + .map_err(|e| format!("Error creating testnet directories: {}", e))?; + } else { + if let Some(p) = outpath.parent() { + if !p.exists() { + fs::create_dir_all(&p) + .map_err(|e| format!("Error creating testnet directories: {}", e))?; + } + } + + let mut outfile = File::create(&outpath) + .map_err(|e| format!("Error while creating file {:?}: {}", outpath, e))?; + io::copy(&mut file, &mut outfile) + .map_err(|e| format!("Error writing file {:?}: {}", outpath, e))?; + } } Ok(()) } - -fn get_file(testnet: &Eth2NetDirectory, filename: &str) -> Result<(), String> { - let url = Handlebars::new() - .render_template( - testnet.url_template, - &json!({"commit": testnet.commit, "file": filename}), - ) - .unwrap(); - - let path = testnet.dir().join(filename); - - let mut file = - File::create(path).map_err(|e| format!("Failed to create {}: {:?}", filename, e))?; - - let request = reqwest::blocking::Client::builder() - .build() - .map_err(|_| "Could not build request client".to_string())? - .get(&url) - .timeout(std::time::Duration::from_secs(120)); - - let contents = request - .send() - .map_err(|e| format!("Failed to download {}: {}", filename, e))? - .error_for_status() - .map_err(|e| format!("Error downloading {}: {}", filename, e))? - .bytes() - .map_err(|e| format!("Failed to read {} response bytes: {}", filename, e))?; - - file.write(&contents) - .map_err(|e| format!("Failed to write to {}: {:?}", filename, e))?; - - Ok(()) -} diff --git a/common/eth2_testnet_config/testnet_altona_a94e00c1a03df851f960fcf44a79f2a6b1d29af1.zip b/common/eth2_testnet_config/testnet_altona_a94e00c1a03df851f960fcf44a79f2a6b1d29af1.zip new file mode 100644 index 0000000000..29b9cd0cef Binary files /dev/null and b/common/eth2_testnet_config/testnet_altona_a94e00c1a03df851f960fcf44a79f2a6b1d29af1.zip differ diff --git a/common/eth2_testnet_config/testnet_medalla_09bbf2c9d108944ac934f94ec6a1d0684ca062a5.zip b/common/eth2_testnet_config/testnet_medalla_09bbf2c9d108944ac934f94ec6a1d0684ca062a5.zip new file mode 100644 index 0000000000..a6cd0a8430 Binary files /dev/null and b/common/eth2_testnet_config/testnet_medalla_09bbf2c9d108944ac934f94ec6a1d0684ca062a5.zip differ