From 9bd9556e4a492224e140e437572264cd2e21bd5f Mon Sep 17 00:00:00 2001 From: Alexander Koval Date: Fri, 10 Dec 2021 17:39:33 +0200 Subject: [PATCH 01/12] Implement basic DockerImage build step --- Cargo.lock | 1001 +++++++++++++++++++++++++++++++- Cargo.toml | 5 +- src/builder/commands/docker.rs | 319 ++++++++++ src/builder/commands/tarcmd.rs | 330 ++++++----- src/builder/error.rs | 8 + src/builder/mod.rs | 1 + src/config/builders.rs | 1 + src/config/read_settings.rs | 14 +- src/config/settings.rs | 3 +- tests/docker.bats | 55 ++ tests/docker/.dockerignore | 1 + tests/docker/Dockerfile | 10 + tests/docker/vagga.yaml | 99 ++++ vagga.yaml | 23 + 14 files changed, 1719 insertions(+), 151 deletions(-) create mode 100644 src/builder/commands/docker.rs create mode 100644 tests/docker.bats create mode 100644 tests/docker/.dockerignore create mode 100644 tests/docker/Dockerfile create mode 100644 tests/docker/vagga.yaml diff --git a/Cargo.lock b/Cargo.lock index 2108260e..8bffd94e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + [[package]] name = "aho-corasick" version = "0.7.18" @@ -50,6 +56,27 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" +[[package]] +name = "async-stream" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171374e7e3b2504e0e5236e3b59260560f9fe94bfe9ac39ba5e4e929c5590625" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "648ed8c8d2ce5409ccd57453d9d1b214b342a0d69376a6feda1fd6cae3299308" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atty" version = "0.2.14" @@ -82,6 +109,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + [[package]] name = "bitflags" version = "1.3.2" @@ -95,7 +128,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a4e37d16930f5459780f5621038b6382b9bb37c19016f39fb6b5808d831f174" dependencies = [ "crypto-mac", - "digest", + "digest 0.9.0", "opaque-debug", ] @@ -110,7 +143,7 @@ dependencies = [ "cc", "cfg-if 1.0.0", "constant_time_eq", - "digest", + "digest 0.9.0", ] [[package]] @@ -122,6 +155,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1d36a02058e76b040de25a4464ba1c80935655595b661505c8b39b664828b95" +dependencies = [ + "generic-array", +] + [[package]] name = "bstr" version = "0.2.17" @@ -131,6 +173,18 @@ dependencies = [ "memchr", ] +[[package]] +name = "bumpalo" +version = "3.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1e260c3a9040a7c19a12468758f4c16f31a81a1fe087482be9570ec864bb6c" + +[[package]] +name = "bytes" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" + [[package]] name = "bzip2" version = "0.3.3" @@ -185,6 +239,22 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +[[package]] +name = "core-foundation" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6888e10551bb93e424d8df1d07f1a8b4fceb0001a3a4b048bfc47554946f47b3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + [[package]] name = "cpufeatures" version = "0.2.1" @@ -203,6 +273,15 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "crypto-common" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567569e659735adb39ff2d4c20600f7cd78be5471f8c58ab162bce3c03fdbc5f" +dependencies = [ + "generic-array", +] + [[package]] name = "crypto-mac" version = "0.8.0" @@ -222,6 +301,17 @@ dependencies = [ "generic-array", ] +[[package]] +name = "digest" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8549e6bfdecd113b7e221fe60b433087f6957387a20f8118ebca9b12af19143d" +dependencies = [ + "block-buffer 0.10.0", + "crypto-common", + "generic-array", +] + [[package]] name = "dir-signature" version = "0.2.9" @@ -230,9 +320,9 @@ dependencies = [ "argparse", "blake2", "blake3", - "digest", + "digest 0.9.0", "env_logger", - "futures", + "futures 0.1.31", "futures-cpupool", "generic-array", "itertools", @@ -240,7 +330,35 @@ dependencies = [ "num_cpus", "openat", "quick-error", - "sha2", + "sha2 0.9.8", +] + +[[package]] +name = "dkregistry" +version = "0.5.1-alpha.0" +source = "git+https://github.com/anti-social/dkregistry-rs?rev=1bfac75#1bfac756be72db994682f621ebb38dce464f21cf" +dependencies = [ + "async-stream", + "base64", + "bytes", + "futures 0.3.18", + "http", + "libflate", + "log", + "mime", + "pin-project", + "regex", + "reqwest", + "serde", + "serde_ignored", + "serde_json", + "sha2 0.10.0", + "strum", + "strum_macros", + "tar", + "thiserror", + "tokio", + "url 2.2.2", ] [[package]] @@ -261,6 +379,15 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +[[package]] +name = "encoding_rs" +version = "0.8.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dc8abb250ffdda33912550faa54c88ec8b998dec0b2c55ab224921ce11df" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "env_logger" version = "0.5.13" @@ -326,6 +453,31 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding 2.1.0", +] + [[package]] name = "fuchsia-cprng" version = "0.1.1" @@ -338,16 +490,105 @@ version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" +[[package]] +name = "futures" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cd0210d8c325c245ff06fd95a3b13689a1a276ac8cfa8e8720cb840bfb84b9e" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fc8cd39e3dbf865f7340dce6a2d401d24fd37c6fe6c4f0ee0de8bfca2252d27" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "629316e42fe7c2a0b9a65b47d159ceaa5453ab14e8f0a3c5eedbb8cd55b4a445" + [[package]] name = "futures-cpupool" version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4" dependencies = [ - "futures", + "futures 0.1.31", "num_cpus", ] +[[package]] +name = "futures-executor" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b808bf53348a36cab739d7e04755909b9fcaaa69b7d7e588b37b6ec62704c97" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e481354db6b5c353246ccf6a728b0c5511d752c08da7260546fc0933869daa11" + +[[package]] +name = "futures-macro" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89f17b21645bc4ed773c69af9c9a0effd4a3f1a3876eadd453469f8854e7fdd" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "996c6442437b62d21a32cd9906f9c41e7dc1e19a9579843fad948696769305af" + +[[package]] +name = "futures-task" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dabf1872aaab32c886832f2276d2f5399887e2bd613698a02359e4ea83f8de12" + +[[package]] +name = "futures-util" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d22213122356472061ac0f1ab2cee28d2bac8491410fd68c2af53d1cedb83e" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + [[package]] name = "generic-array" version = "0.14.4" @@ -385,7 +626,7 @@ dependencies = [ "libc", "libgit2-sys", "log", - "url", + "url 1.7.2", ] [[package]] @@ -401,6 +642,40 @@ dependencies = [ "regex", ] +[[package]] +name = "h2" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d09bbc040ce3758a0d32a8a910562104a853aea429dbe1a998beb065c2eacb2" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "hermit-abi" version = "0.1.19" @@ -410,6 +685,40 @@ dependencies = [ "libc", ] +[[package]] +name = "http" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1323096b05d41827dadeaee54c9981958c0f94e670bc94ed80037d1a7b8b186b" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acd94fdbe1d4ff688b67b04eee2e17bd50995534a61539e45adfefb45e5e5503" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + [[package]] name = "humannum" version = "0.1.0" @@ -429,6 +738,43 @@ dependencies = [ "quick-error", ] +[[package]] +name = "hyper" +version = "0.14.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436ec0091e4f20e655156a30a0df3770fe2900aa301e548e08446ec794b6953c" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "idna" version = "0.1.5" @@ -440,6 +786,42 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "ipnet" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" + [[package]] name = "itertools" version = "0.7.11" @@ -455,6 +837,15 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" +[[package]] +name = "js-sys" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cc9ffccd38c451a86bf13657df244e9c3f37493cce8e5e21e940963777acc84" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -467,6 +858,26 @@ version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8521a1b57e76b1ec69af7599e75e38e7b7fad6610f037db8c79b127201b5d119" +[[package]] +name = "libflate" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16364af76ebb39b5869bb32c81fa93573267cd8c62bb3474e28d78fac3fb141e" +dependencies = [ + "adler32", + "crc32fast", + "libflate_lz77", +] + +[[package]] +name = "libflate_lz77" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39a734c0493409afcd49deee13c006a04e3586b9761a03543c6272c9c51f2f5a" +dependencies = [ + "rle-decode-fast", +] + [[package]] name = "libgit2-sys" version = "0.7.11" @@ -502,6 +913,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "lock_api" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712a4d093c9976e24e7dbca41db895dabcbac38eb5f4045393d17a95bdfb1109" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.14" @@ -534,6 +954,12 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + [[package]] name = "miniz_oxide" version = "0.4.4" @@ -544,6 +970,28 @@ dependencies = [ "autocfg", ] +[[package]] +name = "mio" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8067b404fe97c70829f082dec8bcf4f71225d7eaea1d8645349cb76fa06205cc" +dependencies = [ + "libc", + "log", + "miow", + "ntapi", + "winapi", +] + +[[package]] +name = "miow" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" +dependencies = [ + "winapi", +] + [[package]] name = "mopa" version = "0.2.2" @@ -560,6 +1008,24 @@ dependencies = [ "winapi", ] +[[package]] +name = "native-tls" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "net2" version = "0.2.37" @@ -598,8 +1064,17 @@ dependencies = [ ] [[package]] -name = "num-traits" -version = "0.1.43" +name = "ntapi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +dependencies = [ + "winapi", +] + +[[package]] +name = "num-traits" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" dependencies = [ @@ -634,6 +1109,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "once_cell" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" + [[package]] name = "opaque-debug" version = "0.3.0" @@ -649,6 +1130,64 @@ dependencies = [ "libc", ] +[[package]] +name = "openssl" +version = "0.10.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" +dependencies = [ + "bitflags", + "cfg-if 1.0.0", + "foreign-types", + "libc", + "once_cell", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" + +[[package]] +name = "openssl-sys" +version = "0.9.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df13d165e607909b363a4757a6f133f8a818a74e9d3a98d09c6128e15fa4c73" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi", +] + [[package]] name = "path-filter" version = "0.1.0" @@ -665,6 +1204,44 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "pin-project" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "576bc800220cc65dac09e99e97b08b358cfab6e17078de8dc5fee223bd2d0c08" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e8fe8163d14ce7f0cdac2e040116f22eac817edabff0be91e8aff7e9accf389" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d31d11c69a6b52a174b42bdc0c30e5e11670f90788b2c471c31c1d17d449443" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkg-config" version = "0.3.22" @@ -822,6 +1399,42 @@ dependencies = [ "winapi", ] +[[package]] +name = "reqwest" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bea77bc708afa10e59905c3d4af7c8fd43c9214251673095ff8b14345fcbc5" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "native-tls", + "percent-encoding 2.1.0", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "tokio-util", + "url 2.2.2", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "resolv-conf" version = "0.6.3" @@ -831,12 +1444,24 @@ dependencies = [ "quick-error", ] +[[package]] +name = "rle-decode-fast" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cabe4fa914dec5870285fa7f71f602645da47c486e68486d2b4ceb4a343e90ac" + [[package]] name = "rustc-demangle" version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" +[[package]] +name = "rustversion" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" + [[package]] name = "ryu" version = "1.0.6" @@ -861,6 +1486,45 @@ dependencies = [ "quick-error", ] +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "security-framework" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525bc1abfda2e1998d152c45cf13e696f76d0a4972310b22fac1658b05df7c87" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9dd14d83160b528b7bfd66439110573efcfbe281b17fc2ca9f39f550d619c7e" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" version = "1.0.130" @@ -881,6 +1545,15 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_ignored" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c2c7d39d14f2f2ea82239de71594782f186fd03501ac81f0ce08e674819ff2f" +dependencies = [ + "serde", +] + [[package]] name = "serde_json" version = "1.0.72" @@ -892,19 +1565,42 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha2" version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b69f9a4c9740d74c5baa3fd2e547f9525fa8088a8a958e0ca2409a514e33f5fa" dependencies = [ - "block-buffer", + "block-buffer 0.9.0", "cfg-if 1.0.0", "cpufeatures", - "digest", + "digest 0.9.0", "opaque-debug", ] +[[package]] +name = "sha2" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d964dd36bb15bcf2f2b35694c072feab74969a54f2bbeec7a2d725d2bdcb6" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest 0.10.0", +] + [[package]] name = "signal" version = "0.6.0" @@ -915,12 +1611,62 @@ dependencies = [ "nix 0.11.1", ] +[[package]] +name = "signal-hook-registry" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" + +[[package]] +name = "smallvec" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ecab6c735a6bb4139c0caafd0cc3635748bbb3acf4550e8138122099251f309" + +[[package]] +name = "socket2" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dc90fe6c7be1a323296982db1836d1ea9e47b6839496dde9a541bc496df3516" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "strsim" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strum" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cae14b91c7d11c9a851d3fbc80a963198998c2a64eec840477fa92d8ce9b70bb" + +[[package]] +name = "strum_macros" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bb0dc7ee9c15cea6199cde9a127fa16a4c5819af85395457ad72d68edc85a38" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + [[package]] name = "subtle" version = "2.4.1" @@ -984,6 +1730,26 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "thiserror" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "time" version = "0.1.43" @@ -1009,6 +1775,93 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +[[package]] +name = "tokio" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e992e41e0d2fb9f755b37446f20900f64446ef54874f40a60c78f021ac6144" +dependencies = [ + "autocfg", + "bytes", + "libc", + "memchr", + "mio", + "num_cpus", + "once_cell", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "tokio-macros", + "winapi", +] + +[[package]] +name = "tokio-macros" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9efc1aba077437943f7515666aa2b882dfabfbfdf89c819ea75a8d6e9eaba5e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower-service" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" + +[[package]] +name = "tracing" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105" +dependencies = [ + "cfg-if 1.0.0", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f4ed65637b8390770814083d20756f87bfa2c21bf2f110babdc5438351746e4" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + [[package]] name = "typenum" version = "1.14.0" @@ -1030,6 +1883,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b" + [[package]] name = "unicode-xid" version = "0.2.2" @@ -1052,9 +1911,21 @@ version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a" dependencies = [ - "idna", + "idna 0.1.5", "matches", - "percent-encoding", + "percent-encoding 1.0.1", +] + +[[package]] +name = "url" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +dependencies = [ + "form_urlencoded", + "idna 0.2.3", + "matches", + "percent-encoding 2.1.0", ] [[package]] @@ -1065,12 +1936,14 @@ dependencies = [ "argparse", "blake2", "bzip2", - "digest", + "digest 0.9.0", "dir-signature", + "dkregistry", "docopt", "env_logger", "failure", "flate2", + "futures 0.3.18", "git2", "humantime", "itertools", @@ -1092,13 +1965,14 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "sha2", + "sha2 0.9.8", "signal", "tar", "tempfile", + "tokio", "typenum", "unshare", - "url", + "url 2.2.2", "xz2", "zip", ] @@ -1132,12 +2006,98 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + [[package]] name = "wasi" version = "0.10.2+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" +[[package]] +name = "wasm-bindgen" +version = "0.2.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "632f73e236b219150ea279196e54e610f5dbafa5d61786303d4da54f84e47fce" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a317bf8f9fba2476b4b2c85ef4c4af8ff39c3c7f0cdfeed4f82c34a880aa837b" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e8d7523cb1f2a4c96c1317ca690031b714a51cc14e05f712446691f413f5d39" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56146e7c495528bf6587663bea13a8eb588d39b36b679d83972e1a2dbbdacf9" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7803e0eea25835f8abdc585cd3021b3deb11543c6fe226dcd30b228857c5c5ab" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0237232789cf037d5480773fe568aac745bfe2afbc11a863e97901780a6b47cc" + +[[package]] +name = "web-sys" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38eb105f1c59d9eaa6b5cdc92b859d85b926e82cb2e0945cd0c9259faa6fe9fb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi" version = "0.3.9" @@ -1169,6 +2129,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi", +] + [[package]] name = "xattr" version = "0.2.2" diff --git a/Cargo.toml b/Cargo.toml index 9c1ae32d..1cb0c763 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ rand = "0.5.0" argparse = "0.2.1" log = "0.4.0" env_logger = "0.5.6" -url = "1.0.0" +url = "2.2.2" unshare = { version="0.5.0", optional=true } signal = { version="0.6.0", optional=true } mopa = "0.2.2" @@ -51,6 +51,9 @@ serde_json = "1.0.2" serde_derive = "1.0.11" failure = "0.1.1" resolv-conf = "0.6.0" +dkregistry = { git="https://github.com/anti-social/dkregistry-rs", rev="1bfac75" } +tokio = { version = "1", features = ["full"] } +futures = "0.3.18" [features] default = ["containers"] diff --git a/src/builder/commands/docker.rs b/src/builder/commands/docker.rs new file mode 100644 index 00000000..5e29f5af --- /dev/null +++ b/src/builder/commands/docker.rs @@ -0,0 +1,319 @@ +use std::collections::{BTreeMap, HashSet}; +use std::fs::{remove_dir_all, remove_file}; +use std::io::{ErrorKind, Read}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +#[cfg(feature="containers")] +use dkregistry::v2::Client as RegistryClient; + +#[cfg(feature="containers")] +use futures::stream::StreamExt; + +#[cfg(feature="containers")] +use tar::{Entry, EntryType}; + +#[cfg(feature="containers")] +use tokio::{ + io::AsyncWriteExt, + sync::Semaphore, +}; + +#[cfg(feature="containers")] +use quire::{ + validate as V, + ast::{Ast, ScalarKind, Tag}, +}; + +#[cfg(feature="containers")] +use crate::{ + builder::commands::tarcmd::TarCmd, + capsule::packages as capsule, + container::util::clean_dir, + file_util::{Dir, Lock}, +}; +use crate::build_step::{BuildStep, Config, Digest, Guard, StepError, VersionError}; + +const DEFAULT_REGISTRY_HOST: &str = "registry-1.docker.io"; +const DEFAULT_IMAGE_NAMESPACE: &str = "library"; +const DEFAULT_IMAGE_TAG: &str = "latest"; + +const DOCKER_LAYERS_CACHE_PATH: &str = "/vagga/cache/docker-layers"; +const DOCKER_LAYERS_DOWNLOAD_CONCURRENCY: usize = 4; + +#[derive(Serialize, Deserialize, Debug)] +pub struct DockerImage { + pub registry: String, + pub image: String, + pub tag: String, + pub insecure: Option, + pub path: PathBuf, +} + +impl DockerImage { + pub fn config() -> V::Structure<'static> { + V::Structure::new() + .member("registry", V::Scalar::new().default(DEFAULT_REGISTRY_HOST)) + .member("image", V::Scalar::new()) + .member("tag", V::Scalar::new().default(DEFAULT_IMAGE_TAG)) + .member("insecure", V::Scalar::new().optional()) + .member("path", V::Directory::new().absolute(true).default("/")) + .parser(parse_image) + } +} + +fn parse_image(ast: Ast) -> BTreeMap { + match ast { + Ast::Scalar(pos, _, _, value) => { + let mut map = BTreeMap::new(); + + let (image, registry) = if let Some((registry, image)) = value.split_once('/') { + if registry == "localhost" || registry.contains(|c| c == '.' || c == ':') { + map.insert( + "registry".to_string(), + Ast::Scalar(pos.clone(), Tag::NonSpecific, ScalarKind::Plain, registry.to_string()) + ); + (image, Some(registry)) + } else { + (value.as_str(), None) + } + } else { + (value.as_str(), None) + }; + + let image = if let Some((image, tag)) = image.rsplit_once(':') { + map.insert( + "tag".to_string(), + Ast::Scalar(pos.clone(), Tag::NonSpecific, ScalarKind::Plain, tag.to_string()) + ); + image + } else { + image + }; + + let image = if !image.contains('/') && registry.is_none() { + format!("{}/{}", DEFAULT_IMAGE_NAMESPACE, image) + } else { + image.to_string() + }; + + map.insert( + "image".to_string(), + Ast::Scalar(pos.clone(), Tag::NonSpecific, ScalarKind::Plain, image) + ); + + map + }, + _ => unreachable!(), + } +} + +impl BuildStep for DockerImage { + fn name(&self) -> &'static str { + "DockerImage" + } + + #[cfg(feature="containers")] + fn hash(&self, _cfg: &Config, hash: &mut Digest) -> Result<(), VersionError> { + hash.field("registry", &self.registry); + hash.field("image", &self.image); + hash.field("tag", &self.tag); + hash.opt_field("insecure", &self.insecure); + hash.field("path", &self.path); + Ok(()) + } + + #[cfg(feature="containers")] + fn build(&self, guard: &mut Guard, _build: bool) -> Result<(), StepError> { + let insecure = self.insecure.unwrap_or_else(|| { + is_insecure_registry(&self.registry, &guard.ctx.settings.docker_insecure_registries) + }); + if !insecure { + capsule::ensure(&mut guard.ctx.capsule, &[capsule::Https])?; + } + Dir::new(DOCKER_LAYERS_CACHE_PATH) + .recursive(true) + .create() + .expect("Docker layers cache dir"); + let layer_paths = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("Tokio runtime") + .block_on(download_image(&self.registry, insecure, &self.image, &self.tag)) + .expect("Downloaded layers"); + let dst_path = Path::new("/vagga/root").join(&self.path.strip_prefix("/").unwrap()); + for layer_path in layer_paths.iter() { + TarCmd::new(layer_path, &dst_path) + .preserve_owner(true) + .entry_handler(whiteout_entry_handler) + .unpack()?; + } + Ok(()) + } + + fn is_dependent_on(&self) -> Option<&str> { + None + } +} + +/// See: +/// - https://github.com/moby/moby/blob/v20.10.11/pkg/archive/whiteouts.go +/// - https://github.com/moby/moby/blob/v20.10.11/pkg/archive/diff.go#L131 +#[cfg(feature="containers")] +fn whiteout_entry_handler(entry: &Entry>, dst_path: &Path) -> Result { + let file_name = dst_path.file_name() + .and_then(|fname| fname.to_str()); + let file_name = if let Some(file_name) = file_name { + file_name + } else { + return Ok(false); + }; + + if entry.header().entry_type() != EntryType::Regular { + return Ok(false); + } + + if let Some(whiteout) = file_name.strip_prefix(".wh.") { + let dir = dst_path.parent().unwrap(); + if whiteout == ".wh..opq" { + // TODO: Track and keep files that were unpacked from the current archive + clean_dir(dir, false)? + } else { + let mut whiteout_path = dir.to_path_buf(); + whiteout_path.push(whiteout); + if whiteout_path.is_dir() { + remove_dir_all(&whiteout_path) + .map_err(|e| format!("Cannot remove directory: {}", e))?; + } else { + remove_file(whiteout_path) + .map_err(|e| format!("Cannot delete file: {}", e))?; + } + } + return Ok(true); + } + + Ok(false) +} + +fn is_insecure_registry(registry: &str, insecure_registries: &HashSet) -> bool { + let registry_url = url::Url::parse(&format!("http://{}", registry)).unwrap(); + let registry_host = registry_url.domain().unwrap(); + insecure_registries.contains(registry_host) +} + +#[cfg(feature="containers")] +async fn download_image( + registry: &str, insecure: bool, image: &str, tag: &str +) -> Result, StepError> { + let auth_scope = format!("repository:{}:pull", image); + let client = build_client(registry, insecure, &[&auth_scope]).await?; + + println!("Downloading docker image: {}/{}:{}", registry, image, tag); + let manifest = client.get_manifest(&image, &tag).await?; + + let layers_digests = manifest.layers_digests(None)?; + + let layers_download_semaphore = Arc::new( + Semaphore::new(DOCKER_LAYERS_DOWNLOAD_CONCURRENCY) + ); + let layers_futures = layers_digests.iter() + .map(|digest| { + let image = image.to_string(); + let digest = digest.clone(); + let client = client.clone(); + let sem = layers_download_semaphore.clone(); + tokio::spawn(async move { + if let Ok(_guard) = sem.acquire().await { + info!("Downloading docker layer: {}", &digest); + download_blob(&client, &image, &digest).await + } else { + panic!("Semaphore was closed unexpectedly") + } + }) + }) + .collect::>(); + let mut layers_paths = vec!(); + let mut layers_errors = vec!(); + for layer_res in futures::future::join_all(layers_futures).await.into_iter() { + match layer_res { + Ok(Ok(layer)) => layers_paths.push(layer), + Ok(Err(client_err)) => layers_errors.push(client_err), + Err(join_err) => layers_errors.push(format!("{}", join_err)), + } + } + if !layers_errors.is_empty() { + Err(layers_errors.into()) + } else { + Ok(layers_paths) + } +} + +#[cfg(feature="containers")] +async fn build_client( + registry: &str, insecure: bool, auth_scopes: &[&str] +) -> Result, StepError> { + let client_config = RegistryClient::configure() + .registry(registry) + .insecure_registry(insecure) + .username(None) + .password(None); + let client = client_config.build()?; + + let client = match client.is_auth().await { + Ok(true) => client, + Ok(false) => client.authenticate(auth_scopes).await?, + Err(e) => return Err(e.into()), + }; + Ok(Arc::new(client)) +} + +#[cfg(feature="containers")] +async fn download_blob( + client: &RegistryClient, image: &str, layer_digest: &str +) -> Result { + let digest = layer_digest.split_once(':').unwrap().1; + let short_digest = &digest[..12]; + + let layers_cache = Path::new(DOCKER_LAYERS_CACHE_PATH); + let blob_file_name = format!("{}.tar.gz", digest); + let blob_path = layers_cache.join(&blob_file_name); + match tokio::fs::symlink_metadata(&blob_path).await { + Ok(_) => {} + Err(e) if e.kind() == ErrorKind::NotFound => { + let lock_file_name = format!(".{}.lock", &blob_file_name); + let lock_msg = format!("Another process downloads blob: {}", &short_digest); + let lock_fut = tokio::task::spawn_blocking(move || { + let lockfile = layers_cache.join(lock_file_name); + Lock::exclusive_wait(lockfile, true, &lock_msg) + }); + let _lock = lock_fut.await + .map_err(|e| format!("Error waiting a lock file future: {}", e))? + .map_err(|e| format!("Error taking exclusive lock: {}", e))?; + + match tokio::fs::symlink_metadata(&blob_path).await { + Ok(_) => {} + Err(e) if e.kind() == ErrorKind::NotFound => { + let blob_tmp_file_name = format!(".{}.tmp", &blob_file_name); + let blob_tmp_path = layers_cache.join(&blob_tmp_file_name); + + println!("Downloading docker blob: {}", &short_digest); + let mut blob_stream = client.get_blob_stream(image, layer_digest).await + .expect("Get blob response"); + let mut blob_file = tokio::fs::File::create(&blob_tmp_path).await + .expect("Create layer file"); + while let Some(chunk) = blob_stream.next().await { + let chunk = chunk.expect("Layer chunk"); + blob_file.write_all(&chunk).await + .map_err(|e| format!("Cannot write blob file: {}", e))?; + } + tokio::fs::rename(&blob_tmp_path, &blob_path).await + .map_err(|e| format!("Cannot rename docker blob: {}", e))?; + } + Err(e) => return Err(format!("{}", e)), + } + + } + Err(e) => return Err(format!("{}", e)), + } + Ok(blob_path) +} \ No newline at end of file diff --git a/src/builder/commands/tarcmd.rs b/src/builder/commands/tarcmd.rs index 67a94cc2..114a6d92 100644 --- a/src/builder/commands/tarcmd.rs +++ b/src/builder/commands/tarcmd.rs @@ -4,7 +4,6 @@ use std::fs::{create_dir_all, set_permissions, hard_link}; use std::io::{self, Read, Seek, SeekFrom, BufReader}; use std::os::unix::fs::{PermissionsExt, symlink}; use std::path::{Path, PathBuf}; -use std::u32; #[cfg(feature="containers")] use bzip2::read::BzDecoder; #[cfg(feature="containers")] use flate2::read::GzDecoder; @@ -64,124 +63,211 @@ impl TarInstall { } } - #[cfg(feature="containers")] -pub fn unpack_file(_ctx: &mut Context, src: &Path, tgt: &Path, - includes: &[&Path], excludes: &[&Path], preserve_owner: bool) - -> Result<(), String> -{ +pub struct TarCmd<'a> { + archive: &'a Path, + target_dir: &'a Path, + includes: &'a[&'a Path], + excludes: &'a[&'a Path], + preserve_owner: bool, + entry_handler: fn(&Entry>, &Path) -> Result, +} - info!("Unpacking {:?} -> {:?}", src, tgt); - let read_err = |e| format!("Error reading {:?}: {}", src, e); +const DEFAULT_TAR_INCLUDES: &[&Path] = &[]; +const DEFAULT_TAR_EXCLUDES: &[&Path] = &[]; - let mut file = BufReader::new(File::open(&src).map_err(&read_err)?); +use tar::Entry; - let mut buf = [0u8; 8]; - let nbytes = file.read(&mut buf).map_err(&read_err)?; - file.seek(SeekFrom::Start(0)).map_err(&read_err)?; - let magic = &buf[..nbytes]; - if magic.len() >= 2 && magic[..2] == [0x1f, 0x8b] { - return unpack_stream(GzDecoder::new(file), - src, tgt, includes, excludes, preserve_owner); - } else if magic.len() >= 6 && magic[..6] == - [ 0xFD, b'7', b'z', b'X', b'Z', 0x00] - { - return unpack_stream(XzDecoder::new(file), - src, tgt, includes, excludes, preserve_owner); - } else if magic.len() >= 3 && magic[..3] == [ b'B', b'Z', b'h'] { - return unpack_stream(BzDecoder::new(file), - src, tgt, includes, excludes, preserve_owner); - } else { - return Err(format!("unpacking {:?}: unexpected compression", src)); +fn dummy_entry_handler(_entry: &Entry>, _dst_path: &Path) -> Result { + Ok(false) +} + +impl<'a> TarCmd<'a> { + pub fn new(archive: &'a Path, target_dir: &'a Path) -> Self { + Self { + archive, + target_dir, + includes: DEFAULT_TAR_INCLUDES, + excludes: DEFAULT_TAR_EXCLUDES, + preserve_owner: false, + entry_handler: dummy_entry_handler, + } } -} + pub fn includes(mut self, includes: &'a[&'a Path]) -> Self { + self.includes = includes; + self + } -#[cfg(feature="containers")] -fn unpack_stream(file: F, srcpath: &Path, tgt: &Path, - includes: &[&Path], excludes: &[&Path], preserve_owner: bool) - -> Result<(), String> -{ - let read_err = |e| format!("Error reading {:?}: {}", srcpath, e); - let mut arc = Archive::new(file); - let mut hardlinks = Vec::new(); - - for item in arc.entries().map_err(&read_err)? { - let mut src = item.map_err(&read_err)?; - let path_ref = src.path().map_err(&read_err)? - .to_path_buf(); - let mut orig_path: &Path = &path_ref; - if orig_path.is_absolute() { - orig_path = orig_path.strip_prefix("/").unwrap(); + pub fn excludes(mut self, excludes: &'a[&'a Path]) -> Self { + self.excludes = excludes; + self + } + + pub fn preserve_owner(mut self, preserve_order: bool) -> Self { + self.preserve_owner = preserve_order; + self + } + + pub fn entry_handler( + self, + entry_handler: fn(&Entry>, &Path) -> Result + ) -> TarCmd<'a> { + TarCmd { + archive: self.archive, + target_dir: self.target_dir, + includes: self.includes, + excludes: self.excludes, + preserve_owner: self.preserve_owner, + entry_handler, } - if includes.len() > 0 { - if !includes.iter().any(|x| orig_path.starts_with(x)) { + } + + pub fn unpack(&self) -> Result<(), String> { + info!("Unpacking {:?} -> {:?}", self.archive, self.target_dir); + let read_err = |e| format!("Error reading {:?}: {}", self.archive, e); + + let mut file = BufReader::new( + File::open(&self.archive) + .map_err(&read_err)? + ); + + let mut buf = [0u8; 8]; + let nbytes = file.read(&mut buf).map_err(&read_err)?; + file.seek(SeekFrom::Start(0)).map_err(&read_err)?; + let magic = &buf[..nbytes]; + let reader = if magic.len() >= 2 && magic[..2] == [0x1f, 0x8b] { + Box::new(GzDecoder::new(file)) as Box + } else if magic.len() >= 6 && magic[..6] == + [ 0xFD, b'7', b'z', b'X', b'Z', 0x00] + { + Box::new(XzDecoder::new(file)) as Box + } else if magic.len() >= 3 && magic[..3] == [ b'B', b'Z', b'h'] { + Box::new(BzDecoder::new(file)) as Box + } else { + return Err(format!("unpacking {:?}: unexpected compression", self.archive)); + }; + self.unpack_stream(reader) + } + + fn unpack_stream(&self, file: Box) -> Result<(), String> { + let read_err = |e| format!("Error reading {:?}: {}", self.archive, e); + let mut arc = Archive::new(file); + let mut hardlinks = Vec::new(); + + for item in arc.entries().map_err(&read_err)? { + let mut src = item.map_err(&read_err)?; + let path_ref = src.path().map_err(&read_err)? + .to_path_buf(); + let mut orig_path: &Path = &path_ref; + if orig_path.is_absolute() { + orig_path = orig_path.strip_prefix("/").unwrap(); + } + if self.includes.len() > 0 { + if !self.includes.iter().any(|x| orig_path.starts_with(x)) { + continue; + } + } + if self.excludes.iter().any(|x| orig_path.starts_with(x)) { continue; } - } - if excludes.iter().any(|x| orig_path.starts_with(x)) { - continue; - } - let path = tgt.join(orig_path); - let write_err = |e| format!("Error writing {:?}: {}", path, e); - let entry = src.header().entry_type(); - - // Some archives don't have uids - // TODO(tailhook) should this be handled in tar-rs? - let uid = min(src.header().uid().unwrap_or(0), u32::MAX as u64) as u32; - let gid = min(src.header().gid().unwrap_or(0), u32::MAX as u64) as u32; - - if entry.is_dir() { - let mode = src.header().mode().map_err(&read_err)?; - let mut dir_builder = Dir::new(&path); - dir_builder.recursive(true).mode(mode); - if preserve_owner { - dir_builder.uid(uid).gid(gid); + let path = self.target_dir.join(orig_path); + let write_err = |e| format!("Error writing {:?}: {}", path, e); + let entry = src.header().entry_type(); + + // Some archives don't have uids + // TODO(tailhook) should this be handled in tar-rs? + let uid = min(src.header().uid().unwrap_or(0), u32::MAX as u64) as u32; + let gid = min(src.header().gid().unwrap_or(0), u32::MAX as u64) as u32; + + if (self.entry_handler)(&src, &path)? { + continue; } - dir_builder.create().map_err(&write_err)?; - } else if entry.is_symlink() { - let src = src.link_name().map_err(&read_err)? - .ok_or(format!("Error unpacking {:?}, broken symlink", path))?; - match symlink(&src, &path) { - Ok(_) => {}, - Err(e) => { - if e.kind() == io::ErrorKind::NotFound { - if let Some(parent) = path.parent() { - Dir::new(parent).recursive(true).create() - .map_err(&write_err)?; - symlink(&src, &path).map_err(&write_err)? - } else { - return Err(write_err(e)); + + use tar::EntryType::*; + match entry { + Directory => { + let mode = src.header().mode().map_err(&read_err)?; + let mut dir_builder = Dir::new(&path); + dir_builder.recursive(true).mode(mode); + if self.preserve_owner { + dir_builder.uid(uid).gid(gid); + } + dir_builder.create().map_err(&write_err)?; + } + Regular => { + let mut dest = match File::create(&path) { + Ok(x) => x, + Err(e) => { + if e.kind() == io::ErrorKind::NotFound { + if let Some(parent) = path.parent() { + Dir::new(parent).recursive(true).create() + .map_err(&write_err)?; + File::create(&path).map_err(&write_err)? + } else { + return Err(write_err(e)); + } + } else { + return Err(write_err(e)); + } } - } else { - return Err(write_err(e)); + }; + copy_stream(&mut src, &mut dest) + .map_err(|e| + format!("Error unpacking {:?} -> {:?}: {}", self.archive, path, e) + )?; + let mode = src.header().mode().map_err(&read_err)?; + set_permissions(&path, Permissions::from_mode(mode)) + .map_err(&write_err)?; + if self.preserve_owner { + set_owner_group(&path, uid, gid).map_err(&write_err)?; } } + Symlink => { + let src = src.link_name().map_err(&read_err)? + .ok_or(format!("Error unpacking {:?}, broken symlink", path))?; + match symlink(&src, &path) { + Ok(_) => {}, + Err(e) => { + if e.kind() == io::ErrorKind::NotFound { + if let Some(parent) = path.parent() { + Dir::new(parent).recursive(true).create() + .map_err(&write_err)?; + symlink(&src, &path).map_err(&write_err)? + } else { + return Err(write_err(e)); + } + } else { + return Err(write_err(e)); + } + } + }; + } + Link => { + let link = src.link_name().map_err(&read_err)? + .ok_or(format!("Error unpacking {:?}, broken symlink", path))?; + let link = if link.is_absolute() { + link.strip_prefix("/").unwrap() + } else { + &*link + }; + hardlinks.push((self.target_dir.join(link).to_path_buf(), path.to_path_buf())); + } + _ => {} + } + } + for (src, dst) in hardlinks.into_iter() { + let write_err = |e| { + format!("Error hardlinking {:?} - {:?}: {}", &src, &dst, e) }; - } else if entry.is_hard_link() { - let link = src.link_name().map_err(&read_err)? - .ok_or(format!("Error unpacking {:?}, broken symlink", path))?; - let link = if link.is_absolute() { - link.strip_prefix("/").unwrap() - } else { - &*link - }; - hardlinks.push((tgt.join(link).to_path_buf(), path.to_path_buf())); - } else if entry.is_pax_global_extensions() || - entry.is_pax_local_extensions() || - entry.is_gnu_longname() || - entry.is_gnu_longlink() - { - // nothing to do - } else { - let mut dest = match File::create(&path) { - Ok(x) => x, + match hard_link(&src, &dst) { + Ok(_) => {}, Err(e) => { if e.kind() == io::ErrorKind::NotFound { - if let Some(parent) = path.parent() { + if let Some(parent) = dst.parent() { Dir::new(parent).recursive(true).create() .map_err(&write_err)?; - File::create(&path).map_err(&write_err)? + hard_link(&src, &dst).map_err(&write_err)? } else { return Err(write_err(e)); } @@ -190,39 +276,21 @@ fn unpack_stream(file: F, srcpath: &Path, tgt: &Path, } } }; - copy_stream(&mut src, &mut dest).map_err(|e| - format!("Error unpacking {:?} -> {:?}: {}", - srcpath, path, e))?; - let mode = src.header().mode().map_err(&read_err)?; - set_permissions(&path, Permissions::from_mode(mode)) - .map_err(&write_err)?; - if preserve_owner { - set_owner_group(&path, uid, gid).map_err(&write_err)?; - } } + Ok(()) } - for (src, dst) in hardlinks.into_iter() { - let write_err = |e| { - format!("Error hardlinking {:?} - {:?}: {}", &src, &dst, e) - }; - match hard_link(&src, &dst) { - Ok(_) => {}, - Err(e) => { - if e.kind() == io::ErrorKind::NotFound { - if let Some(parent) = dst.parent() { - Dir::new(parent).recursive(true).create() - .map_err(&write_err)?; - hard_link(&src, &dst).map_err(&write_err)? - } else { - return Err(write_err(e)); - } - } else { - return Err(write_err(e)); - } - } - }; - } - Ok(()) +} + +#[cfg(feature="containers")] +pub fn unpack_file(_ctx: &mut Context, src: &Path, tgt: &Path, + includes: &[&Path], excludes: &[&Path], preserve_owner: bool) + -> Result<(), String> +{ + TarCmd::new(src, tgt) + .includes(includes) + .excludes(excludes) + .preserve_owner(preserve_owner) + .unpack() } #[cfg(feature="containers")] diff --git a/src/builder/error.rs b/src/builder/error.rs index d1eef4a1..308c819f 100644 --- a/src/builder/error.rs +++ b/src/builder/error.rs @@ -100,6 +100,14 @@ quick_error! { from() display("{}", err) } + DockerClientError(err: dkregistry::errors::Error) { + from() + display("{}", err) + } + DownloadDockerLayersErrors(errors: Vec) { + from() + display("{:?}", errors) + } } } diff --git a/src/builder/mod.rs b/src/builder/mod.rs index 3613c75a..44f24286 100644 --- a/src/builder/mod.rs +++ b/src/builder/mod.rs @@ -33,6 +33,7 @@ pub mod commands { pub mod packaging; pub mod tarcmd; pub mod unzip; + pub mod docker; } #[cfg(feature="containers")] pub mod guard; #[cfg(feature="containers")] mod packages; diff --git a/src/config/builders.rs b/src/config/builders.rs index 47044253..1b9c0ede 100644 --- a/src/config/builders.rs +++ b/src/config/builders.rs @@ -126,6 +126,7 @@ define_commands! { composer::ComposerInstall, composer::ComposerDependencies, composer::ComposerConfig, + docker::DockerImage, } pub struct NameVisitor; diff --git a/src/config/read_settings.rs b/src/config/read_settings.rs index 591c5382..04271a10 100644 --- a/src/config/read_settings.rs +++ b/src/config/read_settings.rs @@ -1,5 +1,5 @@ use std::env; -use std::collections::{BTreeMap, HashMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::path::{Path, PathBuf}; use quire::{parse_config, parse_string, Options}; @@ -33,6 +33,7 @@ struct SecureSettings { disable_auto_clean: Option, environ: BTreeMap, propagate_environ: BTreeSet, + docker_insecure_registries: HashSet, } pub fn secure_settings_validator<'a>(has_children: bool) @@ -62,7 +63,8 @@ pub fn secure_settings_validator<'a>(has_children: bool) .member("disable_auto_clean", V::Scalar::new().optional()) .member("environ", V::Mapping::new( V::Scalar::new(), V::Scalar::new())) - .member("propagate_environ", V::Sequence::new(V::Scalar::new())); + .member("propagate_environ", V::Sequence::new(V::Scalar::new())) + .member("docker_insecure_registries", V::Sequence::new(V::Scalar::new())); if has_children { s = s.member("site_settings", V::Mapping::new( V::Scalar::new(), @@ -181,6 +183,9 @@ fn merge_settings(cfg: SecureSettings, project_root: &Path, for item in &cfg.propagate_environ { ext_settings.propagate_environ.insert(item.clone()); } + for registry in &cfg.docker_insecure_registries { + int_settings.docker_insecure_registries.insert(registry.clone()); + } if let Some(cfg) = cfg.site_settings.get(project_root) { if let Some(ref dir) = cfg.storage_dir { ext_settings.storage_dir = Some(dir.clone()); @@ -273,6 +278,11 @@ pub fn read_settings(project_root: &Path) disable_auto_clean: false, environ: BTreeMap::new(), storage_subdir_from_env_var: None, + docker_insecure_registries: { + let mut registries = HashSet::new(); + registries.insert("localhost".to_string()); + registries + }, }; let mut secure_files = vec!(); if let Ok(home) = env::var("_VAGGA_HOME") { diff --git a/src/config/settings.rs b/src/config/settings.rs index 6820528e..b9a3e3cf 100644 --- a/src/config/settings.rs +++ b/src/config/settings.rs @@ -1,5 +1,5 @@ use std::str::FromStr; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashSet}; use libc::{uid_t, gid_t}; use serde_json; @@ -29,6 +29,7 @@ pub struct Settings { pub run_symlinks_as_commands: bool, pub disable_auto_clean: bool, pub storage_subdir_from_env_var: Option, + pub docker_insecure_registries: HashSet, } impl Settings { diff --git a/tests/docker.bats b/tests/docker.bats new file mode 100644 index 00000000..4625c403 --- /dev/null +++ b/tests/docker.bats @@ -0,0 +1,55 @@ +setup() { + load '/bats/bats-support/load.bash' + load '/bats/bats-assert/load.bash' + cd /work/tests/docker +} + +@test "docker: hello-world" { + run vagga _build hello + assert_success + + run vagga hello + assert_success + assert_line "Hello from Docker!" +} + +@test "docker: python" { + run vagga _build python + assert_success + + run vagga zen + assert_success + assert_line "The Zen of Python, by Tim Peters" +} + +@test "docker: java" { + run vagga _build java + assert_success + + run vagga _run java /usr/local/openjdk-17/bin/java -version + assert_success + assert_line -p "17.0.1" +} + +@test "docker: registry" { + run vagga _build buildah + assert_success + run vagga _build registry + assert_success + run vagga _build test-image + assert_equal "$status" 121 + + run vagga buildah --version + assert_success + + run vagga build-test-image + assert_success + + run vagga --isolate-network push-and-build-test-image + assert_success + link=$(readlink .vagga/test-image) + assert_equal "$link" ".roots/test-image.f62f1455/root" + [[ ! -e .vagga/test-image/will-be-deleted ]] + [[ -f .vagga/test-image/test/vagga.yaml ]] + [[ ! -e .vagga/test-image/test/Dockerfile ]] +} diff --git a/tests/docker/.dockerignore b/tests/docker/.dockerignore new file mode 100644 index 00000000..079f8aa1 --- /dev/null +++ b/tests/docker/.dockerignore @@ -0,0 +1 @@ +.vagga diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile new file mode 100644 index 00000000..278c3240 --- /dev/null +++ b/tests/docker/Dockerfile @@ -0,0 +1,10 @@ +FROM busybox + +COPY Dockerfile vagga.yaml /will-be-deleted/ +COPY Dockerfile vagga.yaml /test/ +RUN id +RUN mount +RUN ls -l / +RUN ls -l /will-be-deleted +RUN rm -rf /will-be-deleted +RUN rm -f /test/Dockerfile diff --git a/tests/docker/vagga.yaml b/tests/docker/vagga.yaml new file mode 100644 index 00000000..818022bf --- /dev/null +++ b/tests/docker/vagga.yaml @@ -0,0 +1,99 @@ +containers: + hello: + setup: + - !DockerImage + image: library/hello-world + + python: + setup: + - !DockerImage + image: library/python + tag: 3.10-slim + + java: + setup: + - !DockerImage openjdk:17.0.1-slim-buster + + registry: + setup: + - !DockerImage registry:2.7.1 + - !EnsureDir /var/lib/registry + volumes: + /var/lib/registry: !Persistent registry + + buildah: + setup: + - !Ubuntu focal + - !UbuntuUniverse + - !Install [add-apt-key, ca-certificates] + - !Download + url: https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_20.04/Release.key + path: /tmp/Release.key + - !Sh apt-key add - < /tmp/Release.key + - !UbuntuRepo + url: http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_20.04/ + suite: "" + components: ["/"] + - !Install [buildah, fuse-overlayfs, netcat] + - !Text + /etc/containers/registries.conf: | + unqualified-search-registries = ["docker.io"] + - !Sh | + sed -i 's|#mount_program.*|mount_program = "/usr/bin/fuse-overlayfs"|' /etc/containers/storage.conf + # FIXME + - !Sh | + rm -f /etc/resolv.conf + touch /etc/resolv.conf + - !EnsureDir /usr/lib/vagga + - !EnsureDir /var/lib/containers + - !EnsureDir /var/cache/containers + environ: + PATH: /usr/lib/vagga:/sbin:/bin:/usr/sbin:/usr/bin + volumes: + /usr/lib/vagga: !VaggaBin + /var/lib/containers: !Persistent buildah + /var/cache/containers: !Tmpfs + /var/tmp: !Tmpfs + + test-image: + setup: + - !DockerImage localhost:5000/test + +commands: + hello: !Command + container: hello + run: [/hello] + + zen: !Command + container: python + run: python -m this + + buildah: !Command + container: buildah + run: [buildah] + + build-test-image: !Command + container: buildah + run: | + set -eux + + # By default it uses oci format which is not supported by dkregistry yet + buildah bud --format docker --layers -t localhost:5000/test:latest -f Dockerfile + + push-and-build-test-image: !Supervise + children: + registry: !Command + container: registry + run: [/entrypoint.sh, /etc/docker/registry/config.yml] + build-and-push: !Command + container: buildah + run: | + set -eux + + while ! nc -z localhost 5000; do + sleep 0.5 + done + buildah push --tls-verify=false localhost:5000/test:latest + + export RUST_LOG=dkregistry::v2=trace + vagga _build test-image diff --git a/vagga.yaml b/vagga.yaml index 7c26dd31..7d582d39 100644 --- a/vagga.yaml +++ b/vagga.yaml @@ -11,6 +11,10 @@ containers: environ: &rustenv LD_LIBRARY_PATH: /musl/lib/rustlib/x86_64-unknown-linux-musl/lib PATH: /musl/bin:/usr/local/bin:/usr/bin:/bin + HOME: /work/target + PKG_CONFIG_ALLOW_CROSS: 1 + OPENSSL_STATIC: true + OPENSSL_DIR: /usr/lib/x86_64-linux-musl setup: - !Ubuntu focal - !UbuntuUniverse @@ -29,6 +33,25 @@ containers: - !TarInstall url: "https://static.rust-lang.org/dist/rust-std-1.57.0-wasm32-unknown-unknown.tar.gz" script: "./install.sh --prefix=/usr --components=rust-std-wasm32-unknown-unknown" + + # OpenSSL + # https://qiita.com/liubin/items/6c94f0b61f746c08b74c + - !Sh | + ln -s /usr/include/x86_64-linux-gnu/asm /usr/include/x86_64-linux-musl/asm + ln -s /usr/include/asm-generic /usr/include/x86_64-linux-musl/asm-generic + ln -s /usr/include/linux /usr/include/x86_64-linux-musl/linux + - !TarInstall + url: https://github.com/openssl/openssl/archive/OpenSSL_1_1_1l.tar.gz + script: | + CC="musl-gcc -fPIE -pie" \ + ./Configure no-shared no-async \ + --prefix=/usr/lib/x86_64-linux-musl \ + --openssldir=/usr/lib/x86_64-linux-musl/ssl \ + linux-x86_64 + make depend + make -j$(nproc) + make install_sw + - &bulk !Tar url: "https://github.com/tailhook/bulk/releases/download/v0.4.9/bulk-v0.4.9.tar.gz" sha256: 23471a9986274bb4b7098c03e2eb7e1204171869b72c45385fcee1c64db2d111 From e024ce106ff7e8a08bd2973a75977ac2d25885fb Mon Sep 17 00:00:00 2001 From: Alexander Koval Date: Fri, 4 Feb 2022 19:59:20 +0200 Subject: [PATCH 02/12] Unpack docker layers in parallel with downloading --- src/builder/commands/docker.rs | 143 +++++++++++++++++++++++---------- 1 file changed, 99 insertions(+), 44 deletions(-) diff --git a/src/builder/commands/docker.rs b/src/builder/commands/docker.rs index 5e29f5af..693cad6e 100644 --- a/src/builder/commands/docker.rs +++ b/src/builder/commands/docker.rs @@ -39,7 +39,7 @@ const DEFAULT_IMAGE_NAMESPACE: &str = "library"; const DEFAULT_IMAGE_TAG: &str = "latest"; const DOCKER_LAYERS_CACHE_PATH: &str = "/vagga/cache/docker-layers"; -const DOCKER_LAYERS_DOWNLOAD_CONCURRENCY: usize = 4; +const DOCKER_LAYERS_DOWNLOAD_CONCURRENCY: usize = 2; #[derive(Serialize, Deserialize, Debug)] pub struct DockerImage { @@ -125,29 +125,26 @@ impl BuildStep for DockerImage { #[cfg(feature="containers")] fn build(&self, guard: &mut Guard, _build: bool) -> Result<(), StepError> { - let insecure = self.insecure.unwrap_or_else(|| { + let insecure = self.insecure.unwrap_or_else(|| is_insecure_registry(&self.registry, &guard.ctx.settings.docker_insecure_registries) - }); + ); if !insecure { capsule::ensure(&mut guard.ctx.capsule, &[capsule::Https])?; } Dir::new(DOCKER_LAYERS_CACHE_PATH) .recursive(true) .create() - .expect("Docker layers cache dir"); - let layer_paths = tokio::runtime::Builder::new_current_thread() + .map_err(|e| + format!("Cannot create docker layers cache directory: {}", e) + )?; + let dst_path = Path::new("/vagga/root").join(&self.path.strip_prefix("/").unwrap()); + tokio::runtime::Builder::new_current_thread() .enable_all() .build() - .expect("Tokio runtime") - .block_on(download_image(&self.registry, insecure, &self.image, &self.tag)) - .expect("Downloaded layers"); - let dst_path = Path::new("/vagga/root").join(&self.path.strip_prefix("/").unwrap()); - for layer_path in layer_paths.iter() { - TarCmd::new(layer_path, &dst_path) - .preserve_owner(true) - .entry_handler(whiteout_entry_handler) - .unpack()?; - } + .map_err(|e| format!("Error creating tokio runtime: {}", e))? + .block_on(download_and_unpack_image( + &self.registry, insecure, &self.image, &self.tag, &dst_path + ))?; Ok(()) } @@ -156,6 +153,16 @@ impl BuildStep for DockerImage { } } +fn is_insecure_registry( + registry: &str, insecure_registries: &HashSet +) -> bool { + let registry_host = match registry.split_once(':') { + Some((host, _port)) => host, + None => registry, + }; + insecure_registries.contains(registry_host) +} + /// See: /// - https://github.com/moby/moby/blob/v20.10.11/pkg/archive/whiteouts.go /// - https://github.com/moby/moby/blob/v20.10.11/pkg/archive/diff.go#L131 @@ -195,16 +202,10 @@ fn whiteout_entry_handler(entry: &Entry>, dst_path: &Path) -> Resu Ok(false) } -fn is_insecure_registry(registry: &str, insecure_registries: &HashSet) -> bool { - let registry_url = url::Url::parse(&format!("http://{}", registry)).unwrap(); - let registry_host = registry_url.domain().unwrap(); - insecure_registries.contains(registry_host) -} - #[cfg(feature="containers")] -async fn download_image( - registry: &str, insecure: bool, image: &str, tag: &str -) -> Result, StepError> { +async fn download_and_unpack_image( + registry: &str, insecure: bool, image: &str, tag: &str, dst_path: &Path +) -> Result<(), StepError> { let auth_scope = format!("repository:{}:pull", image); let client = build_client(registry, insecure, &[&auth_scope]).await?; @@ -216,22 +217,55 @@ async fn download_image( let layers_download_semaphore = Arc::new( Semaphore::new(DOCKER_LAYERS_DOWNLOAD_CONCURRENCY) ); - let layers_futures = layers_digests.iter() - .map(|digest| { - let image = image.to_string(); - let digest = digest.clone(); - let client = client.clone(); - let sem = layers_download_semaphore.clone(); - tokio::spawn(async move { - if let Ok(_guard) = sem.acquire().await { - info!("Downloading docker layer: {}", &digest); - download_blob(&client, &image, &digest).await - } else { - panic!("Semaphore was closed unexpectedly") + + use tokio::sync::oneshot; + + let mut layers_futures = vec!(); + let mut unpack_channels = vec!(); + for digest in &layers_digests { + let image = image.to_string(); + let digest = digest.clone(); + let client = client.clone(); + let sem = layers_download_semaphore.clone(); + let (tx, rx) = oneshot::channel(); + unpack_channels.push(rx); + let download_future = tokio::spawn(async move { + if let Ok(_guard) = sem.acquire().await { + println!("Downloading docker layer: {}", &digest); + match download_blob(&client, &image, &digest).await { + Ok(layer_path) => { + if let Err(_) = tx.send((digest.clone(), layer_path)) { + return Err(format!("Error sending downloaded layer")); + } + Ok(()) + } + Err(e) => Err(e) + } + } else { + panic!("Semaphore was closed unexpectedly") + } + }); + layers_futures.push(download_future); + } + + let dst_path = dst_path.to_path_buf(); + let unpack_future = tokio::spawn(async move { + for ch in unpack_channels { + match ch.await { + Ok((digest, layer_path)) => { + let dst_path = dst_path.clone(); + if let Err(e) = unpack_layer(digest, layer_path, dst_path).await { + return Err(e); + } } - }) - }) - .collect::>(); + Err(e) => return Err( + format!("Error waiting downloaded layer: {}", e) + ), + } + } + Ok(()) + }); + let mut layers_paths = vec!(); let mut layers_errors = vec!(); for layer_res in futures::future::join_all(layers_futures).await.into_iter() { @@ -241,13 +275,32 @@ async fn download_image( Err(join_err) => layers_errors.push(format!("{}", join_err)), } } + + unpack_future.await + .map_err(|e| format!("Error waiting unpack future: {}", e))??; + if !layers_errors.is_empty() { Err(layers_errors.into()) } else { - Ok(layers_paths) + Ok(()) } } +async fn unpack_layer( + digest: String, layer_path: PathBuf, dst_path: PathBuf +) -> Result<(), String> { + let unpack_future_res = tokio::task::spawn_blocking(move || { + println!("Unpacking docker layer: {}", digest); + TarCmd::new(&layer_path, &dst_path) + .preserve_owner(true) + .entry_handler(whiteout_entry_handler) + .unpack() + }).await; + unpack_future_res + .map_err(|e| format!("Error waiting a unpack layer future: {}", e))? + .map_err(|e| format!("Error unpacking docker layer: {}", e)) +} + #[cfg(feature="containers")] async fn build_client( registry: &str, insecure: bool, auth_scopes: &[&str] @@ -271,7 +324,9 @@ async fn build_client( async fn download_blob( client: &RegistryClient, image: &str, layer_digest: &str ) -> Result { - let digest = layer_digest.split_once(':').unwrap().1; + let digest = layer_digest.split_once(':') + .ok_or(format!("Invalid layer digest: {}", layer_digest))? + .1; let short_digest = &digest[..12]; let layers_cache = Path::new(DOCKER_LAYERS_CACHE_PATH); @@ -298,11 +353,11 @@ async fn download_blob( println!("Downloading docker blob: {}", &short_digest); let mut blob_stream = client.get_blob_stream(image, layer_digest).await - .expect("Get blob response"); + .map_err(|e| format!("Error getting docker blob response: {}", e))?; let mut blob_file = tokio::fs::File::create(&blob_tmp_path).await - .expect("Create layer file"); + .map_err(|e| format!("Cannot create layer file: {}", e))?; while let Some(chunk) = blob_stream.next().await { - let chunk = chunk.expect("Layer chunk"); + let chunk = chunk.map_err(|e| format!("Error fetching layer chunk: {}", e))?; blob_file.write_all(&chunk).await .map_err(|e| format!("Cannot write blob file: {}", e))?; } From 31ee4d0fe6d5399b77c071f6d3ffbc9f9545b534 Mon Sep 17 00:00:00 2001 From: Alexander Koval Date: Mon, 7 Feb 2022 23:05:54 +0200 Subject: [PATCH 03/12] Cancelling downloading and unpacking futures --- src/builder/commands/docker.rs | 95 ++++++++++++++++++++-------------- tests/docker/Dockerfile | 4 -- tests/docker/vagga.yaml | 1 - 3 files changed, 55 insertions(+), 45 deletions(-) diff --git a/src/builder/commands/docker.rs b/src/builder/commands/docker.rs index 693cad6e..768fd4ab 100644 --- a/src/builder/commands/docker.rs +++ b/src/builder/commands/docker.rs @@ -16,9 +16,10 @@ use tar::{Entry, EntryType}; #[cfg(feature="containers")] use tokio::{ io::AsyncWriteExt, - sync::Semaphore, + sync::{oneshot, Semaphore}, }; + #[cfg(feature="containers")] use quire::{ validate as V, @@ -218,65 +219,79 @@ async fn download_and_unpack_image( Semaphore::new(DOCKER_LAYERS_DOWNLOAD_CONCURRENCY) ); - use tokio::sync::oneshot; + let mut downloaded_layer_txs = vec!(); + let mut downloaded_layer_rxs = vec!(); + for _ in &layers_digests { + let (tx, rx) = oneshot::channel(); + downloaded_layer_rxs.push(rx); + downloaded_layer_txs.push(tx); + } + + let dst_path = dst_path.to_path_buf(); + let unpack_task = tokio::spawn(async move { + for layer_ch in downloaded_layer_rxs { + match layer_ch.await { + Ok((digest, layer_path)) => { + let dst_path = dst_path.clone(); + if let Err(e) = unpack_layer(digest, layer_path, dst_path).await { + return Err(e); + } + } + Err(_) => { + // Channel is dropped when download task was cancelled + }, + } + } + Ok(()) + }); - let mut layers_futures = vec!(); - let mut unpack_channels = vec!(); - for digest in &layers_digests { + let mut layers_tasks = vec!(); + for (digest, tx) in layers_digests.iter().zip(downloaded_layer_txs.into_iter()) { let image = image.to_string(); let digest = digest.clone(); let client = client.clone(); let sem = layers_download_semaphore.clone(); - let (tx, rx) = oneshot::channel(); - unpack_channels.push(rx); - let download_future = tokio::spawn(async move { + let download_task = tokio::spawn(async move { if let Ok(_guard) = sem.acquire().await { - println!("Downloading docker layer: {}", &digest); match download_blob(&client, &image, &digest).await { Ok(layer_path) => { - if let Err(_) = tx.send((digest.clone(), layer_path)) { - return Err(format!("Error sending downloaded layer")); - } + // Unpack task may be cancelled so ignore sending errors + tx.send((digest.clone(), layer_path)).ok(); Ok(()) } - Err(e) => Err(e) + Err(e) => { + Err(e) + } } } else { panic!("Semaphore was closed unexpectedly") } }); - layers_futures.push(download_future); + layers_tasks.push(download_task); } - let dst_path = dst_path.to_path_buf(); - let unpack_future = tokio::spawn(async move { - for ch in unpack_channels { - match ch.await { - Ok((digest, layer_path)) => { - let dst_path = dst_path.clone(); - if let Err(e) = unpack_layer(digest, layer_path, dst_path).await { - return Err(e); - } - } - Err(e) => return Err( - format!("Error waiting downloaded layer: {}", e) - ), - } - } - Ok(()) - }); - - let mut layers_paths = vec!(); let mut layers_errors = vec!(); - for layer_res in futures::future::join_all(layers_futures).await.into_iter() { - match layer_res { - Ok(Ok(layer)) => layers_paths.push(layer), - Ok(Err(client_err)) => layers_errors.push(client_err), - Err(join_err) => layers_errors.push(format!("{}", join_err)), + let mut canceling = false; + for download_layer_future in layers_tasks.into_iter() { + if canceling { + download_layer_future.abort(); + } + match download_layer_future.await { + Ok(Ok(_)) => {}, + Ok(Err(client_err)) => { + canceling = true; + unpack_task.abort(); + layers_errors.push(client_err) + }, + Err(join_err) => { + if !join_err.is_cancelled() { + layers_errors.push(format!("{}", join_err)) + } + }, } } - unpack_future.await + unpack_task.await .map_err(|e| format!("Error waiting unpack future: {}", e))??; if !layers_errors.is_empty() { @@ -351,7 +366,7 @@ async fn download_blob( let blob_tmp_file_name = format!(".{}.tmp", &blob_file_name); let blob_tmp_path = layers_cache.join(&blob_tmp_file_name); - println!("Downloading docker blob: {}", &short_digest); + println!("Downloading docker layer: {}", &layer_digest); let mut blob_stream = client.get_blob_stream(image, layer_digest).await .map_err(|e| format!("Error getting docker blob response: {}", e))?; let mut blob_file = tokio::fs::File::create(&blob_tmp_path).await diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index 278c3240..da5a31d1 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -2,9 +2,5 @@ FROM busybox COPY Dockerfile vagga.yaml /will-be-deleted/ COPY Dockerfile vagga.yaml /test/ -RUN id -RUN mount -RUN ls -l / -RUN ls -l /will-be-deleted RUN rm -rf /will-be-deleted RUN rm -f /test/Dockerfile diff --git a/tests/docker/vagga.yaml b/tests/docker/vagga.yaml index 818022bf..766522c7 100644 --- a/tests/docker/vagga.yaml +++ b/tests/docker/vagga.yaml @@ -95,5 +95,4 @@ commands: done buildah push --tls-verify=false localhost:5000/test:latest - export RUST_LOG=dkregistry::v2=trace vagga _build test-image From 66ebc0d0d2ffa82eb4efe044fcfc21e027571b42 Mon Sep 17 00:00:00 2001 From: Alexander Koval Date: Tue, 8 Feb 2022 16:25:08 +0200 Subject: [PATCH 04/12] Override existing entries when unpacking docker layers --- src/builder/commands/docker.rs | 13 ++++------- src/builder/commands/tarcmd.rs | 40 +++++++++++++++++++++++++++++++--- src/builder/guard.rs | 21 +++++------------- src/file_util.rs | 8 ++++++- tests/docker.bats | 4 ++++ tests/docker/Dockerfile | 14 ++++++++++++ 6 files changed, 72 insertions(+), 28 deletions(-) diff --git a/src/builder/commands/docker.rs b/src/builder/commands/docker.rs index 768fd4ab..83caa86b 100644 --- a/src/builder/commands/docker.rs +++ b/src/builder/commands/docker.rs @@ -1,5 +1,4 @@ use std::collections::{BTreeMap, HashSet}; -use std::fs::{remove_dir_all, remove_file}; use std::io::{ErrorKind, Read}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -31,7 +30,7 @@ use crate::{ builder::commands::tarcmd::TarCmd, capsule::packages as capsule, container::util::clean_dir, - file_util::{Dir, Lock}, + file_util::{Dir, Lock, safe_remove}, }; use crate::build_step::{BuildStep, Config, Digest, Guard, StepError, VersionError}; @@ -189,13 +188,8 @@ fn whiteout_entry_handler(entry: &Entry>, dst_path: &Path) -> Resu } else { let mut whiteout_path = dir.to_path_buf(); whiteout_path.push(whiteout); - if whiteout_path.is_dir() { - remove_dir_all(&whiteout_path) - .map_err(|e| format!("Cannot remove directory: {}", e))?; - } else { - remove_file(whiteout_path) - .map_err(|e| format!("Cannot delete file: {}", e))?; - } + safe_remove(&whiteout_path) + .map_err(|e| format!("Cannot remove {:?} path: {}", &whiteout_path, e))?; } return Ok(true); } @@ -308,6 +302,7 @@ async fn unpack_layer( println!("Unpacking docker layer: {}", digest); TarCmd::new(&layer_path, &dst_path) .preserve_owner(true) + .override_entries(true) .entry_handler(whiteout_entry_handler) .unpack() }).await; diff --git a/src/builder/commands/tarcmd.rs b/src/builder/commands/tarcmd.rs index 114a6d92..575755ef 100644 --- a/src/builder/commands/tarcmd.rs +++ b/src/builder/commands/tarcmd.rs @@ -10,8 +10,12 @@ use std::path::{Path, PathBuf}; #[cfg(feature="containers")] use libmount::BindMount; #[cfg(feature="containers")] use tar::Archive; #[cfg(feature="containers")] use xz2::read::XzDecoder; + use quire::validate as V; +#[cfg(feature="containers")] +use tar::Entry; + #[cfg(feature="containers")] use crate::{ build_step::{BuildStep, VersionError, StepError, Digest, Config, Guard}, @@ -20,7 +24,7 @@ use crate::{ builder::dns::revert_name_files, capsule::download::{maybe_download_and_check_hashsum}, container::mount::{unmount}, - file_util::{Dir, read_visible_entries, copy_stream, set_owner_group}, + file_util::{Dir, read_visible_entries, copy_stream, safe_remove, set_owner_group}, }; @@ -70,18 +74,19 @@ pub struct TarCmd<'a> { includes: &'a[&'a Path], excludes: &'a[&'a Path], preserve_owner: bool, + override_entries: bool, entry_handler: fn(&Entry>, &Path) -> Result, } const DEFAULT_TAR_INCLUDES: &[&Path] = &[]; const DEFAULT_TAR_EXCLUDES: &[&Path] = &[]; -use tar::Entry; - +#[cfg(feature="containers")] fn dummy_entry_handler(_entry: &Entry>, _dst_path: &Path) -> Result { Ok(false) } +#[cfg(feature="containers")] impl<'a> TarCmd<'a> { pub fn new(archive: &'a Path, target_dir: &'a Path) -> Self { Self { @@ -90,6 +95,7 @@ impl<'a> TarCmd<'a> { includes: DEFAULT_TAR_INCLUDES, excludes: DEFAULT_TAR_EXCLUDES, preserve_owner: false, + override_entries: false, entry_handler: dummy_entry_handler, } } @@ -109,6 +115,11 @@ impl<'a> TarCmd<'a> { self } + pub fn override_entries(mut self, override_entries: bool) -> Self { + self.override_entries = override_entries; + self + } + pub fn entry_handler( self, entry_handler: fn(&Entry>, &Path) -> Result @@ -119,6 +130,7 @@ impl<'a> TarCmd<'a> { includes: self.includes, excludes: self.excludes, preserve_owner: self.preserve_owner, + override_entries: self.override_entries, entry_handler, } } @@ -187,6 +199,19 @@ impl<'a> TarCmd<'a> { use tar::EntryType::*; match entry { Directory => { + if self.override_entries { + match path.symlink_metadata() { + Ok(stat) if stat.is_dir() => {} + Ok(_) => { + safe_remove(&path) + .map_err(|e| format!("Cannot remove {:?} path: {}", &path, e))?; + } + Err(e) if e.kind() == io::ErrorKind::NotFound => {} + Err(e) => { + return Err(format!("Cannot stat {:?} path: {}", &path, e)); + } + } + } let mode = src.header().mode().map_err(&read_err)?; let mut dir_builder = Dir::new(&path); dir_builder.recursive(true).mode(mode); @@ -196,6 +221,11 @@ impl<'a> TarCmd<'a> { dir_builder.create().map_err(&write_err)?; } Regular => { + if self.override_entries { + safe_remove(&path) + .map_err(|e| format!("Cannot remove {:?} path: {}", &path, e))?; + } + // TODO: Should we allow truncate a file here? let mut dest = match File::create(&path) { Ok(x) => x, Err(e) => { @@ -224,6 +254,10 @@ impl<'a> TarCmd<'a> { } } Symlink => { + if self.override_entries { + safe_remove(&path) + .map_err(|e| format!("Cannot remove {:?} path: {}", &path, e))?; + } let src = src.link_name().map_err(&read_err)? .ok_or(format!("Error unpacking {:?}, broken symlink", path))?; match symlink(&src, &path) { diff --git a/src/builder/guard.rs b/src/builder/guard.rs index 0b4f81e8..6adc224b 100644 --- a/src/builder/guard.rs +++ b/src/builder/guard.rs @@ -1,5 +1,5 @@ use std::path::Path; -use std::fs::{remove_dir_all, remove_file, symlink_metadata}; +use std::fs::remove_dir_all; use std::collections::HashMap; use crate::build_step::BuildStep; @@ -12,7 +12,7 @@ use crate::container::mount::{ unmount, mount_system_dirs, mount_proc, mount_run, unmount_system_dirs, }; use crate::container::util::{clean_dir, write_container_signature}; -use crate::file_util::{Dir, copy, truncate_file}; +use crate::file_util::{Dir, copy, truncate_file, safe_remove}; use crate::path_util::IterSelfAndParents; @@ -198,19 +198,10 @@ fn remove_all_except(root: &Path, keep_rel_paths: &HashMap<&Path, bool>) remove_all_except(path, keep_rel_paths)?; }, None => { - let file_type = try_msg!( - symlink_metadata(path), - "Error querying file metadata: {path:?}: {err}", path=path - ).file_type(); - if file_type.is_dir() { - try_msg!(clean_dir(path, true), - "Error cleaning dir {path:?}: {err}", - path=path); - } else { - try_msg!(remove_file(path), - "Error removing file {path:?}: {err}", - path=path); - } + try_msg!( + safe_remove(path), + "Cannot remove {path:?}: {err}", path=path + ); }, } } diff --git a/src/file_util.rs b/src/file_util.rs index 907100e0..3db47860 100644 --- a/src/file_util.rs +++ b/src/file_util.rs @@ -684,7 +684,13 @@ pub fn set_times>(path: P, atime: i64, mtime: i64) pub fn safe_remove>(path: P) -> io::Result<()> { let path = path.as_ref(); path.symlink_metadata() - .and_then(|_| fs::remove_file(path)) + .and_then(|stat| { + if stat.is_dir() { + fs::remove_dir_all(path) + } else { + fs::remove_file(path) + } + }) .or_else(|e| { if e.kind() == io::ErrorKind::NotFound { Ok(()) diff --git a/tests/docker.bats b/tests/docker.bats index 4625c403..67ca9d5d 100644 --- a/tests/docker.bats +++ b/tests/docker.bats @@ -52,4 +52,8 @@ setup() { [[ ! -e .vagga/test-image/will-be-deleted ]] [[ -f .vagga/test-image/test/vagga.yaml ]] [[ ! -e .vagga/test-image/test/Dockerfile ]] + assert_equal "$(cat .vagga/test-image/hello.txt)" "Hello world!" + [[ -L .vagga/test-image/hi.txt ]] + assert_equal "$(cat .vagga/test-image/hi.txt)" "Hello world!" + [[ -f .vagga/test-image/empty ]] } diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index da5a31d1..1c3869cf 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -2,5 +2,19 @@ FROM busybox COPY Dockerfile vagga.yaml /will-be-deleted/ COPY Dockerfile vagga.yaml /test/ + +# Test whiteouts RUN rm -rf /will-be-deleted RUN rm -f /test/Dockerfile + +# Override file +RUN echo -n 'Hello' > /hello.txt +RUN echo -n ' world!' >> /hello.txt + +# Override symlink +RUN ln -s nonexistent /hi.txt +RUN rm -r /hi.txt && ln -s hello.txt /hi.txt + +# Override directory +RUN mkdir /empty +RUN rmdir /empty && touch /empty From 415c1887c1e45059523bb683908a8a5322677237 Mon Sep 17 00:00:00 2001 From: Alexander Koval Date: Wed, 9 Feb 2022 12:39:02 +0200 Subject: [PATCH 05/12] Update dkregistry dependency --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8bffd94e..045c6e33 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -336,7 +336,7 @@ dependencies = [ [[package]] name = "dkregistry" version = "0.5.1-alpha.0" -source = "git+https://github.com/anti-social/dkregistry-rs?rev=1bfac75#1bfac756be72db994682f621ebb38dce464f21cf" +source = "git+https://github.com/anti-social/dkregistry-rs?rev=6c132c4#6c132c43002c52081d666d821c49e4a97f58ac55" dependencies = [ "async-stream", "base64", diff --git a/Cargo.toml b/Cargo.toml index 1cb0c763..702615ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,7 +51,7 @@ serde_json = "1.0.2" serde_derive = "1.0.11" failure = "0.1.1" resolv-conf = "0.6.0" -dkregistry = { git="https://github.com/anti-social/dkregistry-rs", rev="1bfac75" } +dkregistry = { git="https://github.com/anti-social/dkregistry-rs", rev="6c132c4" } tokio = { version = "1", features = ["full"] } futures = "0.3.18" From 8d7d045381b3bc230a4009e00ed1e66e192360ba Mon Sep 17 00:00:00 2001 From: Alexander Koval Date: Tue, 15 Feb 2022 20:48:51 +0200 Subject: [PATCH 06/12] Download docker image layers in series --- src/builder/commands/docker.rs | 93 ++++++++++++++-------------------- 1 file changed, 37 insertions(+), 56 deletions(-) diff --git a/src/builder/commands/docker.rs b/src/builder/commands/docker.rs index 83caa86b..cdb01a07 100644 --- a/src/builder/commands/docker.rs +++ b/src/builder/commands/docker.rs @@ -15,7 +15,7 @@ use tar::{Entry, EntryType}; #[cfg(feature="containers")] use tokio::{ io::AsyncWriteExt, - sync::{oneshot, Semaphore}, + sync::oneshot, }; @@ -39,7 +39,6 @@ const DEFAULT_IMAGE_NAMESPACE: &str = "library"; const DEFAULT_IMAGE_TAG: &str = "latest"; const DOCKER_LAYERS_CACHE_PATH: &str = "/vagga/cache/docker-layers"; -const DOCKER_LAYERS_DOWNLOAD_CONCURRENCY: usize = 2; #[derive(Serialize, Deserialize, Debug)] pub struct DockerImage { @@ -209,10 +208,6 @@ async fn download_and_unpack_image( let layers_digests = manifest.layers_digests(None)?; - let layers_download_semaphore = Arc::new( - Semaphore::new(DOCKER_LAYERS_DOWNLOAD_CONCURRENCY) - ); - let mut downloaded_layer_txs = vec!(); let mut downloaded_layer_rxs = vec!(); for _ in &layers_digests { @@ -232,66 +227,52 @@ async fn download_and_unpack_image( } } Err(_) => { - // Channel is dropped when download task was cancelled + // Channel is dropped if download task is cancelled }, } } Ok(()) }); - let mut layers_tasks = vec!(); - for (digest, tx) in layers_digests.iter().zip(downloaded_layer_txs.into_iter()) { - let image = image.to_string(); - let digest = digest.clone(); - let client = client.clone(); - let sem = layers_download_semaphore.clone(); - let download_task = tokio::spawn(async move { - if let Ok(_guard) = sem.acquire().await { - match download_blob(&client, &image, &digest).await { - Ok(layer_path) => { - // Unpack task may be cancelled so ignore sending errors - tx.send((digest.clone(), layer_path)).ok(); - Ok(()) - } - Err(e) => { - Err(e) - } + let image = image.to_string(); + let download_task = tokio::spawn(async move { + for (digest, tx) in layers_digests.iter().zip(downloaded_layer_txs.into_iter()) { + let digest = digest.clone(); + let client = client.clone(); + match download_blob(&client, &image, &digest).await { + Ok(layer_path) => { + // Unpack task may be cancelled so ignore sending errors + tx.send((digest.clone(), layer_path)).ok(); } - } else { - panic!("Semaphore was closed unexpectedly") - } - }); - layers_tasks.push(download_task); - } - - let mut layers_errors = vec!(); - let mut canceling = false; - for download_layer_future in layers_tasks.into_iter() { - if canceling { - download_layer_future.abort(); - } - match download_layer_future.await { - Ok(Ok(_)) => {}, - Ok(Err(client_err)) => { - canceling = true; - unpack_task.abort(); - layers_errors.push(client_err) - }, - Err(join_err) => { - if !join_err.is_cancelled() { - layers_errors.push(format!("{}", join_err)) + Err(e) => { + return Err(e); } - }, + } } - } + Ok(()) + }); - unpack_task.await - .map_err(|e| format!("Error waiting unpack future: {}", e))??; + match download_task.await { + Ok(Ok(_)) => {}, + Ok(Err(client_err)) => { + unpack_task.abort(); + return Err(client_err.into()); + }, + Err(join_err) => { + unpack_task.abort(); + return Err( + format!("Error waiting a download layers task: {}", join_err).into() + ); + }, + } - if !layers_errors.is_empty() { - Err(layers_errors.into()) - } else { - Ok(()) + match unpack_task.await { + Ok(Ok(_)) => Ok(()), + Ok(Err(unpack_err)) => Err(unpack_err.into()), + Err(join_err) if join_err.is_cancelled() => Ok(()), + Err(join_err) => Err( + format!("Error waiting an unpack layers task: {}", join_err).into() + ), } } @@ -307,7 +288,7 @@ async fn unpack_layer( .unpack() }).await; unpack_future_res - .map_err(|e| format!("Error waiting a unpack layer future: {}", e))? + .map_err(|e| format!("Error waiting an unpack layer future: {}", e))? .map_err(|e| format!("Error unpacking docker layer: {}", e)) } From 5d954b69ef53883c758138bd89aa97b6dcfc900c Mon Sep 17 00:00:00 2001 From: Alexander Koval Date: Tue, 15 Feb 2022 22:13:53 +0200 Subject: [PATCH 07/12] Change default docker registry host --- src/builder/commands/docker.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/builder/commands/docker.rs b/src/builder/commands/docker.rs index cdb01a07..646e9f95 100644 --- a/src/builder/commands/docker.rs +++ b/src/builder/commands/docker.rs @@ -34,7 +34,7 @@ use crate::{ }; use crate::build_step::{BuildStep, Config, Digest, Guard, StepError, VersionError}; -const DEFAULT_REGISTRY_HOST: &str = "registry-1.docker.io"; +const DEFAULT_REGISTRY_HOST: &str = "index.docker.io"; const DEFAULT_IMAGE_NAMESPACE: &str = "library"; const DEFAULT_IMAGE_TAG: &str = "latest"; From ba370f5cbf4efc768e885782a3994a37fb0dc4d7 Mon Sep 17 00:00:00 2001 From: Alexander Koval Date: Tue, 15 Feb 2022 22:13:05 +0200 Subject: [PATCH 08/12] Documentation for DockerImage --- docs/build_steps.rst | 41 +++++++++++++++++++++++++++++++++++++++++ docs/settings.rst | 3 +++ 2 files changed, 44 insertions(+) diff --git a/docs/build_steps.rst b/docs/build_steps.rst index 8a77461d..2f290e13 100644 --- a/docs/build_steps.rst +++ b/docs/build_steps.rst @@ -26,6 +26,7 @@ of empty container): * :step:`SubConfig` * :step:`Container` * :step:`Tar` +* :step:`DockerImage` Ubuntu Commands =============== @@ -314,6 +315,46 @@ config if you use :step:`SubConfig` or :step:`Container`) # Container built. Now, everything in BuildDeps(wget and curl) is removed from the container. +Docker Commands +=============== + +.. step:: DockerImage + + .. warning:: Following functionality is experimental and is actively developed. + + Downloads and unpacks a docker image. + + Have 2 flavors. A short one:: + + - !DockerImage elasticsearch:7.13.4 + + And a full one:: + + - !DockerImage + image: elasticsearch + tag: 7.13.4 + + Options: + + registry + (default ``index.docker.io``) A registry hostname. + + image + (required) An image name. If namespace is missing ``library`` namespace will be used. + So ``python`` and ``library/python`` values point to the same image. + + tag + (default ``latest``) A tag of the image. + + insecure + (optional) If ``true``, an insecure client will be used. Only ``localhost`` + registry is considered insecure by default. + See also :opt:`docker-insecure-registries` setting. + + path + (default ``\``) A path inside a container where the image should be unpacked. + + Generic Commands ================ diff --git a/docs/settings.rst b/docs/settings.rst index 0251f1d5..2150007d 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -319,5 +319,8 @@ not to do so. being built simultaneously. It makes little sense to enable it on a workstation. +.. opt:: docker-insecure-registries + (default ``[localhost]``) A list of docker insecure registries. + ``localhost`` is always considered as insecure. From 5a7c513c9e593f7e09a1faba118c404338d032e6 Mon Sep 17 00:00:00 2001 From: Alexander Koval Date: Tue, 15 Feb 2022 22:57:10 +0200 Subject: [PATCH 09/12] Docker registry aliases --- docs/settings.rst | 4 ++++ src/builder/commands/docker.rs | 18 +++++++++++++++--- src/config/read_settings.rs | 14 +++++++++++++- src/config/settings.rs | 3 ++- tests/docker/vagga.yaml | 4 ++-- 5 files changed, 36 insertions(+), 7 deletions(-) diff --git a/docs/settings.rst b/docs/settings.rst index 2150007d..3038e7a0 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -324,3 +324,7 @@ not to do so. (default ``[localhost]``) A list of docker insecure registries. ``localhost`` is always considered as insecure. +.. opt:: docker-registry-aliases + + (default ``{docker.io: index.docker.io}``) A mapping of docker registry aliases. + Alias for ``docker.io`` is always added to the aliases mapping. diff --git a/src/builder/commands/docker.rs b/src/builder/commands/docker.rs index 646e9f95..c0c6eb56 100644 --- a/src/builder/commands/docker.rs +++ b/src/builder/commands/docker.rs @@ -34,7 +34,7 @@ use crate::{ }; use crate::build_step::{BuildStep, Config, Digest, Guard, StepError, VersionError}; -const DEFAULT_REGISTRY_HOST: &str = "index.docker.io"; +pub const DEFAULT_REGISTRY_HOST: &str = "index.docker.io"; const DEFAULT_IMAGE_NAMESPACE: &str = "library"; const DEFAULT_IMAGE_TAG: &str = "latest"; @@ -124,8 +124,20 @@ impl BuildStep for DockerImage { #[cfg(feature="containers")] fn build(&self, guard: &mut Guard, _build: bool) -> Result<(), StepError> { + let registry = if let Some(registry) = guard.ctx.settings.docker_registry_aliases.get(&self.registry) { + registry + } else { + &self.registry + }; + let _image; + let image = if registry == DEFAULT_REGISTRY_HOST && !self.image.contains("/") { + _image = format!("library/{}", &self.image); + &_image + } else { + &self.image + }; let insecure = self.insecure.unwrap_or_else(|| - is_insecure_registry(&self.registry, &guard.ctx.settings.docker_insecure_registries) + is_insecure_registry(registry, &guard.ctx.settings.docker_insecure_registries) ); if !insecure { capsule::ensure(&mut guard.ctx.capsule, &[capsule::Https])?; @@ -142,7 +154,7 @@ impl BuildStep for DockerImage { .build() .map_err(|e| format!("Error creating tokio runtime: {}", e))? .block_on(download_and_unpack_image( - &self.registry, insecure, &self.image, &self.tag, &dst_path + registry, insecure, image, &self.tag, &dst_path ))?; Ok(()) } diff --git a/src/config/read_settings.rs b/src/config/read_settings.rs index 04271a10..defe124e 100644 --- a/src/config/read_settings.rs +++ b/src/config/read_settings.rs @@ -5,6 +5,7 @@ use std::path::{Path, PathBuf}; use quire::{parse_config, parse_string, Options}; use quire::validate as V; +use crate::builder::commands::docker::DEFAULT_REGISTRY_HOST; use crate::config::Settings; use crate::path_util::Expand; @@ -34,6 +35,7 @@ struct SecureSettings { environ: BTreeMap, propagate_environ: BTreeSet, docker_insecure_registries: HashSet, + docker_registry_aliases: HashMap, } pub fn secure_settings_validator<'a>(has_children: bool) @@ -64,7 +66,9 @@ pub fn secure_settings_validator<'a>(has_children: bool) .member("environ", V::Mapping::new( V::Scalar::new(), V::Scalar::new())) .member("propagate_environ", V::Sequence::new(V::Scalar::new())) - .member("docker_insecure_registries", V::Sequence::new(V::Scalar::new())); + .member("docker_insecure_registries", V::Sequence::new(V::Scalar::new())) + .member("docker_registry_aliases", V::Mapping::new( + V::Scalar::new(), V::Scalar::new())); if has_children { s = s.member("site_settings", V::Mapping::new( V::Scalar::new(), @@ -186,6 +190,9 @@ fn merge_settings(cfg: SecureSettings, project_root: &Path, for registry in &cfg.docker_insecure_registries { int_settings.docker_insecure_registries.insert(registry.clone()); } + for (alias, registry) in &cfg.docker_registry_aliases { + int_settings.docker_registry_aliases.insert(alias.clone(), registry.clone()); + } if let Some(cfg) = cfg.site_settings.get(project_root) { if let Some(ref dir) = cfg.storage_dir { ext_settings.storage_dir = Some(dir.clone()); @@ -283,6 +290,11 @@ pub fn read_settings(project_root: &Path) registries.insert("localhost".to_string()); registries }, + docker_registry_aliases: { + let mut aliases = HashMap::new(); + aliases.insert("docker.io".to_string(), DEFAULT_REGISTRY_HOST.to_string()); + aliases + }, }; let mut secure_files = vec!(); if let Ok(home) = env::var("_VAGGA_HOME") { diff --git a/src/config/settings.rs b/src/config/settings.rs index b9a3e3cf..a8e46136 100644 --- a/src/config/settings.rs +++ b/src/config/settings.rs @@ -1,5 +1,5 @@ use std::str::FromStr; -use std::collections::{BTreeMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use libc::{uid_t, gid_t}; use serde_json; @@ -30,6 +30,7 @@ pub struct Settings { pub disable_auto_clean: bool, pub storage_subdir_from_env_var: Option, pub docker_insecure_registries: HashSet, + pub docker_registry_aliases: HashMap, } impl Settings { diff --git a/tests/docker/vagga.yaml b/tests/docker/vagga.yaml index 766522c7..8b0af850 100644 --- a/tests/docker/vagga.yaml +++ b/tests/docker/vagga.yaml @@ -2,7 +2,7 @@ containers: hello: setup: - !DockerImage - image: library/hello-world + image: hello-world python: setup: @@ -12,7 +12,7 @@ containers: java: setup: - - !DockerImage openjdk:17.0.1-slim-buster + - !DockerImage docker.io/library/openjdk:17.0.1-slim-buster registry: setup: From c5536ede4372df01d19f769aa80bf613643add03 Mon Sep 17 00:00:00 2001 From: Alexander Koval Date: Fri, 18 Feb 2022 16:06:59 +0200 Subject: [PATCH 10/12] More documentation for docker-insecure-registries setting --- docs/settings.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/settings.rst b/docs/settings.rst index 3038e7a0..5017372d 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -321,7 +321,8 @@ not to do so. .. opt:: docker-insecure-registries - (default ``[localhost]``) A list of docker insecure registries. + (default ``[localhost]``) A list of docker insecure registries. An insecure + registry works using only an unencrypted ``http`` protocol. ``localhost`` is always considered as insecure. .. opt:: docker-registry-aliases From 4ecaa12bd2599dee505f88819bbba75ca52d4086 Mon Sep 17 00:00:00 2001 From: Alexander Koval Date: Mon, 9 May 2022 22:33:50 +0300 Subject: [PATCH 11/12] Clear destination hardlink when unpacking docker layer --- src/builder/commands/tarcmd.rs | 6 +++++- tests/docker.bats | 2 ++ tests/docker/Dockerfile | 4 ++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/builder/commands/tarcmd.rs b/src/builder/commands/tarcmd.rs index 575755ef..d10452e4 100644 --- a/src/builder/commands/tarcmd.rs +++ b/src/builder/commands/tarcmd.rs @@ -279,7 +279,7 @@ impl<'a> TarCmd<'a> { } Link => { let link = src.link_name().map_err(&read_err)? - .ok_or(format!("Error unpacking {:?}, broken symlink", path))?; + .ok_or(format!("Error unpacking {:?}, broken hardlink", path))?; let link = if link.is_absolute() { link.strip_prefix("/").unwrap() } else { @@ -294,6 +294,10 @@ impl<'a> TarCmd<'a> { let write_err = |e| { format!("Error hardlinking {:?} - {:?}: {}", &src, &dst, e) }; + if self.override_entries { + safe_remove(&dst) + .map_err(|e| format!("Cannot remove {:?} path: {}", &dst, e))?; + } match hard_link(&src, &dst) { Ok(_) => {}, Err(e) => { diff --git a/tests/docker.bats b/tests/docker.bats index 67ca9d5d..78496b73 100644 --- a/tests/docker.bats +++ b/tests/docker.bats @@ -55,5 +55,7 @@ setup() { assert_equal "$(cat .vagga/test-image/hello.txt)" "Hello world!" [[ -L .vagga/test-image/hi.txt ]] assert_equal "$(cat .vagga/test-image/hi.txt)" "Hello world!" + assert_equal "$(cat .vagga/test-image/see-you.txt)" "Bye-bye!" + assert_equal "$(stat -c '%i' .vagga/test-image/see-you.txt)" "$(stat -c '%i' .vagga/test-image/bye-bye.txt)" [[ -f .vagga/test-image/empty ]] } diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index 1c3869cf..c35349a2 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -15,6 +15,10 @@ RUN echo -n ' world!' >> /hello.txt RUN ln -s nonexistent /hi.txt RUN rm -r /hi.txt && ln -s hello.txt /hi.txt +# Override hardlink +RUN echo -n 'Bye!' > /bye.txt && ln /bye.txt /see-you.txt +RUN echo -n 'Bye-bye!' > /bye-bye.txt && ln -f /bye-bye.txt /see-you.txt + # Override directory RUN mkdir /empty RUN rmdir /empty && touch /empty From 83e2d2b08b22a065b132c5c54121cd382dcb2698 Mon Sep 17 00:00:00 2001 From: Alexander Koval Date: Mon, 9 May 2022 22:23:42 +0300 Subject: [PATCH 12/12] Render progress bar when downloading docker layers --- Cargo.lock | 50 +++++++++++++++++++++++++++++++++- Cargo.toml | 3 +- src/builder/commands/docker.rs | 31 +++++++++++++++++++-- 3 files changed, 80 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 045c6e33..4598308e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -233,6 +233,19 @@ dependencies = [ "bitflags", ] +[[package]] +name = "console" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28b32d32ca44b70c3e4acd7db1babf555fa026e385fb95f18028f88848b3c31" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "terminal_size", + "winapi", +] + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -336,7 +349,7 @@ dependencies = [ [[package]] name = "dkregistry" version = "0.5.1-alpha.0" -source = "git+https://github.com/anti-social/dkregistry-rs?rev=6c132c4#6c132c43002c52081d666d821c49e4a97f58ac55" +source = "git+https://github.com/anti-social/dkregistry-rs?rev=e720262#e72026207e7f15d68a58a670c62285bd4c4c1929" dependencies = [ "async-stream", "base64", @@ -379,6 +392,12 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoding_rs" version = "0.8.30" @@ -807,6 +826,18 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "indicatif" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d207dc617c7a380ab07ff572a6e52fa202a2a8f355860ac9c38e23f8196be1b" +dependencies = [ + "console", + "lazy_static", + "number_prefix", + "regex", +] + [[package]] name = "instant" version = "0.1.12" @@ -1100,6 +1131,12 @@ dependencies = [ "libc", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "object" version = "0.27.1" @@ -1730,6 +1767,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "terminal_size" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "thiserror" version = "1.0.30" @@ -1946,6 +1993,7 @@ dependencies = [ "futures 0.3.18", "git2", "humantime", + "indicatif", "itertools", "lazy_static", "libc", diff --git a/Cargo.toml b/Cargo.toml index 702615ee..f79fb15c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,9 +51,10 @@ serde_json = "1.0.2" serde_derive = "1.0.11" failure = "0.1.1" resolv-conf = "0.6.0" -dkregistry = { git="https://github.com/anti-social/dkregistry-rs", rev="6c132c4" } +dkregistry = { git="https://github.com/anti-social/dkregistry-rs", rev="e720262" } tokio = { version = "1", features = ["full"] } futures = "0.3.18" +indicatif = "0.16.2" [features] default = ["containers"] diff --git a/src/builder/commands/docker.rs b/src/builder/commands/docker.rs index c0c6eb56..2fd5d275 100644 --- a/src/builder/commands/docker.rs +++ b/src/builder/commands/docker.rs @@ -9,6 +9,9 @@ use dkregistry::v2::Client as RegistryClient; #[cfg(feature="containers")] use futures::stream::StreamExt; +#[cfg(feature="containers")] +use indicatif::{ProgressBar, ProgressStyle}; + #[cfg(feature="containers")] use tar::{Entry, EntryType}; @@ -355,15 +358,39 @@ async fn download_blob( let blob_tmp_path = layers_cache.join(&blob_tmp_file_name); println!("Downloading docker layer: {}", &layer_digest); - let mut blob_stream = client.get_blob_stream(image, layer_digest).await + let blob_resp = client.get_blob_response(image, layer_digest).await .map_err(|e| format!("Error getting docker blob response: {}", e))?; + let blob_size = blob_resp.size(); + let mut blob_stream = blob_resp.stream(); let mut blob_file = tokio::fs::File::create(&blob_tmp_path).await .map_err(|e| format!("Cannot create layer file: {}", e))?; + + let progress = if let Some(blob_size) = blob_size { + ProgressBar::new(blob_size) + .with_style( + ProgressStyle::default_bar() + .template("{msg}: {percent}%[{bar:40}] {bytes}/{total_bytes} {bytes_per_sec} {eta}") + .progress_chars("=> ") + ) + } else { + ProgressBar::new_spinner() + .with_style( + ProgressStyle::default_bar() + .template("{msg}: [{spinner:40}] {bytes} {bytes_per_sec}") + .progress_chars("=> ") + ) + }; + progress.set_message(short_digest.to_string()); + progress.set_draw_rate(5); + while let Some(chunk) = blob_stream.next().await { let chunk = chunk.map_err(|e| format!("Error fetching layer chunk: {}", e))?; blob_file.write_all(&chunk).await .map_err(|e| format!("Cannot write blob file: {}", e))?; + progress.inc(chunk.len() as u64); } + progress.finish_and_clear(); + tokio::fs::rename(&blob_tmp_path, &blob_path).await .map_err(|e| format!("Cannot rename docker blob: {}", e))?; } @@ -374,4 +401,4 @@ async fn download_blob( Err(e) => return Err(format!("{}", e)), } Ok(blob_path) -} \ No newline at end of file +}