diff --git a/Cargo.lock b/Cargo.lock index 2108260e..4598308e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + [[package]] name = "aho-corasick" version = "0.7.18" @@ -50,6 +56,27 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" +[[package]] +name = "async-stream" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171374e7e3b2504e0e5236e3b59260560f9fe94bfe9ac39ba5e4e929c5590625" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "648ed8c8d2ce5409ccd57453d9d1b214b342a0d69376a6feda1fd6cae3299308" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atty" version = "0.2.14" @@ -82,6 +109,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + [[package]] name = "bitflags" version = "1.3.2" @@ -95,7 +128,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a4e37d16930f5459780f5621038b6382b9bb37c19016f39fb6b5808d831f174" dependencies = [ "crypto-mac", - "digest", + "digest 0.9.0", "opaque-debug", ] @@ -110,7 +143,7 @@ dependencies = [ "cc", "cfg-if 1.0.0", "constant_time_eq", - "digest", + "digest 0.9.0", ] [[package]] @@ -122,6 +155,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1d36a02058e76b040de25a4464ba1c80935655595b661505c8b39b664828b95" +dependencies = [ + "generic-array", +] + [[package]] name = "bstr" version = "0.2.17" @@ -131,6 +173,18 @@ dependencies = [ "memchr", ] +[[package]] +name = "bumpalo" +version = "3.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1e260c3a9040a7c19a12468758f4c16f31a81a1fe087482be9570ec864bb6c" + +[[package]] +name = "bytes" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" + [[package]] name = "bzip2" version = "0.3.3" @@ -179,12 +233,41 @@ dependencies = [ "bitflags", ] +[[package]] +name = "console" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28b32d32ca44b70c3e4acd7db1babf555fa026e385fb95f18028f88848b3c31" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "terminal_size", + "winapi", +] + [[package]] name = "constant_time_eq" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +[[package]] +name = "core-foundation" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6888e10551bb93e424d8df1d07f1a8b4fceb0001a3a4b048bfc47554946f47b3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + [[package]] name = "cpufeatures" version = "0.2.1" @@ -203,6 +286,15 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "crypto-common" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567569e659735adb39ff2d4c20600f7cd78be5471f8c58ab162bce3c03fdbc5f" +dependencies = [ + "generic-array", +] + [[package]] name = "crypto-mac" version = "0.8.0" @@ -222,6 +314,17 @@ dependencies = [ "generic-array", ] +[[package]] +name = "digest" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8549e6bfdecd113b7e221fe60b433087f6957387a20f8118ebca9b12af19143d" +dependencies = [ + "block-buffer 0.10.0", + "crypto-common", + "generic-array", +] + [[package]] name = "dir-signature" version = "0.2.9" @@ -230,9 +333,9 @@ dependencies = [ "argparse", "blake2", "blake3", - "digest", + "digest 0.9.0", "env_logger", - "futures", + "futures 0.1.31", "futures-cpupool", "generic-array", "itertools", @@ -240,7 +343,35 @@ dependencies = [ "num_cpus", "openat", "quick-error", - "sha2", + "sha2 0.9.8", +] + +[[package]] +name = "dkregistry" +version = "0.5.1-alpha.0" +source = "git+https://github.com/anti-social/dkregistry-rs?rev=e720262#e72026207e7f15d68a58a670c62285bd4c4c1929" +dependencies = [ + "async-stream", + "base64", + "bytes", + "futures 0.3.18", + "http", + "libflate", + "log", + "mime", + "pin-project", + "regex", + "reqwest", + "serde", + "serde_ignored", + "serde_json", + "sha2 0.10.0", + "strum", + "strum_macros", + "tar", + "thiserror", + "tokio", + "url 2.2.2", ] [[package]] @@ -261,6 +392,21 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + +[[package]] +name = "encoding_rs" +version = "0.8.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dc8abb250ffdda33912550faa54c88ec8b998dec0b2c55ab224921ce11df" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "env_logger" version = "0.5.13" @@ -326,6 +472,31 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding 2.1.0", +] + [[package]] name = "fuchsia-cprng" version = "0.1.1" @@ -338,16 +509,105 @@ version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" +[[package]] +name = "futures" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cd0210d8c325c245ff06fd95a3b13689a1a276ac8cfa8e8720cb840bfb84b9e" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fc8cd39e3dbf865f7340dce6a2d401d24fd37c6fe6c4f0ee0de8bfca2252d27" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "629316e42fe7c2a0b9a65b47d159ceaa5453ab14e8f0a3c5eedbb8cd55b4a445" + [[package]] name = "futures-cpupool" version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4" dependencies = [ - "futures", + "futures 0.1.31", "num_cpus", ] +[[package]] +name = "futures-executor" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b808bf53348a36cab739d7e04755909b9fcaaa69b7d7e588b37b6ec62704c97" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e481354db6b5c353246ccf6a728b0c5511d752c08da7260546fc0933869daa11" + +[[package]] +name = "futures-macro" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89f17b21645bc4ed773c69af9c9a0effd4a3f1a3876eadd453469f8854e7fdd" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "996c6442437b62d21a32cd9906f9c41e7dc1e19a9579843fad948696769305af" + +[[package]] +name = "futures-task" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dabf1872aaab32c886832f2276d2f5399887e2bd613698a02359e4ea83f8de12" + +[[package]] +name = "futures-util" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d22213122356472061ac0f1ab2cee28d2bac8491410fd68c2af53d1cedb83e" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + [[package]] name = "generic-array" version = "0.14.4" @@ -385,7 +645,7 @@ dependencies = [ "libc", "libgit2-sys", "log", - "url", + "url 1.7.2", ] [[package]] @@ -401,6 +661,40 @@ dependencies = [ "regex", ] +[[package]] +name = "h2" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d09bbc040ce3758a0d32a8a910562104a853aea429dbe1a998beb065c2eacb2" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "hermit-abi" version = "0.1.19" @@ -410,6 +704,40 @@ dependencies = [ "libc", ] +[[package]] +name = "http" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1323096b05d41827dadeaee54c9981958c0f94e670bc94ed80037d1a7b8b186b" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acd94fdbe1d4ff688b67b04eee2e17bd50995534a61539e45adfefb45e5e5503" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + [[package]] name = "humannum" version = "0.1.0" @@ -429,6 +757,43 @@ dependencies = [ "quick-error", ] +[[package]] +name = "hyper" +version = "0.14.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436ec0091e4f20e655156a30a0df3770fe2900aa301e548e08446ec794b6953c" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "idna" version = "0.1.5" @@ -440,6 +805,54 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "indicatif" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d207dc617c7a380ab07ff572a6e52fa202a2a8f355860ac9c38e23f8196be1b" +dependencies = [ + "console", + "lazy_static", + "number_prefix", + "regex", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "ipnet" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" + [[package]] name = "itertools" version = "0.7.11" @@ -455,6 +868,15 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" +[[package]] +name = "js-sys" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cc9ffccd38c451a86bf13657df244e9c3f37493cce8e5e21e940963777acc84" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -467,6 +889,26 @@ version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8521a1b57e76b1ec69af7599e75e38e7b7fad6610f037db8c79b127201b5d119" +[[package]] +name = "libflate" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16364af76ebb39b5869bb32c81fa93573267cd8c62bb3474e28d78fac3fb141e" +dependencies = [ + "adler32", + "crc32fast", + "libflate_lz77", +] + +[[package]] +name = "libflate_lz77" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39a734c0493409afcd49deee13c006a04e3586b9761a03543c6272c9c51f2f5a" +dependencies = [ + "rle-decode-fast", +] + [[package]] name = "libgit2-sys" version = "0.7.11" @@ -502,6 +944,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "lock_api" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712a4d093c9976e24e7dbca41db895dabcbac38eb5f4045393d17a95bdfb1109" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.14" @@ -534,6 +985,12 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + [[package]] name = "miniz_oxide" version = "0.4.4" @@ -545,13 +1002,35 @@ dependencies = [ ] [[package]] -name = "mopa" -version = "0.2.2" +name = "mio" +version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a785740271256c230f57462d3b83e52f998433a7062fc18f96d5999474a9f915" +checksum = "8067b404fe97c70829f082dec8bcf4f71225d7eaea1d8645349cb76fa06205cc" +dependencies = [ + "libc", + "log", + "miow", + "ntapi", + "winapi", +] [[package]] -name = "msdos_time" +name = "miow" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" +dependencies = [ + "winapi", +] + +[[package]] +name = "mopa" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a785740271256c230f57462d3b83e52f998433a7062fc18f96d5999474a9f915" + +[[package]] +name = "msdos_time" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aad9dfe950c057b1bfe9c1f2aa51583a8468ef2a5baba2ebbe06d775efeb7729" @@ -560,6 +1039,24 @@ dependencies = [ "winapi", ] +[[package]] +name = "native-tls" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "net2" version = "0.2.37" @@ -597,6 +1094,15 @@ dependencies = [ "void", ] +[[package]] +name = "ntapi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +dependencies = [ + "winapi", +] + [[package]] name = "num-traits" version = "0.1.43" @@ -625,6 +1131,12 @@ dependencies = [ "libc", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "object" version = "0.27.1" @@ -634,6 +1146,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "once_cell" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" + [[package]] name = "opaque-debug" version = "0.3.0" @@ -649,6 +1167,64 @@ dependencies = [ "libc", ] +[[package]] +name = "openssl" +version = "0.10.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" +dependencies = [ + "bitflags", + "cfg-if 1.0.0", + "foreign-types", + "libc", + "once_cell", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" + +[[package]] +name = "openssl-sys" +version = "0.9.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df13d165e607909b363a4757a6f133f8a818a74e9d3a98d09c6128e15fa4c73" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi", +] + [[package]] name = "path-filter" version = "0.1.0" @@ -665,6 +1241,44 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "pin-project" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "576bc800220cc65dac09e99e97b08b358cfab6e17078de8dc5fee223bd2d0c08" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e8fe8163d14ce7f0cdac2e040116f22eac817edabff0be91e8aff7e9accf389" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d31d11c69a6b52a174b42bdc0c30e5e11670f90788b2c471c31c1d17d449443" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkg-config" version = "0.3.22" @@ -822,6 +1436,42 @@ dependencies = [ "winapi", ] +[[package]] +name = "reqwest" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bea77bc708afa10e59905c3d4af7c8fd43c9214251673095ff8b14345fcbc5" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "native-tls", + "percent-encoding 2.1.0", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "tokio-util", + "url 2.2.2", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "resolv-conf" version = "0.6.3" @@ -831,12 +1481,24 @@ dependencies = [ "quick-error", ] +[[package]] +name = "rle-decode-fast" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cabe4fa914dec5870285fa7f71f602645da47c486e68486d2b4ceb4a343e90ac" + [[package]] name = "rustc-demangle" version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" +[[package]] +name = "rustversion" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" + [[package]] name = "ryu" version = "1.0.6" @@ -861,6 +1523,45 @@ dependencies = [ "quick-error", ] +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "security-framework" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525bc1abfda2e1998d152c45cf13e696f76d0a4972310b22fac1658b05df7c87" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9dd14d83160b528b7bfd66439110573efcfbe281b17fc2ca9f39f550d619c7e" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" version = "1.0.130" @@ -881,6 +1582,15 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_ignored" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c2c7d39d14f2f2ea82239de71594782f186fd03501ac81f0ce08e674819ff2f" +dependencies = [ + "serde", +] + [[package]] name = "serde_json" version = "1.0.72" @@ -892,19 +1602,42 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha2" version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b69f9a4c9740d74c5baa3fd2e547f9525fa8088a8a958e0ca2409a514e33f5fa" dependencies = [ - "block-buffer", + "block-buffer 0.9.0", "cfg-if 1.0.0", "cpufeatures", - "digest", + "digest 0.9.0", "opaque-debug", ] +[[package]] +name = "sha2" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d964dd36bb15bcf2f2b35694c072feab74969a54f2bbeec7a2d725d2bdcb6" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest 0.10.0", +] + [[package]] name = "signal" version = "0.6.0" @@ -915,12 +1648,62 @@ dependencies = [ "nix 0.11.1", ] +[[package]] +name = "signal-hook-registry" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" + +[[package]] +name = "smallvec" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ecab6c735a6bb4139c0caafd0cc3635748bbb3acf4550e8138122099251f309" + +[[package]] +name = "socket2" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dc90fe6c7be1a323296982db1836d1ea9e47b6839496dde9a541bc496df3516" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "strsim" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strum" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cae14b91c7d11c9a851d3fbc80a963198998c2a64eec840477fa92d8ce9b70bb" + +[[package]] +name = "strum_macros" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bb0dc7ee9c15cea6199cde9a127fa16a4c5819af85395457ad72d68edc85a38" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + [[package]] name = "subtle" version = "2.4.1" @@ -984,6 +1767,36 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "terminal_size" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "thiserror" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "time" version = "0.1.43" @@ -1009,6 +1822,93 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +[[package]] +name = "tokio" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e992e41e0d2fb9f755b37446f20900f64446ef54874f40a60c78f021ac6144" +dependencies = [ + "autocfg", + "bytes", + "libc", + "memchr", + "mio", + "num_cpus", + "once_cell", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "tokio-macros", + "winapi", +] + +[[package]] +name = "tokio-macros" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9efc1aba077437943f7515666aa2b882dfabfbfdf89c819ea75a8d6e9eaba5e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower-service" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" + +[[package]] +name = "tracing" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105" +dependencies = [ + "cfg-if 1.0.0", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f4ed65637b8390770814083d20756f87bfa2c21bf2f110babdc5438351746e4" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + [[package]] name = "typenum" version = "1.14.0" @@ -1030,6 +1930,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b" + [[package]] name = "unicode-xid" version = "0.2.2" @@ -1052,9 +1958,21 @@ version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a" dependencies = [ - "idna", + "idna 0.1.5", + "matches", + "percent-encoding 1.0.1", +] + +[[package]] +name = "url" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +dependencies = [ + "form_urlencoded", + "idna 0.2.3", "matches", - "percent-encoding", + "percent-encoding 2.1.0", ] [[package]] @@ -1065,14 +1983,17 @@ dependencies = [ "argparse", "blake2", "bzip2", - "digest", + "digest 0.9.0", "dir-signature", + "dkregistry", "docopt", "env_logger", "failure", "flate2", + "futures 0.3.18", "git2", "humantime", + "indicatif", "itertools", "lazy_static", "libc", @@ -1092,13 +2013,14 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "sha2", + "sha2 0.9.8", "signal", "tar", "tempfile", + "tokio", "typenum", "unshare", - "url", + "url 2.2.2", "xz2", "zip", ] @@ -1132,12 +2054,98 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + [[package]] name = "wasi" version = "0.10.2+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" +[[package]] +name = "wasm-bindgen" +version = "0.2.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "632f73e236b219150ea279196e54e610f5dbafa5d61786303d4da54f84e47fce" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a317bf8f9fba2476b4b2c85ef4c4af8ff39c3c7f0cdfeed4f82c34a880aa837b" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e8d7523cb1f2a4c96c1317ca690031b714a51cc14e05f712446691f413f5d39" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56146e7c495528bf6587663bea13a8eb588d39b36b679d83972e1a2dbbdacf9" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7803e0eea25835f8abdc585cd3021b3deb11543c6fe226dcd30b228857c5c5ab" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0237232789cf037d5480773fe568aac745bfe2afbc11a863e97901780a6b47cc" + +[[package]] +name = "web-sys" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38eb105f1c59d9eaa6b5cdc92b859d85b926e82cb2e0945cd0c9259faa6fe9fb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi" version = "0.3.9" @@ -1169,6 +2177,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi", +] + [[package]] name = "xattr" version = "0.2.2" diff --git a/Cargo.toml b/Cargo.toml index 9c1ae32d..f79fb15c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ rand = "0.5.0" argparse = "0.2.1" log = "0.4.0" env_logger = "0.5.6" -url = "1.0.0" +url = "2.2.2" unshare = { version="0.5.0", optional=true } signal = { version="0.6.0", optional=true } mopa = "0.2.2" @@ -51,6 +51,10 @@ serde_json = "1.0.2" serde_derive = "1.0.11" failure = "0.1.1" resolv-conf = "0.6.0" +dkregistry = { git="https://github.com/anti-social/dkregistry-rs", rev="e720262" } +tokio = { version = "1", features = ["full"] } +futures = "0.3.18" +indicatif = "0.16.2" [features] default = ["containers"] diff --git a/docs/build_steps.rst b/docs/build_steps.rst index 8a77461d..2f290e13 100644 --- a/docs/build_steps.rst +++ b/docs/build_steps.rst @@ -26,6 +26,7 @@ of empty container): * :step:`SubConfig` * :step:`Container` * :step:`Tar` +* :step:`DockerImage` Ubuntu Commands =============== @@ -314,6 +315,46 @@ config if you use :step:`SubConfig` or :step:`Container`) # Container built. Now, everything in BuildDeps(wget and curl) is removed from the container. +Docker Commands +=============== + +.. step:: DockerImage + + .. warning:: Following functionality is experimental and is actively developed. + + Downloads and unpacks a docker image. + + Have 2 flavors. A short one:: + + - !DockerImage elasticsearch:7.13.4 + + And a full one:: + + - !DockerImage + image: elasticsearch + tag: 7.13.4 + + Options: + + registry + (default ``index.docker.io``) A registry hostname. + + image + (required) An image name. If namespace is missing ``library`` namespace will be used. + So ``python`` and ``library/python`` values point to the same image. + + tag + (default ``latest``) A tag of the image. + + insecure + (optional) If ``true``, an insecure client will be used. Only ``localhost`` + registry is considered insecure by default. + See also :opt:`docker-insecure-registries` setting. + + path + (default ``\``) A path inside a container where the image should be unpacked. + + Generic Commands ================ diff --git a/docs/settings.rst b/docs/settings.rst index 0251f1d5..5017372d 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -319,5 +319,13 @@ not to do so. being built simultaneously. It makes little sense to enable it on a workstation. +.. opt:: docker-insecure-registries + (default ``[localhost]``) A list of docker insecure registries. An insecure + registry works using only an unencrypted ``http`` protocol. + ``localhost`` is always considered as insecure. +.. opt:: docker-registry-aliases + + (default ``{docker.io: index.docker.io}``) A mapping of docker registry aliases. + Alias for ``docker.io`` is always added to the aliases mapping. diff --git a/src/builder/commands/docker.rs b/src/builder/commands/docker.rs new file mode 100644 index 00000000..2fd5d275 --- /dev/null +++ b/src/builder/commands/docker.rs @@ -0,0 +1,404 @@ +use std::collections::{BTreeMap, HashSet}; +use std::io::{ErrorKind, Read}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +#[cfg(feature="containers")] +use dkregistry::v2::Client as RegistryClient; + +#[cfg(feature="containers")] +use futures::stream::StreamExt; + +#[cfg(feature="containers")] +use indicatif::{ProgressBar, ProgressStyle}; + +#[cfg(feature="containers")] +use tar::{Entry, EntryType}; + +#[cfg(feature="containers")] +use tokio::{ + io::AsyncWriteExt, + sync::oneshot, +}; + + +#[cfg(feature="containers")] +use quire::{ + validate as V, + ast::{Ast, ScalarKind, Tag}, +}; + +#[cfg(feature="containers")] +use crate::{ + builder::commands::tarcmd::TarCmd, + capsule::packages as capsule, + container::util::clean_dir, + file_util::{Dir, Lock, safe_remove}, +}; +use crate::build_step::{BuildStep, Config, Digest, Guard, StepError, VersionError}; + +pub const DEFAULT_REGISTRY_HOST: &str = "index.docker.io"; +const DEFAULT_IMAGE_NAMESPACE: &str = "library"; +const DEFAULT_IMAGE_TAG: &str = "latest"; + +const DOCKER_LAYERS_CACHE_PATH: &str = "/vagga/cache/docker-layers"; + +#[derive(Serialize, Deserialize, Debug)] +pub struct DockerImage { + pub registry: String, + pub image: String, + pub tag: String, + pub insecure: Option, + pub path: PathBuf, +} + +impl DockerImage { + pub fn config() -> V::Structure<'static> { + V::Structure::new() + .member("registry", V::Scalar::new().default(DEFAULT_REGISTRY_HOST)) + .member("image", V::Scalar::new()) + .member("tag", V::Scalar::new().default(DEFAULT_IMAGE_TAG)) + .member("insecure", V::Scalar::new().optional()) + .member("path", V::Directory::new().absolute(true).default("/")) + .parser(parse_image) + } +} + +fn parse_image(ast: Ast) -> BTreeMap { + match ast { + Ast::Scalar(pos, _, _, value) => { + let mut map = BTreeMap::new(); + + let (image, registry) = if let Some((registry, image)) = value.split_once('/') { + if registry == "localhost" || registry.contains(|c| c == '.' || c == ':') { + map.insert( + "registry".to_string(), + Ast::Scalar(pos.clone(), Tag::NonSpecific, ScalarKind::Plain, registry.to_string()) + ); + (image, Some(registry)) + } else { + (value.as_str(), None) + } + } else { + (value.as_str(), None) + }; + + let image = if let Some((image, tag)) = image.rsplit_once(':') { + map.insert( + "tag".to_string(), + Ast::Scalar(pos.clone(), Tag::NonSpecific, ScalarKind::Plain, tag.to_string()) + ); + image + } else { + image + }; + + let image = if !image.contains('/') && registry.is_none() { + format!("{}/{}", DEFAULT_IMAGE_NAMESPACE, image) + } else { + image.to_string() + }; + + map.insert( + "image".to_string(), + Ast::Scalar(pos.clone(), Tag::NonSpecific, ScalarKind::Plain, image) + ); + + map + }, + _ => unreachable!(), + } +} + +impl BuildStep for DockerImage { + fn name(&self) -> &'static str { + "DockerImage" + } + + #[cfg(feature="containers")] + fn hash(&self, _cfg: &Config, hash: &mut Digest) -> Result<(), VersionError> { + hash.field("registry", &self.registry); + hash.field("image", &self.image); + hash.field("tag", &self.tag); + hash.opt_field("insecure", &self.insecure); + hash.field("path", &self.path); + Ok(()) + } + + #[cfg(feature="containers")] + fn build(&self, guard: &mut Guard, _build: bool) -> Result<(), StepError> { + let registry = if let Some(registry) = guard.ctx.settings.docker_registry_aliases.get(&self.registry) { + registry + } else { + &self.registry + }; + let _image; + let image = if registry == DEFAULT_REGISTRY_HOST && !self.image.contains("/") { + _image = format!("library/{}", &self.image); + &_image + } else { + &self.image + }; + let insecure = self.insecure.unwrap_or_else(|| + is_insecure_registry(registry, &guard.ctx.settings.docker_insecure_registries) + ); + if !insecure { + capsule::ensure(&mut guard.ctx.capsule, &[capsule::Https])?; + } + Dir::new(DOCKER_LAYERS_CACHE_PATH) + .recursive(true) + .create() + .map_err(|e| + format!("Cannot create docker layers cache directory: {}", e) + )?; + let dst_path = Path::new("/vagga/root").join(&self.path.strip_prefix("/").unwrap()); + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|e| format!("Error creating tokio runtime: {}", e))? + .block_on(download_and_unpack_image( + registry, insecure, image, &self.tag, &dst_path + ))?; + Ok(()) + } + + fn is_dependent_on(&self) -> Option<&str> { + None + } +} + +fn is_insecure_registry( + registry: &str, insecure_registries: &HashSet +) -> bool { + let registry_host = match registry.split_once(':') { + Some((host, _port)) => host, + None => registry, + }; + insecure_registries.contains(registry_host) +} + +/// See: +/// - https://github.com/moby/moby/blob/v20.10.11/pkg/archive/whiteouts.go +/// - https://github.com/moby/moby/blob/v20.10.11/pkg/archive/diff.go#L131 +#[cfg(feature="containers")] +fn whiteout_entry_handler(entry: &Entry>, dst_path: &Path) -> Result { + let file_name = dst_path.file_name() + .and_then(|fname| fname.to_str()); + let file_name = if let Some(file_name) = file_name { + file_name + } else { + return Ok(false); + }; + + if entry.header().entry_type() != EntryType::Regular { + return Ok(false); + } + + if let Some(whiteout) = file_name.strip_prefix(".wh.") { + let dir = dst_path.parent().unwrap(); + if whiteout == ".wh..opq" { + // TODO: Track and keep files that were unpacked from the current archive + clean_dir(dir, false)? + } else { + let mut whiteout_path = dir.to_path_buf(); + whiteout_path.push(whiteout); + safe_remove(&whiteout_path) + .map_err(|e| format!("Cannot remove {:?} path: {}", &whiteout_path, e))?; + } + return Ok(true); + } + + Ok(false) +} + +#[cfg(feature="containers")] +async fn download_and_unpack_image( + registry: &str, insecure: bool, image: &str, tag: &str, dst_path: &Path +) -> Result<(), StepError> { + let auth_scope = format!("repository:{}:pull", image); + let client = build_client(registry, insecure, &[&auth_scope]).await?; + + println!("Downloading docker image: {}/{}:{}", registry, image, tag); + let manifest = client.get_manifest(&image, &tag).await?; + + let layers_digests = manifest.layers_digests(None)?; + + let mut downloaded_layer_txs = vec!(); + let mut downloaded_layer_rxs = vec!(); + for _ in &layers_digests { + let (tx, rx) = oneshot::channel(); + downloaded_layer_rxs.push(rx); + downloaded_layer_txs.push(tx); + } + + let dst_path = dst_path.to_path_buf(); + let unpack_task = tokio::spawn(async move { + for layer_ch in downloaded_layer_rxs { + match layer_ch.await { + Ok((digest, layer_path)) => { + let dst_path = dst_path.clone(); + if let Err(e) = unpack_layer(digest, layer_path, dst_path).await { + return Err(e); + } + } + Err(_) => { + // Channel is dropped if download task is cancelled + }, + } + } + Ok(()) + }); + + let image = image.to_string(); + let download_task = tokio::spawn(async move { + for (digest, tx) in layers_digests.iter().zip(downloaded_layer_txs.into_iter()) { + let digest = digest.clone(); + let client = client.clone(); + match download_blob(&client, &image, &digest).await { + Ok(layer_path) => { + // Unpack task may be cancelled so ignore sending errors + tx.send((digest.clone(), layer_path)).ok(); + } + Err(e) => { + return Err(e); + } + } + } + Ok(()) + }); + + match download_task.await { + Ok(Ok(_)) => {}, + Ok(Err(client_err)) => { + unpack_task.abort(); + return Err(client_err.into()); + }, + Err(join_err) => { + unpack_task.abort(); + return Err( + format!("Error waiting a download layers task: {}", join_err).into() + ); + }, + } + + match unpack_task.await { + Ok(Ok(_)) => Ok(()), + Ok(Err(unpack_err)) => Err(unpack_err.into()), + Err(join_err) if join_err.is_cancelled() => Ok(()), + Err(join_err) => Err( + format!("Error waiting an unpack layers task: {}", join_err).into() + ), + } +} + +async fn unpack_layer( + digest: String, layer_path: PathBuf, dst_path: PathBuf +) -> Result<(), String> { + let unpack_future_res = tokio::task::spawn_blocking(move || { + println!("Unpacking docker layer: {}", digest); + TarCmd::new(&layer_path, &dst_path) + .preserve_owner(true) + .override_entries(true) + .entry_handler(whiteout_entry_handler) + .unpack() + }).await; + unpack_future_res + .map_err(|e| format!("Error waiting an unpack layer future: {}", e))? + .map_err(|e| format!("Error unpacking docker layer: {}", e)) +} + +#[cfg(feature="containers")] +async fn build_client( + registry: &str, insecure: bool, auth_scopes: &[&str] +) -> Result, StepError> { + let client_config = RegistryClient::configure() + .registry(registry) + .insecure_registry(insecure) + .username(None) + .password(None); + let client = client_config.build()?; + + let client = match client.is_auth().await { + Ok(true) => client, + Ok(false) => client.authenticate(auth_scopes).await?, + Err(e) => return Err(e.into()), + }; + Ok(Arc::new(client)) +} + +#[cfg(feature="containers")] +async fn download_blob( + client: &RegistryClient, image: &str, layer_digest: &str +) -> Result { + let digest = layer_digest.split_once(':') + .ok_or(format!("Invalid layer digest: {}", layer_digest))? + .1; + let short_digest = &digest[..12]; + + let layers_cache = Path::new(DOCKER_LAYERS_CACHE_PATH); + let blob_file_name = format!("{}.tar.gz", digest); + let blob_path = layers_cache.join(&blob_file_name); + match tokio::fs::symlink_metadata(&blob_path).await { + Ok(_) => {} + Err(e) if e.kind() == ErrorKind::NotFound => { + let lock_file_name = format!(".{}.lock", &blob_file_name); + let lock_msg = format!("Another process downloads blob: {}", &short_digest); + let lock_fut = tokio::task::spawn_blocking(move || { + let lockfile = layers_cache.join(lock_file_name); + Lock::exclusive_wait(lockfile, true, &lock_msg) + }); + let _lock = lock_fut.await + .map_err(|e| format!("Error waiting a lock file future: {}", e))? + .map_err(|e| format!("Error taking exclusive lock: {}", e))?; + + match tokio::fs::symlink_metadata(&blob_path).await { + Ok(_) => {} + Err(e) if e.kind() == ErrorKind::NotFound => { + let blob_tmp_file_name = format!(".{}.tmp", &blob_file_name); + let blob_tmp_path = layers_cache.join(&blob_tmp_file_name); + + println!("Downloading docker layer: {}", &layer_digest); + let blob_resp = client.get_blob_response(image, layer_digest).await + .map_err(|e| format!("Error getting docker blob response: {}", e))?; + let blob_size = blob_resp.size(); + let mut blob_stream = blob_resp.stream(); + let mut blob_file = tokio::fs::File::create(&blob_tmp_path).await + .map_err(|e| format!("Cannot create layer file: {}", e))?; + + let progress = if let Some(blob_size) = blob_size { + ProgressBar::new(blob_size) + .with_style( + ProgressStyle::default_bar() + .template("{msg}: {percent}%[{bar:40}] {bytes}/{total_bytes} {bytes_per_sec} {eta}") + .progress_chars("=> ") + ) + } else { + ProgressBar::new_spinner() + .with_style( + ProgressStyle::default_bar() + .template("{msg}: [{spinner:40}] {bytes} {bytes_per_sec}") + .progress_chars("=> ") + ) + }; + progress.set_message(short_digest.to_string()); + progress.set_draw_rate(5); + + while let Some(chunk) = blob_stream.next().await { + let chunk = chunk.map_err(|e| format!("Error fetching layer chunk: {}", e))?; + blob_file.write_all(&chunk).await + .map_err(|e| format!("Cannot write blob file: {}", e))?; + progress.inc(chunk.len() as u64); + } + progress.finish_and_clear(); + + tokio::fs::rename(&blob_tmp_path, &blob_path).await + .map_err(|e| format!("Cannot rename docker blob: {}", e))?; + } + Err(e) => return Err(format!("{}", e)), + } + + } + Err(e) => return Err(format!("{}", e)), + } + Ok(blob_path) +} diff --git a/src/builder/commands/tarcmd.rs b/src/builder/commands/tarcmd.rs index 67a94cc2..d10452e4 100644 --- a/src/builder/commands/tarcmd.rs +++ b/src/builder/commands/tarcmd.rs @@ -4,15 +4,18 @@ use std::fs::{create_dir_all, set_permissions, hard_link}; use std::io::{self, Read, Seek, SeekFrom, BufReader}; use std::os::unix::fs::{PermissionsExt, symlink}; use std::path::{Path, PathBuf}; -use std::u32; #[cfg(feature="containers")] use bzip2::read::BzDecoder; #[cfg(feature="containers")] use flate2::read::GzDecoder; #[cfg(feature="containers")] use libmount::BindMount; #[cfg(feature="containers")] use tar::Archive; #[cfg(feature="containers")] use xz2::read::XzDecoder; + use quire::validate as V; +#[cfg(feature="containers")] +use tar::Entry; + #[cfg(feature="containers")] use crate::{ build_step::{BuildStep, VersionError, StepError, Digest, Config, Guard}, @@ -21,7 +24,7 @@ use crate::{ builder::dns::revert_name_files, capsule::download::{maybe_download_and_check_hashsum}, container::mount::{unmount}, - file_util::{Dir, read_visible_entries, copy_stream, set_owner_group}, + file_util::{Dir, read_visible_entries, copy_stream, safe_remove, set_owner_group}, }; @@ -64,124 +67,245 @@ impl TarInstall { } } +#[cfg(feature="containers")] +pub struct TarCmd<'a> { + archive: &'a Path, + target_dir: &'a Path, + includes: &'a[&'a Path], + excludes: &'a[&'a Path], + preserve_owner: bool, + override_entries: bool, + entry_handler: fn(&Entry>, &Path) -> Result, +} + +const DEFAULT_TAR_INCLUDES: &[&Path] = &[]; +const DEFAULT_TAR_EXCLUDES: &[&Path] = &[]; #[cfg(feature="containers")] -pub fn unpack_file(_ctx: &mut Context, src: &Path, tgt: &Path, - includes: &[&Path], excludes: &[&Path], preserve_owner: bool) - -> Result<(), String> -{ +fn dummy_entry_handler(_entry: &Entry>, _dst_path: &Path) -> Result { + Ok(false) +} - info!("Unpacking {:?} -> {:?}", src, tgt); - let read_err = |e| format!("Error reading {:?}: {}", src, e); +#[cfg(feature="containers")] +impl<'a> TarCmd<'a> { + pub fn new(archive: &'a Path, target_dir: &'a Path) -> Self { + Self { + archive, + target_dir, + includes: DEFAULT_TAR_INCLUDES, + excludes: DEFAULT_TAR_EXCLUDES, + preserve_owner: false, + override_entries: false, + entry_handler: dummy_entry_handler, + } + } - let mut file = BufReader::new(File::open(&src).map_err(&read_err)?); + pub fn includes(mut self, includes: &'a[&'a Path]) -> Self { + self.includes = includes; + self + } - let mut buf = [0u8; 8]; - let nbytes = file.read(&mut buf).map_err(&read_err)?; - file.seek(SeekFrom::Start(0)).map_err(&read_err)?; - let magic = &buf[..nbytes]; - if magic.len() >= 2 && magic[..2] == [0x1f, 0x8b] { - return unpack_stream(GzDecoder::new(file), - src, tgt, includes, excludes, preserve_owner); - } else if magic.len() >= 6 && magic[..6] == - [ 0xFD, b'7', b'z', b'X', b'Z', 0x00] - { - return unpack_stream(XzDecoder::new(file), - src, tgt, includes, excludes, preserve_owner); - } else if magic.len() >= 3 && magic[..3] == [ b'B', b'Z', b'h'] { - return unpack_stream(BzDecoder::new(file), - src, tgt, includes, excludes, preserve_owner); - } else { - return Err(format!("unpacking {:?}: unexpected compression", src)); + pub fn excludes(mut self, excludes: &'a[&'a Path]) -> Self { + self.excludes = excludes; + self } -} + pub fn preserve_owner(mut self, preserve_order: bool) -> Self { + self.preserve_owner = preserve_order; + self + } -#[cfg(feature="containers")] -fn unpack_stream(file: F, srcpath: &Path, tgt: &Path, - includes: &[&Path], excludes: &[&Path], preserve_owner: bool) - -> Result<(), String> -{ - let read_err = |e| format!("Error reading {:?}: {}", srcpath, e); - let mut arc = Archive::new(file); - let mut hardlinks = Vec::new(); - - for item in arc.entries().map_err(&read_err)? { - let mut src = item.map_err(&read_err)?; - let path_ref = src.path().map_err(&read_err)? - .to_path_buf(); - let mut orig_path: &Path = &path_ref; - if orig_path.is_absolute() { - orig_path = orig_path.strip_prefix("/").unwrap(); + pub fn override_entries(mut self, override_entries: bool) -> Self { + self.override_entries = override_entries; + self + } + + pub fn entry_handler( + self, + entry_handler: fn(&Entry>, &Path) -> Result + ) -> TarCmd<'a> { + TarCmd { + archive: self.archive, + target_dir: self.target_dir, + includes: self.includes, + excludes: self.excludes, + preserve_owner: self.preserve_owner, + override_entries: self.override_entries, + entry_handler, } - if includes.len() > 0 { - if !includes.iter().any(|x| orig_path.starts_with(x)) { + } + + pub fn unpack(&self) -> Result<(), String> { + info!("Unpacking {:?} -> {:?}", self.archive, self.target_dir); + let read_err = |e| format!("Error reading {:?}: {}", self.archive, e); + + let mut file = BufReader::new( + File::open(&self.archive) + .map_err(&read_err)? + ); + + let mut buf = [0u8; 8]; + let nbytes = file.read(&mut buf).map_err(&read_err)?; + file.seek(SeekFrom::Start(0)).map_err(&read_err)?; + let magic = &buf[..nbytes]; + let reader = if magic.len() >= 2 && magic[..2] == [0x1f, 0x8b] { + Box::new(GzDecoder::new(file)) as Box + } else if magic.len() >= 6 && magic[..6] == + [ 0xFD, b'7', b'z', b'X', b'Z', 0x00] + { + Box::new(XzDecoder::new(file)) as Box + } else if magic.len() >= 3 && magic[..3] == [ b'B', b'Z', b'h'] { + Box::new(BzDecoder::new(file)) as Box + } else { + return Err(format!("unpacking {:?}: unexpected compression", self.archive)); + }; + self.unpack_stream(reader) + } + + fn unpack_stream(&self, file: Box) -> Result<(), String> { + let read_err = |e| format!("Error reading {:?}: {}", self.archive, e); + let mut arc = Archive::new(file); + let mut hardlinks = Vec::new(); + + for item in arc.entries().map_err(&read_err)? { + let mut src = item.map_err(&read_err)?; + let path_ref = src.path().map_err(&read_err)? + .to_path_buf(); + let mut orig_path: &Path = &path_ref; + if orig_path.is_absolute() { + orig_path = orig_path.strip_prefix("/").unwrap(); + } + if self.includes.len() > 0 { + if !self.includes.iter().any(|x| orig_path.starts_with(x)) { + continue; + } + } + if self.excludes.iter().any(|x| orig_path.starts_with(x)) { continue; } - } - if excludes.iter().any(|x| orig_path.starts_with(x)) { - continue; - } - let path = tgt.join(orig_path); - let write_err = |e| format!("Error writing {:?}: {}", path, e); - let entry = src.header().entry_type(); - - // Some archives don't have uids - // TODO(tailhook) should this be handled in tar-rs? - let uid = min(src.header().uid().unwrap_or(0), u32::MAX as u64) as u32; - let gid = min(src.header().gid().unwrap_or(0), u32::MAX as u64) as u32; - - if entry.is_dir() { - let mode = src.header().mode().map_err(&read_err)?; - let mut dir_builder = Dir::new(&path); - dir_builder.recursive(true).mode(mode); - if preserve_owner { - dir_builder.uid(uid).gid(gid); + let path = self.target_dir.join(orig_path); + let write_err = |e| format!("Error writing {:?}: {}", path, e); + let entry = src.header().entry_type(); + + // Some archives don't have uids + // TODO(tailhook) should this be handled in tar-rs? + let uid = min(src.header().uid().unwrap_or(0), u32::MAX as u64) as u32; + let gid = min(src.header().gid().unwrap_or(0), u32::MAX as u64) as u32; + + if (self.entry_handler)(&src, &path)? { + continue; } - dir_builder.create().map_err(&write_err)?; - } else if entry.is_symlink() { - let src = src.link_name().map_err(&read_err)? - .ok_or(format!("Error unpacking {:?}, broken symlink", path))?; - match symlink(&src, &path) { - Ok(_) => {}, - Err(e) => { - if e.kind() == io::ErrorKind::NotFound { - if let Some(parent) = path.parent() { - Dir::new(parent).recursive(true).create() - .map_err(&write_err)?; - symlink(&src, &path).map_err(&write_err)? - } else { - return Err(write_err(e)); + + use tar::EntryType::*; + match entry { + Directory => { + if self.override_entries { + match path.symlink_metadata() { + Ok(stat) if stat.is_dir() => {} + Ok(_) => { + safe_remove(&path) + .map_err(|e| format!("Cannot remove {:?} path: {}", &path, e))?; + } + Err(e) if e.kind() == io::ErrorKind::NotFound => {} + Err(e) => { + return Err(format!("Cannot stat {:?} path: {}", &path, e)); + } } - } else { - return Err(write_err(e)); } + let mode = src.header().mode().map_err(&read_err)?; + let mut dir_builder = Dir::new(&path); + dir_builder.recursive(true).mode(mode); + if self.preserve_owner { + dir_builder.uid(uid).gid(gid); + } + dir_builder.create().map_err(&write_err)?; } + Regular => { + if self.override_entries { + safe_remove(&path) + .map_err(|e| format!("Cannot remove {:?} path: {}", &path, e))?; + } + // TODO: Should we allow truncate a file here? + let mut dest = match File::create(&path) { + Ok(x) => x, + Err(e) => { + if e.kind() == io::ErrorKind::NotFound { + if let Some(parent) = path.parent() { + Dir::new(parent).recursive(true).create() + .map_err(&write_err)?; + File::create(&path).map_err(&write_err)? + } else { + return Err(write_err(e)); + } + } else { + return Err(write_err(e)); + } + } + }; + copy_stream(&mut src, &mut dest) + .map_err(|e| + format!("Error unpacking {:?} -> {:?}: {}", self.archive, path, e) + )?; + let mode = src.header().mode().map_err(&read_err)?; + set_permissions(&path, Permissions::from_mode(mode)) + .map_err(&write_err)?; + if self.preserve_owner { + set_owner_group(&path, uid, gid).map_err(&write_err)?; + } + } + Symlink => { + if self.override_entries { + safe_remove(&path) + .map_err(|e| format!("Cannot remove {:?} path: {}", &path, e))?; + } + let src = src.link_name().map_err(&read_err)? + .ok_or(format!("Error unpacking {:?}, broken symlink", path))?; + match symlink(&src, &path) { + Ok(_) => {}, + Err(e) => { + if e.kind() == io::ErrorKind::NotFound { + if let Some(parent) = path.parent() { + Dir::new(parent).recursive(true).create() + .map_err(&write_err)?; + symlink(&src, &path).map_err(&write_err)? + } else { + return Err(write_err(e)); + } + } else { + return Err(write_err(e)); + } + } + }; + } + Link => { + let link = src.link_name().map_err(&read_err)? + .ok_or(format!("Error unpacking {:?}, broken hardlink", path))?; + let link = if link.is_absolute() { + link.strip_prefix("/").unwrap() + } else { + &*link + }; + hardlinks.push((self.target_dir.join(link).to_path_buf(), path.to_path_buf())); + } + _ => {} + } + } + for (src, dst) in hardlinks.into_iter() { + let write_err = |e| { + format!("Error hardlinking {:?} - {:?}: {}", &src, &dst, e) }; - } else if entry.is_hard_link() { - let link = src.link_name().map_err(&read_err)? - .ok_or(format!("Error unpacking {:?}, broken symlink", path))?; - let link = if link.is_absolute() { - link.strip_prefix("/").unwrap() - } else { - &*link - }; - hardlinks.push((tgt.join(link).to_path_buf(), path.to_path_buf())); - } else if entry.is_pax_global_extensions() || - entry.is_pax_local_extensions() || - entry.is_gnu_longname() || - entry.is_gnu_longlink() - { - // nothing to do - } else { - let mut dest = match File::create(&path) { - Ok(x) => x, + if self.override_entries { + safe_remove(&dst) + .map_err(|e| format!("Cannot remove {:?} path: {}", &dst, e))?; + } + match hard_link(&src, &dst) { + Ok(_) => {}, Err(e) => { if e.kind() == io::ErrorKind::NotFound { - if let Some(parent) = path.parent() { + if let Some(parent) = dst.parent() { Dir::new(parent).recursive(true).create() .map_err(&write_err)?; - File::create(&path).map_err(&write_err)? + hard_link(&src, &dst).map_err(&write_err)? } else { return Err(write_err(e)); } @@ -190,39 +314,21 @@ fn unpack_stream(file: F, srcpath: &Path, tgt: &Path, } } }; - copy_stream(&mut src, &mut dest).map_err(|e| - format!("Error unpacking {:?} -> {:?}: {}", - srcpath, path, e))?; - let mode = src.header().mode().map_err(&read_err)?; - set_permissions(&path, Permissions::from_mode(mode)) - .map_err(&write_err)?; - if preserve_owner { - set_owner_group(&path, uid, gid).map_err(&write_err)?; - } } + Ok(()) } - for (src, dst) in hardlinks.into_iter() { - let write_err = |e| { - format!("Error hardlinking {:?} - {:?}: {}", &src, &dst, e) - }; - match hard_link(&src, &dst) { - Ok(_) => {}, - Err(e) => { - if e.kind() == io::ErrorKind::NotFound { - if let Some(parent) = dst.parent() { - Dir::new(parent).recursive(true).create() - .map_err(&write_err)?; - hard_link(&src, &dst).map_err(&write_err)? - } else { - return Err(write_err(e)); - } - } else { - return Err(write_err(e)); - } - } - }; - } - Ok(()) +} + +#[cfg(feature="containers")] +pub fn unpack_file(_ctx: &mut Context, src: &Path, tgt: &Path, + includes: &[&Path], excludes: &[&Path], preserve_owner: bool) + -> Result<(), String> +{ + TarCmd::new(src, tgt) + .includes(includes) + .excludes(excludes) + .preserve_owner(preserve_owner) + .unpack() } #[cfg(feature="containers")] diff --git a/src/builder/error.rs b/src/builder/error.rs index d1eef4a1..308c819f 100644 --- a/src/builder/error.rs +++ b/src/builder/error.rs @@ -100,6 +100,14 @@ quick_error! { from() display("{}", err) } + DockerClientError(err: dkregistry::errors::Error) { + from() + display("{}", err) + } + DownloadDockerLayersErrors(errors: Vec) { + from() + display("{:?}", errors) + } } } diff --git a/src/builder/guard.rs b/src/builder/guard.rs index 0b4f81e8..6adc224b 100644 --- a/src/builder/guard.rs +++ b/src/builder/guard.rs @@ -1,5 +1,5 @@ use std::path::Path; -use std::fs::{remove_dir_all, remove_file, symlink_metadata}; +use std::fs::remove_dir_all; use std::collections::HashMap; use crate::build_step::BuildStep; @@ -12,7 +12,7 @@ use crate::container::mount::{ unmount, mount_system_dirs, mount_proc, mount_run, unmount_system_dirs, }; use crate::container::util::{clean_dir, write_container_signature}; -use crate::file_util::{Dir, copy, truncate_file}; +use crate::file_util::{Dir, copy, truncate_file, safe_remove}; use crate::path_util::IterSelfAndParents; @@ -198,19 +198,10 @@ fn remove_all_except(root: &Path, keep_rel_paths: &HashMap<&Path, bool>) remove_all_except(path, keep_rel_paths)?; }, None => { - let file_type = try_msg!( - symlink_metadata(path), - "Error querying file metadata: {path:?}: {err}", path=path - ).file_type(); - if file_type.is_dir() { - try_msg!(clean_dir(path, true), - "Error cleaning dir {path:?}: {err}", - path=path); - } else { - try_msg!(remove_file(path), - "Error removing file {path:?}: {err}", - path=path); - } + try_msg!( + safe_remove(path), + "Cannot remove {path:?}: {err}", path=path + ); }, } } diff --git a/src/builder/mod.rs b/src/builder/mod.rs index 3613c75a..44f24286 100644 --- a/src/builder/mod.rs +++ b/src/builder/mod.rs @@ -33,6 +33,7 @@ pub mod commands { pub mod packaging; pub mod tarcmd; pub mod unzip; + pub mod docker; } #[cfg(feature="containers")] pub mod guard; #[cfg(feature="containers")] mod packages; diff --git a/src/config/builders.rs b/src/config/builders.rs index 47044253..1b9c0ede 100644 --- a/src/config/builders.rs +++ b/src/config/builders.rs @@ -126,6 +126,7 @@ define_commands! { composer::ComposerInstall, composer::ComposerDependencies, composer::ComposerConfig, + docker::DockerImage, } pub struct NameVisitor; diff --git a/src/config/read_settings.rs b/src/config/read_settings.rs index 591c5382..defe124e 100644 --- a/src/config/read_settings.rs +++ b/src/config/read_settings.rs @@ -1,10 +1,11 @@ use std::env; -use std::collections::{BTreeMap, HashMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::path::{Path, PathBuf}; use quire::{parse_config, parse_string, Options}; use quire::validate as V; +use crate::builder::commands::docker::DEFAULT_REGISTRY_HOST; use crate::config::Settings; use crate::path_util::Expand; @@ -33,6 +34,8 @@ struct SecureSettings { disable_auto_clean: Option, environ: BTreeMap, propagate_environ: BTreeSet, + docker_insecure_registries: HashSet, + docker_registry_aliases: HashMap, } pub fn secure_settings_validator<'a>(has_children: bool) @@ -62,7 +65,10 @@ pub fn secure_settings_validator<'a>(has_children: bool) .member("disable_auto_clean", V::Scalar::new().optional()) .member("environ", V::Mapping::new( V::Scalar::new(), V::Scalar::new())) - .member("propagate_environ", V::Sequence::new(V::Scalar::new())); + .member("propagate_environ", V::Sequence::new(V::Scalar::new())) + .member("docker_insecure_registries", V::Sequence::new(V::Scalar::new())) + .member("docker_registry_aliases", V::Mapping::new( + V::Scalar::new(), V::Scalar::new())); if has_children { s = s.member("site_settings", V::Mapping::new( V::Scalar::new(), @@ -181,6 +187,12 @@ fn merge_settings(cfg: SecureSettings, project_root: &Path, for item in &cfg.propagate_environ { ext_settings.propagate_environ.insert(item.clone()); } + for registry in &cfg.docker_insecure_registries { + int_settings.docker_insecure_registries.insert(registry.clone()); + } + for (alias, registry) in &cfg.docker_registry_aliases { + int_settings.docker_registry_aliases.insert(alias.clone(), registry.clone()); + } if let Some(cfg) = cfg.site_settings.get(project_root) { if let Some(ref dir) = cfg.storage_dir { ext_settings.storage_dir = Some(dir.clone()); @@ -273,6 +285,16 @@ pub fn read_settings(project_root: &Path) disable_auto_clean: false, environ: BTreeMap::new(), storage_subdir_from_env_var: None, + docker_insecure_registries: { + let mut registries = HashSet::new(); + registries.insert("localhost".to_string()); + registries + }, + docker_registry_aliases: { + let mut aliases = HashMap::new(); + aliases.insert("docker.io".to_string(), DEFAULT_REGISTRY_HOST.to_string()); + aliases + }, }; let mut secure_files = vec!(); if let Ok(home) = env::var("_VAGGA_HOME") { diff --git a/src/config/settings.rs b/src/config/settings.rs index 6820528e..a8e46136 100644 --- a/src/config/settings.rs +++ b/src/config/settings.rs @@ -1,5 +1,5 @@ use std::str::FromStr; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap, HashSet}; use libc::{uid_t, gid_t}; use serde_json; @@ -29,6 +29,8 @@ pub struct Settings { pub run_symlinks_as_commands: bool, pub disable_auto_clean: bool, pub storage_subdir_from_env_var: Option, + pub docker_insecure_registries: HashSet, + pub docker_registry_aliases: HashMap, } impl Settings { diff --git a/src/file_util.rs b/src/file_util.rs index 907100e0..3db47860 100644 --- a/src/file_util.rs +++ b/src/file_util.rs @@ -684,7 +684,13 @@ pub fn set_times>(path: P, atime: i64, mtime: i64) pub fn safe_remove>(path: P) -> io::Result<()> { let path = path.as_ref(); path.symlink_metadata() - .and_then(|_| fs::remove_file(path)) + .and_then(|stat| { + if stat.is_dir() { + fs::remove_dir_all(path) + } else { + fs::remove_file(path) + } + }) .or_else(|e| { if e.kind() == io::ErrorKind::NotFound { Ok(()) diff --git a/tests/docker.bats b/tests/docker.bats new file mode 100644 index 00000000..78496b73 --- /dev/null +++ b/tests/docker.bats @@ -0,0 +1,61 @@ +setup() { + load '/bats/bats-support/load.bash' + load '/bats/bats-assert/load.bash' + cd /work/tests/docker +} + +@test "docker: hello-world" { + run vagga _build hello + assert_success + + run vagga hello + assert_success + assert_line "Hello from Docker!" +} + +@test "docker: python" { + run vagga _build python + assert_success + + run vagga zen + assert_success + assert_line "The Zen of Python, by Tim Peters" +} + +@test "docker: java" { + run vagga _build java + assert_success + + run vagga _run java /usr/local/openjdk-17/bin/java -version + assert_success + assert_line -p "17.0.1" +} + +@test "docker: registry" { + run vagga _build buildah + assert_success + run vagga _build registry + assert_success + run vagga _build test-image + assert_equal "$status" 121 + + run vagga buildah --version + assert_success + + run vagga build-test-image + assert_success + + run vagga --isolate-network push-and-build-test-image + assert_success + link=$(readlink .vagga/test-image) + assert_equal "$link" ".roots/test-image.f62f1455/root" + [[ ! -e .vagga/test-image/will-be-deleted ]] + [[ -f .vagga/test-image/test/vagga.yaml ]] + [[ ! -e .vagga/test-image/test/Dockerfile ]] + assert_equal "$(cat .vagga/test-image/hello.txt)" "Hello world!" + [[ -L .vagga/test-image/hi.txt ]] + assert_equal "$(cat .vagga/test-image/hi.txt)" "Hello world!" + assert_equal "$(cat .vagga/test-image/see-you.txt)" "Bye-bye!" + assert_equal "$(stat -c '%i' .vagga/test-image/see-you.txt)" "$(stat -c '%i' .vagga/test-image/bye-bye.txt)" + [[ -f .vagga/test-image/empty ]] +} diff --git a/tests/docker/.dockerignore b/tests/docker/.dockerignore new file mode 100644 index 00000000..079f8aa1 --- /dev/null +++ b/tests/docker/.dockerignore @@ -0,0 +1 @@ +.vagga diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile new file mode 100644 index 00000000..c35349a2 --- /dev/null +++ b/tests/docker/Dockerfile @@ -0,0 +1,24 @@ +FROM busybox + +COPY Dockerfile vagga.yaml /will-be-deleted/ +COPY Dockerfile vagga.yaml /test/ + +# Test whiteouts +RUN rm -rf /will-be-deleted +RUN rm -f /test/Dockerfile + +# Override file +RUN echo -n 'Hello' > /hello.txt +RUN echo -n ' world!' >> /hello.txt + +# Override symlink +RUN ln -s nonexistent /hi.txt +RUN rm -r /hi.txt && ln -s hello.txt /hi.txt + +# Override hardlink +RUN echo -n 'Bye!' > /bye.txt && ln /bye.txt /see-you.txt +RUN echo -n 'Bye-bye!' > /bye-bye.txt && ln -f /bye-bye.txt /see-you.txt + +# Override directory +RUN mkdir /empty +RUN rmdir /empty && touch /empty diff --git a/tests/docker/vagga.yaml b/tests/docker/vagga.yaml new file mode 100644 index 00000000..8b0af850 --- /dev/null +++ b/tests/docker/vagga.yaml @@ -0,0 +1,98 @@ +containers: + hello: + setup: + - !DockerImage + image: hello-world + + python: + setup: + - !DockerImage + image: library/python + tag: 3.10-slim + + java: + setup: + - !DockerImage docker.io/library/openjdk:17.0.1-slim-buster + + registry: + setup: + - !DockerImage registry:2.7.1 + - !EnsureDir /var/lib/registry + volumes: + /var/lib/registry: !Persistent registry + + buildah: + setup: + - !Ubuntu focal + - !UbuntuUniverse + - !Install [add-apt-key, ca-certificates] + - !Download + url: https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_20.04/Release.key + path: /tmp/Release.key + - !Sh apt-key add - < /tmp/Release.key + - !UbuntuRepo + url: http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_20.04/ + suite: "" + components: ["/"] + - !Install [buildah, fuse-overlayfs, netcat] + - !Text + /etc/containers/registries.conf: | + unqualified-search-registries = ["docker.io"] + - !Sh | + sed -i 's|#mount_program.*|mount_program = "/usr/bin/fuse-overlayfs"|' /etc/containers/storage.conf + # FIXME + - !Sh | + rm -f /etc/resolv.conf + touch /etc/resolv.conf + - !EnsureDir /usr/lib/vagga + - !EnsureDir /var/lib/containers + - !EnsureDir /var/cache/containers + environ: + PATH: /usr/lib/vagga:/sbin:/bin:/usr/sbin:/usr/bin + volumes: + /usr/lib/vagga: !VaggaBin + /var/lib/containers: !Persistent buildah + /var/cache/containers: !Tmpfs + /var/tmp: !Tmpfs + + test-image: + setup: + - !DockerImage localhost:5000/test + +commands: + hello: !Command + container: hello + run: [/hello] + + zen: !Command + container: python + run: python -m this + + buildah: !Command + container: buildah + run: [buildah] + + build-test-image: !Command + container: buildah + run: | + set -eux + + # By default it uses oci format which is not supported by dkregistry yet + buildah bud --format docker --layers -t localhost:5000/test:latest -f Dockerfile + + push-and-build-test-image: !Supervise + children: + registry: !Command + container: registry + run: [/entrypoint.sh, /etc/docker/registry/config.yml] + build-and-push: !Command + container: buildah + run: | + set -eux + + while ! nc -z localhost 5000; do + sleep 0.5 + done + buildah push --tls-verify=false localhost:5000/test:latest + + vagga _build test-image diff --git a/vagga.yaml b/vagga.yaml index 7c26dd31..7d582d39 100644 --- a/vagga.yaml +++ b/vagga.yaml @@ -11,6 +11,10 @@ containers: environ: &rustenv LD_LIBRARY_PATH: /musl/lib/rustlib/x86_64-unknown-linux-musl/lib PATH: /musl/bin:/usr/local/bin:/usr/bin:/bin + HOME: /work/target + PKG_CONFIG_ALLOW_CROSS: 1 + OPENSSL_STATIC: true + OPENSSL_DIR: /usr/lib/x86_64-linux-musl setup: - !Ubuntu focal - !UbuntuUniverse @@ -29,6 +33,25 @@ containers: - !TarInstall url: "https://static.rust-lang.org/dist/rust-std-1.57.0-wasm32-unknown-unknown.tar.gz" script: "./install.sh --prefix=/usr --components=rust-std-wasm32-unknown-unknown" + + # OpenSSL + # https://qiita.com/liubin/items/6c94f0b61f746c08b74c + - !Sh | + ln -s /usr/include/x86_64-linux-gnu/asm /usr/include/x86_64-linux-musl/asm + ln -s /usr/include/asm-generic /usr/include/x86_64-linux-musl/asm-generic + ln -s /usr/include/linux /usr/include/x86_64-linux-musl/linux + - !TarInstall + url: https://github.com/openssl/openssl/archive/OpenSSL_1_1_1l.tar.gz + script: | + CC="musl-gcc -fPIE -pie" \ + ./Configure no-shared no-async \ + --prefix=/usr/lib/x86_64-linux-musl \ + --openssldir=/usr/lib/x86_64-linux-musl/ssl \ + linux-x86_64 + make depend + make -j$(nproc) + make install_sw + - &bulk !Tar url: "https://github.com/tailhook/bulk/releases/download/v0.4.9/bulk-v0.4.9.tar.gz" sha256: 23471a9986274bb4b7098c03e2eb7e1204171869b72c45385fcee1c64db2d111