diff --git a/Cargo.lock b/Cargo.lock index 3f8c08c..8426685 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -48,7 +48,7 @@ checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" dependencies = [ "cfg-if 1.0.0", "const-random", - "getrandom 0.2.11", + "getrandom 0.2.12", "once_cell", "version_check", "zerocopy 0.7.32", @@ -78,6 +78,12 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -172,6 +178,12 @@ dependencies = [ "zerocopy 0.6.6", ] +[[package]] +name = "arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" + [[package]] name = "arrow" version = "49.0.0" @@ -221,7 +233,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "hashbrown", + "hashbrown 0.14.3", "num 0.4.1", ] @@ -247,7 +259,7 @@ dependencies = [ "arrow-data", "arrow-schema", "arrow-select", - "base64 0.21.5", + "base64 0.21.6", "chrono", "half", "lexical-core", @@ -346,7 +358,7 @@ dependencies = [ "arrow-data", "arrow-schema", "half", - "hashbrown", + "hashbrown 0.14.3", ] [[package]] @@ -440,11 +452,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6218987c374650fdad0b476bfc675729762c28dfb35f58608a38a2b1ea337dd" dependencies = [ "async-trait", - "base64 0.21.5", + "base64 0.21.6", "bytes", "dyn-clone", "futures", - "getrandom 0.2.11", + "getrandom 0.2.12", "hmac 0.12.1", "http-types", "log", @@ -526,7 +538,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" dependencies = [ "futures-core", - "getrandom 0.2.11", + "getrandom 0.2.12", "instant", "pin-project-lite", "rand 0.8.5", @@ -556,9 +568,18 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.5" +version = "0.21.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +checksum = "c79fed4cdb43e993fcdadc7e58a09fd0e3e649c4436fa11da71c9f1f3ee7feb9" + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] [[package]] name = "bitflags" @@ -674,18 +695,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.13" +version = "4.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52bdc885e4cacc7f7c9eedc1ef6da641603180c783c41a15c264944deeaab642" +checksum = "33e92c5c1a78c62968ec57dbc2440366a2d6e5a23faf829970ff1585dc6b18e2" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.4.12" +version = "4.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb7fb5e4e979aec3be7791562fcba452f94ad85e954da024396433e0e25a79e9" +checksum = "f4323769dc8a61e2c39ad7dc26f6f2800524691a44d74fe3d1071a5c24db6370" dependencies = [ "anstream", "anstyle", @@ -720,7 +741,7 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" dependencies = [ - "crossbeam-utils 0.8.18", + "crossbeam-utils 0.8.19", ] [[package]] @@ -738,7 +759,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom 0.2.11", + "getrandom 0.2.12", "once_cell", "tiny-keccak", ] @@ -759,6 +780,15 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +[[package]] +name = "cpp_demangle" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeaa953eaad386a53111e47172c2fedba671e5684c8dd601a5f474f4f118710f" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "cpufeatures" version = "0.2.12" @@ -768,6 +798,115 @@ dependencies = [ "libc", ] +[[package]] +name = "cranelift-bforest" +version = "0.103.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c22542c0b95bd3302f7ed6839869c561f2324bac2fd5e7e99f5cfa65fdc8b92" +dependencies = [ + "cranelift-entity", +] + +[[package]] +name = "cranelift-codegen" +version = "0.103.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b3db903ef2e9c8a4de2ea6db5db052c7857282952f9df604aa55d169e6000d8" +dependencies = [ + "bumpalo", + "cranelift-bforest", + "cranelift-codegen-meta", + "cranelift-codegen-shared", + "cranelift-control", + "cranelift-entity", + "cranelift-isle", + "gimli", + "hashbrown 0.14.3", + "log", + "regalloc2", + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cranelift-codegen-meta" +version = "0.103.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6590feb5a1d6438f974bf6a5ac4dddf69fca14e1f07f3265d880f69e61a94463" +dependencies = [ + "cranelift-codegen-shared", +] + +[[package]] +name = "cranelift-codegen-shared" +version = "0.103.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7239038c56fafe77fddc8788fc8533dd6c474dc5bdc5637216404f41ba807330" + +[[package]] +name = "cranelift-control" +version = "0.103.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7dc9c595341404d381d27a3d950160856b35b402275f0c3990cd1ad683c8053" +dependencies = [ + "arbitrary", +] + +[[package]] +name = "cranelift-entity" +version = "0.103.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44e3ee532fc4776c69bcedf7e62f9632cbb3f35776fa9a525cdade3195baa3f7" +dependencies = [ + "serde", + "serde_derive", +] + +[[package]] +name = "cranelift-frontend" +version = "0.103.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a612c94d09e653662ec37681dc2d6fd2b9856e6df7147be0afc9aabb0abf19df" +dependencies = [ + "cranelift-codegen", + "log", + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cranelift-isle" +version = "0.103.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85db9830abeb1170b7d29b536ffd55af1d4d26ac8a77570b5d1aca003bf225cc" + +[[package]] +name = "cranelift-native" +version = "0.103.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "301ef0edafeaeda5771a5d2db64ac53e1818ae3111220a185677025fe91db4a1" +dependencies = [ + "cranelift-codegen", + "libc", + "target-lexicon", +] + +[[package]] +name = "cranelift-wasm" +version = "0.103.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "380f0abe8264e4570ac615fc31cef32a3b90a77f7eb97b08331f9dd357b1f500" +dependencies = [ + "cranelift-codegen", + "cranelift-entity", + "cranelift-frontend", + "itertools 0.10.5", + "log", + "smallvec", + "wasmparser", + "wasmtime-types", +] + [[package]] name = "crc32fast" version = "1.3.2" @@ -787,6 +926,25 @@ dependencies = [ "maybe-uninit", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils 0.8.19", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils 0.8.19", +] + [[package]] name = "crossbeam-utils" version = "0.7.2" @@ -800,12 +958,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.18" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3a430a770ebd84726f584a90ee7f020d28db52c6d02138900f22341f866d39c" -dependencies = [ - "cfg-if 1.0.0", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "crunchy" @@ -870,7 +1025,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if 1.0.0", - "hashbrown", + "hashbrown 0.14.3", "lock_api 0.4.11", "once_cell", "parking_lot_core 0.9.9", @@ -886,10 +1041,19 @@ dependencies = [ "uuid 0.8.2", ] +[[package]] +name = "debugid" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" +dependencies = [ + "uuid 1.6.1", +] + [[package]] name = "deltalake-aws" version = "0.1.0" -source = "git+https://github.com/delta-io/delta-rs?branch=main#9264edea89a2fc1c35f4a6b9faab125748ff3651" +source = "git+https://github.com/delta-io/delta-rs?branch=main#f7c303b74218c202ef683f727701a67da5aaaca5" dependencies = [ "async-trait", "backoff", @@ -914,7 +1078,7 @@ dependencies = [ [[package]] name = "deltalake-azure" version = "0.1.0" -source = "git+https://github.com/delta-io/delta-rs?branch=main#9264edea89a2fc1c35f4a6b9faab125748ff3651" +source = "git+https://github.com/delta-io/delta-rs?branch=main#f7c303b74218c202ef683f727701a67da5aaaca5" dependencies = [ "async-trait", "bytes", @@ -932,7 +1096,7 @@ dependencies = [ [[package]] name = "deltalake-core" version = "0.17.0" -source = "git+https://github.com/delta-io/delta-rs?branch=main#9264edea89a2fc1c35f4a6b9faab125748ff3651" +source = "git+https://github.com/delta-io/delta-rs?branch=main#f7c303b74218c202ef683f727701a67da5aaaca5" dependencies = [ "arrow", "arrow-arith", @@ -1029,6 +1193,16 @@ dependencies = [ "time 0.1.45", ] +[[package]] +name = "directories-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339ee130d97a610ea5a5872d2bbb130fdf68884ff09d3028b81bec8a1ac23bbc" +dependencies = [ + "cfg-if 1.0.0", + "dirs-sys-next", +] + [[package]] name = "dirs-next" version = "2.0.0" @@ -1149,6 +1323,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + [[package]] name = "fastrand" version = "1.9.0" @@ -1338,6 +1518,28 @@ dependencies = [ "slab", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "fxprof-processed-profile" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27d12c0aed7f1e24276a241aadc4cb8ea9f83000f34bc062b7cc2d51e3b0fabd" +dependencies = [ + "bitflags 2.4.1", + "debugid 0.8.0", + "fxhash", + "serde", + "serde_json", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1361,9 +1563,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -1377,6 +1579,11 @@ name = "gimli" version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +dependencies = [ + "fallible-iterator", + "indexmap", + "stable_deref_trait", +] [[package]] name = "h2" @@ -1408,11 +1615,24 @@ dependencies = [ "num-traits", ] +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +dependencies = [ + "ahash", +] + [[package]] name = "hashbrown" version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +dependencies = [ + "ahash", + "allocator-api2", +] [[package]] name = "heck" @@ -1622,6 +1842,12 @@ dependencies = [ "cc", ] +[[package]] +name = "id-arena" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" + [[package]] name = "idna" version = "0.5.0" @@ -1639,7 +1865,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.14.3", + "serde", ] [[package]] @@ -1680,6 +1907,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.11.0" @@ -1704,6 +1940,26 @@ version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +[[package]] +name = "ittapi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b996fe614c41395cdaedf3cf408a9534851090959d90d54a535f675550b64b1" +dependencies = [ + "anyhow", + "ittapi-sys", + "log", +] + +[[package]] +name = "ittapi-sys" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5385394064fa2c886205dba02598013ce83d3e92d33dbdc0c52fe0e7bf4fc" +dependencies = [ + "cc", +] + [[package]] name = "jmespatch" version = "0.3.0" @@ -1800,12 +2056,70 @@ dependencies = [ "uuid 0.8.2", ] +[[package]] +name = "kafka-delta-ingest-wasm" +version = "0.2.0" +dependencies = [ + "anyhow", + "apache-avro", + "async-trait", + "base64 0.13.1", + "bytes", + "chrono", + "clap", + "dipstick", + "env_logger", + "futures", + "jmespatch", + "lazy-init", + "lazy_static", + "log", + "maplit", + "rdkafka", + "schema_registry_converter", + "serde", + "serde_json", + "serial_test", + "strum 0.20.0", + "strum_macros 0.20.1", + "tempfile", + "thiserror", + "time 0.3.31", + "tokio", + "tokio-stream", + "tokio-util 0.6.10", + "url", + "utime", + "uuid 0.8.2", + "wasmtime", +] + +[[package]] +name = "kafka-delta-ingest-wasm-sdk" +version = "0.2.1" +dependencies = [ + "hashbrown 0.14.3", + "log", +] + +[[package]] +name = "lazy-init" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f40963626ac12dcaf92afc15e4c3db624858c92fd9f8ba2125eaada3ac2706f" + [[package]] name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "leb128" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" + [[package]] name = "lexical-core" version = "0.8.5" @@ -1872,9 +2186,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.151" +version = "0.2.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" +checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" [[package]] name = "libflate" @@ -1915,9 +2229,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.12" +version = "1.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" +checksum = "295c17e837573c8c821dbaeb3cceb3d745ad082f7572191409e69cbc1b3fd050" dependencies = [ "cc", "libc", @@ -1965,6 +2279,15 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "mach" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +dependencies = [ + "libc", +] + [[package]] name = "maplit" version = "1.0.2" @@ -2000,6 +2323,24 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" +[[package]] +name = "memfd" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2cffa4ad52c6f791f4f8b15f0c05f9824b2ced1160e88cc393d64fff9a8ac64" +dependencies = [ + "rustix", +] + +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.17" @@ -2210,6 +2551,9 @@ version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ + "crc32fast", + "hashbrown 0.14.3", + "indexmap", "memchr", ] @@ -2220,7 +2564,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2524735495ea1268be33d200e1ee97455096a0846295a21548cd2f3541de7050" dependencies = [ "async-trait", - "base64 0.21.5", + "base64 0.21.6", "bytes", "chrono", "futures", @@ -2384,13 +2728,13 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64 0.21.5", + "base64 0.21.6", "brotli", "bytes", "chrono", "flate2", "futures", - "hashbrown", + "hashbrown 0.14.3", "lz4_flex", "num 0.4.1", "num-bigint", @@ -2402,7 +2746,7 @@ dependencies = [ "thrift", "tokio", "twox-hash", - "zstd", + "zstd 0.13.0", ] [[package]] @@ -2526,9 +2870,9 @@ dependencies = [ [[package]] name = "psl" -version = "2.1.13" +version = "2.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe5b9b0f029db58ebf478fa2a45a8c6f2772c979d878c2c495e2eb2f217f41bc" +checksum = "383703acfc34f7a00724846c14dc5ea4407c59e5aedcbbb18a1c0c1a23fe5013" dependencies = [ "psl-types", ] @@ -2539,6 +2883,15 @@ version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac" +[[package]] +name = "psm" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874" +dependencies = [ + "cc", +] + [[package]] name = "quad-rand" version = "0.2.1" @@ -2624,7 +2977,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.11", + "getrandom 0.2.12", ] [[package]] @@ -2646,15 +2999,35 @@ dependencies = [ ] [[package]] -name = "rdkafka" -version = "0.28.0" +name = "rayon" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1de127f294f2dba488ed46760b129d5ecbeabbd337ccbf3739cb29d50db2161c" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" dependencies = [ - "futures", - "libc", - "log", - "rdkafka-sys", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils 0.8.19", +] + +[[package]] +name = "rdkafka" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1de127f294f2dba488ed46760b129d5ecbeabbd337ccbf3739cb29d50db2161c" +dependencies = [ + "futures", + "libc", + "log", + "rdkafka-sys", "serde", "serde_derive", "serde_json", @@ -2696,11 +3069,24 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" dependencies = [ - "getrandom 0.2.11", + "getrandom 0.2.12", "libredox", "thiserror", ] +[[package]] +name = "regalloc2" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad156d539c879b7a24a363a2016d77961786e71f48f2e2fc8302a92abd2429a6" +dependencies = [ + "hashbrown 0.13.2", + "log", + "rustc-hash", + "slice-group-by", + "smallvec", +] + [[package]] name = "regex" version = "1.10.2" @@ -2736,7 +3122,7 @@ version = "0.11.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" dependencies = [ - "base64 0.21.5", + "base64 0.21.6", "bytes", "encoding_rs", "futures-core", @@ -2803,7 +3189,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" dependencies = [ "cc", - "getrandom 0.2.11", + "getrandom 0.2.12", "libc", "spin 0.9.8", "untrusted 0.9.0", @@ -2944,6 +3330,12 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc_version" version = "0.4.0" @@ -3009,7 +3401,7 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64 0.21.5", + "base64 0.21.6", ] [[package]] @@ -3198,7 +3590,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4dd2266fee014a86e250e98e389191ecd23be546b5c42b6a2fb9af2972fadac" dependencies = [ "chrono", - "debugid", + "debugid 0.7.3", "serde", "serde_json", "thiserror", @@ -3268,9 +3660,9 @@ dependencies = [ [[package]] name = "serial_test" -version = "2.0.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e56dd856803e253c8f298af3f4d7eb0ae5e23a737252cd90bb4f3b435033b2d" +checksum = "953ad9342b3aaca7cb43c45c097dd008d4907070394bd0751a0aa8817e5a018d" dependencies = [ "dashmap", "futures", @@ -3282,9 +3674,9 @@ dependencies = [ [[package]] name = "serial_test_derive" -version = "2.0.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" +checksum = "b93fb4adc70021ac1b47f7d45e8cc4169baaa7ea58483bc5b721d19a26202212" dependencies = [ "proc-macro2", "quote", @@ -3345,6 +3737,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "slice-group-by" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" + [[package]] name = "slug" version = "0.1.5" @@ -3411,6 +3809,18 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "sptr" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b9b39299b249ad65f3b7e96443bad61c02ca5cd3589f46cb6d610a0fd6c0d6a" + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "static_assertions" version = "1.1.0" @@ -3509,6 +3919,12 @@ dependencies = [ "libc", ] +[[package]] +name = "target-lexicon" +version = "0.12.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69758bda2e78f098e4ccb393021a0963bb3442eac05f135c30f61b7370bbafae" + [[package]] name = "tempfile" version = "3.9.0" @@ -3727,6 +4143,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + [[package]] name = "toml_datetime" version = "0.6.5" @@ -3851,6 +4276,18 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + +[[package]] +name = "unicode-xid" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" + [[package]] name = "untrusted" version = "0.7.1" @@ -3906,7 +4343,7 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" dependencies = [ - "getrandom 0.2.11", + "getrandom 0.2.12", "serde", ] @@ -3916,7 +4353,7 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" dependencies = [ - "getrandom 0.2.11", + "getrandom 0.2.12", "serde", ] @@ -4063,6 +4500,24 @@ version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" +[[package]] +name = "wasm-encoder" +version = "0.38.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad2b51884de9c7f4fe2fd1043fccb8dcad4b1e29558146ee57a144d15779f3f" +dependencies = [ + "leb128", +] + +[[package]] +name = "wasm-encoder" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "111495d6204760238512f57a9af162f45086504da332af210f2f75dd80b34f1d" +dependencies = [ + "leb128", +] + [[package]] name = "wasm-streams" version = "0.3.0" @@ -4076,6 +4531,320 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasmparser" +version = "0.118.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95ee9723b928e735d53000dec9eae7b07a60e490c85ab54abb66659fc61bfcd9" +dependencies = [ + "indexmap", + "semver", +] + +[[package]] +name = "wasmtime" +version = "16.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8e539fded2495422ea3c4dfa7beeddba45904eece182cf315294009e1a323bf" +dependencies = [ + "anyhow", + "async-trait", + "bincode", + "bumpalo", + "cfg-if 1.0.0", + "fxprof-processed-profile", + "indexmap", + "libc", + "log", + "object", + "once_cell", + "paste", + "rayon", + "serde", + "serde_derive", + "serde_json", + "target-lexicon", + "wasm-encoder 0.38.1", + "wasmparser", + "wasmtime-cache", + "wasmtime-component-macro", + "wasmtime-cranelift", + "wasmtime-environ", + "wasmtime-fiber", + "wasmtime-jit", + "wasmtime-runtime", + "wat", + "windows-sys 0.48.0", +] + +[[package]] +name = "wasmtime-asm-macros" +version = "16.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "660ba9143e15a2acd921820df221b73aee256bd3ca2d208d73d8adc9587ccbb9" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "wasmtime-cache" +version = "16.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3ce373743892002f9391c6741ef0cb0335b55ec899d874f311222b7e36f4594" +dependencies = [ + "anyhow", + "base64 0.21.6", + "bincode", + "directories-next", + "log", + "rustix", + "serde", + "serde_derive", + "sha2 0.10.8", + "toml", + "windows-sys 0.48.0", + "zstd 0.11.2+zstd.1.5.2", +] + +[[package]] +name = "wasmtime-component-macro" +version = "16.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12ef32643324e564e1c359e9044daa06cbf90d7e2d6c99a738d17a12959f01a5" +dependencies = [ + "anyhow", + "proc-macro2", + "quote", + "syn 2.0.48", + "wasmtime-component-util", + "wasmtime-wit-bindgen", + "wit-parser", +] + +[[package]] +name = "wasmtime-component-util" +version = "16.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c87d06c18d21a4818f354c00a85f4ebc62b2270961cd022968452b0e4dbed9d" + +[[package]] +name = "wasmtime-cranelift" +version = "16.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d648c8b4064a7911093b02237cd5569f71ca171d3a0a486bf80600b19e1cba2" +dependencies = [ + "anyhow", + "cfg-if 1.0.0", + "cranelift-codegen", + "cranelift-control", + "cranelift-entity", + "cranelift-frontend", + "cranelift-native", + "cranelift-wasm", + "gimli", + "log", + "object", + "target-lexicon", + "thiserror", + "wasmparser", + "wasmtime-cranelift-shared", + "wasmtime-environ", + "wasmtime-versioned-export-macros", +] + +[[package]] +name = "wasmtime-cranelift-shared" +version = "16.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290a89027688782da8ff60b12bb95695494b1874e0d0ba2ba387d23dace6d70c" +dependencies = [ + "anyhow", + "cranelift-codegen", + "cranelift-control", + "cranelift-native", + "gimli", + "object", + "target-lexicon", + "wasmtime-environ", +] + +[[package]] +name = "wasmtime-environ" +version = "16.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61eb64fb3e0da883e2df4a13a81d6282e072336e6cb6295021d0f7ab2e352754" +dependencies = [ + "anyhow", + "cranelift-entity", + "gimli", + "indexmap", + "log", + "object", + "serde", + "serde_derive", + "target-lexicon", + "thiserror", + "wasmparser", + "wasmtime-types", +] + +[[package]] +name = "wasmtime-fiber" +version = "16.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ecf1d3a838b0956b71ad3f8cb80069a228339775bf02dd35d86a5a68bbe443" +dependencies = [ + "anyhow", + "cc", + "cfg-if 1.0.0", + "rustix", + "wasmtime-asm-macros", + "wasmtime-versioned-export-macros", + "windows-sys 0.48.0", +] + +[[package]] +name = "wasmtime-jit" +version = "16.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f485336add49267d8859e8f8084d2d4b9a4b1564496b6f30ba5b168d50c10ceb" +dependencies = [ + "addr2line", + "anyhow", + "bincode", + "cfg-if 1.0.0", + "cpp_demangle", + "gimli", + "ittapi", + "log", + "object", + "rustc-demangle", + "rustix", + "serde", + "serde_derive", + "target-lexicon", + "wasmtime-environ", + "wasmtime-jit-debug", + "wasmtime-jit-icache-coherence", + "wasmtime-runtime", + "windows-sys 0.48.0", +] + +[[package]] +name = "wasmtime-jit-debug" +version = "16.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65e119affec40edb2fab9044f188759a00c2df9c3017278d047012a2de1efb4f" +dependencies = [ + "object", + "once_cell", + "rustix", + "wasmtime-versioned-export-macros", +] + +[[package]] +name = "wasmtime-jit-icache-coherence" +version = "16.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b6d197fcc34ad32ed440e1f9552fd57d1f377d9699d31dee1b5b457322c1f8a" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "wasmtime-runtime" +version = "16.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "794b2bb19b99ef8322ff0dd9fe1ba7e19c41036dfb260b3f99ecce128c42ff92" +dependencies = [ + "anyhow", + "cc", + "cfg-if 1.0.0", + "indexmap", + "libc", + "log", + "mach", + "memfd", + "memoffset", + "paste", + "psm", + "rustix", + "sptr", + "wasm-encoder 0.38.1", + "wasmtime-asm-macros", + "wasmtime-environ", + "wasmtime-fiber", + "wasmtime-jit-debug", + "wasmtime-versioned-export-macros", + "wasmtime-wmemcheck", + "windows-sys 0.48.0", +] + +[[package]] +name = "wasmtime-types" +version = "16.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d995db8bb56f2cd8d2dc0ed5ffab94ffb435283b0fe6747f80f7aab40b2d06a1" +dependencies = [ + "cranelift-entity", + "serde", + "serde_derive", + "thiserror", + "wasmparser", +] + +[[package]] +name = "wasmtime-versioned-export-macros" +version = "16.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f55c5565959287c21dd0f4277ae3518dd2ae62679f655ee2dbc4396e19d210db" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "wasmtime-wit-bindgen" +version = "16.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f328b2d4a690270324756e886ed5be3a4da4c00be0eea48253f4595ad068062b" +dependencies = [ + "anyhow", + "heck 0.4.1", + "indexmap", + "wit-parser", +] + +[[package]] +name = "wasmtime-wmemcheck" +version = "16.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67761d8f8c0b3c13a5d34356274b10a40baba67fe9cfabbfc379a8b414e45de2" + +[[package]] +name = "wast" +version = "70.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee4bc54bbe1c6924160b9f75e374a1d07532e7580eb632c0ee6cdd109bb217e" +dependencies = [ + "leb128", + "memchr", + "unicode-width", + "wasm-encoder 0.39.0", +] + +[[package]] +name = "wat" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f0dce8cdc288c717cf01e461a1e451a7b8445d53451123536ba576e423a101a" +dependencies = [ + "wast", +] + [[package]] name = "web-sys" version = "0.3.66" @@ -4293,6 +5062,23 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "wit-parser" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df4913a2219096373fd6512adead1fb77ecdaa59d7fc517972a7d30b12f625be" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", +] + [[package]] name = "xml-rs" version = "0.8.19" @@ -4352,13 +5138,32 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe 5.0.2+zstd.1.5.2", +] + [[package]] name = "zstd" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" dependencies = [ - "zstd-safe", + "zstd-safe 7.0.0", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 978f3a8..b7af30f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,79 +1,15 @@ -[package] -name = "kafka-delta-ingest" -version = "0.2.0" -authors = ["R. Tyler Croy ", "Christian Williams "] -edition = "2018" - -[dependencies] -anyhow = "1" -async-trait = "0.1" -apache-avro = "^0.14" -base64 = "0.13" -bytes = "1" -chrono = "0.4.31" -clap = { version = "4", features = ["color"] } -dipstick = "0.9" -env_logger = "0" -futures = "0.3" -jmespatch = { version = "0.3", features = ["sync"] } -lazy_static = "1" -log = "0" -maplit = "1" -rdkafka = { version = "0.28", features = ["ssl-vendored"] } -schema_registry_converter = { version = "3.1.0", features = ["easy", "json", "avro"] } -serde = { version = "1", features = ["derive"] } -serde_json = "1" -strum = "0.20" -strum_macros = "0.20" -thiserror = "1" -tokio = { version = "1", features = ["full"] } -tokio-stream = { version = "0", features = ["fs"] } -tokio-util = "0.6.3" -uuid = { version = "0.8", features = ["serde", "v4"] } -url = "2.3" - -#deltalake = { version = "0.16.5", features = ["arrow", "json", "parquet"], optional = true } -deltalake-core = { git = "https://github.com/delta-io/delta-rs", branch = "main", features = ["json"]} -deltalake-aws = { git = "https://github.com/delta-io/delta-rs", branch = "main", optional = true } -deltalake-azure = { git = "https://github.com/delta-io/delta-rs", branch = "main", optional = true } - -# s3 feature enabled -dynamodb_lock = { version = "0.6.0", optional = true } -rusoto_core = { version = "0.47", default-features = false, features = ["rustls"], optional = true } -rusoto_credential = { version = "0.47", optional = true } -rusoto_s3 = { version = "0.47", default-features = false, features = ["rustls"], optional = true } - -# sentry -sentry = { version = "0.23.0", optional = true } - -# azure feature enabled, mostly used for tests -azure_core = { version = "0.18.0", optional = true } -azure_storage = { version = "0.18.0", optional = true } -azure_storage_blobs = { version = "0.18.0", optional = true } - -[features] -default = [] -sentry-ext = ["sentry"] -dynamic-linking = [ "rdkafka/dynamic-linking" ] -azure = [ - "deltalake-azure", - "azure_core", - "azure_storage", - "azure_storage_blobs" -] -s3 = [ - "deltalake-aws", - "dynamodb_lock", - "rusoto_core", - "rusoto_credential", - "rusoto_s3", +[workspace] +members = [ + "kafka-delta-ingest", + "kafka-delta-ingest-wasm", + "kafka-delta-ingest-wasm-sdk", ] -[dev-dependencies] -utime = "0.3" -serial_test = "*" -tempfile = "3" -time = "0.3.20" +resolver = "2" + +[profile.release-with-debug] +inherits = "release" +debug = true -[profile.release] -lto = true +[workspace.dependencies] +log = "0" \ No newline at end of file diff --git a/bin/clean-example-data.sh b/bin/clean-example-data.sh index d389e07..5e3c6d4 100755 --- a/bin/clean-example-data.sh +++ b/bin/clean-example-data.sh @@ -1,6 +1,6 @@ #!/bin/bash -WEB_REQUESTS_DIR=tests/data/web_requests +WEB_REQUESTS_DIR=kafka-delta-ingest/tests/data/web_requests find $WEB_REQUESTS_DIR/_delta_log -type f -not -name '00000000000000000000.json' -exec rm {} + find $WEB_REQUESTS_DIR -type d -name 'date=*' -exec rm -rf {} + diff --git a/bin/consume-example-json.sh b/bin/consume-example-json.sh index 1b0f362..35a7d46 100755 --- a/bin/consume-example-json.sh +++ b/bin/consume-example-json.sh @@ -4,7 +4,7 @@ export AWS_ENDPOINT_URL=http://0.0.0.0:4566 export AWS_ACCESS_KEY_ID=test export AWS_SECRET_ACCESS_KEY=test -RUST_LOG=debug ./target/debug/kafka-delta-ingest ingest web_requests ./tests/data/web_requests \ +RUST_LOG=debug ./target/debug/kafka-delta-ingest ingest web_requests ./tests/tests/data/web_requests \ -l 60 \ -a web_requests \ -K "auto.offset.reset=earliest" \ diff --git a/bin/extract-example-json.sh b/bin/extract-example-json.sh index 84cdf8b..ff2d218 100755 --- a/bin/extract-example-json.sh +++ b/bin/extract-example-json.sh @@ -1,4 +1,4 @@ #!/bin/bash -tar -xzvf tests/json/web_requests-100K.json.tar.gz -C tests/json +tar -xzvf kafka-delta-ingest/tests/json/web_requests-100K.json.tar.gz -C kafka-delta-ingest/tests/json diff --git a/docker-compose.yml b/docker-compose.yml index fa6d878..eb09413 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,5 @@ --- -version: '3.9' +version: "3.9" services: kafka: image: docker.redpanda.com/redpandadata/redpanda:v23.1.13 @@ -7,12 +7,12 @@ services: - redpanda - start - --smp - - '1' + - "1" - --reserve-memory - 0M - --overprovisioned - --node-id - - '0' + - "0" - --kafka-addr - PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092 - --advertise-kafka-addr @@ -41,7 +41,7 @@ services: - "8081:8081" environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'redpanda:29092' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: "redpanda:29092" SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 localstack: @@ -55,7 +55,7 @@ services: - DOCKER_HOST=unix:///var/run/docker.sock - HOST_TMP_FOLDER=${TMPDIR} healthcheck: - test: [ "CMD", "curl", "-f", "http://localhost:4566/health" ] + test: ["CMD", "curl", "-f", "http://localhost:4566/health"] azurite: image: mcr.microsoft.com/azure-storage/azurite ports: @@ -75,4 +75,4 @@ services: - /localstack-setup_emails.sh volumes: - ./bin/localstack-setup_emails.sh:/localstack-setup_emails.sh - - "./tests/data/emails/:/data/emails" + - "./kafka-delta-ingest/tests/data/emails/:/data/emails" diff --git a/kafka-delta-ingest-wasm-sdk/Cargo.toml b/kafka-delta-ingest-wasm-sdk/Cargo.toml new file mode 100644 index 0000000..19d3ff7 --- /dev/null +++ b/kafka-delta-ingest-wasm-sdk/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "kafka-delta-ingest-wasm-sdk" +version = "0.2.1" +authors = ["KyJah Keys "] +rust-version = "1.61" +description = "WebAssembly for Proxies" +readme = "README.md" +license = "Apache-2.0" +repository = "https://github.com/delta-io/kafka-delta-ingest" +edition = "2018" +build = "build.rs" + +[lib] +crate-type = ["cdylib"] +bench = false + +[dependencies] +hashbrown = "0.14" +log = { workspace = true } + +[profile.release] +lto = true +opt-level = 3 +codegen-units = 1 +panic = "abort" +strip = "debuginfo" \ No newline at end of file diff --git a/kafka-delta-ingest-wasm-sdk/build.rs b/kafka-delta-ingest-wasm-sdk/build.rs new file mode 100644 index 0000000..b2d9edb --- /dev/null +++ b/kafka-delta-ingest-wasm-sdk/build.rs @@ -0,0 +1,19 @@ +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-env-changed=RUSTFLAGS"); + + if let Some(target_os) = std::env::var_os("CARGO_CFG_TARGET_OS") { + if target_os != "wasi" { + return; + } + } + + if let Some(rustflags) = std::env::var_os("CARGO_ENCODED_RUSTFLAGS") { + for flag in rustflags.to_string_lossy().split('\x1f') { + if flag.ends_with("wasi-exec-model=reactor") { + println!("cargo:rustc-cfg=wasi_exec_model_reactor"); + return; + } + } + } +} diff --git a/kafka-delta-ingest-wasm-sdk/src/allocator.rs b/kafka-delta-ingest-wasm-sdk/src/allocator.rs new file mode 100644 index 0000000..2bdc9a3 --- /dev/null +++ b/kafka-delta-ingest-wasm-sdk/src/allocator.rs @@ -0,0 +1,15 @@ +use std::mem::MaybeUninit; + +#[cfg_attr( + all(target_arch = "wasm32", target_os = "unknown"), + export_name = "malloc" +)] +#[no_mangle] +pub extern "C" fn proxy_on_memory_allocate(size: usize) -> *mut u8 { + let mut vec: Vec> = Vec::with_capacity(size); + unsafe { + vec.set_len(size); + } + let slice = vec.into_boxed_slice(); + Box::into_raw(slice) as *mut u8 +} diff --git a/kafka-delta-ingest-wasm-sdk/src/dispatcher.rs b/kafka-delta-ingest-wasm-sdk/src/dispatcher.rs new file mode 100644 index 0000000..a3e99d7 --- /dev/null +++ b/kafka-delta-ingest-wasm-sdk/src/dispatcher.rs @@ -0,0 +1,194 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::traits::*; +use crate::types::*; +use hashbrown::HashMap; +use std::cell::{Cell, RefCell}; + +thread_local! { +static DISPATCHER: Dispatcher = Dispatcher::new(); +} + +pub(crate) fn set_root_context(callback: NewRootContext) { + DISPATCHER.with(|dispatcher| dispatcher.set_root_context(callback)); +} + +pub(crate) fn set_message_context(callback: NewMessageContext) { + DISPATCHER.with(|dispatcher| dispatcher.set_message_context(callback)); +} + +struct NoopRoot; + +impl Context for NoopRoot {} +impl RootContext for NoopRoot {} + +struct Dispatcher { + new_root: Cell>, + roots: RefCell>>, + new_message: Cell>, + messages: RefCell>>, + active_id: Cell, +} + +impl Dispatcher { + fn new() -> Dispatcher { + Dispatcher { + new_root: Cell::new(None), + roots: RefCell::new(HashMap::new()), + new_message: Cell::new(None), + messages: RefCell::new(HashMap::new()), + active_id: Cell::new(0), + } + } + + fn set_root_context(&self, callback: NewRootContext) { + self.new_root.set(Some(callback)); + } + + fn set_message_context(&self, callback: NewMessageContext) { + self.new_message.set(Some(callback)); + } + + fn create_root_context(&self, context_id: u32) { + let new_context = match self.new_root.get() { + Some(f) => f(context_id), + None => Box::new(NoopRoot), + }; + if self + .roots + .borrow_mut() + .insert(context_id, new_context) + .is_some() + { + panic!("duplicate context_id") + } + } + + fn create_message_context(&self, context_id: u32, root_context_id: u32) { + let new_context = match self.roots.borrow().get(&root_context_id) { + Some(root_context) => match self.new_message.get() { + Some(f) => f(context_id, root_context_id), + None => match root_context.create_message_context(context_id) { + Some(stream_context) => stream_context, + None => panic!("create_message_context returned None"), + }, + }, + None => panic!("invalid root_context_id"), + }; + if self + .messages + .borrow_mut() + .insert(context_id, new_context) + .is_some() + { + panic!("duplicate context_id") + } + } + + fn on_create_context(&self, context_id: u32, root_context_id: u32) { + if root_context_id == 0 { + self.create_root_context(context_id); + } else if self.new_message.get().is_some() { + self.create_message_context(context_id, root_context_id); + } else if let Some(root_context) = self.roots.borrow().get(&root_context_id) { + match root_context.get_type() { + Some(ContextType::MessageContext) => { + self.create_message_context(context_id, root_context_id) + } + None => panic!("missing ContextType on root_context"), + } + } else { + panic!("invalid root_context_id and missing constructors"); + } + } + + fn on_log(&self, context_id: u32) { + if let Some(message) = self.messages.borrow_mut().get_mut(&context_id) { + self.active_id.set(context_id); + message.on_log() + } else if let Some(root) = self.roots.borrow_mut().get_mut(&context_id) { + self.active_id.set(context_id); + root.on_log() + } else { + panic!("invalid context_id") + } + } + + fn on_vm_start(&self, context_id: u32, vm_configuration_size: usize) -> bool { + if let Some(root) = self.roots.borrow_mut().get_mut(&context_id) { + self.active_id.set(context_id); + root.on_vm_start(vm_configuration_size) + } else { + panic!("invalid context_id") + } + } + + fn on_configure(&self, context_id: u32, plugin_configuration_size: usize) -> bool { + if let Some(root) = self.roots.borrow_mut().get_mut(&context_id) { + self.active_id.set(context_id); + root.on_configure(plugin_configuration_size) + } else { + panic!("invalid context_id") + } + } + + fn on_tick(&self, context_id: u32) { + if let Some(root) = self.roots.borrow_mut().get_mut(&context_id) { + self.active_id.set(context_id); + root.on_tick() + } else { + panic!("invalid context_id") + } + } + + fn on_message_received(&self, context_id: u32, body_size: usize) -> Action { + if let Some(message) = self.messages.borrow_mut().get_mut(&context_id) { + self.active_id.set(context_id); + message.on_message_received(body_size) + } else { + panic!("invalid context_id") + } + } +} + +#[no_mangle] +pub extern "C" fn proxy_on_context_create(context_id: u32, root_context_id: u32) { + DISPATCHER.with(|dispatcher| dispatcher.on_create_context(context_id, root_context_id)) +} + +#[no_mangle] +pub extern "C" fn proxy_on_log(context_id: u32) { + DISPATCHER.with(|dispatcher| dispatcher.on_log(context_id)) +} + +#[no_mangle] +pub extern "C" fn proxy_on_vm_start(context_id: u32, vm_configuration_size: usize) -> bool { + DISPATCHER.with(|dispatcher| dispatcher.on_vm_start(context_id, vm_configuration_size)) +} + +#[no_mangle] +pub extern "C" fn proxy_on_configure(context_id: u32, plugin_configuration_size: usize) -> bool { + DISPATCHER.with(|dispatcher| dispatcher.on_configure(context_id, plugin_configuration_size)) +} + +#[no_mangle] +pub extern "C" fn proxy_on_tick(context_id: u32) { + DISPATCHER.with(|dispatcher| dispatcher.on_tick(context_id)) +} + +#[no_mangle] +pub extern "C" fn proxy_on_message_received(context_id: u32, body_size: usize) -> Action { + DISPATCHER.with(|dispatcher| dispatcher.on_message_received(context_id, body_size)) +} diff --git a/kafka-delta-ingest-wasm-sdk/src/hostcalls.rs b/kafka-delta-ingest-wasm-sdk/src/hostcalls.rs new file mode 100644 index 0000000..48799c5 --- /dev/null +++ b/kafka-delta-ingest-wasm-sdk/src/hostcalls.rs @@ -0,0 +1,249 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::types::*; +use std::ptr::null_mut; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +extern "C" { + fn proxy_log(level: LogLevel, message_data: *const u8, message_size: usize) -> Status; +} + +pub fn log(level: LogLevel, message: &str) -> Result<(), Status> { + unsafe { + match proxy_log(level, message.as_ptr(), message.len()) { + Status::Ok => Ok(()), + status => panic!("unexpected status: {}", status as u32), + } + } +} + +extern "C" { + fn proxy_get_log_level(return_level: *mut LogLevel) -> Status; +} + +pub fn get_log_level() -> Result { + let mut return_level: LogLevel = LogLevel::Trace; + unsafe { + match proxy_get_log_level(&mut return_level) { + Status::Ok => Ok(return_level), + status => panic!("unexpected status: {}", status as u32), + } + } +} + +extern "C" { + fn proxy_get_current_time_nanoseconds(return_time: *mut u64) -> Status; +} + +pub fn get_current_time() -> Result { + let mut return_time: u64 = 0; + unsafe { + match proxy_get_current_time_nanoseconds(&mut return_time) { + Status::Ok => Ok(UNIX_EPOCH + Duration::from_nanos(return_time)), + status => panic!("unexpected status: {}", status as u32), + } + } +} + +extern "C" { + fn proxy_set_tick_period_milliseconds(period: u32) -> Status; +} + +pub fn set_tick_period(period: Duration) -> Result<(), Status> { + unsafe { + match proxy_set_tick_period_milliseconds(period.as_millis() as u32) { + Status::Ok => Ok(()), + status => panic!("unexpected status: {}", status as u32), + } + } +} + +extern "C" { + fn proxy_set_output(buffer_data: *const u8, buffer_size: usize) -> Status; +} + +pub fn set_output(value: &[u8]) -> Result<(), Status> { + unsafe { + match proxy_set_output(value.as_ptr(), value.len()) { + Status::Ok => Ok(()), + status => panic!("unexpected status: {}", status as u32), + } + } +} + +extern "C" { + fn proxy_get_message_body( + return_buffer_data: *mut *mut u8, + return_buffer_size: *mut usize, + ) -> Status; +} + +pub fn get_body() -> Result, Status> { + let mut return_data: *mut u8 = null_mut(); + let mut return_size: usize = 0; + unsafe { + match proxy_get_message_body(&mut return_data, &mut return_size) { + Status::Ok => { + if !return_data.is_null() { + Ok(Some(Vec::from_raw_parts( + return_data, + return_size, + return_size, + ))) + } else { + Ok(None) + } + } + Status::NotFound => Ok(None), + status => panic!("unexpected status: {}", status as u32), + } + } +} + +extern "C" { + fn proxy_get_buffer_bytes( + buffer_type: BufferType, + start: usize, + max_size: usize, + return_buffer_data: *mut *mut u8, + return_buffer_size: *mut usize, + ) -> Status; +} + +pub fn get_buffer( + buffer_type: BufferType, + start: usize, + max_size: usize, +) -> Result, Status> { + let mut return_data: *mut u8 = null_mut(); + let mut return_size: usize = 0; + unsafe { + match proxy_get_buffer_bytes( + buffer_type, + start, + max_size, + &mut return_data, + &mut return_size, + ) { + Status::Ok => { + if !return_data.is_null() { + Ok(Some(Vec::from_raw_parts( + return_data, + return_size, + return_size, + ))) + } else { + Ok(None) + } + } + Status::NotFound => Ok(None), + status => panic!("unexpected status: {}", status as u32), + } + } +} + +extern "C" { + fn proxy_get_property( + property: i32, + return_value_data: *mut *mut u8, + return_value_size: *mut usize, + ) -> Status; +} + +pub fn get_property(path: MapType) -> Result, Status> { + let mut return_data: *mut u8 = null_mut(); + let mut return_size: usize = 0; + unsafe { + match proxy_get_property(path as i32, &mut return_data, &mut return_size) { + Status::Ok => { + if !return_data.is_null() { + Ok(Some(Vec::from_raw_parts( + return_data, + return_size, + return_size, + ))) + } else { + Ok(None) + } + } + Status::NotFound => Ok(None), + status => panic!("unexpected status: {}", status as u32), + } + } +} + +extern "C" { + fn proxy_define_metric( + metric_type: MetricType, + name_data: *const u8, + name_size: usize, + return_id: *mut u32, + ) -> Status; +} + +pub fn define_metric(metric_type: MetricType, name: &str) -> Result { + let mut return_id: u32 = 0; + unsafe { + match proxy_define_metric(metric_type, name.as_ptr(), name.len(), &mut return_id) { + Status::Ok => Ok(return_id), + status => panic!("unexpected status: {}", status as u32), + } + } +} + +extern "C" { + fn proxy_get_metric(metric_id: u32, return_value: *mut u64) -> Status; +} + +pub fn get_metric(metric_id: u32) -> Result { + let mut return_value: u64 = 0; + unsafe { + match proxy_get_metric(metric_id, &mut return_value) { + Status::Ok => Ok(return_value), + Status::NotFound => Err(Status::NotFound), + Status::BadArgument => Err(Status::BadArgument), + status => panic!("unexpected status: {}", status as u32), + } + } +} + +extern "C" { + fn proxy_record_metric(metric_id: u32, value: u64) -> Status; +} + +pub fn record_metric(metric_id: u32, value: u64) -> Result<(), Status> { + unsafe { + match proxy_record_metric(metric_id, value) { + Status::Ok => Ok(()), + Status::NotFound => Err(Status::NotFound), + status => panic!("unexpected status: {}", status as u32), + } + } +} + +extern "C" { + fn proxy_increment_metric(metric_id: u32, offset: i64) -> Status; +} + +pub fn increment_metric(metric_id: u32, offset: i64) -> Result<(), Status> { + unsafe { + match proxy_increment_metric(metric_id, offset) { + Status::Ok => Ok(()), + Status::NotFound => Err(Status::NotFound), + Status::BadArgument => Err(Status::BadArgument), + status => panic!("unexpected status: {}", status as u32), + } + } +} diff --git a/kafka-delta-ingest-wasm-sdk/src/lib.rs b/kafka-delta-ingest-wasm-sdk/src/lib.rs new file mode 100644 index 0000000..cdd2caa --- /dev/null +++ b/kafka-delta-ingest-wasm-sdk/src/lib.rs @@ -0,0 +1,53 @@ +pub mod hostcalls; +pub mod traits; +pub mod types; + +mod allocator; +mod dispatcher; +mod logger; + +// For crate-type="cdylib". +#[cfg(not(wasi_exec_model_reactor))] +#[macro_export] +macro_rules! main { + ($code:block) => { + #[cfg(target_os = "wasi")] + extern "C" { + fn __wasm_call_ctors(); + } + + #[no_mangle] + pub extern "C" fn _initialize() { + #[cfg(target_os = "wasi")] + unsafe { + __wasm_call_ctors(); + } + + $code; + } + }; +} + +// For crate-type="bin" with RUSTFLAGS="-Z wasi-exec-model=reactor". +#[cfg(wasi_exec_model_reactor)] +#[macro_export] +macro_rules! main { + ($code:block) => { + pub fn main() -> Result<(), Box> { + $code; + Ok(()) + } + }; +} + +pub fn set_log_level(level: types::LogLevel) { + logger::set_log_level(level); +} + +pub fn set_root_context(callback: types::NewRootContext) { + dispatcher::set_root_context(callback); +} + +pub fn set_message_context(callback: types::NewMessageContext) { + dispatcher::set_message_context(callback); +} diff --git a/kafka-delta-ingest-wasm-sdk/src/logger.rs b/kafka-delta-ingest-wasm-sdk/src/logger.rs new file mode 100644 index 0000000..050a356 --- /dev/null +++ b/kafka-delta-ingest-wasm-sdk/src/logger.rs @@ -0,0 +1,71 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::hostcalls; +use crate::types::LogLevel; +use std::panic; +use std::sync::atomic::{AtomicBool, Ordering}; + +struct Logger; + +static LOGGER: Logger = Logger; +static INITIALIZED: AtomicBool = AtomicBool::new(false); + +pub(crate) fn set_log_level(level: LogLevel) { + if !INITIALIZED.load(Ordering::Relaxed) { + log::set_logger(&LOGGER).unwrap(); + panic::set_hook(Box::new(|panic_info| { + hostcalls::log(LogLevel::Critical, &panic_info.to_string()).unwrap(); + })); + INITIALIZED.store(true, Ordering::Relaxed); + } + LOGGER.set_log_level(level); +} + +impl Logger { + pub fn set_log_level(&self, level: LogLevel) { + let filter = match level { + LogLevel::Trace => log::LevelFilter::Trace, + LogLevel::Debug => log::LevelFilter::Debug, + LogLevel::Info => log::LevelFilter::Info, + LogLevel::Warn => log::LevelFilter::Warn, + LogLevel::Error => log::LevelFilter::Error, + LogLevel::Critical => log::LevelFilter::Off, + }; + log::set_max_level(filter); + } +} + +impl log::Log for Logger { + fn enabled(&self, metadata: &log::Metadata) -> bool { + metadata.level() <= log::max_level() + } + + fn log(&self, record: &log::Record) { + if !self.enabled(record.metadata()) { + return; + } + let level = match record.level() { + log::Level::Trace => LogLevel::Trace, + log::Level::Debug => LogLevel::Debug, + log::Level::Info => LogLevel::Info, + log::Level::Warn => LogLevel::Warn, + log::Level::Error => LogLevel::Error, + }; + let message = record.args().to_string(); + hostcalls::log(level, &message).unwrap(); + } + + fn flush(&self) {} +} diff --git a/kafka-delta-ingest-wasm-sdk/src/traits.rs b/kafka-delta-ingest-wasm-sdk/src/traits.rs new file mode 100644 index 0000000..274004b --- /dev/null +++ b/kafka-delta-ingest-wasm-sdk/src/traits.rs @@ -0,0 +1,73 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::hostcalls; +use crate::types::*; +use std::time::{Duration, SystemTime}; + +pub trait Context { + fn get_current_time(&self) -> SystemTime { + hostcalls::get_current_time().unwrap() + } + + fn get_property(&self, path: MapType) -> Option { + hostcalls::get_property(path).unwrap() + } +} + +pub trait MessageContext: Context { + fn on_message_received(&mut self, _body_size: usize) -> Action { + Action::Use + } + + fn set_message_body(&self, _start: usize, _size: usize, value: &[u8]) { + hostcalls::set_output(value).unwrap(); + } + + fn on_log(&mut self) {} +} + +pub trait RootContext: Context { + fn on_vm_start(&mut self, _vm_configuration_size: usize) -> bool { + true + } + + fn get_vm_configuration(&self) -> Option { + hostcalls::get_buffer(BufferType::VmConfiguration, 0, usize::MAX).unwrap() + } + + fn on_configure(&mut self, _plugin_configuration_size: usize) -> bool { + true + } + + fn get_plugin_configuration(&self) -> Option { + hostcalls::get_buffer(BufferType::PluginConfiguration, 0, usize::MAX).unwrap() + } + + fn set_tick_period(&self, period: Duration) { + hostcalls::set_tick_period(period).unwrap() + } + + fn on_tick(&mut self) {} + + fn on_log(&mut self) {} + + fn create_message_context(&self, _context_id: u32) -> Option> { + None + } + + fn get_type(&self) -> Option { + None + } +} diff --git a/kafka-delta-ingest-wasm-sdk/src/types.rs b/kafka-delta-ingest-wasm-sdk/src/types.rs new file mode 100644 index 0000000..1f73ab2 --- /dev/null +++ b/kafka-delta-ingest-wasm-sdk/src/types.rs @@ -0,0 +1,85 @@ +use crate::traits::*; + +pub type NewRootContext = fn(context_id: u32) -> Box; +pub type NewMessageContext = fn(context_id: u32, root_context_id: u32) -> Box; + +#[repr(u32)] +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] +pub enum LogLevel { + Trace = 0, + Debug = 1, + Info = 2, + Warn = 3, + Error = 4, + Critical = 5, +} + +#[repr(u32)] +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] +#[non_exhaustive] +pub enum Action { + Use = 0, + Discard = 1, +} + +#[repr(u32)] +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] +#[non_exhaustive] +pub enum Status { + Ok = 0, + NotFound = 1, + BadArgument = 2, + ParseFailure = 4, + Empty = 7, + CasMismatch = 8, + InternalFailure = 10, +} + +#[repr(u32)] +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] +#[non_exhaustive] +pub enum ContextType { + MessageContext = 0, +} + +#[repr(u32)] +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] +#[non_exhaustive] +pub enum StreamType { + HttpRequest = 0, + HttpResponse = 1, + Downstream = 2, + Upstream = 3, +} + +#[repr(u32)] +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] +#[non_exhaustive] +pub enum BufferType { + MessageBody = 0, + VmConfiguration = 1, + PluginConfiguration = 2, +} + +#[repr(u32)] +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] +#[non_exhaustive] +pub enum MapType { + MessageHeaders = 0, + MessageTopic = 1, + MessageKey = 2, + MessagePartition = 3, + MessageOffset = 4, + MessageTimestamp = 5, +} + +#[repr(u32)] +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] +#[non_exhaustive] +pub enum MetricType { + Counter = 0, + Gauge = 1, + Histogram = 2, +} + +pub type Bytes = Vec; diff --git a/kafka-delta-ingest-wasm/Cargo.toml b/kafka-delta-ingest-wasm/Cargo.toml new file mode 100644 index 0000000..c264020 --- /dev/null +++ b/kafka-delta-ingest-wasm/Cargo.toml @@ -0,0 +1,50 @@ +[package] +name = "kafka-delta-ingest-wasm" +version = "0.2.0" +authors = ["R. Tyler Croy ", "Christian Williams "] +edition = "2018" + +[lib] +crate-type = ["cdylib"] +bench = false + +[dependencies] +anyhow = "1" +async-trait = "0.1" +apache-avro = "^0.14" +base64 = "0.13" +bytes = "1" +chrono = "0.4.31" +clap = { version = "4", features = ["color"] } +dipstick = "0.9" +env_logger = "0" +futures = "0.3" +jmespatch = { version = "0.3", features = ["sync"] } +lazy_static = "1" +log = { workspace = true } +maplit = "1" +rdkafka = { version = "0.28", features = ["ssl-vendored"] } +schema_registry_converter = { version = "3.1.0", features = ["easy", "json", "avro"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +strum = "0.20" +strum_macros = "0.20" +thiserror = "1" +tokio = { version = "1", features = ["full"] } +tokio-stream = { version = "0", features = ["fs"] } +tokio-util = "0.6.3" +uuid = { version = "0.8", features = ["serde", "v4"] } +url = "2.3" + +# wasm feature enabled +lazy-init = {version = "0.5.1" } +wasmtime = { version="16.0.0" } + +[dev-dependencies] +utime = "0.3" +serial_test = "*" +tempfile = "3" +time = "0.3.20" + +[profile.release] +lto = true diff --git a/kafka-delta-ingest-wasm/src/lib.rs b/kafka-delta-ingest-wasm/src/lib.rs new file mode 100644 index 0000000..076f7b7 --- /dev/null +++ b/kafka-delta-ingest-wasm/src/lib.rs @@ -0,0 +1,309 @@ +use std::cell::{Cell, RefCell}; +use std::collections::HashMap; + +use std::sync::{Arc, RwLock}; + +use rdkafka::message::OwnedMessage; +use rdkafka::Message; +use wasmtime::{ + AsContext, AsContextMut, Caller, Engine, Extern, Instance, Linker, Module, Store, TypedFunc, +}; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("webassembly startup error: {error}")] + Startup { error: String }, + #[error("webassembly runtime error: {error}")] + Runtime { error: String }, + #[error("invalide json from webassembly content: {source}")] + Json { + #[from] + source: serde_json::Error, + }, +} + +impl From for Error { + fn from(value: wasmtime::Error) -> Self { + Error::Startup { + error: format!("{}", value), + } + } +} + +impl From for Error { + fn from(value: wasmtime::MemoryAccessError) -> Self { + Error::Startup { + error: format!("{}", value), + } + } +} + +impl From for Error { + fn from(value: wasmtime::UnknownImportError) -> Self { + Error::Startup { + error: format!("{}", value), + } + } +} + +#[derive(Clone, Default)] +struct HostContext { + current_context_id: Cell, + messages: Arc>>, +} + +#[derive(Clone)] +struct MessageContext { + message: OwnedMessage, + response: RefCell>>, +} + +impl MessageContext { + fn new(message: OwnedMessage) -> Self { + Self { + message, + response: RefCell::new(None), + } + } +} + +impl HostContext { + fn add_message(&self, context_id: u32, message: OwnedMessage) { + let mut map_ref = self.messages.write().unwrap(); + map_ref.insert(context_id, MessageContext::new(message)); + } + + fn remove_message(&self, context_id: u32) -> Option { + let mut map_ref = self.messages.write().unwrap(); + map_ref.remove(&context_id) + } + + fn save_response(&self, context_id: u32, data: Vec) { + let mut map_ref = self.messages.write().unwrap(); + if let Some(message_context) = map_ref.get_mut(&context_id) { + message_context.response.replace(Some(data)); + } + } +} + +pub struct WasmTransformer { + engine: Engine, + module: Module, + linker: Linker, +} + +impl WasmTransformer { + thread_local! { + static HOST: RefCell> = RefCell::new(None); + } + + pub fn try_from_file(file: impl AsRef) -> Result { + let engine = Engine::default(); + + let module = Module::from_file(&engine, file)?; + let mut linker = Linker::new(&engine); + linker.func_wrap( + "host", + "proxy_set_output", + |mut env: Caller, offset: i32, length: i32| { + let memory = match env.get_export("memory") { + Some(Extern::Memory(memory)) => memory, + _ => panic!("no memory exported from wasm module"), + }; + let mut data = vec![0; length as usize]; + memory + .read(env.as_context(), offset as usize, &mut data) + .unwrap(); + env.data().save_response(1, data); + }, + )?; + linker.func_wrap( + "host", + "proxy_get_message_body", + |mut env: Caller, offset: i32, _length: i32| { + let memory = match env.get_export("memory") { + Some(Extern::Memory(memory)) => memory, + _ => panic!("no memory exported from wasm module"), + }; + let store = env.as_context_mut(); + let data = store.data().clone(); + let map = data.messages.read().unwrap(); + let payload = match map.get(&1) { + Some(message) => message.message.payload().unwrap(), + None => return, + }; + memory.write(store, offset as usize, payload).unwrap(); + }, + )?; + Ok(Self { + engine, + module, + linker, + }) + } + + pub fn process(&self, message: OwnedMessage) -> Result, Error> { + WasmTransformer::HOST.with(|cell| { + let mut borrowed = cell.borrow_mut(); + match borrowed.as_mut() { + Some(host) => host.process(message), + None => { + let new_host = self.new_wasm_host()?; + borrowed.replace(new_host); + borrowed.as_mut().unwrap().process(message) + } + } + }) + } + + fn new_wasm_host(&self) -> Result { + let engine = self.engine.clone(); + let module = self.module.clone(); + let linker = self.linker.clone(); + let data = HostContext::default(); + let mut store = Store::new(&engine, data); + let instance = linker.instantiate(&mut store, &module)?; + let on_message_received = + instance.get_typed_func::<(i32, u32), i32>(&mut store, "proxy_on_message_received")?; + let on_context_create = + instance.get_typed_func::<(i32, i32), ()>(&mut store, "proxy_on_context_create")?; + on_context_create.call(&mut store, (1, 0)).unwrap(); + on_context_create.call(&mut store, (1, 1))?; + Ok(WasmHost { + _instance: instance, + store, + _module: module, + on_message_received, + _on_context_create: on_context_create, + }) + } +} + +struct WasmHost { + _instance: Instance, + store: Store, + _module: Module, + on_message_received: TypedFunc<(i32, u32), i32>, + _on_context_create: TypedFunc<(i32, i32), ()>, +} + +impl WasmHost { + pub fn process(&mut self, message: OwnedMessage) -> Result, Error> { + let context_id = 1; + let mut_context = self.store.as_context_mut(); + let length = message.payload().unwrap_or_default().len(); + let host_context = mut_context.data().clone(); + host_context.current_context_id.replace(context_id); + host_context.add_message(context_id, message); + + match self + .on_message_received + .call(mut_context, (1, length as u32))? + { + // discard + 1 => Ok(None), + _ => { + let message_context = host_context.remove_message(context_id).unwrap(); + let message_content = message_context.response.borrow(); + let content = message_content + .as_deref() + .unwrap_or(message_context.message.payload().unwrap()); + let json = serde_json::de::from_slice::(content)?; + Ok(Some(json)) + } + } + } +} + +#[cfg(test)] +mod tests { + + use rdkafka::message::OwnedHeaders; + use serde_json::Value; + + use super::*; + + #[test] + fn proxy_sample_keep() { + let ctx = WasmTransformer::try_from_file("tests/proxy_sample.wat") + .expect("failed to load wat file"); + let message = OwnedMessage::new( + Some(vec![5, 4, 3, 2, 1]), + Some("test.into".as_bytes().to_vec()), + "test".into(), + rdkafka::Timestamp::NotAvailable, + 0, + 0, + Some(OwnedHeaders::new()), + ); + let result = ctx.process(message).unwrap(); + let content = result.expect("content should return"); + match content { + Value::Object(_) => {} + _ => panic!("should have returned an object"), + } + } + + #[test] + fn proxy_sample_ignore() { + let ctx = WasmTransformer::try_from_file("tests/proxy_sample_ignore.wat") + .expect("failed to load wat file"); + let message = OwnedMessage::new( + Some(vec![5, 4, 3, 2, 1]), + Some("test.into".as_bytes().to_vec()), + "test".into(), + rdkafka::Timestamp::NotAvailable, + 0, + 0, + Some(OwnedHeaders::new()), + ); + if ctx.process(message).unwrap().is_some() { + panic!("response should be none") + } + } + + #[test] + fn proxy_sample_multiple_threads() { + let ctx = Arc::new( + WasmTransformer::try_from_file("tests/proxy_sample.wat") + .expect("failed to load wat file"), + ); + let clone = ctx.clone(); + let first = std::thread::spawn(move || { + let message = OwnedMessage::new( + Some(vec![5, 4, 3, 2, 1]), + Some("test.into".as_bytes().to_vec()), + "test".into(), + rdkafka::Timestamp::NotAvailable, + 0, + 0, + Some(OwnedHeaders::new()), + ); + let result = ctx.process(message).unwrap(); + let content = result.expect("content should return"); + match content { + Value::Object(_) => {} + _ => panic!("should have returned an object"), + }; + }); + let second = std::thread::spawn(move || { + let message = OwnedMessage::new( + Some(vec![5, 4, 3, 2, 1]), + Some("test.into".as_bytes().to_vec()), + "test".into(), + rdkafka::Timestamp::NotAvailable, + 0, + 0, + Some(OwnedHeaders::new()), + ); + let result = clone.process(message).unwrap(); + let content = result.expect("content should return"); + match content { + Value::Object(_) => {} + _ => panic!("should have returned an object"), + }; + }); + first.join().unwrap(); + second.join().unwrap(); + } +} diff --git a/kafka-delta-ingest-wasm/tests/proxy_sample.wat b/kafka-delta-ingest-wasm/tests/proxy_sample.wat new file mode 100644 index 0000000..ba6e761 --- /dev/null +++ b/kafka-delta-ingest-wasm/tests/proxy_sample.wat @@ -0,0 +1,16 @@ +(module + (func $set_output (import "host" "proxy_set_output") (param i32) (param i32)) + (func $get_body (import "host" "proxy_get_message_body") (param i32) (param i32)) + + (memory (export "memory") 2 3) + + ;; Write 'hello world\n' to memory at an offset of 8 bytes + ;; Note the trailing newline which is required for the text to appear + (data (i32.const 8) "{}") + (func $create_context (export "proxy_on_context_create") (param $x i32) (param $y i32)) + (func $receive_message (export "proxy_on_message_received") (param $x i32) (param $y i32) (result i32) + i32.const 8 + i32.const 2 + call $set_output + i32.const 0) +) \ No newline at end of file diff --git a/kafka-delta-ingest-wasm/tests/proxy_sample_failure.wat b/kafka-delta-ingest-wasm/tests/proxy_sample_failure.wat new file mode 100644 index 0000000..36acc23 --- /dev/null +++ b/kafka-delta-ingest-wasm/tests/proxy_sample_failure.wat @@ -0,0 +1,16 @@ +(module + (func $set_output (import "host" "proxy_set_output") (param i32) (param i32)) + (func $get_body (import "host" "proxy_get_message_body") (param i32) (param i32)) + + (memory (export "memory") 2 3) + + ;; Write 'hello world\n' to memory at an offset of 8 bytes + ;; Note the trailing newline which is required for the text to appear + (data (i32.const 8) "{}") + (func $create_context (export "proxy_on_context_create") (param $x i32) (param $y i32)) + (func $receive_message (export "proxy_on_message_received") (param $x i32) (param $y i32) (result i32) + i32.const 8 + i32.const 2 + call $set_output + i32.const 1) +) \ No newline at end of file diff --git a/kafka-delta-ingest/Cargo.lock b/kafka-delta-ingest/Cargo.lock new file mode 100644 index 0000000..961d2e4 --- /dev/null +++ b/kafka-delta-ingest/Cargo.lock @@ -0,0 +1,5101 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "RustyXML" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b5ace29ee3216de37c0546865ad08edef58b0f9e76838ed8959a84a990e58c5" + +[[package]] +name = "addr" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "936697e9caf938eb2905036100edf8e1269da8291f8a02f5fe7b37073784eec0" +dependencies = [ + "no-std-net", + "psl", + "psl-types", +] + +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli 0.28.1", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + +[[package]] +name = "ahash" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a824f2aa7e75a0c98c5a504fceb80649e9c35265d44525b5f94de4771a395cd" +dependencies = [ + "getrandom 0.2.11", + "once_cell", + "version_check", +] + +[[package]] +name = "ahash" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +dependencies = [ + "cfg-if 1.0.0", + "const-random", + "getrandom 0.2.11", + "once_cell", + "version_check", + "zerocopy 0.7.29", +] + +[[package]] +name = "aho-corasick" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" + +[[package]] +name = "anstyle-parse" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3a318f1f38d2418400f8209655bfd825785afd25aa30bb7ba6cc792e4596748" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +dependencies = [ + "anstyle", + "windows-sys 0.52.0", +] + +[[package]] +name = "anyhow" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" + +[[package]] +name = "apache-avro" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf4144857f9e4d7dd6cc4ba4c78efd2a46bad682b029bd0d91e76a021af1b2a" +dependencies = [ + "byteorder", + "digest 0.10.7", + "lazy_static", + "libflate", + "log", + "num-bigint", + "quad-rand", + "rand 0.8.5", + "regex", + "serde", + "serde_json", + "strum 0.24.1", + "strum_macros 0.24.3", + "thiserror", + "typed-builder", + "uuid 1.6.1", + "zerocopy 0.6.5", +] + +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + +[[package]] +name = "arrow" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fab9e93ba8ce88a37d5a30dce4b9913b75413dc1ac56cb5d72e5a840543f829" +dependencies = [ + "ahash 0.8.6", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc1d4e368e87ad9ee64f28b9577a3834ce10fe2703a26b28417d485bbbdff956" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "num 0.4.1", +] + +[[package]] +name = "arrow-array" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d02efa7253ede102d45a4e802a129e83bcc3f49884cab795b1ac223918e4318d" +dependencies = [ + "ahash 0.8.6", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.14.3", + "num 0.4.1", +] + +[[package]] +name = "arrow-buffer" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda119225204141138cb0541c692fbfef0e875ba01bfdeaed09e9d354f9d6195" +dependencies = [ + "bytes", + "half", + "num 0.4.1", +] + +[[package]] +name = "arrow-cast" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d825d51b9968868d50bc5af92388754056796dbc62a4e25307d588a1fc84dee" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "chrono", + "half", + "lexical-core", + "num 0.4.1", +] + +[[package]] +name = "arrow-csv" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43ef855dc6b126dc197f43e061d4de46b9d4c033aa51c2587657f7508242cef1" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "chrono", + "csv", + "csv-core", + "lazy_static", + "lexical-core", + "regex", +] + +[[package]] +name = "arrow-data" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "475a4c3699c8b4095ca61cecf15da6f67841847a5f5aac983ccb9a377d02f73a" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num 0.4.1", +] + +[[package]] +name = "arrow-ipc" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1248005c8ac549f869b7a840859d942bf62471479c1a2d82659d453eebcd166a" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "flatbuffers", +] + +[[package]] +name = "arrow-json" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f03d7e3b04dd688ccec354fe449aed56b831679f03e44ee2c1cfc4045067b69c" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "indexmap 2.1.0", + "lexical-core", + "num 0.4.1", + "serde", + "serde_json", +] + +[[package]] +name = "arrow-ord" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03b87aa408ea6a6300e49eb2eba0c032c88ed9dc19e0a9948489c55efdca71f4" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "half", + "num 0.4.1", +] + +[[package]] +name = "arrow-row" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "114a348ab581e7c9b6908fcab23cb39ff9f060eb19e72b13f8fb8eaa37f65d22" +dependencies = [ + "ahash 0.8.6", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", + "hashbrown 0.14.3", +] + +[[package]] +name = "arrow-schema" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d1d179c117b158853e0101bfbed5615e86fe97ee356b4af901f1c5001e1ce4b" +dependencies = [ + "serde", +] + +[[package]] +name = "arrow-select" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5c71e003202e67e9db139e5278c79f5520bb79922261dfe140e4637ee8b6108" +dependencies = [ + "ahash 0.8.6", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num 0.4.1", +] + +[[package]] +name = "arrow-string" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4cebbb282d6b9244895f4a9a912e55e57bce112554c7fa91fcec5459cb421ab" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "num 0.4.1", + "regex", + "regex-syntax 0.7.5", +] + +[[package]] +name = "async-channel" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +dependencies = [ + "concurrent-queue", + "event-listener 2.5.3", + "futures-core", +] + +[[package]] +name = "async-lock" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7125e42787d53db9dd54261812ef17e937c95a51e4d291373b670342fa44310c" +dependencies = [ + "event-listener 4.0.0", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-trait" +version = "0.1.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "atomic_refcell" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41e67cd8309bbd06cd603a9e693a784ac2e5d1e955f11286e355089fcab3047c" + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "azure_core" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6218987c374650fdad0b476bfc675729762c28dfb35f58608a38a2b1ea337dd" +dependencies = [ + "async-trait", + "base64 0.21.5", + "bytes", + "dyn-clone", + "futures", + "getrandom 0.2.11", + "hmac 0.12.1", + "http-types", + "log", + "once_cell", + "paste", + "pin-project", + "quick-xml 0.31.0", + "rand 0.8.5", + "reqwest", + "rustc_version", + "serde", + "serde_json", + "sha2 0.10.8", + "time 0.3.30", + "url", + "uuid 1.6.1", +] + +[[package]] +name = "azure_storage" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade8f2653e408de88b9eafec9f48c3c26b94026375e88adbd34523a7dd9795a1" +dependencies = [ + "RustyXML", + "async-lock", + "async-trait", + "azure_core", + "bytes", + "log", + "serde", + "serde_derive", + "time 0.3.30", + "url", + "uuid 1.6.1", +] + +[[package]] +name = "azure_storage_blobs" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "025701c7cc5b523100f0f3b2b01723564ec5a86c03236521c06826337047e872" +dependencies = [ + "RustyXML", + "azure_core", + "azure_storage", + "azure_svc_blobstorage", + "bytes", + "futures", + "log", + "serde", + "serde_derive", + "serde_json", + "time 0.3.30", + "url", + "uuid 1.6.1", +] + +[[package]] +name = "azure_svc_blobstorage" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76051e5bb67cea1055abe5e530a0878feac7e0ab4cbbcb4a6adc953a58993389" +dependencies = [ + "azure_core", + "bytes", + "futures", + "log", + "once_cell", + "serde", + "serde_json", + "time 0.3.30", +] + +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "base64" +version = "0.21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" + +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "brotli" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bumpalo" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" + +[[package]] +name = "bytecheck" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6372023ac861f6e6dc89c8344a8f398fb42aaba2b5dbc649ca0c0e9dbcb627" +dependencies = [ + "bytecheck_derive", + "ptr_meta", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7ec4c6f261935ad534c0c22dbef2201b45918860eb1c574b972bd213a76af61" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "cc" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "jobserver", + "libc", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-targets 0.48.5", +] + +[[package]] +name = "clap" +version = "4.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_lex" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + +[[package]] +name = "concurrent-queue" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" +dependencies = [ + "crossbeam-utils 0.8.18", +] + +[[package]] +name = "const-random" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aaf16c9c2c612020bcfd042e170f6e32de9b9d75adb5277cdbbd2e2c8c8299a" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.11", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + +[[package]] +name = "corosensei" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80128832c58ea9cbd041d2a759ec449224487b2c1e400453d99d244eead87a8e" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "libc", + "scopeguard", + "windows-sys 0.33.0", +] + +[[package]] +name = "cpufeatures" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +dependencies = [ + "libc", +] + +[[package]] +name = "cranelift-bforest" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2ab4512dfd3a6f4be184403a195f76e81a8a9f9e6c898e19d2dc3ce20e0115" +dependencies = [ + "cranelift-entity", +] + +[[package]] +name = "cranelift-codegen" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98b022ed2a5913a38839dfbafe6cf135342661293b08049843362df4301261dc" +dependencies = [ + "arrayvec", + "bumpalo", + "cranelift-bforest", + "cranelift-codegen-meta", + "cranelift-codegen-shared", + "cranelift-egraph", + "cranelift-entity", + "cranelift-isle", + "gimli 0.26.2", + "log", + "regalloc2", + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cranelift-codegen-meta" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "639307b45434ad112a98f8300c0f0ab085cbefcd767efcdef9ef19d4c0756e74" +dependencies = [ + "cranelift-codegen-shared", +] + +[[package]] +name = "cranelift-codegen-shared" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "278e52e29c53fcf32431ef08406c295699a70306d05a0715c5b1bf50e33a9ab7" + +[[package]] +name = "cranelift-egraph" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624b54323b06e675293939311943ba82d323bb340468ce1889be5da7932c8d73" +dependencies = [ + "cranelift-entity", + "fxhash", + "hashbrown 0.12.3", + "indexmap 1.9.3", + "log", + "smallvec", +] + +[[package]] +name = "cranelift-entity" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a59bcbca89c3f1b70b93ab3cbba5e5e0cbf3e63dadb23c7525cb142e21a9d4c" + +[[package]] +name = "cranelift-frontend" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d70abacb8cfef3dc8ff7e8836e9c1d70f7967dfdac824a4cd5e30223415aca6" +dependencies = [ + "cranelift-codegen", + "log", + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cranelift-isle" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "393bc73c451830ff8dbb3a07f61843d6cb41a084f9996319917c0b291ed785bb" + +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "crossbeam-channel" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b153fe7cbef478c567df0f972e02e6d736db11affe43dfc9c56a9374d1adfb87" +dependencies = [ + "crossbeam-utils 0.7.2", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fca89a0e215bab21874660c67903c5f143333cab1da83d041c7ded6053774751" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch", + "crossbeam-utils 0.8.18", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e3681d554572a651dda4186cd47240627c3d0114d45a95f6ad27f2f22e7548d" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "crossbeam-utils 0.8.18", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adc6598521bb5a83d491e8c1fe51db7296019d2ca3cb93cc6c2a20369a4d78a2" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils 0.8.18", +] + +[[package]] +name = "crossbeam-utils" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +dependencies = [ + "autocfg", + "cfg-if 0.1.10", + "lazy_static", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3a430a770ebd84726f584a90ee7f020d28db52c6d02138900f22341f866d39c" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-mac" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" +dependencies = [ + "generic-array", + "subtle", +] + +[[package]] +name = "csv" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + +[[package]] +name = "ct-logs" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8" +dependencies = [ + "sct 0.6.1", +] + +[[package]] +name = "darling" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "darling_macro" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "dashmap" +version = "5.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" +dependencies = [ + "cfg-if 1.0.0", + "hashbrown 0.14.3", + "lock_api 0.4.11", + "once_cell", + "parking_lot_core 0.9.9", +] + +[[package]] +name = "debugid" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6ee87af31d84ef885378aebca32be3d682b0e0dc119d5b4860a2c5bb5046730" +dependencies = [ + "serde", + "uuid 0.8.2", +] + +[[package]] +name = "deltalake" +version = "0.16.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb694b21358cfa35ec1ccf9443269d6e21a9afba5e942dceb50019748271e811" +dependencies = [ + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "async-trait", + "bytes", + "cfg-if 1.0.0", + "chrono", + "dynamodb_lock", + "errno", + "futures", + "itertools", + "lazy_static", + "libc", + "log", + "num-bigint", + "num-traits", + "num_cpus", + "object_store", + "once_cell", + "parking_lot 0.12.1", + "parquet", + "percent-encoding", + "rand 0.8.5", + "regex", + "rusoto_core", + "rusoto_credential", + "rusoto_dynamodb", + "rusoto_sts", + "serde", + "serde_json", + "thiserror", + "tokio", + "url", + "uuid 1.6.1", +] + +[[package]] +name = "deranged" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc" +dependencies = [ + "powerfmt", + "serde", +] + +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "deunicode" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a1abaf4d861455be59f64fd2b55606cb151fce304ede7165f410243ce96bde6" + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common", + "subtle", +] + +[[package]] +name = "dipstick" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e838d77faec4b6802f67954313f7586a9794867b1472839eb355eda084de9621" +dependencies = [ + "atomic_refcell", + "crossbeam-channel", + "lazy_static", + "log", + "minreq", + "num 0.2.1", + "parking_lot 0.10.2", + "time 0.1.45", +] + +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if 1.0.0", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + +[[package]] +name = "dyn-clone" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "545b22097d44f8a9581187cdf93de7a71e4722bf51200cfaba810865b49a495d" + +[[package]] +name = "dynamodb_lock" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c4242838952a07117a4444cdf59fdcb47f181525f2dd6ae4ca6281954d09c77" +dependencies = [ + "async-trait", + "log", + "maplit", + "rusoto_core", + "rusoto_dynamodb", + "thiserror", + "tokio", + "uuid 1.6.1", +] + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "encoding_rs" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "enum-iterator" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eeac5c5edb79e4e39fe8439ef35207780a11f69c52cbe424ce3dfad4cb78de6" +dependencies = [ + "enum-iterator-derive", +] + +[[package]] +name = "enum-iterator-derive" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c134c37760b27a871ba422106eedbb8247da973a09e82558bf26d619c882b159" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "enumset" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "226c0da7462c13fb57e5cc9e0dc8f0635e7d27f276a3a7fd30054647f669007d" +dependencies = [ + "enumset_derive", +] + +[[package]] +name = "enumset_derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08b6c6ab82d70f08844964ba10c7babb716de2ecaeab9be5717918a5177d3af" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "env_logger" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95b3f3e67048839cb0d0781f445682a35113da7121f7c949db0e2be96a4fbece" +dependencies = [ + "humantime", + "is-terminal", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "event-listener" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "770d968249b5d99410d61f5bf89057f3199a077a04d087092f58e7d10692baae" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" +dependencies = [ + "event-listener 4.0.0", + "pin-project-lite", +] + +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + +[[package]] +name = "flatbuffers" +version = "23.5.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640" +dependencies = [ + "bitflags 1.3.2", + "rustc_version", +] + +[[package]] +name = "flate2" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" + +[[package]] +name = "futures-executor" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" + +[[package]] +name = "futures-lite" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" +dependencies = [ + "fastrand 1.9.0", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + +[[package]] +name = "futures-macro" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "futures-sink" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" + +[[package]] +name = "futures-task" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" + +[[package]] +name = "futures-util" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", +] + +[[package]] +name = "gimli" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" +dependencies = [ + "fallible-iterator", + "indexmap 1.9.3", + "stable_deref_trait", +] + +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + +[[package]] +name = "h2" +version = "0.3.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap 2.1.0", + "slab", + "tokio", + "tokio-util 0.7.10", + "tracing", +] + +[[package]] +name = "half" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc52e53916c08643f1b56ec082790d1e86a32e58dc5268f897f313fbae7b4872" +dependencies = [ + "cfg-if 1.0.0", + "crunchy", + "num-traits", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash 0.7.7", +] + +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" +dependencies = [ + "crypto-mac", + "digest 0.9.0", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "hostname" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +dependencies = [ + "libc", + "match_cfg", + "winapi", +] + +[[package]] +name = "http" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "http-types" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e9b187a72d63adbfba487f48095306ac823049cb504ee195541e91c7775f5ad" +dependencies = [ + "anyhow", + "async-channel", + "base64 0.13.1", + "futures-lite", + "infer", + "pin-project-lite", + "rand 0.7.3", + "serde", + "serde_json", + "serde_qs", + "serde_urlencoded", + "url", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "hyper" +version = "0.14.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.4.10", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" +dependencies = [ + "ct-logs", + "futures-util", + "hyper", + "log", + "rustls 0.19.1", + "rustls-native-certs", + "tokio", + "tokio-rustls 0.22.0", + "webpki", +] + +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http", + "hyper", + "rustls 0.21.9", + "tokio", + "tokio-rustls 0.24.1", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +dependencies = [ + "equivalent", + "hashbrown 0.14.3", +] + +[[package]] +name = "infer" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac" + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + +[[package]] +name = "is-terminal" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +dependencies = [ + "hermit-abi", + "rustix", + "windows-sys 0.48.0", +] + +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "jmespatch" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7acf91a732ade34d8eda2dee9500a051833f14f0d3d10d77c149845d6ac6a5f0" +dependencies = [ + "lazy_static", + "serde", + "serde_json", + "slug", +] + +[[package]] +name = "jobserver" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "json-pointer" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fe841b94e719a482213cee19dd04927cf412f26d8dc84c5a446c081e49c2997" +dependencies = [ + "serde_json", +] + +[[package]] +name = "jsonway" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "effcb749443c905fbaef49d214f8b1049c240e0adb7af9baa0e201e625e4f9de" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "kafka-delta-ingest" +version = "0.2.0" +dependencies = [ + "anyhow", + "apache-avro", + "async-trait", + "azure_core", + "azure_storage", + "azure_storage_blobs", + "base64 0.13.1", + "bytes", + "chrono", + "clap", + "deltalake", + "dipstick", + "dynamodb_lock", + "env_logger", + "futures", + "jmespatch", + "lazy-init", + "lazy_static", + "log", + "maplit", + "rdkafka", + "rusoto_core", + "rusoto_credential", + "rusoto_s3", + "schema_registry_converter", + "sentry", + "serde", + "serde_json", + "serial_test", + "strum 0.20.0", + "strum_macros 0.20.1", + "tempfile", + "thiserror", + "time 0.3.30", + "tokio", + "tokio-stream", + "tokio-util 0.6.10", + "url", + "utime", + "uuid 0.8.2", + "wasmer", +] + +[[package]] +name = "lazy-init" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f40963626ac12dcaf92afc15e4c3db624858c92fd9f8ba2125eaada3ac2706f" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "leb128" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" + +[[package]] +name = "lexical-core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" +dependencies = [ + "lexical-parse-integer", + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-parse-integer" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-util" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lexical-write-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" +dependencies = [ + "lexical-util", + "lexical-write-integer", + "static_assertions", +] + +[[package]] +name = "lexical-write-integer" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "libc" +version = "0.2.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" + +[[package]] +name = "libflate" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ff4ae71b685bbad2f2f391fe74f6b7659a34871c08b210fdc039e43bee07d18" +dependencies = [ + "adler32", + "crc32fast", + "libflate_lz77", +] + +[[package]] +name = "libflate_lz77" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a52d3a8bfc85f250440e4424db7d857e241a3aebbbe301f3eb606ab15c39acbf" +dependencies = [ + "rle-decode-fast", +] + +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + +[[package]] +name = "libredox" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" +dependencies = [ + "bitflags 2.4.1", + "libc", + "redox_syscall 0.4.1", +] + +[[package]] +name = "libz-sys" +version = "1.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" + +[[package]] +name = "lock_api" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "lock_api" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "lz4" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e9e2dd86df36ce760a60f6ff6ad526f7ba1f14ba0356f8254fb6905e6494df1" +dependencies = [ + "libc", + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "mach" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +dependencies = [ + "libc", +] + +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + +[[package]] +name = "md-5" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15" +dependencies = [ + "block-buffer 0.9.0", + "digest 0.9.0", + "opaque-debug", +] + +[[package]] +name = "memchr" +version = "2.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" + +[[package]] +name = "memmap2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +dependencies = [ + "libc", +] + +[[package]] +name = "memmap2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d28bba84adfe6646737845bc5ebbfa2c08424eb1c37e94a1fd2a82adb56a872" +dependencies = [ + "libc", +] + +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + +[[package]] +name = "minreq" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb3371dfc7b772c540da1380123674a8e20583aca99907087d990ca58cf44203" +dependencies = [ + "log", +] + +[[package]] +name = "mio" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" +dependencies = [ + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.48.0", +] + +[[package]] +name = "more-asserts" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7843ec2de400bcbc6a6328c958dc38e5359da6e93e72e37bc5246bf1ae776389" + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "no-std-net" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bcece43b12349917e096cddfa66107277f123e6c96a5aea78711dc601a47152" +dependencies = [ + "serde", +] + +[[package]] +name = "num" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36" +dependencies = [ + "num-complex 0.2.4", + "num-integer", + "num-iter", + "num-rational 0.2.4", + "num-traits", +] + +[[package]] +name = "num" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05180d69e3da0e530ba2a1dae5110317e49e3b7f3d41be227dc5f92e49ee7af" +dependencies = [ + "num-bigint", + "num-complex 0.4.4", + "num-integer", + "num-iter", + "num-rational 0.4.1", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ba157ca0885411de85d6ca030ba7e2a83a28636056c7c699b07c8b6f7383214" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +dependencies = [ + "autocfg", + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "num_enum" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f646caf906c20226733ed5b1374287eb97e3c2a5c227ce668c1f2ce20ae57c9" +dependencies = [ + "num_enum_derive", +] + +[[package]] +name = "num_enum_derive" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "object" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +dependencies = [ + "memchr", +] + +[[package]] +name = "object_store" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f930c88a43b1c3f6e776dfe495b4afab89882dbc81530c632db2ed65451ebcb4" +dependencies = [ + "async-trait", + "base64 0.21.5", + "bytes", + "chrono", + "futures", + "humantime", + "hyper", + "itertools", + "parking_lot 0.12.1", + "percent-encoding", + "quick-xml 0.30.0", + "rand 0.8.5", + "reqwest", + "ring 0.16.20", + "serde", + "serde_json", + "snafu", + "tokio", + "tracing", + "url", + "walkdir", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "openssl" +version = "0.10.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b8419dc8cc6d866deb801274bba2e6f8f6108c1bb7fcc10ee5ab864931dbb45" +dependencies = [ + "bitflags 2.4.1", + "cfg-if 1.0.0", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-src" +version = "300.1.6+3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439fac53e092cd7442a3660c85dde4643ab3b5bd39040912388dcdabf6b88085" +dependencies = [ + "cc", +] + +[[package]] +name = "openssl-sys" +version = "0.9.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3eaad34cdd97d81de97964fc7f29e2d104f483840d906ef56daa1912338460b" +dependencies = [ + "cc", + "libc", + "openssl-src", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "parking" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" + +[[package]] +name = "parking_lot" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" +dependencies = [ + "lock_api 0.3.4", + "parking_lot_core 0.7.3", +] + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api 0.4.11", + "parking_lot_core 0.9.9", +] + +[[package]] +name = "parking_lot_core" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b93f386bb233083c799e6e642a9d73db98c24a5deeb95ffc85bf281255dffc98" +dependencies = [ + "cfg-if 0.1.10", + "cloudabi", + "libc", + "redox_syscall 0.1.57", + "smallvec", + "winapi", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "redox_syscall 0.4.1", + "smallvec", + "windows-targets 0.48.5", +] + +[[package]] +name = "parquet" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0463cc3b256d5f50408c49a4be3a16674f4c8ceef60941709620a062b1f6bf4d" +dependencies = [ + "ahash 0.8.6", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64 0.21.5", + "brotli", + "bytes", + "chrono", + "flate2", + "futures", + "hashbrown 0.14.3", + "lz4", + "num 0.4.1", + "num-bigint", + "object_store", + "paste", + "seq-macro", + "serde_json", + "snap", + "thrift", + "tokio", + "twox-hash", + "zstd", +] + +[[package]] +name = "paste" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "phf" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526" +dependencies = [ + "phf_shared", + "rand 0.7.3", +] + +[[package]] +name = "phf_shared" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "proc-macro-crate" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" +dependencies = [ + "once_cell", + "toml_edit", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "psl" +version = "2.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdc9f7a25d234ba11af714be527b60c8bc7ce1310011ef8dc032bd5ab1a7eadd" +dependencies = [ + "psl-types", +] + +[[package]] +name = "psl-types" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac" + +[[package]] +name = "ptr_meta" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "quad-rand" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "658fa1faf7a4cc5f057c9ee5ef560f717ad9d8dc66d975267f709624d6e1ab88" + +[[package]] +name = "quick-xml" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quick-xml" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", + "rand_pcg", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.11", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_pcg" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rayon" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils 0.8.18", +] + +[[package]] +name = "rdkafka" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1de127f294f2dba488ed46760b129d5ecbeabbd337ccbf3739cb29d50db2161c" +dependencies = [ + "futures", + "libc", + "log", + "rdkafka-sys", + "serde", + "serde_derive", + "serde_json", + "slab", + "tokio", +] + +[[package]] +name = "rdkafka-sys" +version = "4.7.0+2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55e0d2f9ba6253f6ec72385e453294f8618e9e15c2c6aba2a5c01ccf9622d615" +dependencies = [ + "libc", + "libz-sys", + "num_enum", + "openssl-sys", + "pkg-config", +] + +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_users" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" +dependencies = [ + "getrandom 0.2.11", + "libredox", + "thiserror", +] + +[[package]] +name = "regalloc2" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300d4fbfb40c1c66a78ba3ddd41c1110247cf52f97b87d0f2fc9209bd49b030c" +dependencies = [ + "fxhash", + "log", + "slice-group-by", + "smallvec", +] + +[[package]] +name = "regex" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax 0.8.2", +] + +[[package]] +name = "regex-automata" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.8.2", +] + +[[package]] +name = "regex-syntax" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" + +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + +[[package]] +name = "region" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76e189c2369884dce920945e2ddf79b3dff49e071a167dd1817fa9c4c00d512e" +dependencies = [ + "bitflags 1.3.2", + "libc", + "mach", + "winapi", +] + +[[package]] +name = "rend" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2571463863a6bd50c32f94402933f03457a3fbaf697a707c5be741e459f08fd" +dependencies = [ + "bytecheck", +] + +[[package]] +name = "reqwest" +version = "0.11.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" +dependencies = [ + "base64 0.21.5", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-rustls 0.24.2", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls 0.21.9", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "system-configuration", + "tokio", + "tokio-native-tls", + "tokio-rustls 0.24.1", + "tokio-util 0.7.10", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", + "winreg", +] + +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin 0.5.2", + "untrusted 0.7.1", + "web-sys", + "winapi", +] + +[[package]] +name = "ring" +version = "0.17.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" +dependencies = [ + "cc", + "getrandom 0.2.11", + "libc", + "spin 0.9.8", + "untrusted 0.9.0", + "windows-sys 0.48.0", +] + +[[package]] +name = "rkyv" +version = "0.7.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527a97cdfef66f65998b5f3b637c26f5a5ec09cc52a3f9932313ac645f4190f5" +dependencies = [ + "bitvec", + "bytecheck", + "bytes", + "hashbrown 0.12.3", + "indexmap 1.9.3", + "ptr_meta", + "rend", + "rkyv_derive", + "seahash", + "tinyvec", + "uuid 1.6.1", +] + +[[package]] +name = "rkyv_derive" +version = "0.7.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5c462a1328c8e67e4d6dbad1eb0355dd43e8ab432c6e227a43657f16ade5033" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "rle-decode-fast" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" + +[[package]] +name = "rusoto_core" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b4f000e8934c1b4f70adde180056812e7ea6b1a247952db8ee98c94cd3116cc" +dependencies = [ + "async-trait", + "base64 0.13.1", + "bytes", + "crc32fast", + "futures", + "http", + "hyper", + "hyper-rustls 0.22.1", + "lazy_static", + "log", + "rusoto_credential", + "rusoto_signature", + "rustc_version", + "serde", + "serde_json", + "tokio", + "xml-rs", +] + +[[package]] +name = "rusoto_credential" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a46b67db7bb66f5541e44db22b0a02fed59c9603e146db3a9e633272d3bac2f" +dependencies = [ + "async-trait", + "chrono", + "dirs-next", + "futures", + "hyper", + "serde", + "serde_json", + "shlex", + "tokio", + "zeroize", +] + +[[package]] +name = "rusoto_dynamodb" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7935e1f9ca57c4ee92a4d823dcd698eb8c992f7e84ca21976ae72cd2b03016e7" +dependencies = [ + "async-trait", + "bytes", + "futures", + "rusoto_core", + "serde", + "serde_json", +] + +[[package]] +name = "rusoto_s3" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "048c2fe811a823ad5a9acc976e8bf4f1d910df719dcf44b15c3e96c5b7a51027" +dependencies = [ + "async-trait", + "bytes", + "futures", + "rusoto_core", + "xml-rs", +] + +[[package]] +name = "rusoto_signature" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6264e93384b90a747758bcc82079711eacf2e755c3a8b5091687b5349d870bcc" +dependencies = [ + "base64 0.13.1", + "bytes", + "chrono", + "digest 0.9.0", + "futures", + "hex", + "hmac 0.11.0", + "http", + "hyper", + "log", + "md-5", + "percent-encoding", + "pin-project-lite", + "rusoto_credential", + "rustc_version", + "serde", + "sha2 0.9.9", + "tokio", +] + +[[package]] +name = "rusoto_sts" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7edd42473ac006fd54105f619e480b0a94136e7f53cf3fb73541363678fd92" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "futures", + "rusoto_core", + "serde_urlencoded", + "xml-rs", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "0.38.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfeae074e687625746172d639330f1de242a178bf3189b51e35a7a21573513ac" +dependencies = [ + "bitflags 2.4.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" +dependencies = [ + "base64 0.13.1", + "log", + "ring 0.16.20", + "sct 0.6.1", + "webpki", +] + +[[package]] +name = "rustls" +version = "0.21.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" +dependencies = [ + "log", + "ring 0.17.7", + "rustls-webpki", + "sct 0.7.1", +] + +[[package]] +name = "rustls-native-certs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092" +dependencies = [ + "openssl-probe", + "rustls 0.19.1", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.5", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring 0.17.7", + "untrusted 0.9.0", +] + +[[package]] +name = "rustversion" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +dependencies = [ + "windows-sys 0.48.0", +] + +[[package]] +name = "schema_registry_converter" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ea1c251bea837ece3126074d9e01c5868e5ae6135946404ceb20b5b2e264f7c" +dependencies = [ + "apache-avro", + "byteorder", + "dashmap", + "futures", + "reqwest", + "serde", + "serde_json", + "tokio", + "url", + "valico", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sct" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" +dependencies = [ + "ring 0.16.20", + "untrusted 0.7.1", +] + +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring 0.17.7", + "untrusted 0.9.0", +] + +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "self_cell" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58bf37232d3bb9a2c4e641ca2a11d83b5062066f88df7fed36c28772046d65ba" + +[[package]] +name = "semver" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" + +[[package]] +name = "sentry" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "546b9b6f76c26c60ffbcf0b7136e15169fe13d43949b4aadb7c1edc1c3f3a26f" +dependencies = [ + "httpdate", + "reqwest", + "sentry-backtrace", + "sentry-contexts", + "sentry-core", + "sentry-panic", + "tokio", +] + +[[package]] +name = "sentry-backtrace" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cd0cba2aff36ac98708f7a6e7abbdde82dbaf180d5870c41084dc1b473648b9" +dependencies = [ + "backtrace", + "lazy_static", + "regex", + "sentry-core", +] + +[[package]] +name = "sentry-contexts" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bacf1c62427c6c97b896640d0c4dd204bbd3b79dd192d7cb40891aa5ee11d58" +dependencies = [ + "hostname", + "lazy_static", + "libc", + "regex", + "rustc_version", + "sentry-core", + "uname", +] + +[[package]] +name = "sentry-core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a957270c9a430218f8031c866493061a27e35a70250e9527f093563a33ce6b" +dependencies = [ + "chrono", + "lazy_static", + "rand 0.8.5", + "sentry-types", + "serde", + "serde_json", +] + +[[package]] +name = "sentry-panic" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692bf989f0c99f025e33d7f58e62822c3771f56d189698c66dcc863122255d95" +dependencies = [ + "sentry-backtrace", + "sentry-core", +] + +[[package]] +name = "sentry-types" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4dd2266fee014a86e250e98e389191ecd23be546b5c42b6a2fb9af2972fadac" +dependencies = [ + "chrono", + "debugid", + "serde", + "serde_json", + "thiserror", + "url", + "uuid 0.8.2", +] + +[[package]] +name = "seq-macro" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" + +[[package]] +name = "serde" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde-wasm-bindgen" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b4c031cd0d9014307d82b8abf653c0290fbdaeb4c02d00c63cf52f728628bf" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + +[[package]] +name = "serde_derive" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "serde_json" +version = "1.0.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_qs" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7715380eec75f029a4ef7de39a9200e0a63823176b759d055b613f5a87df6a6" +dependencies = [ + "percent-encoding", + "serde", + "thiserror", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serial_test" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e56dd856803e253c8f298af3f4d7eb0ae5e23a737252cd90bb4f3b435033b2d" +dependencies = [ + "dashmap", + "futures", + "lazy_static", + "log", + "parking_lot 0.12.1", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "sha2" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if 1.0.0", + "cpufeatures", + "digest 0.9.0", + "opaque-debug", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest 0.10.7", +] + +[[package]] +name = "shared-buffer" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6c99835bad52957e7aa241d3975ed17c1e5f8c92026377d117a606f36b84b16" +dependencies = [ + "bytes", + "memmap2 0.6.2", +] + +[[package]] +name = "shlex" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "simdutf8" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" + +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "slice-group-by" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" + +[[package]] +name = "slug" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bd94acec9c8da640005f8e135a39fc0372e74535e6b368b7a04b875f784c8c4" +dependencies = [ + "deunicode", + "wasm-bindgen", +] + +[[package]] +name = "smallvec" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" + +[[package]] +name = "snafu" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" +dependencies = [ + "doc-comment", + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "snap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" + +[[package]] +name = "socket2" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "socket2" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "strum" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7318c509b5ba57f18533982607f24070a55d353e90d4cae30c467cdb2ad5ac5c" + +[[package]] +name = "strum" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" + +[[package]] +name = "strum_macros" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee8bc6b87a5112aeeab1f4a9f7ab634fe6cbefc4850006df31267f4cfb9e3149" +dependencies = [ + "heck 0.3.3", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "strum_macros" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "rustversion", + "syn 1.0.109", +] + +[[package]] +name = "subtle" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "target-lexicon" +version = "0.12.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69758bda2e78f098e4ccb393021a0963bb3442eac05f135c30f61b7370bbafae" + +[[package]] +name = "tempfile" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +dependencies = [ + "cfg-if 1.0.0", + "fastrand 2.0.1", + "redox_syscall 0.4.1", + "rustix", + "windows-sys 0.48.0", +] + +[[package]] +name = "termcolor" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thiserror" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "thrift" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" +dependencies = [ + "byteorder", + "integer-encoding", + "ordered-float", +] + +[[package]] +name = "time" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi", +] + +[[package]] +name = "time" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" +dependencies = [ + "deranged", + "itoa", + "js-sys", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" +dependencies = [ + "time-core", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "num_cpus", + "parking_lot 0.12.1", + "pin-project-lite", + "signal-hook-registry", + "socket2 0.5.5", + "tokio-macros", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-macros" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" +dependencies = [ + "rustls 0.19.1", + "tokio", + "webpki", +] + +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.9", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "toml_datetime" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" + +[[package]] +name = "toml_edit" +version = "0.19.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" +dependencies = [ + "indexmap 2.1.0", + "toml_datetime", + "winnow", +] + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "twox-hash" +version = "1.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" +dependencies = [ + "cfg-if 1.0.0", + "static_assertions", +] + +[[package]] +name = "typed-builder" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89851716b67b937e393b3daa8423e67ddfc4bbbf1654bcf05488e95e0828db0c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "uname" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b72f89f0ca32e4db1c04e2a72f5345d59796d4866a1ee0609084569f73683dc8" +dependencies = [ + "libc", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" + +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "uritemplate-next" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcde98d1fc3f528255b1ecb22fb688ee0d23deb672a8c57127df10b98b4bd18c" +dependencies = [ + "regex", +] + +[[package]] +name = "url" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + +[[package]] +name = "utime" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91baa0c65eabd12fcbdac8cc35ff16159cab95cae96d0222d6d0271db6193cef" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "uuid" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +dependencies = [ + "getrandom 0.2.11", + "serde", +] + +[[package]] +name = "uuid" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" +dependencies = [ + "getrandom 0.2.11", + "serde", +] + +[[package]] +name = "valico" +version = "3.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "647856408e327686b6640397d19f60fac3e64c3bfaa6afc409da63ef7da45edb" +dependencies = [ + "addr", + "base64 0.13.1", + "chrono", + "json-pointer", + "jsonway", + "percent-encoding", + "phf", + "phf_codegen", + "regex", + "serde", + "serde_json", + "uritemplate-next", + "url", + "uuid 0.8.2", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "waker-fn" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" + +[[package]] +name = "walkdir" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.39", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" + +[[package]] +name = "wasm-encoder" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ba64e81215916eaeb48fee292f29401d69235d62d8b8fd92a7b2844ec5ae5f7" +dependencies = [ + "leb128", +] + +[[package]] +name = "wasm-streams" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "wasmer" +version = "4.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5467c7a23f9be04d5691590bea509dbea27e5ba5810d0020bef908456a495f33" +dependencies = [ + "bytes", + "cfg-if 1.0.0", + "derivative", + "indexmap 1.9.3", + "js-sys", + "more-asserts", + "rustc-demangle", + "serde", + "serde-wasm-bindgen", + "shared-buffer", + "target-lexicon", + "thiserror", + "wasm-bindgen", + "wasmer-compiler", + "wasmer-compiler-cranelift", + "wasmer-derive", + "wasmer-types", + "wasmer-vm", + "wat", + "winapi", +] + +[[package]] +name = "wasmer-compiler" +version = "4.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "510ad01a668d774f3a103a7c219bbc0970be93e8f1b27e2fdb48d1f4ccd1deff" +dependencies = [ + "backtrace", + "bytes", + "cfg-if 1.0.0", + "enum-iterator", + "enumset", + "lazy_static", + "leb128", + "memmap2 0.5.10", + "more-asserts", + "region", + "rkyv", + "self_cell", + "shared-buffer", + "smallvec", + "thiserror", + "wasmer-types", + "wasmer-vm", + "wasmparser", + "winapi", +] + +[[package]] +name = "wasmer-compiler-cranelift" +version = "4.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54bf93078990d83960d798de3c5935bddaba771fc2fefb9ed6bab9c0bbdea5c1" +dependencies = [ + "cranelift-codegen", + "cranelift-entity", + "cranelift-frontend", + "gimli 0.26.2", + "more-asserts", + "rayon", + "smallvec", + "target-lexicon", + "tracing", + "wasmer-compiler", + "wasmer-types", +] + +[[package]] +name = "wasmer-derive" +version = "4.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b374fd34d97b1c091d8675f9bc472df52dc6787d139d3762d42c7dc84813a9b" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "wasmer-types" +version = "4.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0caf1c87937b52aba8e9f920a278e1beda282f7439612c0b48f51a58e7a87bab" +dependencies = [ + "bytecheck", + "enum-iterator", + "enumset", + "indexmap 1.9.3", + "more-asserts", + "rkyv", + "target-lexicon", + "thiserror", +] + +[[package]] +name = "wasmer-vm" +version = "4.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58315c25492bc72a33f47a7d7fb0869a0106fc0164ec051e349a9e1eddba9a01" +dependencies = [ + "backtrace", + "cc", + "cfg-if 1.0.0", + "corosensei", + "crossbeam-queue", + "dashmap", + "derivative", + "enum-iterator", + "fnv", + "indexmap 1.9.3", + "lazy_static", + "libc", + "mach", + "memoffset", + "more-asserts", + "region", + "scopeguard", + "thiserror", + "wasmer-types", + "winapi", +] + +[[package]] +name = "wasmparser" +version = "0.95.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ea896273ea99b15132414be1da01ab0d8836415083298ecaffbe308eaac87a" +dependencies = [ + "indexmap 1.9.3", + "url", +] + +[[package]] +name = "wast" +version = "64.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a259b226fd6910225aa7baeba82f9d9933b6d00f2ce1b49b80fa4214328237cc" +dependencies = [ + "leb128", + "memchr", + "unicode-width", + "wasm-encoder", +] + +[[package]] +name = "wat" +version = "1.0.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53253d920ab413fca1c7dc2161d601c79b4fdf631d0ba51dd4343bf9b556c3f6" +dependencies = [ + "wast", +] + +[[package]] +name = "web-sys" +version = "0.3.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring 0.16.20", + "untrusted 0.7.1", +] + +[[package]] +name = "webpki-roots" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.51.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43dbb096663629518eb1dfa72d80243ca5a6aca764cae62a2df70af760a9be75" +dependencies = [ + "windows_aarch64_msvc 0.33.0", + "windows_i686_gnu 0.33.0", + "windows_i686_msvc 0.33.0", + "windows_x86_64_gnu 0.33.0", + "windows_x86_64_msvc 0.33.0", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd761fd3eb9ab8cc1ed81e56e567f02dd82c4c837e48ac3b2181b9ffc5060807" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + +[[package]] +name = "windows_i686_gnu" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab0cf703a96bab2dc0c02c0fa748491294bf9b7feb27e1f4f96340f208ada0e" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + +[[package]] +name = "windows_i686_msvc" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cfdbe89cc9ad7ce618ba34abc34bbb6c36d99e96cae2245b7943cd75ee773d0" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4dd9b0c0e9ece7bb22e84d70d01b71c6d6248b81a3c60d11869451b4cb24784" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff1e4aa646495048ec7f3ffddc411e1d829c026a2ec62b39da15c1055e406eaa" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + +[[package]] +name = "winnow" +version = "0.5.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67b5f0a4e7a27a64c651977932b9dc5667ca7fc31ac44b03ed37a0cf42fdfff" +dependencies = [ + "memchr", +] + +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if 1.0.0", + "windows-sys 0.48.0", +] + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "xml-rs" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fcb9cbac069e033553e8bb871be2fbdffcab578eb25bd0f7c508cedc6dcd75a" + +[[package]] +name = "zerocopy" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96f8f25c15a0edc9b07eb66e7e6e97d124c0505435c382fde1ab7ceb188aa956" +dependencies = [ + "byteorder", + "zerocopy-derive 0.6.5", +] + +[[package]] +name = "zerocopy" +version = "0.7.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d075cf85bbb114e933343e087b92f2146bac0d55b534cbb8188becf0039948e" +dependencies = [ + "zerocopy-derive 0.7.29", +] + +[[package]] +name = "zerocopy-derive" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "855e0f6af9cd72b87d8a6c586f3cb583f5cdcc62c2c80869d8cd7e96fdf7ee20" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86cd5ca076997b97ef09d3ad65efe811fa68c9e874cb636ccb211223a813b0c2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "zeroize" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" + +[[package]] +name = "zstd" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "6.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.9+zstd.1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/kafka-delta-ingest/Cargo.toml b/kafka-delta-ingest/Cargo.toml new file mode 100644 index 0000000..6416709 --- /dev/null +++ b/kafka-delta-ingest/Cargo.toml @@ -0,0 +1,80 @@ +[package] +name = "kafka-delta-ingest" +version = "0.2.0" +authors = ["R. Tyler Croy ", "Christian Williams "] +edition = "2018" + +[dependencies] +anyhow = "1" +async-trait = "0.1" +apache-avro = "^0.14" +base64 = "0.13" +bytes = "1" +chrono = "0.4.31" +clap = { version = "4", features = ["color"] } +dipstick = "0.9" +env_logger = "0" +futures = "0.3" +jmespatch = { version = "0.3", features = ["sync"] } +lazy_static = "1" +log = "0" +maplit = "1" +rdkafka = { version = "0.28", features = ["ssl-vendored"] } +schema_registry_converter = { version = "3.1.0", features = ["easy", "json", "avro"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +strum = "0.20" +strum_macros = "0.20" +thiserror = "1" +tokio = { version = "1", features = ["full"] } +tokio-stream = { version = "0", features = ["fs"] } +tokio-util = "0.6.3" +uuid = { version = "0.8", features = ["serde", "v4"] } +url = "2.3" + +#deltalake = { version = "0.16.5", features = ["arrow", "json", "parquet"], optional = true } +deltalake-core = { git = "https://github.com/delta-io/delta-rs", branch = "main", features = ["json"]} +deltalake-aws = { git = "https://github.com/delta-io/delta-rs", branch = "main", optional = true } +deltalake-azure = { git = "https://github.com/delta-io/delta-rs", branch = "main", optional = true } + +# s3 feature enabled +dynamodb_lock = { version = "0.6.0", optional = true } +rusoto_core = { version = "0.47", default-features = false, features = ["rustls"], optional = true } +rusoto_credential = { version = "0.47", optional = true } +rusoto_s3 = { version = "0.47", default-features = false, features = ["rustls"], optional = true } + +# sentry +sentry = { version = "0.23.0", optional = true } + +# azure feature enabled, mostly used for tests +azure_core = { version = "0.18.0", optional = true } +azure_storage = { version = "0.18.0", optional = true } +azure_storage_blobs = { version = "0.18.0", optional = true } + +[features] +default = [] +sentry-ext = ["sentry"] +dynamic-linking = [ "rdkafka/dynamic-linking" ] +azure = [ + "deltalake-azure", + "azure_core", + "azure_storage", + "azure_storage_blobs" +] +s3 = [ + "deltalake-aws", + "dynamodb_lock", + "rusoto_core", + "rusoto_credential", + "rusoto_s3", +] +wasm = [] + +[dev-dependencies] +utime = "0.3" +serial_test = "*" +tempfile = "3" +time = "0.3.20" + +[profile.release] +lto = true diff --git a/build.rs b/kafka-delta-ingest/build.rs similarity index 100% rename from build.rs rename to kafka-delta-ingest/build.rs diff --git a/src/coercions.rs b/kafka-delta-ingest/src/coercions.rs similarity index 100% rename from src/coercions.rs rename to kafka-delta-ingest/src/coercions.rs diff --git a/src/cursor.rs b/kafka-delta-ingest/src/cursor.rs similarity index 100% rename from src/cursor.rs rename to kafka-delta-ingest/src/cursor.rs diff --git a/src/dead_letters.rs b/kafka-delta-ingest/src/dead_letters.rs similarity index 99% rename from src/dead_letters.rs rename to kafka-delta-ingest/src/dead_letters.rs index a4e709e..d1acec7 100644 --- a/src/dead_letters.rs +++ b/kafka-delta-ingest/src/dead_letters.rs @@ -254,7 +254,7 @@ impl DeltaSinkDeadLetterQueue { dynamo_lock_options::DYNAMO_LOCK_PARTITION_KEY_VALUE.to_string() => std::env::var(env_vars::DEAD_LETTER_DYNAMO_LOCK_PARTITION_KEY_VALUE) .unwrap_or_else(|_| "kafka_delta_ingest-dead_letters".to_string()), }; - #[cfg(feature = "azure")] + #[cfg(all(feature = "azure", not(feature = "s3")))] let opts = HashMap::default(); let table = crate::delta_helpers::load_table(table_uri, opts.clone()).await?; diff --git a/src/delta_helpers.rs b/kafka-delta-ingest/src/delta_helpers.rs similarity index 100% rename from src/delta_helpers.rs rename to kafka-delta-ingest/src/delta_helpers.rs diff --git a/src/lib.rs b/kafka-delta-ingest/src/lib.rs similarity index 100% rename from src/lib.rs rename to kafka-delta-ingest/src/lib.rs diff --git a/src/main.rs b/kafka-delta-ingest/src/main.rs similarity index 100% rename from src/main.rs rename to kafka-delta-ingest/src/main.rs diff --git a/src/metrics.rs b/kafka-delta-ingest/src/metrics.rs similarity index 100% rename from src/metrics.rs rename to kafka-delta-ingest/src/metrics.rs diff --git a/src/offsets.rs b/kafka-delta-ingest/src/offsets.rs similarity index 100% rename from src/offsets.rs rename to kafka-delta-ingest/src/offsets.rs diff --git a/src/serialization.rs b/kafka-delta-ingest/src/serialization.rs similarity index 100% rename from src/serialization.rs rename to kafka-delta-ingest/src/serialization.rs diff --git a/src/transforms.rs b/kafka-delta-ingest/src/transforms.rs similarity index 100% rename from src/transforms.rs rename to kafka-delta-ingest/src/transforms.rs diff --git a/src/value_buffers.rs b/kafka-delta-ingest/src/value_buffers.rs similarity index 98% rename from src/value_buffers.rs rename to kafka-delta-ingest/src/value_buffers.rs index 6e61725..9169ae5 100644 --- a/src/value_buffers.rs +++ b/kafka-delta-ingest/src/value_buffers.rs @@ -179,7 +179,7 @@ mod tests { values.sort(); - let expected: Vec = vec!["\"0:0\"", "\"0:1\"", "\"0:2\"", "\"1:0\"", "\"1:1\""] + let expected: Vec = ["\"0:0\"", "\"0:1\"", "\"0:2\"", "\"1:0\"", "\"1:1\""] .iter() .map(|s| s.to_string()) .collect(); diff --git a/src/writer.rs b/kafka-delta-ingest/src/writer.rs similarity index 99% rename from src/writer.rs rename to kafka-delta-ingest/src/writer.rs index bcb7657..bff577f 100644 --- a/src/writer.rs +++ b/kafka-delta-ingest/src/writer.rs @@ -1196,7 +1196,7 @@ mod tests { ("date", ColumnValueStat::Value(v)) => { assert_eq!("2021-06-22", v.as_str().unwrap()) } - _ => assert!(false, "Key should not be present"), + _ => panic!("Key should not be present"), } } @@ -1224,7 +1224,7 @@ mod tests { ("date", ColumnValueStat::Value(v)) => { assert_eq!("2021-06-22", v.as_str().unwrap()) } - _ => assert!(false, "Key should not be present"), + _ => panic!("Key should not be present"), } } @@ -1250,7 +1250,7 @@ mod tests { ("some_list", ColumnCountStat::Value(v)) => assert_eq!(100, *v), ("some_nested_list", ColumnCountStat::Value(v)) => assert_eq!(0, *v), ("date", ColumnCountStat::Value(v)) => assert_eq!(0, *v), - _ => assert!(false, "Key should not be present"), + _ => panic!("Key should not be present"), } } } diff --git a/kafka-delta-ingest/tests/buffer_flush_tests.rs b/kafka-delta-ingest/tests/buffer_flush_tests.rs new file mode 100644 index 0000000..71fb987 --- /dev/null +++ b/kafka-delta-ingest/tests/buffer_flush_tests.rs @@ -0,0 +1,161 @@ +#[allow(dead_code)] +mod helpers; + +use log::info; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use serial_test::serial; +use tokio::time::{sleep, Duration}; + +use kafka_delta_ingest::IngestOptions; + +#[tokio::test] +#[serial] +async fn test_flush_when_latency_expires() { + let (topic, table, producer, kdi, token, rt) = helpers::create_and_run_kdi( + "flush_when_latency_expires", + json!({ + "id": "integer", + "date": "string", + }), + vec!["date"], + 1, + Some(IngestOptions { + app_id: "flush_when_latency_expires".to_string(), + // buffer for 5 seconds before flush + allowed_latency: 5, + // large value - avoid flushing on num messages + max_messages_per_batch: 5000, + // large value - avoid flushing on file size + min_bytes_per_file: 1000000, + kafka_brokers: helpers::test_broker(), + ..Default::default() + }), + ) + .await; + + for m in create_generator(1).take(10) { + info!("Writing test message"); + helpers::send_json(&producer, &topic, &serde_json::to_value(m).unwrap()).await; + } + + // wait for latency flush + helpers::wait_until_version_created(&table, 1); + + for m in create_generator(11).take(10) { + info!("Writing test message"); + helpers::send_json(&producer, &topic, &serde_json::to_value(m).unwrap()).await; + } + + // wait for latency flush + helpers::wait_until_version_created(&table, 2); + + let v1_rows: Vec = helpers::read_table_content_at_version_as(&table, 1).await; + let v2_rows: Vec = helpers::read_table_content_as(&table).await; + + assert_eq!(v1_rows.len(), 10); + assert_eq!(v2_rows.len(), 20); + + token.cancel(); + kdi.await.unwrap(); + rt.shutdown_background(); +} + +#[tokio::test] +#[serial] +async fn test_dont_write_an_empty_buffer() { + let (topic, table, producer, kdi, token, rt) = helpers::create_and_run_kdi( + "dont_write_an_empty_buffer", + json!({ + "id": "integer", + "date": "string", + }), + vec!["date"], + 1, + Some(IngestOptions { + app_id: "dont_write_an_empty_buffer".to_string(), + // buffer for 5 seconds before flush + allowed_latency: 5, + kafka_brokers: helpers::test_broker(), + ..Default::default() + }), + ) + .await; + // write one version so we can make sure the stream is up and running. + + for m in create_generator(1).take(10) { + info!("Writing test message"); + helpers::send_json(&producer, &topic, &serde_json::to_value(m).unwrap()).await; + } + + // wait for latency flush + helpers::wait_until_version_created(&table, 1); + + // wait for the latency timer to trigger + sleep(Duration::from_secs(6)).await; + + // verify that an empty version _was not_ created. + // i.e. we should still be at version 1 + + let t = deltalake_core::open_table(&table).await.unwrap(); + + assert_eq!(1, t.version()); + + token.cancel(); + kdi.await.unwrap(); + rt.shutdown_background(); +} + +#[tokio::test] +#[serial] +async fn test_flush_on_size_without_latency_expiration() { + let (topic, table, producer, kdi, token, rt) = helpers::create_and_run_kdi( + "flush_on_size_without_latency_expiration", + json!({ + "id": "integer", + "date": "string", + }), + vec!["date"], + 1, + Some(IngestOptions { + app_id: "flush_on_size_without_latency_expiration".to_string(), + // buffer for an hour + allowed_latency: 3600, + // create a record batch when we have 10 messages + max_messages_per_batch: 10, + // tiny buffer size for write flush + min_bytes_per_file: 20, + kafka_brokers: helpers::test_broker(), + ..Default::default() + }), + ) + .await; + + for m in create_generator(1).take(10) { + info!("Writing test message"); + helpers::send_json(&producer, &topic, &serde_json::to_value(m).unwrap()).await; + } + + helpers::wait_until_version_created(&table, 1); + + let data: Vec = helpers::read_table_content_at_version_as(&table, 1).await; + + assert_eq!(data.len(), 10); + + token.cancel(); + kdi.await.unwrap(); + rt.shutdown_background(); +} + +#[derive(Clone, Serialize, Deserialize, Debug)] +struct TestMsg { + id: u64, + date: String, +} + +fn create_generator(staring_id: u64) -> impl Iterator { + std::iter::successors(Some(staring_id), |n| Some(*n + 1)).map(|n| TestMsg { + id: n, + date: "2022-06-03".to_string(), + }) +} diff --git a/tests/data/.gitignore b/kafka-delta-ingest/tests/data/.gitignore similarity index 100% rename from tests/data/.gitignore rename to kafka-delta-ingest/tests/data/.gitignore diff --git a/tests/data/default_schema.avro b/kafka-delta-ingest/tests/data/default_schema.avro similarity index 100% rename from tests/data/default_schema.avro rename to kafka-delta-ingest/tests/data/default_schema.avro diff --git a/tests/data/emails/.gitignore b/kafka-delta-ingest/tests/data/emails/.gitignore similarity index 100% rename from tests/data/emails/.gitignore rename to kafka-delta-ingest/tests/data/emails/.gitignore diff --git a/tests/data/emails/_delta_log/00000000000000000000.json b/kafka-delta-ingest/tests/data/emails/_delta_log/00000000000000000000.json similarity index 100% rename from tests/data/emails/_delta_log/00000000000000000000.json rename to kafka-delta-ingest/tests/data/emails/_delta_log/00000000000000000000.json diff --git a/tests/data/example/.gitignore b/kafka-delta-ingest/tests/data/example/.gitignore similarity index 100% rename from tests/data/example/.gitignore rename to kafka-delta-ingest/tests/data/example/.gitignore diff --git a/tests/data/example/_delta_log/00000000000000000000.json b/kafka-delta-ingest/tests/data/example/_delta_log/00000000000000000000.json similarity index 100% rename from tests/data/example/_delta_log/00000000000000000000.json rename to kafka-delta-ingest/tests/data/example/_delta_log/00000000000000000000.json diff --git a/tests/data/web_requests/.gitignore b/kafka-delta-ingest/tests/data/web_requests/.gitignore similarity index 100% rename from tests/data/web_requests/.gitignore rename to kafka-delta-ingest/tests/data/web_requests/.gitignore diff --git a/tests/data/web_requests/_delta_log/00000000000000000000.json b/kafka-delta-ingest/tests/data/web_requests/_delta_log/00000000000000000000.json similarity index 100% rename from tests/data/web_requests/_delta_log/00000000000000000000.json rename to kafka-delta-ingest/tests/data/web_requests/_delta_log/00000000000000000000.json diff --git a/tests/data/zero_offset/.gitignore b/kafka-delta-ingest/tests/data/zero_offset/.gitignore similarity index 100% rename from tests/data/zero_offset/.gitignore rename to kafka-delta-ingest/tests/data/zero_offset/.gitignore diff --git a/tests/data/zero_offset/_delta_log/00000000000000000000.json b/kafka-delta-ingest/tests/data/zero_offset/_delta_log/00000000000000000000.json similarity index 100% rename from tests/data/zero_offset/_delta_log/00000000000000000000.json rename to kafka-delta-ingest/tests/data/zero_offset/_delta_log/00000000000000000000.json diff --git a/tests/dead_letter_tests.rs b/kafka-delta-ingest/tests/dead_letter_tests.rs similarity index 100% rename from tests/dead_letter_tests.rs rename to kafka-delta-ingest/tests/dead_letter_tests.rs diff --git a/kafka-delta-ingest/tests/delta_partitions_tests.rs b/kafka-delta-ingest/tests/delta_partitions_tests.rs new file mode 100644 index 0000000..0e872a2 --- /dev/null +++ b/kafka-delta-ingest/tests/delta_partitions_tests.rs @@ -0,0 +1,139 @@ +#[allow(dead_code)] +mod helpers; + +use deltalake_core::kernel::{Action, Add}; +use deltalake_core::protocol::{DeltaOperation, SaveMode}; +use kafka_delta_ingest::writer::*; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::collections::HashMap; + +#[derive(Debug, Serialize, Deserialize)] +struct TestMsg { + id: u32, + color: Option, +} + +impl TestMsg { + fn new(id: u32, color: &str) -> Self { + Self { + id, + color: Some(color.to_string()), + } + } + + fn new_color_null(id: u32) -> Self { + Self { id, color: None } + } +} + +#[tokio::test] +async fn test_delta_partitions() { + let table_path = helpers::create_local_table( + json!({ + "id": "integer", + "color": "string", + }), + vec!["color"], + "test_delta_partitions", + ); + + let mut table = deltalake_core::open_table(&table_path).await.unwrap(); + let mut delta_writer = DataWriter::for_table(&table, HashMap::new()).unwrap(); + + let batch1 = vec![ + TestMsg::new(1, "red"), + TestMsg::new(2, "red"), + TestMsg::new(3, "blue"), + TestMsg::new(4, "red"), + ]; + + let batch2 = vec![ + TestMsg::new(5, "blue"), + TestMsg::new(6, "red"), + TestMsg::new(7, "blue"), + TestMsg::new(8, "blue"), + TestMsg::new_color_null(9), + ]; + + delta_writer.write(msgs_to_values(batch1)).await.unwrap(); + delta_writer.write(msgs_to_values(batch2)).await.unwrap(); + + let result = delta_writer + .write_parquet_files(&table.table_uri()) + .await + .unwrap(); + + for add in result.iter() { + match add + .partition_values + .get("color") + .unwrap() + .clone() + .as_deref() + { + Some("red") => { + assert!(add.path.starts_with("color=red")); + assert_eq!(&get_stats_value(add, "numRecords"), "4"); + assert_eq!(msg(get_stats_value(add, "minValues")).id, 1); + assert_eq!(msg(get_stats_value(add, "maxValues")).id, 6); + } + Some("blue") => { + assert!(add.path.starts_with("color=blue")); + assert_eq!(&get_stats_value(add, "numRecords"), "4"); + assert_eq!(msg(get_stats_value(add, "minValues")).id, 3); + assert_eq!(msg(get_stats_value(add, "maxValues")).id, 8); + } + None => { + assert!(add.path.starts_with("color=__HIVE_DEFAULT_PARTITION__")); + assert_eq!(&get_stats_value(add, "numRecords"), "1"); + assert_eq!(msg(get_stats_value(add, "minValues")).id, 9); + assert_eq!(msg(get_stats_value(add, "maxValues")).id, 9); + } + other => { + panic!("{:?}", other); + } + } + } + + let operation = DeltaOperation::Write { + mode: SaveMode::Append, + partition_by: None, + predicate: None, + }; + + let version = deltalake_core::operations::transaction::commit( + table.log_store().clone().as_ref(), + &result.iter().cloned().map(Action::Add).collect(), + operation, + &table.state, + None, + ) + .await + .expect("Failed to create transaction"); + + deltalake_core::checkpoints::create_checkpoint(&table) + .await + .unwrap(); + + let table = deltalake_core::open_table(&table_path).await.unwrap(); + assert_eq!(table.version(), version); + + std::fs::remove_dir_all(&table_path).unwrap(); +} + +fn msgs_to_values(values: Vec) -> Vec { + values + .iter() + .map(|j| serde_json::to_value(j).unwrap()) + .collect() +} + +fn get_stats_value(add: &Add, key: &str) -> String { + let v: Value = serde_json::from_str(add.stats.as_ref().unwrap()).unwrap(); + v.as_object().unwrap().get(key).unwrap().to_string() +} + +fn msg(s: String) -> TestMsg { + serde_json::from_str(&s).unwrap() +} diff --git a/tests/deserialization_tests.rs b/kafka-delta-ingest/tests/deserialization_tests.rs similarity index 100% rename from tests/deserialization_tests.rs rename to kafka-delta-ingest/tests/deserialization_tests.rs diff --git a/kafka-delta-ingest/tests/emails_azure_blob_tests.rs b/kafka-delta-ingest/tests/emails_azure_blob_tests.rs new file mode 100644 index 0000000..d85f111 --- /dev/null +++ b/kafka-delta-ingest/tests/emails_azure_blob_tests.rs @@ -0,0 +1,155 @@ +#![cfg(feature = "azure")] +#[allow(dead_code)] +mod helpers; + +use std::collections::HashMap; +use std::env; +use std::thread; +use std::time::Duration; +use time::OffsetDateTime; + +use serial_test::serial; +use uuid::Uuid; + +use kafka_delta_ingest::IngestOptions; + +use helpers::*; + +use azure_storage::{prelude::BlobSasPermissions, shared_access_signature::SasProtocol}; + +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn when_both_workers_started_simultaneously_azure() { + run_emails_s3_tests(false).await; +} + +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn when_rebalance_happens_azure() { + run_emails_s3_tests(true).await; +} + +async fn run_emails_s3_tests(initiate_rebalance: bool) { + deltalake_azure::register_handlers(None); + helpers::init_logger(); + let topic = format!("emails_azure-{}", Uuid::new_v4()); + let table = prepare_table(&topic).await; + let options = create_options(); + let scope = TestScope::new(&topic, &table, options).await; + + let w1 = scope.create_and_start(helpers::WORKER_1).await; + + // in order to initiate rebalance we first send messages, + // ensure that worker 1 consumes some of them and then create worker 2, + // otherwise, to proceed without rebalance the two workers has to be created simultaneously + let w2 = if initiate_rebalance { + scope.send_messages(TEST_TOTAL_MESSAGES).await; + thread::sleep(Duration::from_secs(1)); + scope.create_and_start(helpers::WORKER_2).await + } else { + let w = scope.create_and_start(helpers::WORKER_2).await; + thread::sleep(Duration::from_secs(4)); + scope.send_messages(TEST_TOTAL_MESSAGES).await; + w + }; + + // this will end up with more app_ids than actual, + // since we're not sure which partitions will get each worker + let partitions = create_partitions_app_ids(helpers::TEST_PARTITIONS); + + // wait until the destination table will get every expected message, we check this summing up + // the each offset of each partition to get the TOTAL_MESSAGES value + scope + .wait_on_total_offset(partitions, helpers::TEST_TOTAL_MESSAGES) + .await; + + println!("Waiting on workers futures to exit..."); + // wait until workers are completely stopped + let w1_result = w1.await; + println!("Worker 1 finished - {:?}", w1_result); + + let w2_result = w2.await; + println!("Worker 2 finished - {:?}", w2_result); + + scope.validate_data().await; + scope.shutdown(); +} + +async fn prepare_table(topic: &str) -> String { + let container_client = azure_storage_blobs::prelude::ClientBuilder::emulator() + .container_client(helpers::test_s3_bucket()); + let source_blob = + container_client.blob_client(format!("emails/_delta_log/00000000000000000000.json")); + let sas_url = { + let now = OffsetDateTime::now_utc(); + let later = now + time::Duration::hours(1); + let sas = source_blob + .shared_access_signature( + BlobSasPermissions { + read: true, + ..Default::default() + }, + later, + ) + .await + .unwrap() + .start(now) + .protocol(SasProtocol::HttpHttps); + source_blob.generate_signed_blob_url(&sas).unwrap() + }; + container_client + .blob_client(format!("{}/_delta_log/00000000000000000000.json", topic)) + .copy_from_url(sas_url) + .await + .unwrap(); + + format!("az://{}/{}", helpers::test_s3_bucket(), topic) +} + +fn create_partitions_app_ids(num_p: i32) -> Vec { + let mut vector = Vec::new(); + for n in 0..num_p { + vector.push(format!("{}-{}", helpers::TEST_APP_ID, n)); + } + vector +} + +fn create_options() -> IngestOptions { + env::set_var("AZURE_STORAGE_USE_EMULATOR", "true"); + env::set_var("AZURE_ACCOUNT_NAME", "devstoreaccount1"); + env::set_var( + "AZURE_ACCESS_KEY", + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + ); + env::set_var("AZURE_STORAGE_CONTAINER_NAME", "tests"); + env::set_var("AZURE_STORAGE_ALLOW_HTTP", "1"); + env::set_var("AZURITE_BLOB_STORAGE_URL", "http://127.0.0.1:10000"); + env::set_var( + "AZURE_STORAGE_CONNECTION_STRING", + "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;"); + + let mut additional_kafka_settings = HashMap::new(); + additional_kafka_settings.insert("auto.offset.reset".to_string(), "earliest".to_string()); + let additional_kafka_settings = Some(additional_kafka_settings); + + let allowed_latency = 2; + let max_messages_per_batch = 10; + let min_bytes_per_file = 370; + + let mut transforms = HashMap::new(); + transforms.insert("date".to_string(), "substr(timestamp,`0`,`10`)".to_string()); + transforms.insert("_kafka_offset".to_string(), "kafka.offset".to_string()); + + IngestOptions { + transforms, + kafka_brokers: helpers::test_broker(), + consumer_group_id: TEST_CONSUMER_GROUP_ID.to_string(), + app_id: TEST_APP_ID.to_string(), + additional_kafka_settings, + allowed_latency, + max_messages_per_batch, + min_bytes_per_file, + write_checkpoints: true, + ..Default::default() + } +} diff --git a/kafka-delta-ingest/tests/emails_s3_tests.rs b/kafka-delta-ingest/tests/emails_s3_tests.rs new file mode 100644 index 0000000..8d5e652 --- /dev/null +++ b/kafka-delta-ingest/tests/emails_s3_tests.rs @@ -0,0 +1,171 @@ +#![cfg(feature = "s3")] + +#[allow(dead_code)] +mod helpers; + +use std::collections::HashMap; +use std::env; +use std::io::Read; +use std::thread; +use std::time::Duration; + +use serial_test::serial; +use uuid::Uuid; + +use kafka_delta_ingest::IngestOptions; +use rusoto_core::Region; +use rusoto_s3::{CopyObjectRequest, PutObjectRequest, S3}; + +use helpers::*; + +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn when_both_workers_started_simultaneously() { + run_emails_s3_tests(false).await; +} + +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn when_rebalance_happens() { + run_emails_s3_tests(true).await; +} + +async fn run_emails_s3_tests(initiate_rebalance: bool) { + deltalake_aws::register_handlers(None); + helpers::init_logger(); + let topic = format!("emails_s3-{}", Uuid::new_v4()); + let table = prepare_table(&topic).await; + let options = create_options(helpers::WORKER_1); + let scope = TestScope::new(&topic, &table, options).await; + + let w1 = scope.create_and_start(WORKER_1).await; + + // in order to initiate rebalance we first send messages, + // ensure that worker 1 consumes some of them and then create worker 2, + // otherwise, to proceed without rebalance the two workers has to be created simultaneously + let w2 = if initiate_rebalance { + scope.send_messages(TEST_TOTAL_MESSAGES).await; + thread::sleep(Duration::from_secs(1)); + scope.create_and_start(WORKER_2).await + } else { + let w = scope.create_and_start(WORKER_2).await; + thread::sleep(Duration::from_secs(4)); + scope.send_messages(TEST_TOTAL_MESSAGES).await; + w + }; + + // this will end up with more app_ids than actual, + // since we're not sure which partitions will get each worker + let partitions = create_partitions_app_ids(TEST_PARTITIONS); + + // wait until the destination table will get every expected message, we check this summing up + // the each offset of each partition to get the TOTAL_MESSAGES value + scope + .wait_on_total_offset(partitions, TEST_TOTAL_MESSAGES) + .await; + + println!("Waiting on workers futures to exit..."); + // wait until workers are completely stopped + let w1_result = w1.await; + println!("Worker 1 finished - {:?}", w1_result); + + let w2_result = w2.await; + println!("Worker 2 finished - {:?}", w2_result); + + scope.validate_data().await; + scope.shutdown(); +} + +fn create_options(name: &str) -> IngestOptions { + env::set_var("AWS_ENDPOINT_URL", helpers::test_aws_endpoint()); + env::set_var("AWS_S3_LOCKING_PROVIDER", "dynamodb"); + env::set_var("AWS_REGION", "us-east-2"); + env::set_var("AWS_STORAGE_ALLOW_HTTP", "true"); + env::set_var("DELTA_DYNAMO_TABLE_NAME", "locks"); + env::set_var("DYNAMO_LOCK_OWNER_NAME", name); + env::set_var("DYNAMO_LOCK_PARTITION_KEY_VALUE", "emails_s3_tests"); + env::set_var("DYNAMO_LOCK_REFRESH_PERIOD_MILLIS", "100"); + env::set_var("DYNAMO_LOCK_ADDITIONAL_TIME_TO_WAIT_MILLIS", "100"); + env::set_var("DYNAMO_LOCK_LEASE_DURATION", "2"); + + let mut additional_kafka_settings = HashMap::new(); + additional_kafka_settings.insert("auto.offset.reset".to_string(), "earliest".to_string()); + let additional_kafka_settings = Some(additional_kafka_settings); + + let allowed_latency = 2; + let max_messages_per_batch = 10; + let min_bytes_per_file = 370; + + let mut transforms = HashMap::new(); + transforms.insert("date".to_string(), "substr(timestamp,`0`,`10`)".to_string()); + transforms.insert("_kafka_offset".to_string(), "kafka.offset".to_string()); + + IngestOptions { + transforms, + kafka_brokers: helpers::test_broker(), + consumer_group_id: helpers::TEST_CONSUMER_GROUP_ID.to_string(), + app_id: helpers::TEST_APP_ID.to_string(), + additional_kafka_settings, + allowed_latency, + max_messages_per_batch, + min_bytes_per_file, + write_checkpoints: true, + ..Default::default() + } +} + +async fn prepare_table(topic: &str) -> String { + match env::var("AWS_ACCESS_KEY_ID") { + Err(_) => env::set_var("AWS_ACCESS_KEY_ID", "test"), + Ok(_) => {} + } + match env::var("AWS_SECRET_ACCESS_KEY") { + Err(_) => env::set_var("AWS_SECRET_ACCESS_KEY", "test"), + Ok(_) => {} + } + + let s3 = rusoto_s3::S3Client::new(Region::Custom { + name: "custom".to_string(), + endpoint: helpers::test_aws_endpoint(), + }); + + /* + * Copy the local fixture to create a simple delta table in storage. + */ + let mut buf = vec![]; + let _original_log = + std::fs::File::open("tests/data/emails/_delta_log/00000000000000000000.json") + .unwrap() + .read_to_end(&mut buf); + + s3.put_object(PutObjectRequest { + bucket: helpers::test_s3_bucket(), + body: Some(buf.into()), + key: "emails/_delta_log/00000000000000000000.json".into(), + ..Default::default() + }) + .await + .unwrap(); + + s3.copy_object(CopyObjectRequest { + bucket: helpers::test_s3_bucket(), + key: format!("{}/_delta_log/00000000000000000000.json", topic), + copy_source: format!( + "/{}/emails/_delta_log/00000000000000000000.json", + helpers::test_s3_bucket(), + ), + ..Default::default() + }) + .await + .unwrap(); + + format!("s3://{}/{}", helpers::test_s3_bucket(), topic) +} + +fn create_partitions_app_ids(num_p: i32) -> Vec { + let mut vector = Vec::new(); + for n in 0..num_p { + vector.push(format!("{}-{}", TEST_APP_ID, n)); + } + vector +} diff --git a/kafka-delta-ingest/tests/helpers/mod.rs b/kafka-delta-ingest/tests/helpers/mod.rs new file mode 100644 index 0000000..ce00f65 --- /dev/null +++ b/kafka-delta-ingest/tests/helpers/mod.rs @@ -0,0 +1,660 @@ +use std::collections::HashMap; +use std::env; +use std::fs::{File, OpenOptions}; +use std::io::prelude::*; +use std::io::{BufReader, Cursor}; +use std::path::Path as FilePath; +use std::sync::Arc; +use std::time::Duration; + +use bytes::Buf; +use chrono::prelude::*; +use deltalake_core::kernel::{Action, Add, Metadata, Protocol, Remove, Txn}; +use deltalake_core::parquet::{ + file::reader::{FileReader, SerializedFileReader}, + record::RowAccessor, +}; +use deltalake_core::storage::ObjectStoreRef; +use deltalake_core::{DeltaTable, Path}; +use kafka_delta_ingest::{start_ingest, IngestOptions}; +use rdkafka::admin::{AdminClient, AdminOptions, NewTopic, TopicReplication}; +use rdkafka::client::DefaultClientContext; +use rdkafka::producer::{FutureProducer, FutureRecord}; +use rdkafka::util::{DefaultRuntime, Timeout}; +use rdkafka::ClientConfig; +use serde::de::DeserializeOwned; +use serde_json::{json, Value}; +use tokio::runtime::Runtime; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +/* + * Return the KAFKA_BROKERS set in the environment or default ot the local machine's port 9092 + */ +pub fn test_broker() -> String { + env::var("KAFKA_BROKERS").unwrap_or("0.0.0.0:9092".into()) +} + +pub fn test_aws_endpoint() -> String { + env::var("AWS_ENDPOINT_URL").unwrap_or("http://0.0.0.0:4566".into()) +} + +pub fn test_s3_bucket() -> String { + env::var("AWS_S3_BUCKET").unwrap_or("tests".into()) +} + +pub async fn create_topic(topic: &str, num_partitions: i32) { + let admin_client: AdminClient<_> = ClientConfig::new() + .set("bootstrap.servers", test_broker()) + .create() + .unwrap(); + + let new_topic = NewTopic::new(topic, num_partitions, TopicReplication::Fixed(1)); + + admin_client + .create_topics(&[new_topic], &AdminOptions::default()) + .await + .unwrap(); +} + +pub async fn delete_topic(topic: &str) { + let admin_client: AdminClient<_> = ClientConfig::new() + .set("bootstrap.servers", test_broker()) + .create() + .unwrap(); + + admin_client + .delete_topics(&[topic], &AdminOptions::default()) + .await + .unwrap(); +} + +pub fn create_producer() -> FutureProducer { + ClientConfig::new() + .set("bootstrap.servers", test_broker()) + .create() + .unwrap() +} + +pub async fn send_json(producer: &FutureProducer, topic: &str, json: &Value) -> (i32, i64) { + let json = serde_json::to_string(json).unwrap(); + + let record: FutureRecord = FutureRecord::to(topic).payload(&json); + producer.send(record, Timeout::Never).await.unwrap() +} + +pub async fn send_encoded(producer: &FutureProducer, topic: &str, content: Vec) -> (i32, i64) { + let record: FutureRecord> = FutureRecord::to(topic).payload(&content); + producer.send(record, Timeout::Never).await.unwrap() +} + +pub async fn send_kv_json( + producer: &FutureProducer, + topic: &str, + key: String, + json: &Value, +) -> (i32, i64) { + let json = serde_json::to_string(json).unwrap(); + + let record: FutureRecord = FutureRecord::to(topic).payload(&json).key(&key); + producer.send(record, Timeout::Never).await.unwrap() +} + +pub async fn send_bytes(producer: &FutureProducer, topic: &str, bytes: &Vec) { + let record: FutureRecord> = FutureRecord::to(topic).payload(bytes); + let _ = producer.send(record, Timeout::Never).await; +} + +// Example parquet read is taken from https://docs.rs/parquet/4.1.0/parquet/arrow/index.html#example-of-reading-parquet-file-into-arrow-record-batch +// TODO Research whether it's possible to read parquet data from bytes but not from file +pub async fn read_files_from_store(table: &DeltaTable) -> Vec { + let s3 = table.object_store().clone(); + let paths = table.get_files_iter(); + let tmp = format!(".test-{}.tmp", Uuid::new_v4()); + let mut list = Vec::new(); + + for path in paths { + let get_result = s3.get(&path).await.unwrap(); + let bytes = get_result.bytes().await.unwrap(); + { + let mut file = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(&tmp) + .unwrap(); + file.write_all(bytes.chunk()).unwrap(); + file.flush().unwrap(); + drop(file) + } + + let reader = SerializedFileReader::new(File::open(&tmp).unwrap()).unwrap(); + + let row_iter = reader.get_row_iter(None).unwrap(); + + for record in row_iter { + if let Ok(record) = record { + list.push(record.get_string(0).unwrap().parse::().unwrap()); + } + } + } + + std::fs::remove_file(tmp).unwrap(); + + list.sort(); + list +} + +fn parse_type(schema: &Value) -> Value { + match schema { + Value::String(_) => schema.clone(), + Value::Object(_) => json!({ + "type": "struct", + "fields": parse_fields(schema), + }), + Value::Array(v) if v.len() == 1 => json!({ + "type": "array", + "elementType": parse_type(v.first().unwrap()), + "containsNull": true, + }), + _ => panic!("Unsupported type {}", schema), + } +} + +fn parse_fields(schema: &Value) -> Value { + let iter = schema.as_object().unwrap().iter().map(|(name, value)| { + json!({ + "name": name, + "type": parse_type(value), + "metadata": {}, + "nullable": true, + }) + }); + Value::Array(iter.collect()) +} + +pub fn create_metadata_action_json(schema: Value, partitions: &[&str]) -> String { + let schema = json!({ + "type": "struct", + "fields": parse_fields(&schema), + }); + + json!({ + "metaData": { + "id": "ec285dbc-6479-4cc1-b038-1de97afabf9b", + "format": {"provider":"parquet","options":{}}, + "schemaString": schema.to_string(), + "partitionColumns": partitions, + "createdTime": 1621845641001u64, + "configuration": {}, + } + }) + .to_string() +} + +pub async fn cleanup_kdi(topic: &str, table: &str) { + delete_topic(topic).await; + std::fs::create_dir_all(table).unwrap(); +} + +pub async fn create_and_run_kdi( + app_id: &str, + schema: Value, + delta_partitions: Vec<&str>, + kafka_num_partitions: i32, + opts: Option, +) -> ( + String, + String, + FutureProducer, + JoinHandle<()>, + Arc, + Runtime, +) { + init_logger(); + + #[cfg(feature = "s3")] + deltalake_aws::register_handlers(None); + #[cfg(feature = "azure")] + deltalake_azure::register_handlers(None); + println!("OPTS!: {:?}", opts); + let topic = format!("{}-{}", app_id, Uuid::new_v4()); + let table = create_local_table(schema, delta_partitions, &topic); + create_topic(&topic, kafka_num_partitions).await; + + let opts = opts + .map(|o| IngestOptions { + app_id: app_id.to_string(), + ..o + }) + .unwrap_or_else(|| IngestOptions { + app_id: app_id.to_string(), + allowed_latency: 10, + max_messages_per_batch: 1, + min_bytes_per_file: 20, + ..IngestOptions::default() + }); + + let (kdi, token, rt) = create_kdi(&topic, &table, opts); + let producer = create_producer(); + (topic, table, producer, kdi, token, rt) +} + +pub fn create_local_table(schema: Value, partitions: Vec<&str>, table_name: &str) -> String { + let path = format!("./tests/data/gen/{}-{}", table_name, Uuid::new_v4()); + create_local_table_in(schema, partitions, &path); + path +} + +pub fn create_local_table_in(schema: Value, partitions: Vec<&str>, path: &str) { + let v0 = format!("{}/_delta_log/00000000000000000000.json", &path); + + std::fs::create_dir_all(FilePath::new(&v0).parent().unwrap()).unwrap(); + + let mut file = File::create(v0).unwrap(); + + writeln!(file, r#"{{"commitInfo":{{"timestamp":1621845641000,"operation":"CREATE TABLE","operationParameters":{{"isManaged":"false","description":null,"partitionBy":"[]","properties":"{{}}"}},"isBlindAppend":true}}}}"#).unwrap(); + writeln!( + file, + r#"{{"protocol":{{"minReaderVersion":1,"minWriterVersion":2}}}}"# + ) + .unwrap(); + writeln!(file, "{}", create_metadata_action_json(schema, &partitions)).unwrap(); +} + +pub fn create_kdi_with( + topic: &str, + table: &str, + worker_name: Option, + options: IngestOptions, +) -> (JoinHandle<()>, Arc, Runtime) { + let app_id = options.app_id.to_string(); + let worker_name = worker_name.unwrap_or(app_id.clone()); + + env::set_var("AWS_ENDPOINT_URL", test_aws_endpoint()); + env::set_var("AWS_S3_LOCKING_PROVIDER", "dynamodb"); + env::set_var("AWS_REGION", "us-east-2"); + env::set_var("DYNAMO_LOCK_TABLE_NAME", "locks"); + env::set_var("DYNAMO_LOCK_OWNER_NAME", Uuid::new_v4().to_string()); + env::set_var("DYNAMO_LOCK_PARTITION_KEY_VALUE", app_id.clone()); + env::set_var("DYNAMO_LOCK_REFRESH_PERIOD_MILLIS", "100"); + env::set_var("DYNAMO_LOCK_ADDITIONAL_TIME_TO_WAIT_MILLIS", "100"); + env::set_var("DYNAMO_LOCK_LEASE_DURATION", "2"); + + let rt = create_runtime(&worker_name); + let token = Arc::new(CancellationToken::new()); + + let run_loop = { + let token = token.clone(); + let topic = topic.to_string(); + let table = table.to_string(); + rt.spawn(async move { + start_ingest(topic, table, options, token.clone()) + .await + .unwrap() + }) + }; + + (run_loop, token, rt) +} + +pub fn create_kdi( + topic: &str, + table: &str, + options: IngestOptions, +) -> (JoinHandle<()>, Arc, Runtime) { + create_kdi_with(topic, table, None, options) +} + +pub fn create_runtime(name: &str) -> Runtime { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .thread_name(name) + .thread_stack_size(3 * 1024 * 1024) + .enable_io() + .enable_time() + .build() + .expect("Tokio runtime error") +} + +pub fn init_logger() { + // Any time the test_aws_endpoint() is being used the ability to hit HTTP hosts + // needs to be enabled + env::set_var("AWS_ALLOW_HTTP", "true"); + + let _ = env_logger::Builder::new() + .format(|buf, record| { + let thread_name = std::thread::current() + .name() + .unwrap_or("UNKNOWN") + .to_string(); + writeln!( + buf, + "{} {} [{}] - {}: {}", + Local::now().format("%Y-%m-%dT%H:%M:%S"), + record.module_path().unwrap(), + record.level(), + thread_name, + record.args(), + ) + }) + // .filter(Some("dipstick"), log::LevelFilter::Info) + // .filter(Some("rusoto_core"), log::LevelFilter::Info) + // .filter(Some("deltalake"), log::LevelFilter::Info) + .filter(None, log::LevelFilter::Info) + .try_init(); +} + +pub fn wait_until_file_created(path: &FilePath) { + let start_time = Local::now(); + loop { + if path.exists() { + return; + } + + let now = Local::now(); + let poll_time = now - start_time; + std::thread::sleep(Duration::from_secs(1)); + + if poll_time > chrono::Duration::seconds(180) { + panic!("File was not created before timeout"); + } + } +} + +pub fn wait_until_version_created(table: &str, version: i64) { + let path = format!("{}/_delta_log/{:020}.json", table, version); + wait_until_file_created(FilePath::new(&path)); +} + +pub async fn expect_termination_within(kdi: JoinHandle<()>, seconds: i64) { + let start_time = Local::now(); + let timelimit = chrono::Duration::seconds(seconds); + + loop { + if kdi.is_finished() { + kdi.await.unwrap(); + return; + } + let now = Local::now(); + let poll_time = now - start_time; + + if poll_time > timelimit { + panic!("KDI did not terminate within timeout",); + } + } +} + +pub async fn read_table_content_as(table_uri: &str) -> Vec { + read_table_content_as_jsons(table_uri) + .await + .iter() + .map(|v| serde_json::from_value(v.clone()).unwrap()) + .collect() +} + +pub async fn read_table_content_at_version_as( + table_uri: &str, + version: i64, +) -> Vec { + read_table_content_at_version_as_jsons(table_uri, version) + .await + .iter() + .map(|v| serde_json::from_value(v.clone()).unwrap()) + .collect() +} + +pub async fn read_table_content_as_jsons(table_uri: &str) -> Vec { + let table = deltalake_core::open_table(table_uri).await.unwrap(); + let store = table.object_store().clone(); + json_listify_table_content(table, store).await +} + +pub async fn read_table_content_at_version_as_jsons(table_uri: &str, version: i64) -> Vec { + let table = deltalake_core::open_table_with_version(table_uri, version) + .await + .unwrap(); + let store = table.object_store().clone(); + + json_listify_table_content(table, store).await +} + +async fn json_listify_table_content(table: DeltaTable, store: ObjectStoreRef) -> Vec { + let tmp = format!(".test-{}.tmp", Uuid::new_v4()); + let mut list = Vec::new(); + for file in table.get_files_iter() { + let get_result = store.get(&file).await.unwrap(); + let bytes = get_result.bytes().await.unwrap(); + let mut file = File::create(&tmp).unwrap(); + file.write_all(bytes.chunk()).unwrap(); + drop(file); + let reader = SerializedFileReader::new(File::open(&tmp).unwrap()).unwrap(); + let row_iter = reader.get_row_iter(None).unwrap(); + + for record in row_iter { + list.push(record.unwrap().to_json_value()); + } + } + + if !list.is_empty() { + std::fs::remove_file(tmp).unwrap(); + } + + list +} + +pub fn commit_file_path(table: &str, version: i64) -> String { + format!("{}/_delta_log/{:020}.json", table, version) +} + +pub async fn inspect_table(path: &str) { + let table = deltalake_core::open_table(path).await.unwrap(); + println!("Inspecting table {}", path); + for (k, v) in table.get_app_transaction_version().iter() { + println!(" {}: {}", k, v); + } + let store = table.object_store().clone(); + + for version in 1..=table.version() { + let log_path = format!("{}/_delta_log/{:020}.json", path, version); + let get_result = store.get(&Path::parse(&log_path).unwrap()).await.unwrap(); + let bytes = get_result.bytes().await.unwrap(); + let reader = BufReader::new(Cursor::new(bytes.chunk())); + + println!("Version {}:", version); + + for line in reader.lines() { + let action: Action = serde_json::from_str(line.unwrap().as_str()).unwrap(); + match action { + Action::Txn(t) => { + println!(" Txn: {}: {}", t.app_id, t.version) + } + Action::Add(a) => { + let stats = a.get_stats().unwrap().unwrap(); + println!(" Add: {}. Records: {}", &a.path, stats.num_records); + let full_path = format!("{}/{}", &path, &a.path); + let parquet_bytes = store + .get(&Path::parse(&full_path).unwrap()) + .await + .unwrap() + .bytes() + .await + .unwrap(); + let reader = SerializedFileReader::new(parquet_bytes).unwrap(); + for record in reader.get_row_iter(None).unwrap() { + println!(" - {}", record.unwrap().to_json_value()) + } + } + _ => println!("Unknown action {:?}", action), + } + } + } + println!(); + println!("Checkpoints:"); + for version in 1..=table.version() { + if version % 10 == 0 { + println!("Version: {}", version); + let log_path = format!("{}/_delta_log/{:020}.checkpoint.parquet", path, version); + let bytes = store + .get(&Path::parse(&log_path).unwrap()) + .await + .unwrap() + .bytes() + .await + .unwrap(); + let reader = SerializedFileReader::new(bytes).unwrap(); + let mut i = 0; + for record in reader.get_row_iter(None).unwrap() { + let json = record.unwrap().to_json_value(); + if let Some(m) = parse_json_field::(&json, "metaData") { + println!(" {}. metaData: {}", i, m.id); + } + if let Some(p) = parse_json_field::(&json, "protocol") { + println!( + " {}. protocol: minReaderVersion={}, minWriterVersion={}", + i, p.min_reader_version, p.min_writer_version + ); + } + if let Some(t) = parse_json_field::(&json, "txn") { + println!(" {}. txn: appId={}, version={}", i, t.app_id, t.version); + } + if let Some(r) = parse_json_field::(&json, "remove") { + println!(" {}. remove: {}", i, r.path); + } + if let Some(a) = parse_json_field::(&json, "add") { + let records = a + .get_stats() + .ok() + .flatten() + .map(|s| s.num_records) + .unwrap_or(-1); + println!(" {}. add[{}]: {}", i, records, a.path); + } + + i += 1; + } + println!() + } + } +} + +fn parse_json_field(value: &Value, key: &str) -> Option { + value + .as_object() + .and_then(|v| v.get(key)) + .and_then(|v| serde_json::from_value::(v.clone()).ok()) +} + +pub const TEST_APP_ID: &str = "emails_test"; +pub const TEST_CONSUMER_GROUP_ID: &str = "kafka_delta_ingest_emails"; +pub const TEST_PARTITIONS: i32 = 4; +pub const TEST_TOTAL_MESSAGES: i32 = 200; + +pub const WORKER_1: &str = "WORKER-1"; +pub const WORKER_2: &str = "WORKER-2"; + +pub struct TestScope { + pub topic: String, + pub table: String, + pub workers_token: Arc, + pub runtime: HashMap<&'static str, Runtime>, + options: IngestOptions, +} + +impl TestScope { + pub async fn new(topic: &str, table: &str, options: IngestOptions) -> Self { + let workers_token = Arc::new(CancellationToken::new()); + let mut runtime = HashMap::new(); + runtime.insert(WORKER_1, create_runtime(WORKER_1)); + runtime.insert(WORKER_2, create_runtime(WORKER_2)); + + println!("Topic: {}", &topic); + println!("Table: {}", &table); + create_topic(topic, TEST_PARTITIONS).await; + + Self { + topic: topic.into(), + table: table.into(), + workers_token, + runtime, + options, + } + } + + pub fn shutdown(self) { + for (_, rt) in self.runtime { + rt.shutdown_background() + } + } + + pub async fn create_and_start(&self, name: &str) -> JoinHandle<()> { + let rt = self.runtime.get(name).unwrap(); + let topic = self.topic.clone(); + let table = self.table.clone(); + let token = self.workers_token.clone(); + let options = self.options.clone(); + rt.spawn(async move { + let res = start_ingest(topic, table, options, token.clone()).await; + res.unwrap_or_else(|e| println!("AN ERROR OCCURED: {}", e)); + println!("Ingest process exited"); + + token.cancel(); + }) + } + + pub async fn send_messages(&self, amount: i32) { + let producer = create_producer(); + let now: DateTime = Utc::now(); + + println!("Sending {} messages to {}", amount, &self.topic); + for n in 0..amount { + let json = &json!({ + "id": n.to_string(), + "sender": format!("sender-{}@example.com", n), + "recipient": format!("recipient-{}@example.com", n), + "timestamp": (now + chrono::Duration::seconds(1)).to_rfc3339_opts(SecondsFormat::Secs, true), + }); + send_json(&producer, &self.topic, json).await; + } + println!("All messages are sent"); + } + + pub async fn wait_on_total_offset(&self, apps: Vec, offset: i32) { + let mut table = deltalake_core::open_table(&self.table).await.unwrap(); + let expected_total = offset - TEST_PARTITIONS; + loop { + table.update().await.unwrap(); + let mut total = 0; + for key in apps.iter() { + total += table + .get_app_transaction_version() + .get(key) + .copied() + .unwrap_or(0); + } + + if total >= expected_total as i64 { + self.workers_token.cancel(); + println!("All messages are in delta"); + return; + } + + println!("Expecting offsets in delta {}/{}...", total, expected_total); + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + + pub async fn validate_data(&self) { + let table = deltalake_core::open_table(&self.table).await.unwrap(); + let result = read_files_from_store(&table).await; + let r: Vec = (0..TEST_TOTAL_MESSAGES).collect(); + println!("Got messages {}/{}", result.len(), TEST_TOTAL_MESSAGES); + + if result.len() != TEST_TOTAL_MESSAGES as usize { + inspect_table(&self.table).await; + } + + assert_eq!(result, r); + } +} diff --git a/tests/json/web_requests-100.json b/kafka-delta-ingest/tests/json/web_requests-100.json similarity index 100% rename from tests/json/web_requests-100.json rename to kafka-delta-ingest/tests/json/web_requests-100.json diff --git a/tests/json/web_requests-100K.json.tar.gz b/kafka-delta-ingest/tests/json/web_requests-100K.json.tar.gz similarity index 100% rename from tests/json/web_requests-100K.json.tar.gz rename to kafka-delta-ingest/tests/json/web_requests-100K.json.tar.gz diff --git a/kafka-delta-ingest/tests/offset_tests.rs b/kafka-delta-ingest/tests/offset_tests.rs new file mode 100644 index 0000000..17dc277 --- /dev/null +++ b/kafka-delta-ingest/tests/offset_tests.rs @@ -0,0 +1,397 @@ +use deltalake_core::DeltaTable; +use log::{debug, info}; +use rdkafka::{producer::Producer, util::Timeout}; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use serial_test::serial; +use std::path::Path; +use uuid::Uuid; + +use kafka_delta_ingest::{AutoOffsetReset, IngestOptions}; +#[allow(dead_code)] +mod helpers; + +#[derive(Debug, Serialize, Deserialize)] +struct TestMsg { + id: u64, + color: String, +} + +impl TestMsg { + fn new(id: u64) -> Self { + Self { + id, + color: "default".to_string(), + } + } +} + +#[tokio::test] +async fn zero_offset_issue() { + let table = "./tests/data/zero_offset"; + helpers::init_logger(); + let topic = format!("zero_offset_issue_{}", Uuid::new_v4()); + + helpers::create_topic(&topic, 1).await; + + let (kdi, token, rt) = helpers::create_kdi( + &topic, + table, + IngestOptions { + app_id: "zero_offset".to_string(), + allowed_latency: 5, + max_messages_per_batch: 1, + min_bytes_per_file: 20, + ..Default::default() + }, + ); + + { + // check that there's only 1 record in table + let table = deltalake_core::open_table(table).await.unwrap(); + assert_eq!(table.version(), 1); + assert_eq!(count_records(table), 1); + } + + let producer = helpers::create_producer(); + + // submit 3 messages in kafka, but only 2nd and 3rd should go in as msg 0:0 already in delta + for i in 0..3 { + helpers::send_json( + &producer, + &topic, + &serde_json::to_value(TestMsg::new(i)).unwrap(), + ) + .await; + } + + let v2 = Path::new("./tests/data/zero_offset/_delta_log/00000000000000000002.json"); + let v3 = Path::new("./tests/data/zero_offset/_delta_log/00000000000000000003.json"); + + helpers::wait_until_file_created(v2); + helpers::wait_until_file_created(v3); + token.cancel(); + // if it succeeds then it means that we successfully seeked into offset 0, e.g. Offset::Beginning + kdi.await.unwrap(); + rt.shutdown_background(); + + // check that there's only 3 records + let table = deltalake_core::open_table(table).await.unwrap(); + assert_eq!(table.version(), 3); + assert_eq!(count_records(table), 3); + + //cleanup + std::fs::remove_file(v2).unwrap(); + std::fs::remove_file(v3).unwrap(); +} + +fn count_records(table: DeltaTable) -> i64 { + let mut count = 0; + for x in table.get_stats() { + count += x.as_ref().unwrap().as_ref().unwrap().num_records; + } + count +} + +#[tokio::test] +#[serial] +async fn test_start_from_explicit() { + helpers::init_logger(); + + let table = helpers::create_local_table( + json!({ + "id": "integer", + "color": "string", + }), + vec!["color"], + "starting_offsets_explicit", + ); + + let topic = format!("starting_offsets_explicit_{}", uuid::Uuid::new_v4()); + helpers::create_topic(&topic, 1).await; + + let producer = helpers::create_producer(); + + // Send messages to Kafka before starting kafka-delta-ingest + for m in create_generator(1).take(10) { + info!("Writing test message"); + helpers::send_json(&producer, &topic, &serde_json::to_value(m).unwrap()).await; + } + + producer.flush(Timeout::Never); + + debug!("Sent test messages to Kafka"); + + // Start ingest + let (kdi, token, rt) = helpers::create_kdi( + &topic, + &table, + IngestOptions { + app_id: "starting_offsets_explicit".to_string(), + allowed_latency: 20, + max_messages_per_batch: 10, + min_bytes_per_file: 10, + seek_offsets: Some(vec![(0, 3)]), // starting offset is goign to be 4 + ..Default::default() + }, + ); + + // Wait for the rebalance assignment + std::thread::sleep(std::time::Duration::from_secs(8)); + + // Send messages to Kafka before starting kafka-delta-ingest + for m in create_generator(11).take(5) { + info!("Writing test message"); + helpers::send_json(&producer, &topic, &serde_json::to_value(m).unwrap()).await; + } + + info!("Waiting for version 1"); + helpers::wait_until_version_created(&table, 1); + + token.cancel(); + kdi.await.unwrap(); + rt.shutdown_background(); + + let written_ids: Vec = helpers::read_table_content_as_jsons(&table) + .await + .iter() + .map(|v| serde_json::from_value::(v.clone()).unwrap().id) + .collect(); + + assert_eq!((5u64..15).collect::>(), written_ids); + + helpers::cleanup_kdi(&topic, &table).await; +} + +#[tokio::test] +#[serial] +async fn test_start_from_earliest() { + helpers::init_logger(); + + let table = helpers::create_local_table( + json!({ + "id": "integer", + "color": "string", + }), + vec!["color"], + "starting_offsets_earliest", + ); + + let topic = format!("starting_offsets_earliest{}", uuid::Uuid::new_v4()); + helpers::create_topic(&topic, 3).await; + + let producer = helpers::create_producer(); + + let messages: Vec = create_generator(1).take(10).collect(); + + // Send messages to Kafka before starting kafka-delta-ingest + for m in messages.iter().take(15) { + info!("Writing test message"); + helpers::send_json(&producer, &topic, &serde_json::to_value(m).unwrap()).await; + } + + // Start ingest + let (kdi, token, rt) = helpers::create_kdi( + &topic, + &table, + IngestOptions { + app_id: "starting_offsets_earliest".to_string(), + allowed_latency: 10, + max_messages_per_batch: 10, + min_bytes_per_file: 10, + auto_offset_reset: AutoOffsetReset::Earliest, + ..Default::default() + }, + ); + + info!("Waiting for version 1"); + helpers::wait_until_version_created(&table, 1); + + token.cancel(); + kdi.await.unwrap(); + rt.shutdown_background(); + + let mut written_ids: Vec = helpers::read_table_content_as_jsons(&table) + .await + .iter() + .map(|v| serde_json::from_value::(v.clone()).unwrap().id) + .collect(); + written_ids.sort(); + + assert_eq!((1u64..11).collect::>(), written_ids); + + helpers::cleanup_kdi(&topic, &table).await; +} + +#[tokio::test] +#[serial] +async fn test_start_from_latest() { + helpers::init_logger(); + + let table = helpers::create_local_table( + json! ({ + "id": "integer", + "color": "string", + }), + vec!["color"], + "starting_offsets_latest", + ); + + let topic = format!("starting_offsets_latest{}", uuid::Uuid::new_v4()); + helpers::create_topic(&topic, 1).await; + + let producer = helpers::create_producer(); + + // Send messages to Kafka before starting kafka-delta-ingest + // offsets for this first set should be 0...4 + for m in create_generator(1).take(5) { + info!("Writing test message"); + helpers::send_json(&producer, &topic, &serde_json::to_value(m).unwrap()).await; + } + + producer.flush(Timeout::Never); + + // Start ingest + let (kdi, token, rt) = helpers::create_kdi( + &topic, + &table, + IngestOptions { + app_id: "starting_offsets_latest".to_string(), + allowed_latency: 10, + max_messages_per_batch: 10, + min_bytes_per_file: 10, + auto_offset_reset: AutoOffsetReset::Latest, + ..Default::default() + }, + ); + + // Wait for the rebalance assignment so the position of latest is clear. + // Precise starting offset in a production environment will depend on message rate, but, "latest is what latest is". + std::thread::sleep(std::time::Duration::from_secs(3)); + + // Send on message to trigger seek to latest + // This skips a message to account for the seek + for m in create_generator(6).take(1) { + info!("Writing test message"); + helpers::send_json(&producer, &topic, &serde_json::to_value(m).unwrap()).await; + } + + // Wait for the rebalance assignment so the position of latest is clear. + std::thread::sleep(std::time::Duration::from_secs(8)); + + // These 10 messages should be in the delta log + for m in create_generator(7).take(10) { + info!("Writing test message"); + helpers::send_json(&producer, &topic, &serde_json::to_value(m).unwrap()).await; + } + + info!("Waiting for version 1"); + helpers::wait_until_version_created(&table, 1); + + token.cancel(); + kdi.await.unwrap(); + rt.shutdown_background(); + + let mut written_ids: Vec = helpers::read_table_content_as_jsons(&table) + .await + .iter() + .map(|v| serde_json::from_value::(v.clone()).unwrap().id) + .collect(); + written_ids.sort(); + + // ids should be 7 -16 (offsets 6-15) + assert_eq!((7u64..17).collect::>(), written_ids); + + helpers::cleanup_kdi(&topic, &table).await; +} + +fn create_generator(starting_id: u64) -> impl Iterator { + std::iter::successors(Some(starting_id), |n| Some(*n + 1)).map(|n| TestMsg { + id: n, + color: "red".to_string(), + }) +} + +#[derive(Debug, Serialize, Deserialize)] +struct Msg { + id: u32, + city: String, +} + +impl Msg { + fn new(id: u32) -> Self { + Self { + id, + city: "default".to_string(), + } + } +} + +#[tokio::test] +async fn end_at_initial_offsets() { + helpers::init_logger(); + let topic = format!("end_at_offset_{}", Uuid::new_v4()); + + let table = helpers::create_local_table( + json!({ + "id": "integer", + "city": "string", + }), + vec!["city"], + &topic, + ); + let table = table.as_str(); + + helpers::create_topic(&topic, 3).await; + + let producer = helpers::create_producer(); + // submit 15 messages in kafka + for i in 0..15 { + helpers::send_json( + &producer, + &topic, + &serde_json::to_value(Msg::new(i)).unwrap(), + ) + .await; + } + + let (kdi, _token, rt) = helpers::create_kdi( + &topic, + table, + IngestOptions { + app_id: topic.clone(), + allowed_latency: 5, + max_messages_per_batch: 20, + min_bytes_per_file: 20, + end_at_last_offsets: true, + ..Default::default() + }, + ); + + helpers::wait_until_version_created(table, 1); + + { + // check that there's 3 records in table + let table = deltalake_core::open_table(table).await.unwrap(); + assert_eq!(table.version(), 1); + assert_eq!(count_records(table), 15); + } + + // messages in kafka + for i in 16..31 { + helpers::send_json( + &producer, + &topic, + &serde_json::to_value(Msg::new(i)).unwrap(), + ) + .await; + } + + helpers::expect_termination_within(kdi, 10).await; + rt.shutdown_background(); + + // check that there's only 3 records + let table = deltalake_core::open_table(table).await.unwrap(); + assert_eq!(table.version(), 1); + assert_eq!(count_records(table), 15); +} diff --git a/kafka-delta-ingest/tests/schema_update_tests.rs b/kafka-delta-ingest/tests/schema_update_tests.rs new file mode 100644 index 0000000..beb3355 --- /dev/null +++ b/kafka-delta-ingest/tests/schema_update_tests.rs @@ -0,0 +1,125 @@ +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use serial_test::serial; +use std::fs::File; +use std::io::prelude::*; + +#[allow(dead_code)] +mod helpers; + +#[derive(Debug, Serialize, Deserialize, Clone)] +struct MsgV1 { + id: u32, + date: String, +} + +#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone)] +struct MsgV2 { + id: u32, + color: Option, + date: String, +} + +#[tokio::test] +#[serial] +async fn schema_update_test() { + let (topic, table, producer, kdi, token, rt) = helpers::create_and_run_kdi( + "schema_update", + json!({ + "id": "integer", + "date": "string", + }), + vec!["date"], + 1, + None, + ) + .await; + + let msg_v1 = MsgV1 { + id: 1, + date: "default".to_string(), + }; + + let msg_v2_1 = MsgV2 { + id: 2, + color: Some("red".to_string()), + date: "default".to_string(), + }; + + let msg_v2_2 = MsgV2 { + id: 3, + color: Some("blue".to_string()), + date: "default".to_string(), + }; + + // send msg v1 + helpers::send_json( + &producer, + &topic, + &serde_json::to_value(msg_v1.clone()).unwrap(), + ) + .await; + helpers::wait_until_version_created(&table, 1); + + // update delta schema with new col 'color' + let new_schema = json!({ + "id": "integer", + "color": "string", + "date": "string", + }); + alter_schema(&table, 2, new_schema, vec!["date"]); + + // send few messages with new schema + helpers::send_json( + &producer, + &topic, + &serde_json::to_value(msg_v2_1.clone()).unwrap(), + ) + .await; + helpers::send_json( + &producer, + &topic, + &serde_json::to_value(msg_v2_2.clone()).unwrap(), + ) + .await; + helpers::wait_until_version_created(&table, 4); + + token.cancel(); + kdi.await.unwrap(); + rt.shutdown_background(); + + // retrieve data from the table + let content: Vec = helpers::read_table_content_as_jsons(&table) + .await + .iter() + .map(|v| serde_json::from_value(v.clone()).unwrap()) + .collect(); + + // convert msg v1 to v2 + let expected = vec![ + MsgV2 { + id: msg_v1.id, + color: None, + date: msg_v1.date.clone(), + }, + msg_v2_1, + msg_v2_2, + ]; + + // and compare the results + assert_eq!(content, expected); + + helpers::cleanup_kdi(&topic, &table).await; +} + +fn alter_schema(table: &str, version: i64, schema: Value, partitions: Vec<&str>) { + let schema = helpers::create_metadata_action_json(schema, &partitions); + let tmp = format!("{}/_delta_log/temp.json", table); + { + let mut file = File::create(&tmp).unwrap(); + writeln!(file, "{}", schema).unwrap(); + file.flush().unwrap(); + } + // rename is atomic, but create+write is not + std::fs::rename(tmp, format!("{}/_delta_log/{:020}.json", table, version)).unwrap(); +} diff --git a/tests/data/zero_offset/_delta_log/00000000000000000001.json b/tests/data/zero_offset/_delta_log/00000000000000000001.json deleted file mode 100644 index 0c0a7b8..0000000 --- a/tests/data/zero_offset/_delta_log/00000000000000000001.json +++ /dev/null @@ -1,2 +0,0 @@ -{"txn":{"appId":"zero_offset-0","version":0,"lastUpdated":1628003413471}} -{"add":{"path":"color=default/part-00000-0f324a9e-6066-4f14-970e-c174ca2299c6-c000.snappy.parquet","size":710,"partitionValues":{"color":"default"},"modificationTime":1628003413471,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"color\":\"default\",\"id\":0},\"maxValues\":{\"color\":\"default\",\"id\":0},\"nullCount\":{\"id\":0,\"color\":0}}","tags":null}} \ No newline at end of file