diff --git a/Cargo.lock b/Cargo.lock index acf04c8..48e1b05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -174,7 +174,7 @@ dependencies = [ "strum_macros 0.24.3", "thiserror", "typed-builder", - "uuid 1.6.1", + "uuid 1.8.0", "zerocopy 0.6.6", ] @@ -192,9 +192,9 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa285343fba4d829d49985bdc541e3789cf6000ed0e84be7c039438df4a4e78c" +checksum = "219d05930b81663fd3b32e3bde8ce5bff3c4d23052a99f11a8fa50a3b47b2658" dependencies = [ "arrow-arith", "arrow-array", @@ -213,9 +213,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "753abd0a5290c1bcade7c6623a556f7d1659c5f4148b140b5b63ce7bd1a45705" +checksum = "0272150200c07a86a390be651abdd320a2d12e84535f0837566ca87ecd8f95e0" dependencies = [ "arrow-array", "arrow-buffer", @@ -228,9 +228,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d390feeb7f21b78ec997a4081a025baef1e2e0d6069e181939b61864c9779609" +checksum = "8010572cf8c745e242d1b632bd97bd6d4f40fefed5ed1290a8f433abaa686fea" dependencies = [ "ahash", "arrow-buffer", @@ -245,9 +245,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69615b061701bcdffbc62756bc7e85c827d5290b472b580c972ebbbf690f5aa4" +checksum = "0d0a2432f0cba5692bf4cb757469c66791394bac9ec7ce63c1afe74744c37b27" dependencies = [ "bytes", "half", @@ -256,28 +256,30 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e448e5dd2f4113bf5b74a1f26531708f5edcacc77335b7066f9398f4bcf4cdef" +checksum = "9abc10cd7995e83505cc290df9384d6e5412b207b79ce6bdff89a10505ed2cba" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", - "base64 0.21.5", + "atoi", + "base64 0.22.1", "chrono", "comfy-table", "half", "lexical-core", "num 0.4.1", + "ryu", ] [[package]] name = "arrow-csv" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46af72211f0712612f5b18325530b9ad1bfbdc87290d5fbfd32a7da128983781" +checksum = "95cbcba196b862270bf2a5edb75927380a7f3a163622c61d40cbba416a6305f2" dependencies = [ "arrow-array", "arrow-buffer", @@ -294,9 +296,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67d644b91a162f3ad3135ce1184d0a31c28b816a581e08f29e8e9277a574c64e" +checksum = "2742ac1f6650696ab08c88f6dd3f0eb68ce10f8c253958a18c943a68cd04aec5" dependencies = [ "arrow-buffer", "arrow-schema", @@ -306,9 +308,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03dea5e79b48de6c2e04f03f62b0afea7105be7b77d134f6c5414868feefb80d" +checksum = "a42ea853130f7e78b9b9d178cb4cd01dee0f78e64d96c2949dc0a915d6d9e19d" dependencies = [ "arrow-array", "arrow-buffer", @@ -321,9 +323,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8950719280397a47d37ac01492e3506a8a724b3fb81001900b866637a829ee0f" +checksum = "eaafb5714d4e59feae964714d724f880511500e3569cc2a94d02456b403a2a49" dependencies = [ "arrow-array", "arrow-buffer", @@ -341,9 +343,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ed9630979034077982d8e74a942b7ac228f33dd93a93b615b4d02ad60c260be" +checksum = "e3e6b61e3dc468f503181dccc2fc705bdcc5f2f146755fa5b56d0a6c5943f412" dependencies = [ "arrow-array", "arrow-buffer", @@ -356,9 +358,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "007035e17ae09c4e8993e4cb8b5b96edf0afb927cd38e2dff27189b274d83dcf" +checksum = "848ee52bb92eb459b811fb471175ea3afcf620157674c8794f539838920f9228" dependencies = [ "ahash", "arrow-array", @@ -371,18 +373,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ff3e9c01f7cd169379d269f926892d0e622a704960350d09d331be3ec9e0029" +checksum = "02d9483aaabe910c4781153ae1b6ae0393f72d9ef757d38d09d450070cf2e528" dependencies = [ "serde", ] [[package]] name = "arrow-select" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce20973c1912de6514348e064829e50947e35977bb9d7fb637dc99ea9ffd78c" +checksum = "849524fa70e0e3c5ab58394c770cb8f514d0122d20de08475f7b472ed8075830" dependencies = [ "ahash", "arrow-array", @@ -394,15 +396,16 @@ dependencies = [ [[package]] name = "arrow-string" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00f3b37f2aeece31a2636d1b037dabb69ef590e03bdc7eb68519b51ec86932a7" +checksum = "9373cb5a021aee58863498c37eb484998ef13377f69989c6c5ccfbd258236cdb" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", + "memchr", "num 0.4.1", "regex", "regex-syntax", @@ -459,6 +462,15 @@ dependencies = [ "syn 2.0.48", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic_refcell" version = "0.1.13" @@ -533,7 +545,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "tracing", - "uuid 1.6.1", + "uuid 1.8.0", ] [[package]] @@ -815,7 +827,7 @@ dependencies = [ "sha2 0.10.8", "time 0.3.31", "url", - "uuid 1.6.1", + "uuid 1.8.0", ] [[package]] @@ -834,7 +846,7 @@ dependencies = [ "serde_derive", "time 0.3.31", "url", - "uuid 1.6.1", + "uuid 1.8.0", ] [[package]] @@ -855,7 +867,7 @@ dependencies = [ "serde_json", "time 0.3.31", "url", - "uuid 1.6.1", + "uuid 1.8.0", ] [[package]] @@ -915,6 +927,12 @@ version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "base64-simd" version = "0.8.0" @@ -1077,9 +1095,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.34" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1335,9 +1353,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "35.0.0" +version = "37.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4328f5467f76d890fe3f924362dbc3a838c6a733f762b32d87f9e0b7bef5fb49" +checksum = "85069782056753459dc47e386219aa1fdac5b731f26c28abb8c0ffd4b7c5ab11" dependencies = [ "ahash", "arrow", @@ -1351,8 +1369,11 @@ dependencies = [ "chrono", "dashmap", "datafusion-common", + "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", + "datafusion-functions", + "datafusion-functions-array", "datafusion-optimizer", "datafusion-physical-expr", "datafusion-physical-plan", @@ -1376,16 +1397,16 @@ dependencies = [ "tokio", "tokio-util 0.7.10", "url", - "uuid 1.6.1", + "uuid 1.8.0", "xz2", "zstd", ] [[package]] name = "datafusion-common" -version = "35.0.0" +version = "37.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29a7752143b446db4a2cccd9a6517293c6b97e8c39e520ca43ccd07135a4f7e" +checksum = "309d9040751f6dc9e33c85dce6abb55a46ef7ea3644577dd014611c379447ef3" dependencies = [ "ahash", "arrow", @@ -1394,6 +1415,7 @@ dependencies = [ "arrow-schema", "chrono", "half", + "instant", "libc", "num_cpus", "object_store", @@ -1401,11 +1423,20 @@ dependencies = [ "sqlparser", ] +[[package]] +name = "datafusion-common-runtime" +version = "37.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e4a44d8ef1b1e85d32234e6012364c411c3787859bb3bba893b0332cb03dfd" +dependencies = [ + "tokio", +] + [[package]] name = "datafusion-execution" -version = "35.0.0" +version = "37.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d447650af16e138c31237f53ddaef6dd4f92f0e2d3f2f35d190e16c214ca496" +checksum = "06a3a29ae36bcde07d179cc33b45656a8e7e4d023623e320e48dcf1200eeee95" dependencies = [ "arrow", "chrono", @@ -1424,25 +1455,71 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "35.0.0" +version = "37.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8d19598e48a498850fb79f97a9719b1f95e7deb64a7a06f93f313e8fa1d524b" +checksum = "2a3542aa322029c2121a671ce08000d4b274171070df13f697b14169ccf4f628" dependencies = [ "ahash", "arrow", "arrow-array", + "chrono", "datafusion-common", "paste", "sqlparser", - "strum 0.25.0", - "strum_macros 0.25.3", + "strum 0.26.2", + "strum_macros 0.26.2", +] + +[[package]] +name = "datafusion-functions" +version = "37.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd221792c666eac174ecc09e606312844772acc12cbec61a420c2fca1ee70959" +dependencies = [ + "arrow", + "base64 0.22.1", + "blake2", + "blake3", + "chrono", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "hex", + "itertools 0.12.0", + "log", + "md-5 0.10.6", + "regex", + "sha2 0.10.8", + "unicode-segmentation", + "uuid 1.8.0", +] + +[[package]] +name = "datafusion-functions-array" +version = "37.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e501801e84d9c6ef54caaebcda1b18a6196a24176c12fb70e969bc0572e03c55" +dependencies = [ + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "itertools 0.12.0", + "log", + "paste", ] [[package]] name = "datafusion-optimizer" -version = "35.0.0" +version = "37.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b7feb0391f1fc75575acb95b74bfd276903dc37a5409fcebe160bc7ddff2010" +checksum = "76bd7f5087817deb961764e8c973d243b54f8572db414a8f0a8f33a48f991e0a" dependencies = [ "arrow", "async-trait", @@ -1458,9 +1535,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "35.0.0" +version = "37.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e911bca609c89a54e8f014777449d8290327414d3e10c57a3e3c2122e38878d0" +checksum = "5cabc0d9aaa0f5eb1b472112f16223c9ffd2fb04e58cbf65c0a331ee6e993f96" dependencies = [ "ahash", "arrow", @@ -1468,11 +1545,13 @@ dependencies = [ "arrow-buffer", "arrow-ord", "arrow-schema", - "base64 0.21.5", + "arrow-string", + "base64 0.22.1", "blake2", "blake3", "chrono", "datafusion-common", + "datafusion-execution", "datafusion-expr", "half", "hashbrown", @@ -1487,14 +1566,13 @@ dependencies = [ "regex", "sha2 0.10.8", "unicode-segmentation", - "uuid 1.6.1", ] [[package]] name = "datafusion-physical-plan" -version = "35.0.0" +version = "37.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96b546b8a02e9c2ab35ac6420d511f12a4701950c1eb2e568c122b4fefb0be3" +checksum = "17c0523e9c8880f2492a88bbd857dde02bed1ed23f3e9211a89d3d7ec3b44af9" dependencies = [ "ahash", "arrow", @@ -1504,6 +1582,7 @@ dependencies = [ "async-trait", "chrono", "datafusion-common", + "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", "datafusion-physical-expr", @@ -1518,14 +1597,13 @@ dependencies = [ "pin-project-lite", "rand 0.8.5", "tokio", - "uuid 1.6.1", ] [[package]] name = "datafusion-proto" -version = "35.0.0" +version = "37.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5742f993d1812d6bb3cdc4ce2a0aa99e10f6dc0daa11dd69b0ff57f2d8e7518c" +checksum = "db73393e42f35e165d31399192fbf41691967d428ebed47875ad34239fbcfc16" dependencies = [ "arrow", "chrono", @@ -1538,16 +1616,18 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "35.0.0" +version = "37.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d18d36f260bbbd63aafdb55339213a23d540d3419810575850ef0a798a6b768" +checksum = "49eb54b42227136f6287573f2434b1de249fe1b8e6cd6cc73a634e4a3ec29356" dependencies = [ "arrow", + "arrow-array", "arrow-schema", "datafusion-common", "datafusion-expr", "log", "sqlparser", + "strum 0.26.2", ] [[package]] @@ -1562,8 +1642,9 @@ dependencies = [ [[package]] name = "deltalake-aws" -version = "0.1.0" -source = "git+https://github.com/delta-io/delta-rs?branch=main#abafd2d0cb8dde32ffa990dc30fb97a5581688ec" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c15dda219ed925c28e2387712f9a77ed7253a80b53fce59849067bc0918eb2" dependencies = [ "async-trait", "aws-config", @@ -1583,13 +1664,14 @@ dependencies = [ "tokio", "tracing", "url", - "uuid 1.6.1", + "uuid 1.8.0", ] [[package]] name = "deltalake-azure" -version = "0.1.0" -source = "git+https://github.com/delta-io/delta-rs?branch=main#abafd2d0cb8dde32ffa990dc30fb97a5581688ec" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e074599ebb06706867093e06f29ca4eb77a310a8ca24c83066063d4062acbbfe" dependencies = [ "async-trait", "bytes", @@ -1606,8 +1688,9 @@ dependencies = [ [[package]] name = "deltalake-core" -version = "0.17.1" -source = "git+https://github.com/delta-io/delta-rs?branch=main#abafd2d0cb8dde32ffa990dc30fb97a5581688ec" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e8dc1bcd91be689ee7f6ce363798dc2e9b6954be9d597f884de11ba27b33add" dependencies = [ "arrow", "arrow-arith", @@ -1628,6 +1711,8 @@ dependencies = [ "datafusion", "datafusion-common", "datafusion-expr", + "datafusion-functions", + "datafusion-functions-array", "datafusion-physical-expr", "datafusion-proto", "datafusion-sql", @@ -1660,7 +1745,7 @@ dependencies = [ "tokio", "tracing", "url", - "uuid 1.6.1", + "uuid 1.8.0", "z85", ] @@ -1762,7 +1847,7 @@ dependencies = [ "rusoto_dynamodb", "thiserror", "tokio", - "uuid 1.6.1", + "uuid 1.8.0", ] [[package]] @@ -2371,6 +2456,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", ] [[package]] @@ -2952,9 +3040,9 @@ dependencies = [ [[package]] name = "object_store" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d139f545f64630e2e3688fd9f81c470888ab01edeb72d13b4e86c566f1130000" +checksum = "b8718f8b65fdf67a45108d1548347d4af7d71fb81ce727bbf9e3b2535e079db3" dependencies = [ "async-trait", "base64 0.21.5", @@ -2964,6 +3052,7 @@ dependencies = [ "humantime", "hyper", "itertools 0.12.0", + "md-5 0.10.6", "parking_lot 0.12.1", "percent-encoding", "quick-xml", @@ -3115,9 +3204,9 @@ dependencies = [ [[package]] name = "parquet" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "547b92ebf0c1177e3892f44c8f79757ee62e678d564a9834189725f2c5b7a750" +checksum = "096795d4f47f65fd3ee1ec5a98b77ab26d602f2cc785b0e4be5443add17ecc32" dependencies = [ "ahash", "arrow-array", @@ -3127,7 +3216,7 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64 0.21.5", + "base64 0.22.1", "brotli", "bytes", "chrono", @@ -4240,9 +4329,9 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "sqlparser" -version = "0.41.0" +version = "0.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc2c25a6c66789625ef164b4c7d2e548d627902280c13710d33da8222169964" +checksum = "aaf9c7ff146298ffda83a200f8d5084f08dcee1edfc135fcc1d646a45d50ffd6" dependencies = [ "log", "sqlparser_derive", @@ -4288,8 +4377,14 @@ name = "strum" version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" + +[[package]] +name = "strum" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29" dependencies = [ - "strum_macros 0.25.3", + "strum_macros 0.26.2", ] [[package]] @@ -4330,6 +4425,19 @@ dependencies = [ "syn 2.0.48", ] +[[package]] +name = "strum_macros" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6cf59daf282c0a494ba14fd21610a0325f9f90ec9d1231dea26bcb1d696c946" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.48", +] + [[package]] name = "subtle" version = "2.4.1" @@ -4499,9 +4607,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.35.1" +version = "1.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" +checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" dependencies = [ "backtrace", "bytes", @@ -4664,7 +4772,7 @@ version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "static_assertions", ] @@ -4794,9 +4902,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.6.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" +checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" dependencies = [ "getrandom 0.2.11", "serde", diff --git a/Cargo.toml b/Cargo.toml index b3e6f55..6681efe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,11 +32,10 @@ tokio-util = "0.6.3" uuid = { version = "0.8", features = ["serde", "v4"] } url = "2.3" -#deltalake = { version = "0.16.5", features = ["arrow", "json", "parquet"], optional = true } # datafusion feature is required for writer version 2 -deltalake-core = { git = "https://github.com/delta-io/delta-rs", branch = "main", features = ["json", "datafusion"]} -deltalake-aws = { git = "https://github.com/delta-io/delta-rs", branch = "main", optional = true } -deltalake-azure = { git = "https://github.com/delta-io/delta-rs", branch = "main", optional = true } +deltalake-core = { version = "~0.17.3", features = ["json", "datafusion"]} +deltalake-aws = { version = "~0.1.1", optional = true } +deltalake-azure = { version = "~0.1.2", optional = true } # s3 feature enabled dynamodb_lock = { version = "0.6.0", optional = true } diff --git a/src/writer.rs b/src/writer.rs index fa56854..83517cc 100644 --- a/src/writer.rs +++ b/src/writer.rs @@ -1159,7 +1159,8 @@ mod tests { .await .unwrap(); assert_eq!(add.len(), 1); - let stats = add[0].get_stats().unwrap().unwrap(); + let stats: deltalake_core::protocol::Stats = + serde_json::from_str(&add[0].stats.as_ref().unwrap()).expect("Failed to parse stats"); let min_max_keys = vec!["meta", "some_int", "some_string", "some_bool"]; let mut null_count_keys = vec!["some_list", "some_nested_list"]; @@ -1241,12 +1242,12 @@ mod tests { let timestamp = producer.get("timestamp").unwrap().as_value().unwrap(); assert_eq!(0, timestamp); } - ("some_int", ColumnCountStat::Value(v)) => assert_eq!(100, v), - ("some_bool", ColumnCountStat::Value(v)) => assert_eq!(100, v), - ("some_string", ColumnCountStat::Value(v)) => assert_eq!(100, v), - ("some_list", ColumnCountStat::Value(v)) => assert_eq!(100, v), - ("some_nested_list", ColumnCountStat::Value(v)) => assert_eq!(0, v), - ("date", ColumnCountStat::Value(v)) => assert_eq!(0, v), + ("some_int", ColumnCountStat::Value(v)) => assert_eq!(100, *v), + ("some_bool", ColumnCountStat::Value(v)) => assert_eq!(100, *v), + ("some_string", ColumnCountStat::Value(v)) => assert_eq!(100, *v), + ("some_list", ColumnCountStat::Value(v)) => assert_eq!(100, *v), + ("some_nested_list", ColumnCountStat::Value(v)) => assert_eq!(0, *v), + ("date", ColumnCountStat::Value(v)) => assert_eq!(0, *v), _ => assert!(false, "Key should not be present"), } } diff --git a/tests/delta_partitions_tests.rs b/tests/delta_partitions_tests.rs index f262ba7..d376f87 100644 --- a/tests/delta_partitions_tests.rs +++ b/tests/delta_partitions_tests.rs @@ -40,7 +40,7 @@ async fn test_delta_partitions() { "test_delta_partitions", ); - let mut table = deltalake_core::open_table(&table_path).await.unwrap(); + let table = deltalake_core::open_table(&table_path).await.unwrap(); let mut delta_writer = DataWriter::for_table(&table, HashMap::new()).unwrap(); let batch1 = vec![ diff --git a/tests/helpers/mod.rs b/tests/helpers/mod.rs index b3deebd..751a01a 100644 --- a/tests/helpers/mod.rs +++ b/tests/helpers/mod.rs @@ -471,7 +471,7 @@ pub async fn inspect_table(path: &str) { println!(" Txn: {}: {}", t.app_id, t.version) } Action::Add(a) => { - let stats = a.get_stats().unwrap().unwrap(); + let stats = a.get_stats_parsed().unwrap().unwrap(); println!(" Add: {}. Records: {}", &a.path, stats.num_records); let full_path = format!("{}/{}", &path, &a.path); let parquet_bytes = store @@ -524,7 +524,7 @@ pub async fn inspect_table(path: &str) { } if let Some(a) = parse_json_field::(&json, "add") { let records = a - .get_stats() + .get_stats_parsed() .ok() .flatten() .map(|s| s.num_records) diff --git a/tests/offset_tests.rs b/tests/offset_tests.rs index 17dc277..03a8057 100644 --- a/tests/offset_tests.rs +++ b/tests/offset_tests.rs @@ -87,8 +87,11 @@ async fn zero_offset_issue() { fn count_records(table: DeltaTable) -> i64 { let mut count = 0; - for x in table.get_stats() { - count += x.as_ref().unwrap().as_ref().unwrap().num_records; + + if let Ok(adds) = table.state.unwrap().file_actions() { + for add in adds.iter() { + count += add.get_stats_parsed().unwrap().unwrap().num_records; + } } count }