diff --git a/CHANGELOG.md b/CHANGELOG.md index ae5782e26d..239f130ea1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.32.0] - 2020-10-31 +### Added +- Support for `flatbuffers` metadata encoding. +- Using `flatbuffers` format for achieving true stable hashes of metadata. +- Aligned metadata with ODF `v0.16.0`. + ## [0.31.0] - 2020-10-08 ### Added - `pull` command now supports `--force-uncacheable` flag for refreshing uncacheable datasets. diff --git a/Cargo.lock b/Cargo.lock index 48e6bdc286..933e9728f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7,10 +7,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "aho-corasick" -version = "0.7.13" +version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -54,13 +54,13 @@ dependencies = [ [[package]] name = "arrayvec" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "arrow" -version = "2.0.0-SNAPSHOT" -source = "git+https://github.com/apache/arrow#109f701d805b761710f8b08f76be17ff16b3c3fd" +version = "3.0.0-SNAPSHOT" +source = "git+https://github.com/apache/arrow#687dceadadc65fc19c42d3f0b36f4ad88c4e2168" dependencies = [ "chrono 0.4.19 (registry+https://github.com/rust-lang/crates.io-index)", "csv 1.1.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -70,10 +70,10 @@ dependencies = [ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "num 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.116 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.116 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.58 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.117 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.117 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.59 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -82,7 +82,7 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "hermit-abi 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -96,6 +96,11 @@ name = "base64" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "bitflags" version = "1.2.1" @@ -107,7 +112,7 @@ version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "arrayref 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "arrayvec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "arrayvec 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", "constant_time_eq 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -132,13 +137,13 @@ dependencies = [ [[package]] name = "bstr" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "regex-automata 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.116 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.117 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -157,7 +162,7 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bzip2-sys 0.1.9+1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -166,8 +171,8 @@ version = "0.1.9+1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.61 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", - "pkg-config 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -193,10 +198,10 @@ name = "chrono" version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", - "num-integer 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.116 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", + "num-integer 0.1.44 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.117 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.44 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -225,15 +230,14 @@ dependencies = [ [[package]] name = "console" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "encode_unicode 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "terminal_size 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", - "termios 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-width 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "winapi-util 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -246,10 +250,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "crc32fast" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -276,11 +280,11 @@ name = "csv" version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bstr 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)", + "bstr 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)", "csv-core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "itoa 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "ryu 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.116 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.117 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -288,16 +292,16 @@ name = "csv-core" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "curl" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "curl-sys 0.4.36+curl-7.71.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "curl-sys 0.4.38+curl-7.73.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-probe 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-sys 0.9.58 (registry+https://github.com/rust-lang/crates.io-index)", "schannel 0.1.19 (registry+https://github.com/rust-lang/crates.io-index)", @@ -307,15 +311,15 @@ dependencies = [ [[package]] name = "curl-sys" -version = "0.4.36+curl-7.71.1" +version = "0.4.38+curl-7.73.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.61 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", "libnghttp2-sys 0.1.4+1.41.0 (registry+https://github.com/rust-lang/crates.io-index)", "libz-sys 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-sys 0.9.58 (registry+https://github.com/rust-lang/crates.io-index)", - "pkg-config 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", "vcpkg 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -339,7 +343,7 @@ dependencies = [ "proc-macro2 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", "strsim 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -349,7 +353,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "darling_core 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -357,7 +361,7 @@ name = "dirs" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", "redox_users 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -376,7 +380,7 @@ name = "dirs-sys" version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", "redox_users 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -402,7 +406,7 @@ version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.57 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -415,14 +419,23 @@ dependencies = [ "smallvec 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "flatbuffers" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "flate2" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "crc32fast 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", "miniz_oxide 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -452,7 +465,7 @@ version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", "wasi 0.9.0+wasi-snapshot-preview1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -471,7 +484,7 @@ name = "hermit-abi" version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -513,18 +526,18 @@ name = "indicatif" version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "console 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", + "console 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "number_prefix 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "indoc" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "unindent 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "unindent 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -558,7 +571,7 @@ name = "jobserver" version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -571,30 +584,28 @@ dependencies = [ [[package]] name = "kamu" -version = "0.31.0" +version = "0.32.0" dependencies = [ "cfg-if 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.4.19 (registry+https://github.com/rust-lang/crates.io-index)", - "curl 0.4.33 (registry+https://github.com/rust-lang/crates.io-index)", - "curl-sys 0.4.36+curl-7.71.1 (registry+https://github.com/rust-lang/crates.io-index)", + "curl 0.4.34 (registry+https://github.com/rust-lang/crates.io-index)", + "curl-sys 0.4.38+curl-7.73.0 (registry+https://github.com/rust-lang/crates.io-index)", "filetime 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", - "flatbuffers 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "flate2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", "fs_extra 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "hex 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "indoc 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "intervals-general 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "indoc 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "kamu-test 0.31.0", - "parquet 2.0.0-SNAPSHOT (git+https://github.com/apache/arrow)", + "kamu-test 0.32.0", + "opendatafabric 0.32.0", + "parquet 3.0.0-SNAPSHOT (git+https://github.com/apache/arrow)", "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "ringbuf 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.116 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.117 (registry+https://github.com/rust-lang/crates.io-index)", "serde_with 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_yaml 0.8.13 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_yaml 0.8.14 (registry+https://github.com/rust-lang/crates.io-index)", "signal-hook 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", "slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)", "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -606,20 +617,21 @@ dependencies = [ [[package]] name = "kamu-cli" -version = "0.31.0" +version = "0.32.0" dependencies = [ "chrono 0.4.19 (registry+https://github.com/rust-lang/crates.io-index)", "chrono-humanize 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.33.3 (registry+https://github.com/rust-lang/crates.io-index)", - "console 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", + "console 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "humansize 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "indicatif 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", - "indoc 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "indoc 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "kamu 0.31.0", + "kamu 0.32.0", "num-format 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "parquet 2.0.0-SNAPSHOT (git+https://github.com/apache/arrow)", + "opendatafabric 0.32.0", + "parquet 3.0.0-SNAPSHOT (git+https://github.com/apache/arrow)", "prettytable-rs 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "read_input 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -634,11 +646,11 @@ dependencies = [ [[package]] name = "kamu-test" -version = "0.31.0" +version = "0.32.0" dependencies = [ "chrono 0.4.19 (registry+https://github.com/rust-lang/crates.io-index)", - "kamu 0.31.0", - "serde_yaml 0.8.13 (registry+https://github.com/rust-lang/crates.io-index)", + "opendatafabric 0.32.0", + "serde_yaml 0.8.14 (registry+https://github.com/rust-lang/crates.io-index)", "url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -649,7 +661,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "libc" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -658,7 +670,7 @@ version = "0.1.4+1.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.61 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -667,8 +679,8 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.61 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", - "pkg-config 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", "vcpkg 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -690,7 +702,7 @@ name = "lz4" version = "1.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", "lz4-sys 1.9.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -700,7 +712,7 @@ version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.61 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -715,7 +727,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "memchr" -version = "2.3.3" +version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -738,11 +750,11 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "num-bigint 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "num-complex 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "num-integer 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", - "num-iter 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", - "num-rational 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", + "num-complex 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "num-integer 0.1.44 (registry+https://github.com/rust-lang/crates.io-index)", + "num-iter 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", + "num-rational 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -751,16 +763,16 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "autocfg 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "num-integer 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", + "num-integer 0.1.44 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "num-complex" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "num-traits 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -774,37 +786,37 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "autocfg 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "num-iter" -version = "0.1.41" +version = "0.1.42" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "autocfg 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "num-integer 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", + "num-integer 0.1.44 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "num-rational" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "autocfg 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "num-bigint 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "num-integer 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", + "num-integer 0.1.44 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "num-traits" -version = "0.2.12" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "autocfg 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -816,7 +828,7 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "hermit-abi 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -826,18 +838,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "opendatafabric" -version = "0.31.0" +version = "0.32.0" dependencies = [ "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.4.19 (registry+https://github.com/rust-lang/crates.io-index)", - "flatbuffers 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "flatbuffers 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "hex 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "indoc 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "indoc 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "intervals-general 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.116 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.117 (registry+https://github.com/rust-lang/crates.io-index)", "serde_with 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_yaml 0.8.13 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_yaml 0.8.14 (registry+https://github.com/rust-lang/crates.io-index)", "thiserror 1.0.21 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -848,7 +860,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "openssl-src" -version = "111.11.0+1.1.1h" +version = "111.12.0+1.1.1h" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.61 (registry+https://github.com/rust-lang/crates.io-index)", @@ -861,9 +873,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "autocfg 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "cc 1.0.61 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", - "openssl-src 111.11.0+1.1.1h (registry+https://github.com/rust-lang/crates.io-index)", - "pkg-config 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl-src 111.12.0+1.1.1h (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", "vcpkg 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -872,15 +884,16 @@ name = "ordered-float" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "num-traits 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "parquet" -version = "2.0.0-SNAPSHOT" -source = "git+https://github.com/apache/arrow#109f701d805b761710f8b08f76be17ff16b3c3fd" +version = "3.0.0-SNAPSHOT" +source = "git+https://github.com/apache/arrow#687dceadadc65fc19c42d3f0b36f4ad88c4e2168" dependencies = [ - "arrow 2.0.0-SNAPSHOT (git+https://github.com/apache/arrow)", + "arrow 3.0.0-SNAPSHOT (git+https://github.com/apache/arrow)", + "base64 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "brotli 3.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.4.19 (registry+https://github.com/rust-lang/crates.io-index)", @@ -908,7 +921,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "pkg-config" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -955,7 +968,7 @@ name = "rand" version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -965,7 +978,7 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", @@ -977,7 +990,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "getrandom 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", "rand_chacha 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1051,12 +1064,12 @@ dependencies = [ [[package]] name = "regex" -version = "1.3.9" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "aho-corasick 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.6.18 (registry+https://github.com/rust-lang/crates.io-index)", + "aho-corasick 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.20 (registry+https://github.com/rust-lang/crates.io-index)", "thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1070,7 +1083,7 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.18" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -1103,7 +1116,7 @@ version = "0.2.36" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "gcc 0.3.55 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.23 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.44 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1130,31 +1143,31 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.116" +version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "serde_derive 1.0.116 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.117 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "serde_derive" -version = "1.0.116" +version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "serde_json" -version = "1.0.58" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "indexmap 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "itoa 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "ryu 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.116 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.117 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1162,7 +1175,7 @@ name = "serde_with" version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "serde 1.0.116 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.117 (registry+https://github.com/rust-lang/crates.io-index)", "serde_with_macros 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1174,17 +1187,17 @@ dependencies = [ "darling 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "serde_yaml" -version = "0.8.13" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "dtoa 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "linked-hash-map 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.116 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.117 (registry+https://github.com/rust-lang/crates.io-index)", "yaml-rust 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1198,7 +1211,7 @@ name = "signal-hook" version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", "signal-hook-registry 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1208,7 +1221,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "arc-swap 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1255,7 +1268,7 @@ version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.57 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1272,7 +1285,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "syn" -version = "1.0.42" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1291,7 +1304,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.57 (registry+https://github.com/rust-lang/crates.io-index)", "remove_dir_all 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1322,18 +1335,10 @@ name = "terminal_size" version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "termios" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "textwrap" version = "0.11.0" @@ -1357,7 +1362,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1393,7 +1398,7 @@ name = "time" version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", "wasi 0.10.0+wasi-snapshot-preview1 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1431,7 +1436,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "unindent" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -1449,7 +1454,7 @@ name = "users" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1492,7 +1497,7 @@ dependencies = [ "log 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", "wasm-bindgen-shared 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1512,7 +1517,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", "wasm-bindgen-backend 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", "wasm-bindgen-shared 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1587,7 +1592,7 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bzip2 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crc32fast 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "flate2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", "podio 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.44 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1606,7 +1611,7 @@ name = "zstd-safe" version = "2.0.5+zstd.1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", "zstd-sys 1.4.17+zstd.1.4.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1618,28 +1623,29 @@ dependencies = [ "cc 1.0.61 (registry+https://github.com/rust-lang/crates.io-index)", "glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)", ] [metadata] "checksum adler 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" -"checksum aho-corasick 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)" = "043164d8ba5c4c3035fec9bbee8647c0261d788f3474306f93bb65901cae0e86" +"checksum aho-corasick 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)" = "b476ce7103678b0c6d3d395dbbae31d48ff910bd28be979ba5d48c6351131d0d" "checksum alloc-no-stdlib 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5192ec435945d87bc2f70992b4d818154b5feede43c09fb7592146374eac90a6" "checksum alloc-stdlib 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "697ed7edc0f1711de49ce108c541623a0af97c6c60b2f6e2b65229847ac843c2" "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" "checksum arc-swap 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "4d25d88fd6b8041580a654f9d0c581a047baee2b3efee13275f2fc392fc75034" "checksum arrayref 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" "checksum arrayvec 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9" -"checksum arrayvec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" -"checksum arrow 2.0.0-SNAPSHOT (git+https://github.com/apache/arrow)" = "" +"checksum arrayvec 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" +"checksum arrow 3.0.0-SNAPSHOT (git+https://github.com/apache/arrow)" = "" "checksum atty 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" "checksum autocfg 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" "checksum base64 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" +"checksum base64 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" "checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" "checksum blake2b_simd 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)" = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a" "checksum brotli 3.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7f29919120f08613aadcd4383764e00526fc9f18b6c0895814faeed0dd78613e" "checksum brotli-decompressor 2.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1052e1c3b8d4d80eb84a8b94f0a1498797b5fb96314c001156a1c761940ef4ec" -"checksum bstr 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "31accafdb70df7871592c058eca3985b71104e15ac32f64706022c58867da931" +"checksum bstr 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "473fc6b38233f9af7baa94fb5852dca389e3d95b8e21c8e3719301462c5d9faf" "checksum bumpalo 3.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" "checksum byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" "checksum bzip2 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "42b7c3cbf0fa9c1b82308d57191728ca0256cb821220f4e2fd410a72ade26e3b" @@ -1650,15 +1656,15 @@ dependencies = [ "checksum chrono 0.4.19 (registry+https://github.com/rust-lang/crates.io-index)" = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" "checksum chrono-humanize 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f0a4c32145b4db85fe1c4f2b125a4f9493769df424f5f84baf6b04ea8eaf33c9" "checksum clap 2.33.3 (registry+https://github.com/rust-lang/crates.io-index)" = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" -"checksum console 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0b1aacfaffdbff75be81c15a399b4bedf78aaefe840e8af1d299ac2ade885d2" +"checksum console 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a50aab2529019abfabfa93f1e6c41ef392f91fbf179b347a7e96abb524884a08" "checksum constant_time_eq 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" -"checksum crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" +"checksum crc32fast 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" "checksum crossbeam-channel 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b153fe7cbef478c567df0f972e02e6d736db11affe43dfc9c56a9374d1adfb87" "checksum crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" "checksum csv 1.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "00affe7f6ab566df61b4be3ce8cf16bc2576bca0963ceb0955e45d514bf9a279" "checksum csv-core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" -"checksum curl 0.4.33 (registry+https://github.com/rust-lang/crates.io-index)" = "78baca05127a115136a9898e266988fc49ca7ea2c839f60fc6e1fc9df1599168" -"checksum curl-sys 0.4.36+curl-7.71.1 (registry+https://github.com/rust-lang/crates.io-index)" = "68cad94adeb0c16558429c3c34a607acc9ea58e09a7b66310aabc9788fc5d721" +"checksum curl 0.4.34 (registry+https://github.com/rust-lang/crates.io-index)" = "e268162af1a5fe89917ae25ba3b0a77c8da752bdc58e7dbb4f15b91fbd33756e" +"checksum curl-sys 0.4.38+curl-7.73.0 (registry+https://github.com/rust-lang/crates.io-index)" = "498ecfb4f59997fd40023d62a9f1e506e768b2baeb59a1d311eb9751cdcd7e3f" "checksum darling 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0d706e75d87e35569db781a9b5e2416cff1236a47ed380831f959382ccd5f858" "checksum darling_core 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f0c960ae2da4de88a91b2d920c2a7233b400bc33cb28453a2987822d8392519b" "checksum darling_macro 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72" @@ -1670,6 +1676,7 @@ dependencies = [ "checksum encode_unicode 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" "checksum filetime 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)" = "3ed85775dcc68644b5c950ac06a2b23768d3bc9390464151aaf27136998dcf9e" "checksum flatbuffers 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a788f068dd10687940565bf4b5480ee943176cbd114b12e811074bcf7c04e4b9" +"checksum flatbuffers 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "dbf606a9d6674ad82094652789cf69b0e6e598c2d3c9137c28970eec13f11644" "checksum flate2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)" = "da80be589a72651dcda34d8b35bcdc9b7254ad06325611074d9cc0fbb19f60ee" "checksum fnv 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" "checksum fs_extra 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" @@ -1685,7 +1692,7 @@ dependencies = [ "checksum idna 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" "checksum indexmap 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "55e2e4c765aa53a0424761bf9f41aa7a6ac1efa87238f59560640e27fca028f2" "checksum indicatif 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4" -"checksum indoc 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "644defcefee68d7805653a682e99a2e2a5014a1fc3cc9be7059a215844eeea6f" +"checksum indoc 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e5a75aeaaef0ce18b58056d306c27b07436fbb34b8816c53094b76dd81803136" "checksum integer-encoding 1.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "4f4ebd0bd29be0f11973e9b3e219005661042a019fd757798c36a47c87852625" "checksum intervals-general 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6261104c6643ef332fd8a7c1a1e630da692335c701bb5048c32c36826b5bd24e" "checksum itertools 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" @@ -1693,7 +1700,7 @@ dependencies = [ "checksum jobserver 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)" = "5c71313ebb9439f74b00d9d2dcec36440beaf57a6aa0623068441dd7cd81a7f2" "checksum js-sys 0.3.45 (registry+https://github.com/rust-lang/crates.io-index)" = "ca059e81d9486668f12d455a4ea6daa600bd408134cd17e3d3fb5a32d1f016f8" "checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -"checksum libc 0.2.79 (registry+https://github.com/rust-lang/crates.io-index)" = "2448f6066e80e3bfc792e9c98bf705b4b0fc6e8ef5b43e5889aff0eaa9c58743" +"checksum libc 0.2.80 (registry+https://github.com/rust-lang/crates.io-index)" = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614" "checksum libnghttp2-sys 0.1.4+1.41.0 (registry+https://github.com/rust-lang/crates.io-index)" = "03624ec6df166e79e139a2310ca213283d6b3c30810c54844f307086d4488df1" "checksum libz-sys 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "602113192b08db8f38796c4e85c39e960c145965140e918018bcde1952429655" "checksum linked-hash-map 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a" @@ -1702,27 +1709,27 @@ dependencies = [ "checksum lz4-sys 1.9.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dca79aa95d8b3226213ad454d328369853be3a1382d89532a854f4d69640acae" "checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" "checksum maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" -"checksum memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" +"checksum memchr 2.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" "checksum miniz_oxide 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d" "checksum nodrop 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" "checksum num 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ab3e176191bc4faad357e3122c4747aa098ac880e88b168f106386128736cf4a" "checksum num-bigint 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b7f3fc75e3697059fb1bc465e3d8cca6cf92f56854f201158b3f9c77d5a3cfa0" -"checksum num-complex 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b05ad05bd8977050b171b3f6b48175fea6e0565b7981059b486075e1026a9fb5" +"checksum num-complex 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "747d632c0c558b87dbabbe6a82f3b4ae03720d0646ac5b7b4dae89394be5f2c5" "checksum num-format 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bafe4179722c2894288ee77a9f044f02811c86af699344c498b0840c698a2465" -"checksum num-integer 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b" -"checksum num-iter 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e6b7c748f995c4c29c5f5ae0248536e04a5739927c74ec0fa564805094b9f" -"checksum num-rational 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a5b4d7360f362cfb50dde8143501e6940b22f644be75a4cc90b2d81968908138" -"checksum num-traits 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)" = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611" +"checksum num-integer 0.1.44 (registry+https://github.com/rust-lang/crates.io-index)" = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +"checksum num-iter 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "b2021c8337a54d21aca0d59a92577a029af9431cb59b909b03252b9c164fad59" +"checksum num-rational 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e5fa6d5f418879385b213d905f7cf5bf4aa553d4c380f0152d1d4f2749186fa9" +"checksum num-traits 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" "checksum num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" "checksum number_prefix 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" "checksum openssl-probe 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" -"checksum openssl-src 111.11.0+1.1.1h (registry+https://github.com/rust-lang/crates.io-index)" = "380fe324132bea01f45239fadfec9343adb044615f29930d039bec1ae7b9fa5b" +"checksum openssl-src 111.12.0+1.1.1h (registry+https://github.com/rust-lang/crates.io-index)" = "858a4132194f8570a7ee9eb8629e85b23cbc4565f2d4a162e87556e5956abf61" "checksum openssl-sys 0.9.58 (registry+https://github.com/rust-lang/crates.io-index)" = "a842db4709b604f0fe5d1170ae3565899be2ad3d9cbc72dedc789ac0511f78de" "checksum ordered-float 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3741934be594d77de1c8461ebcbbe866f585ea616a9753aa78f2bdc69f0e4579" -"checksum parquet 2.0.0-SNAPSHOT (git+https://github.com/apache/arrow)" = "" +"checksum parquet 3.0.0-SNAPSHOT (git+https://github.com/apache/arrow)" = "" "checksum parquet-format 2.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a5bc6b23543b5dedc8f6cce50758a35e5582e148e0cfa26bd0cacd569cda5b71" "checksum percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" -"checksum pkg-config 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)" = "d36492546b6af1463394d46f0c834346f31548646f6ba10849802c9c9a27ac33" +"checksum pkg-config 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)" = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" "checksum podio 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "b18befed8bc2b61abc79a457295e7e838417326da1586050b919414073977f19" "checksum ppv-lite86 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)" = "c36fa947111f5c62a733b652544dd0016a43ce89619538a8ef92724a6f501a20" "checksum prettytable-rs 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0fd04b170004fa2daccf418a7f8253aaf033c27760b5f225889024cf66d7ac2e" @@ -1740,9 +1747,9 @@ dependencies = [ "checksum read_input 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b57518cc6538a2eb7dce826e24fa51d0b7cf8e744ee10c7f56259cdec40050e5" "checksum redox_syscall 0.1.57 (registry+https://github.com/rust-lang/crates.io-index)" = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" "checksum redox_users 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" -"checksum regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" +"checksum regex 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8963b85b8ce3074fecffde43b4b0dded83ce2f367dc8d363afc56679f3ee820b" "checksum regex-automata 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4" -"checksum regex-syntax 0.6.18 (registry+https://github.com/rust-lang/crates.io-index)" = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" +"checksum regex-syntax 0.6.20 (registry+https://github.com/rust-lang/crates.io-index)" = "8cab7a364d15cde1e505267766a2d3c4e22a843e1a601f0fa7564c0f82ced11c" "checksum remove_dir_all 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" "checksum ringbuf 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "605dc50025dcaaeb8ce2b6b3583432b446c80c90ecd8463569dba8a040e87886" "checksum rust-argon2 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9dab61250775933275e84053ac235621dfb739556d5c54a2f2e9313b7cf43a19" @@ -1750,12 +1757,12 @@ dependencies = [ "checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" "checksum ryu 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" "checksum schannel 0.1.19 (registry+https://github.com/rust-lang/crates.io-index)" = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" -"checksum serde 1.0.116 (registry+https://github.com/rust-lang/crates.io-index)" = "96fe57af81d28386a513cbc6858332abc6117cfdb5999647c6444b8f43a370a5" -"checksum serde_derive 1.0.116 (registry+https://github.com/rust-lang/crates.io-index)" = "f630a6370fd8e457873b4bd2ffdae75408bc291ba72be773772a4c2a065d9ae8" -"checksum serde_json 1.0.58 (registry+https://github.com/rust-lang/crates.io-index)" = "a230ea9107ca2220eea9d46de97eddcb04cd00e92d13dda78e478dd33fa82bd4" +"checksum serde 1.0.117 (registry+https://github.com/rust-lang/crates.io-index)" = "b88fa983de7720629c9387e9f517353ed404164b1e482c970a90c1a4aaf7dc1a" +"checksum serde_derive 1.0.117 (registry+https://github.com/rust-lang/crates.io-index)" = "cbd1ae72adb44aab48f325a02444a5fc079349a8d804c1fc922aed3f7454c74e" +"checksum serde_json 1.0.59 (registry+https://github.com/rust-lang/crates.io-index)" = "dcac07dbffa1c65e7f816ab9eba78eb142c6d44410f4eeba1e26e4f5dfa56b95" "checksum serde_with 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8bac272128fb3b1e98872dca27a05c18d8b78b9bd089d3edb7b5871501b50bce" "checksum serde_with_macros 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3c747a9ab2e833b807f74f6b6141530655010bfa9c9c06d5508bce75c8f8072f" -"checksum serde_yaml 0.8.13 (registry+https://github.com/rust-lang/crates.io-index)" = "ae3e2dd40a7cdc18ca80db804b7f461a39bb721160a85c9a1fa30134bf3c02a5" +"checksum serde_yaml 0.8.14 (registry+https://github.com/rust-lang/crates.io-index)" = "f7baae0a99f1a324984bcdc5f0718384c1f69775f1c7eec8b859b71b443e3fd7" "checksum shlex 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" "checksum signal-hook 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "604508c1418b99dfe1925ca9224829bb2a8a9a04dda655cc01fcad46f4ab05ed" "checksum signal-hook-registry 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a3e12110bc539e657a646068aaf5eb5b63af9d0c1f7b29c97113fad80e15f035" @@ -1767,13 +1774,12 @@ dependencies = [ "checksum socket2 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "b1fa70dc5c8104ec096f4fe7ede7a221d35ae13dcd19ba1ad9a81d2cab9a1c44" "checksum strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" "checksum strsim 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c" -"checksum syn 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)" = "9c51d92969d209b54a98397e1b91c8ae82d8c87a7bb87df0b29aa2ad81454228" +"checksum syn 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)" = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac" "checksum take_mut 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" "checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" "checksum term 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "edd106a334b7657c10b7c540a0106114feadeb4dc314513e97df481d5d966f42" "checksum term 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c0863a3345e70f61d613eab32ee046ccd1bcc5f9105fe402c61fcd0c13eeb8b5" "checksum terminal_size 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "9a14cd9f8c72704232f0bfc8455c0e861f0ad4eb60cc9ec8a170e231414c1e13" -"checksum termios 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6f0fcee7b24a25675de40d5bb4de6e41b0df07bc9856295e7e2b3a3600c400c2" "checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" "checksum thiserror 1.0.21 (registry+https://github.com/rust-lang/crates.io-index)" = "318234ffa22e0920fe9a40d7b8369b5f649d490980cf7aadcf1eb91594869b42" "checksum thiserror-impl 1.0.21 (registry+https://github.com/rust-lang/crates.io-index)" = "cae2447b6282786c3493999f40a9be2a6ad20cb8bd268b0a0dbf5a065535c0ab" @@ -1786,7 +1792,7 @@ dependencies = [ "checksum unicode-normalization 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6fb19cf769fa8c6a80a162df694621ebeb4dafb606470b2b2fce0be40a98a977" "checksum unicode-width 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" "checksum unicode-xid 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" -"checksum unindent 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "af41d708427f8fd0e915dcebb2cae0f0e6acb2a939b2d399c265c39a38a18942" +"checksum unindent 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "f14ee04d9415b52b3aeab06258a3f07093182b88ba0f9b8d203f211a7a7d41c7" "checksum url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "829d4a8476c35c9bf0bbce5a3b23f4106f79728039b726d292bb93bc106787cb" "checksum users 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "24cc0f6d6f267b73e5a2cadf007ba8f9bc39c6a6f9666f8cf25ea809a153b032" "checksum vcpkg 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "6454029bf181f092ad1b853286f23e2c507d8e8194d01d92da4a55c274a5508c" diff --git a/kamu-cli/Cargo.toml b/kamu-cli/Cargo.toml index 0de8378946..c15766b2ce 100644 --- a/kamu-cli/Cargo.toml +++ b/kamu-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "kamu-cli" -version = "0.31.0" +version = "0.32.0" description = "Decentralized data management tool" authors = ["Sergii Mikhtoniuk "] license = "MPL-2.0" @@ -13,6 +13,7 @@ categories = ["command-line-utilities"] [dependencies] +opendatafabric = { path = "../opendatafabric" } kamu = { path = "../kamu-core" } # UI diff --git a/kamu-cli/examples/pull-root-multi.rs b/kamu-cli/examples/pull-root-multi.rs index 67e3e2f65d..fd43b2778b 100644 --- a/kamu-cli/examples/pull-root-multi.rs +++ b/kamu-cli/examples/pull-root-multi.rs @@ -1,6 +1,7 @@ use kamu::domain::*; use kamu_cli::commands::*; use kamu_cli::output::OutputConfig; +use opendatafabric::*; use chrono::{DateTime, Utc}; use std::cell::RefCell; @@ -21,17 +22,11 @@ fn main() { cmd.run().unwrap(); } -fn rand_hash() -> String { - use rand::distributions::Standard; - use rand::Rng; - use std::fmt::Write; - - let mut res = String::with_capacity(64); - rand::thread_rng() - .sample_iter::(&Standard) - .take(32) - .for_each(|b| write!(&mut res, "{:02x}", b).unwrap()); - res +fn rand_hash() -> Sha3_256 { + use rand::RngCore; + let mut hash = [0u8; 32]; + rand::thread_rng().fill_bytes(&mut hash); + Sha3_256::new(hash) } pub struct TestPullService; diff --git a/kamu-cli/examples/pull-root-single.rs b/kamu-cli/examples/pull-root-single.rs index 9b357594c9..04e2c944b6 100644 --- a/kamu-cli/examples/pull-root-single.rs +++ b/kamu-cli/examples/pull-root-single.rs @@ -1,6 +1,7 @@ use kamu::domain::*; use kamu_cli::commands::*; use kamu_cli::output::OutputConfig; +use opendatafabric::*; use chrono::{DateTime, Utc}; use std::cell::RefCell; @@ -66,7 +67,10 @@ impl PullService for TestPullService { sleep(1000); - let hash = "a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a"; + let hash = + Sha3_256::from_str("a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a") + .unwrap(); + let result = IngestResult::Updated { block_hash: hash.to_owned(), has_more: false, diff --git a/kamu-cli/src/commands/add_command.rs b/kamu-cli/src/commands/add_command.rs index 08f4b5f5ac..5459a425b9 100644 --- a/kamu-cli/src/commands/add_command.rs +++ b/kamu-cli/src/commands/add_command.rs @@ -1,6 +1,6 @@ use super::{Command, Error}; use kamu::domain::*; -use kamu::infra::serde::yaml::*; +use opendatafabric::*; use std::cell::RefCell; use std::io::Read; diff --git a/kamu-cli/src/commands/complete_command.rs b/kamu-cli/src/commands/complete_command.rs index 4286590372..f0bf5a0915 100644 --- a/kamu-cli/src/commands/complete_command.rs +++ b/kamu-cli/src/commands/complete_command.rs @@ -8,7 +8,7 @@ use std::path; use std::rc::Rc; pub struct CompleteCommand { - metadata_repo: Rc>, + metadata_repo: Option>>, app: clap::App<'static, 'static>, input: String, current: usize, @@ -18,7 +18,7 @@ pub struct CompleteCommand { // but we have to do this until clap supports custom completer functions impl CompleteCommand { pub fn new( - metadata_repo: Rc>, + metadata_repo: Option>>, app: clap::App<'static, 'static>, input: String, current: usize, @@ -32,9 +32,11 @@ impl CompleteCommand { } fn complete_dataset(&self, prefix: &str) { - for dataset_id in self.metadata_repo.borrow().get_all_datasets() { - if dataset_id.starts_with(prefix) { - println!("{}", dataset_id); + if let Some(repo) = self.metadata_repo.as_ref() { + for dataset_id in repo.borrow().get_all_datasets() { + if dataset_id.starts_with(prefix) { + println!("{}", dataset_id); + } } } } diff --git a/kamu-cli/src/commands/delete_command.rs b/kamu-cli/src/commands/delete_command.rs index 1bb7be61e3..b476928057 100644 --- a/kamu-cli/src/commands/delete_command.rs +++ b/kamu-cli/src/commands/delete_command.rs @@ -1,5 +1,6 @@ use super::{Command, Error}; use kamu::domain::*; +use opendatafabric::*; use std::cell::RefCell; use std::convert::TryFrom; diff --git a/kamu-cli/src/commands/depgraph_command.rs b/kamu-cli/src/commands/depgraph_command.rs index 2ffa013bbb..7f2157fcd4 100644 --- a/kamu-cli/src/commands/depgraph_command.rs +++ b/kamu-cli/src/commands/depgraph_command.rs @@ -1,6 +1,6 @@ use super::{Command, Error}; use kamu::domain::*; -use kamu::infra::serde::yaml::DatasetKind; +use kamu::infra::DatasetKind; use std::cell::RefCell; use std::rc::Rc; diff --git a/kamu-cli/src/commands/list_command.rs b/kamu-cli/src/commands/list_command.rs index b928aa8358..32024d572f 100644 --- a/kamu-cli/src/commands/list_command.rs +++ b/kamu-cli/src/commands/list_command.rs @@ -1,6 +1,7 @@ use super::{Command, Error}; use crate::output::*; use kamu::domain::*; +use opendatafabric::*; use chrono::{DateTime, Utc}; use chrono_humanize::HumanTime; diff --git a/kamu-cli/src/commands/log_command.rs b/kamu-cli/src/commands/log_command.rs index 15a2ce44d6..f52432d57f 100644 --- a/kamu-cli/src/commands/log_command.rs +++ b/kamu-cli/src/commands/log_command.rs @@ -1,6 +1,6 @@ use super::{Command, Error}; use kamu::domain::*; -use kamu::infra::serde::yaml::*; +use opendatafabric::*; use console::style; use std::cell::RefCell; diff --git a/kamu-cli/src/commands/pull_command.rs b/kamu-cli/src/commands/pull_command.rs index e90986bdba..b6028f3a36 100644 --- a/kamu-cli/src/commands/pull_command.rs +++ b/kamu-cli/src/commands/pull_command.rs @@ -1,6 +1,7 @@ use super::{Command, Error}; use crate::output::OutputConfig; use kamu::domain::*; +use opendatafabric::*; use std::backtrace::BacktraceStatus; use std::cell::RefCell; @@ -369,7 +370,7 @@ impl IngestListener for PrettyIngestProgress { IngestResult::Updated { ref block_hash, has_more: _, - } => console::style(format!("Committed new block {}", &block_hash[..8])).green(), + } => console::style(format!("Committed new block {}", block_hash.short())).green(), }; self.curr_progress .finish_with_message(&Self::spinner_message( @@ -446,7 +447,7 @@ impl TransformListener for PrettyTransformProgress { console::style("Dataset is up-to-date".to_owned()).yellow() } TransformResult::Updated { ref block_hash } => { - console::style(format!("Committed new block {}", &block_hash[..8])).green() + console::style(format!("Committed new block {}", block_hash.short())).green() } }; self.curr_progress diff --git a/kamu-cli/src/commands/set_watermark_command.rs b/kamu-cli/src/commands/set_watermark_command.rs index 1bbacb556a..1fca5208a2 100644 --- a/kamu-cli/src/commands/set_watermark_command.rs +++ b/kamu-cli/src/commands/set_watermark_command.rs @@ -1,5 +1,6 @@ use super::{Command, Error}; use kamu::domain::*; +use opendatafabric::*; use chrono::DateTime; use std::cell::RefCell; diff --git a/kamu-cli/src/main.rs b/kamu-cli/src/main.rs index 97b6203a36..50a89b0d27 100644 --- a/kamu-cli/src/main.rs +++ b/kamu-cli/src/main.rs @@ -1,10 +1,10 @@ #![feature(backtrace)] -use kamu::domain::*; use kamu::infra::*; use kamu_cli::cli_parser; use kamu_cli::commands::*; use kamu_cli::output::*; +use opendatafabric::DatasetIDBuf; use clap::value_t_or_exit; use console::style; @@ -67,7 +67,11 @@ fn main() { submatches.is_present("recursive"), )), ("complete", Some(submatches)) => Box::new(CompleteCommand::new( - metadata_repo.clone(), + if in_workspace(&workspace_layout) { + Some(metadata_repo.clone()) + } else { + None + }, cli_parser::cli(BINARY_NAME, VERSION), submatches.value_of("input").unwrap().into(), submatches.value_of("current").unwrap().parse().unwrap(), @@ -151,7 +155,7 @@ fn main() { _ => unimplemented!(), }; - let result = if command.needs_workspace() && !workspace_layout.kamu_root_dir.is_dir() { + let result = if command.needs_workspace() && !in_workspace(&workspace_layout) { Err(Error::NotInWorkspace) } else { command.run() @@ -186,6 +190,10 @@ fn find_workspace_rec(p: &Path) -> Option { } } +fn in_workspace(workspace_layout: &WorkspaceLayout) -> bool { + workspace_layout.kamu_root_dir.is_dir() +} + fn configure_logging(output_config: &OutputConfig, workspace_layout: &WorkspaceLayout) -> Logger { let raw_logger = if output_config.verbosity_level > 0 { // Log into stderr for verbose output @@ -226,7 +234,7 @@ fn configure_logging(output_config: &OutputConfig, workspace_layout: &WorkspaceL } fn configure_output_format(matches: &clap::ArgMatches<'_>) -> OutputConfig { - let is_tty = console::Term::stdout().is_term(); + let is_tty = console::Term::stdout().features().is_attended(); let verbosity_level = matches.occurrences_of("v") as u8; if verbosity_level > 0 { diff --git a/kamu-core-test/Cargo.toml b/kamu-core-test/Cargo.toml index dbfb92a92c..112e7c8799 100644 --- a/kamu-core-test/Cargo.toml +++ b/kamu-core-test/Cargo.toml @@ -1,11 +1,11 @@ [package] name = "kamu-test" -version = "0.31.0" +version = "0.32.0" authors = ["Sergii Mikhtoniuk "] edition = "2018" [dependencies] -kamu = { path = "../kamu-core" } +opendatafabric = { path = "../opendatafabric" } chrono = { version = "*", features = ["serde"] } url = "*" diff --git a/kamu-core-test/src/metadata_factory.rs b/kamu-core-test/src/metadata_factory.rs index 406a55dc57..09395ec81f 100644 --- a/kamu-core-test/src/metadata_factory.rs +++ b/kamu-core-test/src/metadata_factory.rs @@ -1,7 +1,6 @@ -use kamu::domain::*; -use kamu::infra::serde::yaml::*; +use opendatafabric::*; -use chrono::{DateTime, SubsecRound, Utc}; +use chrono::{DateTime, Utc}; use std::convert::TryFrom; use std::path::Path; @@ -177,9 +176,9 @@ impl MetadataBlockBuilder { fn new() -> Self { Self { v: MetadataBlock { - block_hash: "".to_owned(), - prev_block_hash: "".to_owned(), - system_time: Utc::now().round_subsecs(3), + block_hash: Sha3_256::zero(), + prev_block_hash: None, + system_time: Utc::now(), output_slice: None, output_watermark: None, input_slices: None, @@ -188,9 +187,8 @@ impl MetadataBlockBuilder { } } - pub fn prev(mut self, prev_block_hash: &str) -> Self { - self.v.prev_block_hash.clear(); - self.v.prev_block_hash.push_str(prev_block_hash); + pub fn prev(mut self, prev_block_hash: &Sha3_256) -> Self { + self.v.prev_block_hash = Some(*prev_block_hash); self } diff --git a/kamu-core/Cargo.toml b/kamu-core/Cargo.toml index ee2da456e9..ab75ad8c84 100644 --- a/kamu-core/Cargo.toml +++ b/kamu-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "kamu" -version = "0.31.0" +version = "0.32.0" authors = ["Sergii Mikhtoniuk "] edition = "2018" @@ -8,13 +8,12 @@ edition = "2018" skip_docker_tests = [] [dependencies] +opendatafabric = { path = "../opendatafabric" } + # Domain chrono = { version = "*", features = ["serde"] } -intervals-general = "*" -rust-crypto = "*" # Data and metadata hashing # Serialization -flatbuffers = "*" hex = "*" serde = { version = "*", features = ["derive"] } serde_with = "*" diff --git a/kamu-core/src/domain/dataset_id.rs b/kamu-core/src/domain/dataset_id.rs deleted file mode 100644 index 30b9e53d6d..0000000000 --- a/kamu-core/src/domain/dataset_id.rs +++ /dev/null @@ -1,265 +0,0 @@ -use std::borrow; -use std::cmp; -use std::convert::{AsRef, TryFrom}; -use std::fmt; -use std::ops; - -use super::grammar::DatasetIDGrammar; - -//////////////////////////////////////////////////////////////////////////////// -// DatasetID (reference type) -//////////////////////////////////////////////////////////////////////////////// - -#[derive(Debug)] -pub struct DatasetID(str); - -impl DatasetID { - pub fn new_unchecked + ?Sized>(s: &S) -> &DatasetID { - unsafe { &*(s.as_ref() as *const str as *const DatasetID) } - } - - pub fn try_from + ?Sized>(s: &S) -> Result<&DatasetID, InvalidDatasetID> { - match DatasetIDGrammar::match_dataset_id(s.as_ref()) { - Some((_, "")) => Ok(DatasetID::new_unchecked(s)), - _ => Err(InvalidDatasetID { - invalid_id: String::from(s.as_ref()), - }), - } - } - - pub fn as_str(&self) -> &str { - &self.0 - } -} - -impl ops::Deref for DatasetID { - type Target = str; - - fn deref(&self) -> &str { - &self.0 - } -} - -impl AsRef for DatasetID { - fn as_ref(&self) -> &str { - &self.0 - } -} - -impl AsRef for DatasetID { - fn as_ref(&self) -> &std::path::Path { - self.0.as_ref() - } -} - -impl ToOwned for DatasetID { - type Owned = DatasetIDBuf; - - fn to_owned(&self) -> DatasetIDBuf { - DatasetIDBuf::from(self) - } -} - -impl cmp::PartialEq for DatasetID { - fn eq(&self, other: &Self) -> bool { - self.0 == other.0 - } -} - -impl cmp::PartialEq for DatasetID { - fn eq(&self, other: &str) -> bool { - &self.0 == other - } -} - -impl cmp::Eq for DatasetID {} - -impl cmp::Ord for DatasetID { - fn cmp(&self, other: &Self) -> cmp::Ordering { - self.0.cmp(&other.0) - } -} - -impl cmp::PartialOrd for DatasetID { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl fmt::Display for DatasetID { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", &self.0) - } -} - -impl std::hash::Hash for DatasetID { - fn hash(&self, state: &mut H) { - self.0.hash(state) - } -} - -//////////////////////////////////////////////////////////////////////////////// -// DatasetIDBuf (buffer type) -//////////////////////////////////////////////////////////////////////////////// - -#[derive(Debug, Clone)] -pub struct DatasetIDBuf(String); - -impl DatasetIDBuf { - pub fn new() -> Self { - Self(String::new()) - } -} - -impl Default for DatasetIDBuf { - fn default() -> Self { - Self::new() - } -} - -impl From<&DatasetID> for DatasetIDBuf { - fn from(id: &DatasetID) -> Self { - Self(String::from(id as &str)) - } -} - -// TODO: Why TryFrom not enough? -impl std::str::FromStr for DatasetIDBuf { - type Err = InvalidDatasetID; - fn from_str(s: &str) -> Result { - Self::try_from(s) - } -} - -// TODO: Replace with AsRef matcher -// See: https://github.com/rust-lang/rust/issues/50133 -impl TryFrom<&str> for DatasetIDBuf { - type Error = InvalidDatasetID; - fn try_from(s: &str) -> Result { - let id = DatasetID::try_from(s)?; - Ok(Self::from(id)) - } -} -impl TryFrom<&std::ffi::OsString> for DatasetIDBuf { - type Error = InvalidDatasetID; - fn try_from(s: &std::ffi::OsString) -> Result { - Self::try_from(s.to_str().unwrap()) - } -} - -impl ops::Deref for DatasetIDBuf { - type Target = DatasetID; - - fn deref(&self) -> &DatasetID { - DatasetID::new_unchecked(&self.0) - } -} - -impl AsRef for DatasetIDBuf { - fn as_ref(&self) -> &str { - self.0.as_ref() - } -} - -impl AsRef for DatasetIDBuf { - fn as_ref(&self) -> &std::path::Path { - self.0.as_ref() - } -} - -impl AsRef for DatasetIDBuf { - fn as_ref(&self) -> &DatasetID { - DatasetID::new_unchecked(&self.0) - } -} - -impl borrow::Borrow for DatasetIDBuf { - fn borrow(&self) -> &DatasetID { - self - } -} - -impl cmp::PartialEq for DatasetIDBuf { - fn eq(&self, other: &Self) -> bool { - self.0 == other.0 - } -} - -impl cmp::PartialEq for DatasetIDBuf { - fn eq(&self, other: &DatasetID) -> bool { - self.0 == other.0 - } -} - -impl cmp::PartialEq for DatasetIDBuf { - fn eq(&self, other: &str) -> bool { - &self.0 == other - } -} - -impl cmp::Eq for DatasetIDBuf {} - -impl cmp::Ord for DatasetIDBuf { - fn cmp(&self, other: &Self) -> cmp::Ordering { - self.0.cmp(&other.0) - } -} - -impl cmp::PartialOrd for DatasetIDBuf { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl fmt::Display for DatasetIDBuf { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", &self.0) - } -} - -impl std::hash::Hash for DatasetIDBuf { - fn hash(&self, state: &mut H) { - self.0.hash(state) - } -} - -impl serde::Serialize for DatasetIDBuf { - fn serialize(&self, serializer: S) -> Result { - serializer.serialize_str(self) - } -} - -impl<'de> serde::Deserialize<'de> for DatasetIDBuf { - fn deserialize>(deserializer: D) -> Result { - deserializer.deserialize_string(DatasetIDBufSerdeVisitor) - } -} - -struct DatasetIDBufSerdeVisitor; - -impl<'de> serde::de::Visitor<'de> for DatasetIDBufSerdeVisitor { - type Value = DatasetIDBuf; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("a DatasetID string") - } - - fn visit_str(self, v: &str) -> Result { - DatasetIDBuf::try_from(v).map_err(serde::de::Error::custom) - } -} - -//////////////////////////////////////////////////////////////////////////////// -// Errors -//////////////////////////////////////////////////////////////////////////////// - -#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)] -pub struct InvalidDatasetID { - invalid_id: String, -} - -impl fmt::Display for InvalidDatasetID { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "Invalid DatasetID: {}", self.invalid_id) - } -} diff --git a/kamu-core/src/domain/engine.rs b/kamu-core/src/domain/engine.rs index 3a6d89be1e..544653d28a 100644 --- a/kamu-core/src/domain/engine.rs +++ b/kamu-core/src/domain/engine.rs @@ -1,10 +1,11 @@ -use crate::domain::*; -use crate::infra::serde::yaml::formats::datetime_rfc3339_opt; -use crate::infra::serde::yaml::*; +use opendatafabric::serde::yaml::formats::datetime_rfc3339_opt; +use opendatafabric::serde::yaml::generated::*; +use opendatafabric::*; use ::serde::{Deserialize, Serialize}; +use ::serde_with::serde_as; +use ::serde_with::skip_serializing_none; use chrono::{DateTime, Utc}; -use serde_with::skip_serializing_none; use std::backtrace::Backtrace; use std::collections::BTreeMap; use std::path::PathBuf; @@ -28,7 +29,9 @@ pub struct IngestRequest { pub ingest_path: PathBuf, #[serde(default, with = "datetime_rfc3339_opt")] pub event_time: Option>, + #[serde(with = "DatasetSourceRootDef")] pub source: DatasetSourceRoot, + #[serde(with = "DatasetVocabularyDef")] pub dataset_vocab: DatasetVocabulary, pub checkpoints_dir: PathBuf, pub data_dir: PathBuf, @@ -38,16 +41,20 @@ pub struct IngestRequest { #[serde(deny_unknown_fields, rename_all = "camelCase")] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct IngestResponse { + #[serde(with = "MetadataBlockDef")] pub block: MetadataBlock, } #[skip_serializing_none] +#[serde_as] #[serde(deny_unknown_fields, rename_all = "camelCase")] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ExecuteQueryRequest { #[serde(rename = "datasetID")] pub dataset_id: DatasetIDBuf, + #[serde(with = "DatasetSourceDerivativeDef")] pub source: DatasetSourceDerivative, + #[serde_as(as = "BTreeMap<_, DatasetVocabularyDef>")] pub dataset_vocabs: BTreeMap, pub input_slices: BTreeMap, pub data_dirs: BTreeMap, @@ -58,6 +65,7 @@ pub struct ExecuteQueryRequest { #[serde(deny_unknown_fields, rename_all = "camelCase")] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ExecuteQueryResponse { + #[serde(with = "MetadataBlockDef")] pub block: MetadataBlock, pub data_file_name: Option, } diff --git a/kamu-core/src/domain/grammar.rs b/kamu-core/src/domain/grammar.rs deleted file mode 100644 index 8640d0e4d2..0000000000 --- a/kamu-core/src/domain/grammar.rs +++ /dev/null @@ -1,62 +0,0 @@ -pub struct DatasetIDGrammar; - -impl DatasetIDGrammar { - fn match_zero_or_many( - s: &str, - matcher: impl Fn(&str) -> Option<(&str, &str)>, - ) -> Option<(&str, &str)> { - let mut len = 0; - let mut tail = s; - loop { - match matcher(tail) { - Some((head, ntail)) => { - len += head.len(); - tail = ntail; - } - _ => break Some((&s[0..len], &s[len..s.len()])), - } - } - } - - fn match_char(s: &str, c: char) -> Option<(&str, &str)> { - if s.len() > 0 && s.as_bytes()[0] == (c as u8) { - Some((&s[0..1], &s[1..s.len()])) - } else { - None - } - } - - fn match_alphanums(s: &str) -> Option<(&str, &str)> { - let alnums = s.bytes().take_while(|b| b.is_ascii_alphanumeric()).count(); - - if alnums == 0 { - None - } else { - Some((&s[0..alnums], &s[alnums..s.len()])) - } - } - - fn match_subdomain(s: &str) -> Option<(&str, &str)> { - let (h, t) = Self::match_alphanums(s)?; - - let (hh, tt) = Self::match_zero_or_many(t, |s| { - let (_, t) = Self::match_char(s, '-')?; - let (h, tt) = Self::match_alphanums(t)?; - Some((&s[0..h.len() + 1], tt)) - })?; - - Some((&s[0..h.len() + hh.len()], tt)) - } - - pub fn match_dataset_id(s: &str) -> Option<(&str, &str)> { - let (h, t) = Self::match_subdomain(s)?; - - let (hh, tt) = Self::match_zero_or_many(t, |s| { - let (_, t) = Self::match_char(s, '.')?; - let (h, tt) = Self::match_subdomain(t)?; - Some((&s[0..h.len() + 1], tt)) - })?; - - Some((&s[0..h.len() + hh.len()], tt)) - } -} diff --git a/kamu-core/src/domain/ingest_service.rs b/kamu-core/src/domain/ingest_service.rs index 9c71e1b8e3..d5efe1fe30 100644 --- a/kamu-core/src/domain/ingest_service.rs +++ b/kamu-core/src/domain/ingest_service.rs @@ -1,5 +1,5 @@ use super::EngineError; -use crate::domain::{DatasetID, DatasetIDBuf}; +use opendatafabric::{DatasetID, DatasetIDBuf, Sha3_256}; use std::backtrace::Backtrace; use std::path::Path; @@ -47,7 +47,10 @@ impl Default for IngestOptions { #[derive(Debug)] pub enum IngestResult { UpToDate, - Updated { block_hash: String, has_more: bool }, + Updated { + block_hash: Sha3_256, + has_more: bool, + }, } /////////////////////////////////////////////////////////////////////////////// diff --git a/kamu-core/src/domain/metadata_chain.rs b/kamu-core/src/domain/metadata_chain.rs index 74ce159d7c..eeb21dc708 100644 --- a/kamu-core/src/domain/metadata_chain.rs +++ b/kamu-core/src/domain/metadata_chain.rs @@ -1,5 +1,4 @@ -// TODO: Use abstraction -use crate::infra::serde::yaml::MetadataBlock; +use opendatafabric::{MetadataBlock, Sha3_256}; #[derive(Debug)] pub enum BlockRef { @@ -9,9 +8,9 @@ pub enum BlockRef { // TODO: Separate mutable and immutable traits // See: https://github.com/rust-lang/rfcs/issues/2035 pub trait MetadataChain: Send { - fn read_ref(&self, r: &BlockRef) -> Option; + fn read_ref(&self, r: &BlockRef) -> Option; - fn get_block(&self, block_hash: &str) -> Option; + fn get_block(&self, block_hash: &Sha3_256) -> Option; fn iter_blocks(&self) -> Box> { self.iter_blocks_ref(&BlockRef::Head) @@ -19,9 +18,9 @@ pub trait MetadataChain: Send { fn iter_blocks_ref(&self, r: &BlockRef) -> Box>; - fn append_ref(&mut self, r: &BlockRef, block: MetadataBlock) -> String; + fn append_ref(&mut self, r: &BlockRef, block: MetadataBlock) -> Sha3_256; - fn append(&mut self, block: MetadataBlock) -> String { + fn append(&mut self, block: MetadataBlock) -> Sha3_256 { self.append_ref(&BlockRef::Head, block) } } diff --git a/kamu-core/src/domain/metadata_repository.rs b/kamu-core/src/domain/metadata_repository.rs index 86053ed011..e0f583203b 100644 --- a/kamu-core/src/domain/metadata_repository.rs +++ b/kamu-core/src/domain/metadata_repository.rs @@ -1,6 +1,6 @@ use super::*; - -use crate::infra::serde::yaml::*; +use crate::infra::DatasetSummary; +use opendatafabric::*; pub trait MetadataRepository { fn get_all_datasets<'s>(&'s self) -> Box + 's>; diff --git a/kamu-core/src/domain/mod.rs b/kamu-core/src/domain/mod.rs index 67a4154714..aab694a1eb 100644 --- a/kamu-core/src/domain/mod.rs +++ b/kamu-core/src/domain/mod.rs @@ -3,14 +3,6 @@ mod error; pub use error::*; -mod grammar; - -mod dataset_id; -pub use dataset_id::*; - -mod time_interval; -pub use time_interval::*; - // Services mod engine; diff --git a/kamu-core/src/domain/pull_service.rs b/kamu-core/src/domain/pull_service.rs index a04cc9dfe6..892de05d71 100644 --- a/kamu-core/src/domain/pull_service.rs +++ b/kamu-core/src/domain/pull_service.rs @@ -1,6 +1,7 @@ use super::ingest_service::*; use super::transform_service::*; -use crate::domain::{DatasetID, DatasetIDBuf, DomainError}; +use crate::domain::DomainError; +use opendatafabric::{DatasetID, DatasetIDBuf, Sha3_256}; use chrono::{DateTime, Utc}; use std::sync::{Arc, Mutex}; @@ -49,7 +50,7 @@ impl Default for PullOptions { #[derive(Debug)] pub enum PullResult { UpToDate, - Updated { block_hash: String }, + Updated { block_hash: Sha3_256 }, } /////////////////////////////////////////////////////////////////////////////// diff --git a/kamu-core/src/domain/resource_loader.rs b/kamu-core/src/domain/resource_loader.rs index ea865578eb..0c51d302c6 100644 --- a/kamu-core/src/domain/resource_loader.rs +++ b/kamu-core/src/domain/resource_loader.rs @@ -1,4 +1,4 @@ -use crate::infra::serde::yaml::DatasetSnapshot; +use opendatafabric::DatasetSnapshot; use std::backtrace::Backtrace; use thiserror::Error; diff --git a/kamu-core/src/domain/transform_service.rs b/kamu-core/src/domain/transform_service.rs index 319d020fae..e005258e14 100644 --- a/kamu-core/src/domain/transform_service.rs +++ b/kamu-core/src/domain/transform_service.rs @@ -1,5 +1,5 @@ use super::EngineError; -use crate::domain::{DatasetID, DatasetIDBuf}; +use opendatafabric::{DatasetID, DatasetIDBuf, Sha3_256}; use std::backtrace::Backtrace; use std::sync::{Arc, Mutex}; @@ -26,7 +26,7 @@ pub trait TransformService { #[derive(Debug)] pub enum TransformResult { UpToDate, - Updated { block_hash: String }, + Updated { block_hash: Sha3_256 }, } /////////////////////////////////////////////////////////////////////////////// diff --git a/kamu-core/src/infra/dataset_layout.rs b/kamu-core/src/infra/dataset_layout.rs index 0f4890fac7..488fd1130e 100644 --- a/kamu-core/src/infra/dataset_layout.rs +++ b/kamu-core/src/infra/dataset_layout.rs @@ -1,5 +1,5 @@ use super::VolumeLayout; -use crate::domain::DatasetID; +use opendatafabric::DatasetID; use std::path::PathBuf; /// Describes the layout of the dataset on disk diff --git a/kamu-core/src/infra/serde/yaml/dtos_extra.rs b/kamu-core/src/infra/dataset_summary.rs similarity index 59% rename from kamu-core/src/infra/serde/yaml/dtos_extra.rs rename to kamu-core/src/infra/dataset_summary.rs index dd2897d082..f3dcb2e7b0 100644 --- a/kamu-core/src/infra/serde/yaml/dtos_extra.rs +++ b/kamu-core/src/infra/dataset_summary.rs @@ -1,19 +1,10 @@ -use super::dtos_odf::DatasetVocabulary; -use super::formats::datetime_rfc3339_opt; -use crate::domain::*; use chrono::{DateTime, Utc}; +use opendatafabric::serde::yaml::formats::datetime_rfc3339_opt; +use opendatafabric::serde::yaml::generated::DatasetVocabularyDef; +use opendatafabric::{DatasetIDBuf, DatasetVocabulary}; use serde::{Deserialize, Serialize}; use serde_with::skip_serializing_none; -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct Manifest { - pub api_version: i32, - pub kind: String, - pub content: T, -} - #[skip_serializing_none] #[serde(deny_unknown_fields, rename_all = "camelCase")] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -34,14 +25,6 @@ pub struct DatasetSummary { pub last_pulled: Option>, pub num_records: u64, pub data_size: u64, + #[serde(with = "DatasetVocabularyDef")] pub vocab: DatasetVocabulary, } - -impl Default for DatasetVocabulary { - fn default() -> Self { - Self { - system_time_column: None, - event_time_column: None, - } - } -} diff --git a/kamu-core/src/infra/engine/engine_flink.rs b/kamu-core/src/infra/engine/engine_flink.rs index c0a7e2b657..aa970a7069 100644 --- a/kamu-core/src/infra/engine/engine_flink.rs +++ b/kamu-core/src/infra/engine/engine_flink.rs @@ -1,5 +1,4 @@ use crate::domain::*; -use crate::infra::serde::yaml::*; use crate::infra::utils::docker_client::*; use crate::infra::*; @@ -246,8 +245,7 @@ impl FlinkEngine { info!(self.logger, "Writing request"; "request" => ?manifest, "path" => ?path); let file = File::create(&path)?; - serde_yaml::to_writer(file, &manifest) - .map_err(|e| EngineError::internal(e))?; + serde_yaml::to_writer(file, &manifest).map_err(|e| EngineError::internal(e))?; Ok(()) } diff --git a/kamu-core/src/infra/engine/engine_spark.rs b/kamu-core/src/infra/engine/engine_spark.rs index 72050d6e96..112608e7bf 100644 --- a/kamu-core/src/infra/engine/engine_spark.rs +++ b/kamu-core/src/infra/engine/engine_spark.rs @@ -1,5 +1,4 @@ use crate::domain::*; -use crate::infra::serde::yaml::*; use crate::infra::utils::docker_client::*; use crate::infra::*; @@ -154,8 +153,7 @@ impl SparkEngine { info!(self.logger, "Writing request"; "request" => ?manifest, "path" => ?path); let file = File::create(&path)?; - serde_yaml::to_writer(file, &manifest) - .map_err(|e| EngineError::internal(e))?; + serde_yaml::to_writer(file, &manifest).map_err(|e| EngineError::internal(e))?; Ok(()) } diff --git a/kamu-core/src/infra/ingest/checkpointing_executor.rs b/kamu-core/src/infra/ingest/checkpointing_executor.rs index fe6e051544..08df99873e 100644 --- a/kamu-core/src/infra/ingest/checkpointing_executor.rs +++ b/kamu-core/src/infra/ingest/checkpointing_executor.rs @@ -1,4 +1,4 @@ -use crate::infra::serde::yaml::*; +use crate::infra::Manifest; use std::fs::File; use std::path::Path; diff --git a/kamu-core/src/infra/ingest/fetch_service.rs b/kamu-core/src/infra/ingest/fetch_service.rs index a06b70168e..86d1c727a5 100644 --- a/kamu-core/src/infra/ingest/fetch_service.rs +++ b/kamu-core/src/infra/ingest/fetch_service.rs @@ -1,11 +1,11 @@ use super::*; use crate::domain::*; -use crate::infra::serde::yaml::formats::{datetime_rfc3339, datetime_rfc3339_opt}; -use crate::infra::serde::yaml::*; +use opendatafabric::serde::yaml::formats::{datetime_rfc3339, datetime_rfc3339_opt}; +use opendatafabric::*; +use ::serde::{Deserialize, Serialize}; +use ::serde_with::skip_serializing_none; use chrono::{DateTime, SubsecRound, TimeZone, Utc}; -use serde::{Deserialize, Serialize}; -use serde_with::skip_serializing_none; use slog::{info, Logger}; use std::io::prelude::*; use std::path::{Path, PathBuf}; diff --git a/kamu-core/src/infra/ingest/ingest_task.rs b/kamu-core/src/infra/ingest/ingest_task.rs index 36a569a558..8b308a54fc 100644 --- a/kamu-core/src/infra/ingest/ingest_task.rs +++ b/kamu-core/src/infra/ingest/ingest_task.rs @@ -1,8 +1,8 @@ use super::*; use crate::domain::*; -use crate::infra::serde::yaml::formats::datetime_rfc3339; -use crate::infra::serde::yaml::*; use crate::infra::*; +use opendatafabric::serde::yaml::formats::datetime_rfc3339; +use opendatafabric::*; use ::serde::{Deserialize, Serialize}; use ::serde_with::skip_serializing_none; @@ -252,7 +252,7 @@ impl IngestTask { &mut self, fetch_result: ExecutionResult, read_result: ExecutionResult, - prev_hash: String, + prev_hash: Sha3_256, ) -> Result, IngestError> { let checkpoint_path = self.layout.cache_dir.join("commit.yaml"); @@ -271,12 +271,12 @@ impl IngestTask { } let new_block = MetadataBlock { - prev_block_hash: prev_hash, + prev_block_hash: Some(prev_hash), ..read_result.checkpoint.last_block }; let hash = self.meta_chain.borrow_mut().append(new_block); - info!(self.logger, "Committed new block"; "hash" => &hash); + info!(self.logger, "Committed new block"; "hash" => hash.to_string()); Ok(ExecutionResult { was_up_to_date: false, @@ -317,5 +317,5 @@ pub struct CommitCheckpoint { pub for_read_at: DateTime, #[serde(with = "datetime_rfc3339")] pub for_fetched_at: DateTime, - pub last_hash: String, + pub last_hash: Sha3_256, } diff --git a/kamu-core/src/infra/ingest/prep_service.rs b/kamu-core/src/infra/ingest/prep_service.rs index fac5a4a1f0..ae997a1ae1 100644 --- a/kamu-core/src/infra/ingest/prep_service.rs +++ b/kamu-core/src/infra/ingest/prep_service.rs @@ -1,11 +1,11 @@ use super::*; use crate::domain::*; -use crate::infra::serde::yaml::formats::datetime_rfc3339; -use crate::infra::serde::yaml::*; +use opendatafabric::serde::yaml::formats::datetime_rfc3339; +use opendatafabric::*; +use ::serde::{Deserialize, Serialize}; +use ::serde_with::skip_serializing_none; use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; -use serde_with::skip_serializing_none; use std::fs::File; use std::io::prelude::*; use std::io::Error as IOError; diff --git a/kamu-core/src/infra/ingest/read_service.rs b/kamu-core/src/infra/ingest/read_service.rs index 878c5ebbdb..d2824c53f6 100644 --- a/kamu-core/src/infra/ingest/read_service.rs +++ b/kamu-core/src/infra/ingest/read_service.rs @@ -1,12 +1,13 @@ use super::*; use crate::domain::*; -use crate::infra::serde::yaml::formats::datetime_rfc3339; -use crate::infra::serde::yaml::*; use crate::infra::*; +use opendatafabric::serde::yaml::formats::datetime_rfc3339; +use opendatafabric::serde::yaml::generated::MetadataBlockDef; +use opendatafabric::*; use ::serde::{Deserialize, Serialize}; +use ::serde_with::skip_serializing_none; use chrono::{DateTime, Utc}; -use serde_with::skip_serializing_none; use std::path::Path; use std::sync::{Arc, Mutex}; @@ -66,5 +67,6 @@ pub struct ReadCheckpoint { pub last_read: DateTime, #[serde(with = "datetime_rfc3339")] pub for_prepared_at: DateTime, + #[serde(with = "MetadataBlockDef")] pub last_block: MetadataBlock, } diff --git a/kamu-core/src/infra/ingest_service_impl.rs b/kamu-core/src/infra/ingest_service_impl.rs index 06d90ef7b3..4303f22616 100644 --- a/kamu-core/src/infra/ingest_service_impl.rs +++ b/kamu-core/src/infra/ingest_service_impl.rs @@ -1,6 +1,7 @@ use super::ingest::*; use crate::domain::*; use crate::infra::*; +use opendatafabric::*; use slog::{info, o, Logger}; use std::cell::RefCell; diff --git a/kamu-core/src/infra/manifest.rs b/kamu-core/src/infra/manifest.rs new file mode 100644 index 0000000000..7cac5d48a6 --- /dev/null +++ b/kamu-core/src/infra/manifest.rs @@ -0,0 +1,11 @@ +use serde::{Deserialize, Serialize}; +use serde_with::skip_serializing_none; + +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Manifest { + pub api_version: i32, + pub kind: String, + pub content: T, +} diff --git a/kamu-core/src/infra/metadata_chain_impl.rs b/kamu-core/src/infra/metadata_chain_impl.rs index a4a9706ba8..a754cbc80b 100644 --- a/kamu-core/src/infra/metadata_chain_impl.rs +++ b/kamu-core/src/infra/metadata_chain_impl.rs @@ -1,9 +1,8 @@ use crate::domain::*; -use crate::infra::serde::yaml::*; use crate::infra::*; +use opendatafabric::serde::yaml::*; +use opendatafabric::*; -use crypto::digest::Digest; -use crypto::sha3::Sha3; use std::io::prelude::*; use std::path::{Path, PathBuf}; @@ -11,6 +10,7 @@ pub struct MetadataChainImpl { meta_path: PathBuf, } +// TODO: Error handling is very poor here impl MetadataChainImpl { pub fn new(meta_path: &Path) -> Self { Self { @@ -21,82 +21,51 @@ impl MetadataChainImpl { pub fn create( meta_path: &Path, first_block: MetadataBlock, - ) -> Result<(Self, String), InfraError> { + ) -> Result<(Self, Sha3_256), InfraError> { std::fs::create_dir(&meta_path)?; std::fs::create_dir(meta_path.join("blocks"))?; std::fs::create_dir(meta_path.join("refs"))?; - let mut chain = Self::new(meta_path); - let first_block_hashed = chain.hashed(first_block); - - chain.write_block(&first_block_hashed)?; - chain.write_ref(&BlockRef::Head, &first_block_hashed.block_hash)?; - - Ok((chain, first_block_hashed.block_hash)) - } - pub fn block_hash(block: &MetadataBlock) -> String { - let mut digest = Sha3::sha3_256(); - // TODO: use generated hashers - digest.input_str(&block.prev_block_hash); - digest.result_str() - } + let hash = chain.write_block(&first_block)?; + chain.write_ref(&BlockRef::Head, &hash)?; - fn hashed(&self, block: MetadataBlock) -> MetadataBlock { - assert!(block.block_hash.is_empty(), "Got an already hashed block"); - let mut b = block; - b.block_hash = Self::block_hash(&b); - b + Ok((chain, hash)) } fn read_block(path: &Path) -> MetadataBlock { - let file = std::fs::File::open(&path) + // TODO: Use mmap + let buffer = std::fs::read(path) .unwrap_or_else(|e| panic!("Failed to open the block file at {}: {}", path.display(), e)); - let manifest: Manifest = serde_yaml::from_reader(&file).unwrap_or_else(|e| { - panic!( - "Failed to deserialize the MetadataBlock at {}: {}", - path.display(), - e - ) - }); - - assert_eq!(manifest.kind, "MetadataBlock"); - manifest.content + YamlMetadataBlockDeserializer + .read_manifest(&buffer) + .unwrap_or_else(|e| panic!("Failed to open the block file at {}: {}", path.display(), e)) } - fn write_block(&mut self, block: &MetadataBlock) -> Result<(), InfraError> { - assert!( - !block.block_hash.is_empty(), - "Attempt to write non-hashed block" - ); + fn write_block(&mut self, block: &MetadataBlock) -> Result { + let (hash, buffer) = YamlMetadataBlockSerializer.write_manifest(block).unwrap(); - let file = std::fs::File::with_options() + let mut file = std::fs::File::with_options() .write(true) .create_new(true) - .open(self.block_path(&block.block_hash))?; - - let manifest = Manifest { - api_version: 1, - kind: "MetadataBlock".to_owned(), - content: block, - }; + .open(self.block_path(&hash))?; + file.write_all(&buffer)?; - serde_yaml::to_writer(file, &manifest)?; - Ok(()) + Ok(hash) } // TODO: atomicity - fn write_ref(&mut self, r: &BlockRef, hash: &str) -> Result<(), InfraError> { + fn write_ref(&mut self, r: &BlockRef, hash: &Sha3_256) -> Result<(), InfraError> { let mut file = std::fs::File::create(self.ref_path(r))?; - file.write_all(hash.as_bytes())?; + file.write_all(hash.to_string().as_bytes())?; file.sync_all()?; Ok(()) } - fn block_path(&self, hash: &str) -> PathBuf { + fn block_path(&self, hash: &Sha3_256) -> PathBuf { let mut p = self.meta_path.join("blocks"); - p.push(hash); + p.push(hash.to_string()); p } @@ -112,19 +81,18 @@ impl MetadataChainImpl { } impl MetadataChain for MetadataChainImpl { - fn read_ref(&self, r: &BlockRef) -> Option { + fn read_ref(&self, r: &BlockRef) -> Option { let path = self.ref_path(r); if !path.exists() { None } else { - Some( - std::fs::read_to_string(&path) - .unwrap_or_else(|e| panic!("Failed to read ref at {}: {}", path.display(), e)), - ) + let s = std::fs::read_to_string(&path) + .unwrap_or_else(|e| panic!("Failed to read ref at {}: {}", path.display(), e)); + Some(Sha3_256::from_str(&s).unwrap()) } } - fn get_block(&self, block_hash: &str) -> Option { + fn get_block(&self, block_hash: &Sha3_256) -> Option { let path = self.block_path(block_hash); if !path.exists() { None @@ -146,13 +114,14 @@ impl MetadataChain for MetadataChainImpl { }) } - fn append_ref(&mut self, r: &BlockRef, block: MetadataBlock) -> String { + fn append_ref(&mut self, r: &BlockRef, block: MetadataBlock) -> Sha3_256 { let last_hash = self .read_ref(r) .unwrap_or_else(|| panic!("Ref {:?} does not exist", r)); assert_eq!( - block.prev_block_hash, last_hash, + block.prev_block_hash, + Some(last_hash), "New block doesn't specify correct prev block hash" ); @@ -165,12 +134,10 @@ impl MetadataChain for MetadataChainImpl { last_block.system_time, ); - let block_hashed = self.hashed(block); - - self.write_block(&block_hashed).unwrap(); - self.write_ref(r, &block_hashed.block_hash).unwrap(); + let hash = self.write_block(&block).unwrap(); + self.write_ref(r, &hash).unwrap(); - block_hashed.block_hash + hash } } @@ -179,15 +146,15 @@ struct BlockReader { } impl BlockReader { - fn read_block(&self, hash: &str) -> MetadataBlock { - let path = self.blocks_dir.join(hash); + fn read_block(&self, hash: &Sha3_256) -> MetadataBlock { + let path = self.blocks_dir.join(hash.to_string()); MetadataChainImpl::read_block(&path) } } struct MetadataBlockIter { reader: BlockReader, - next_hash: Option, + next_hash: Option, } impl Iterator for MetadataBlockIter { @@ -198,7 +165,7 @@ impl Iterator for MetadataBlockIter { None => None, Some(ref hash) => { let block = self.reader.read_block(hash); - self.next_hash = Some(block.prev_block_hash.clone()).filter(|h| !h.is_empty()); + self.next_hash = block.prev_block_hash; Some(block) } } diff --git a/kamu-core/src/infra/metadata_repository_impl.rs b/kamu-core/src/infra/metadata_repository_impl.rs index f71bf1083c..100c0b21a2 100644 --- a/kamu-core/src/infra/metadata_repository_impl.rs +++ b/kamu-core/src/infra/metadata_repository_impl.rs @@ -1,6 +1,6 @@ use super::*; use crate::domain::*; -use crate::infra::serde::yaml::*; +use opendatafabric::*; use chrono::Utc; use std::collections::HashSet; @@ -110,8 +110,8 @@ impl MetadataRepository for MetadataRepositoryImpl { }; let first_block = MetadataBlock { - block_hash: "".to_owned(), - prev_block_hash: "".to_owned(), + block_hash: Sha3_256::zero(), + prev_block_hash: None, system_time: Utc::now(), source: Some(snapshot.source), output_slice: None, diff --git a/kamu-core/src/infra/mod.rs b/kamu-core/src/infra/mod.rs index cb8d3e618e..4f4026fc27 100644 --- a/kamu-core/src/infra/mod.rs +++ b/kamu-core/src/infra/mod.rs @@ -3,7 +3,6 @@ pub use engine::*; pub mod explore; pub mod ingest; -pub mod serde; pub mod utils; mod error; @@ -12,6 +11,12 @@ pub use error::*; mod dataset_layout; pub use dataset_layout::*; +mod dataset_summary; +pub use dataset_summary::*; + +mod manifest; +pub use manifest::*; + mod ingest_service_impl; pub use ingest_service_impl::*; diff --git a/kamu-core/src/infra/pull_service_impl.rs b/kamu-core/src/infra/pull_service_impl.rs index fdce5c124f..8b74674335 100644 --- a/kamu-core/src/infra/pull_service_impl.rs +++ b/kamu-core/src/infra/pull_service_impl.rs @@ -1,5 +1,5 @@ use crate::domain::*; -use crate::infra::serde::yaml::*; +use opendatafabric::*; use chrono::prelude::*; use slog::{info, Logger}; @@ -192,10 +192,10 @@ impl PullService for PullServiceImpl { .borrow_mut() .get_metadata_chain(dataset_id)?; - let last_hash = chain.read_ref(&BlockRef::Head).unwrap(); + let last_hash = chain.read_ref(&BlockRef::Head); let new_block = MetadataBlock { - block_hash: "".to_owned(), + block_hash: Sha3_256::zero(), prev_block_hash: last_hash, system_time: Utc::now(), output_slice: None, diff --git a/kamu-core/src/infra/resource_loader_impl.rs b/kamu-core/src/infra/resource_loader_impl.rs index 973ff159d1..3dea37b3b9 100644 --- a/kamu-core/src/infra/resource_loader_impl.rs +++ b/kamu-core/src/infra/resource_loader_impl.rs @@ -1,5 +1,6 @@ use crate::domain::*; -use crate::infra::serde::yaml::*; +use opendatafabric::serde::yaml::*; +use opendatafabric::*; use std::path::Path; use std::time::Duration; @@ -36,10 +37,10 @@ impl ResourceLoaderImpl { match h.response_code()? { 200 => { - let manifest: Manifest = - serde_yaml::from_slice(&buffer).map_err(|e| ResourceError::serde(e))?; - assert_eq!(manifest.kind, "DatasetSnapshot"); - Ok(manifest.content) + let snapshot = YamlDatasetSnapshotDeserializer + .read_manifest(&buffer) + .map_err(|e| ResourceError::serde(e))?; + Ok(snapshot) } 404 => Err(ResourceError::not_found(url.as_str().to_owned(), None)), _ => Err(ResourceError::unreachable(url.as_str().to_owned(), None)), @@ -52,11 +53,11 @@ impl ResourceLoader for ResourceLoaderImpl { &self, path: &Path, ) -> Result { - let file = std::fs::File::open(path).map_err(|e| ResourceError::internal(e))?; - let manifest: Manifest = - serde_yaml::from_reader(file).map_err(|e| ResourceError::serde(e))?; - assert_eq!(manifest.kind, "DatasetSnapshot"); - Ok(manifest.content) + let buffer = std::fs::read(path).map_err(|e| ResourceError::internal(e))?; + let snapshot = YamlDatasetSnapshotDeserializer + .read_manifest(&buffer) + .map_err(|e| ResourceError::serde(e))?; + Ok(snapshot) } fn load_dataset_snapshot_from_url(&self, url: &Url) -> Result { diff --git a/kamu-core/src/infra/serde/mod.rs b/kamu-core/src/infra/serde/mod.rs deleted file mode 100644 index 1284627481..0000000000 --- a/kamu-core/src/infra/serde/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod yaml; diff --git a/kamu-core/src/infra/serde/yaml/dtos_odf.rs b/kamu-core/src/infra/serde/yaml/dtos_odf.rs deleted file mode 100644 index 804b4d8ff7..0000000000 --- a/kamu-core/src/infra/serde/yaml/dtos_odf.rs +++ /dev/null @@ -1,379 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// WARNING: This file is auto-generated from Open Data Fabric Schemas -// See: http://opendatafabric.org/ -//////////////////////////////////////////////////////////////////////////////// - -use super::formats::{datetime_rfc3339, datetime_rfc3339_opt}; -use crate::domain::DatasetIDBuf; -use crate::domain::TimeInterval; -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; -use serde_with::skip_serializing_none; - -//////////////////////////////////////////////////////////////////////////////// -// DatasetSource -// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#datasetsource-schema -//////////////////////////////////////////////////////////////////////////////// - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase", tag = "kind")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum DatasetSource { - #[serde(rename_all = "camelCase")] - Root(DatasetSourceRoot), - #[serde(rename_all = "camelCase")] - Derivative(DatasetSourceDerivative), -} - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct DatasetSourceRoot { - pub fetch: FetchStep, - pub prepare: Option>, - pub read: ReadStep, - pub preprocess: Option, - pub merge: MergeStrategy, -} - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct DatasetSourceDerivative { - pub inputs: Vec, - pub transform: Transform, -} - -//////////////////////////////////////////////////////////////////////////////// -// DatasetVocabulary -// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#datasetvocabulary-schema -//////////////////////////////////////////////////////////////////////////////// - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct DatasetVocabulary { - pub system_time_column: Option, - pub event_time_column: Option, -} - -//////////////////////////////////////////////////////////////////////////////// -// SourceCaching -// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#sourcecaching-schema -//////////////////////////////////////////////////////////////////////////////// - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase", tag = "kind")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum SourceCaching { - #[serde(rename_all = "camelCase")] - Forever, -} - -//////////////////////////////////////////////////////////////////////////////// -// TemporalTable -// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#temporaltable-schema -//////////////////////////////////////////////////////////////////////////////// - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct TemporalTable { - pub id: String, - pub primary_key: Vec, -} - -//////////////////////////////////////////////////////////////////////////////// -// DatasetSnapshot -// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#datasetsnapshot-schema -//////////////////////////////////////////////////////////////////////////////// - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct DatasetSnapshot { - pub id: DatasetIDBuf, - pub source: DatasetSource, - pub vocab: Option, -} - -//////////////////////////////////////////////////////////////////////////////// -// DataSlice -// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#dataslice-schema -//////////////////////////////////////////////////////////////////////////////// - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct DataSlice { - pub hash: String, - pub interval: TimeInterval, - pub num_records: i64, -} - -//////////////////////////////////////////////////////////////////////////////// -// SqlQueryStep -// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#sqlquerystep-schema -//////////////////////////////////////////////////////////////////////////////// - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct SqlQueryStep { - pub alias: Option, - pub query: String, -} - -//////////////////////////////////////////////////////////////////////////////// -// MergeStrategy -// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#mergestrategy-schema -//////////////////////////////////////////////////////////////////////////////// - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase", tag = "kind")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum MergeStrategy { - #[serde(rename_all = "camelCase")] - Append, - #[serde(rename_all = "camelCase")] - Ledger(MergeStrategyLedger), - #[serde(rename_all = "camelCase")] - Snapshot(MergeStrategySnapshot), -} - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct MergeStrategyLedger { - pub primary_key: Vec, -} - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct MergeStrategySnapshot { - pub primary_key: Vec, - pub compare_columns: Option>, - pub observation_column: Option, - pub obsv_added: Option, - pub obsv_changed: Option, - pub obsv_removed: Option, -} - -//////////////////////////////////////////////////////////////////////////////// -// MetadataBlock -// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#metadatablock-schema -//////////////////////////////////////////////////////////////////////////////// - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct MetadataBlock { - pub block_hash: String, - pub prev_block_hash: String, - #[serde(with = "datetime_rfc3339")] - pub system_time: DateTime, - pub output_slice: Option, - #[serde(default, with = "datetime_rfc3339_opt")] - pub output_watermark: Option>, - pub input_slices: Option>, - pub source: Option, -} - -//////////////////////////////////////////////////////////////////////////////// -// ReadStep -// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#readstep-schema -//////////////////////////////////////////////////////////////////////////////// - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase", tag = "kind")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum ReadStep { - #[serde(rename_all = "camelCase")] - Csv(ReadStepCsv), - #[serde(rename_all = "camelCase")] - JsonLines(ReadStepJsonLines), - #[serde(rename_all = "camelCase")] - GeoJson(ReadStepGeoJson), - #[serde(rename_all = "camelCase")] - EsriShapefile(ReadStepEsriShapefile), -} - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct ReadStepCsv { - pub schema: Option>, - pub separator: Option, - pub encoding: Option, - pub quote: Option, - pub escape: Option, - pub comment: Option, - pub header: Option, - pub enforce_schema: Option, - pub infer_schema: Option, - pub ignore_leading_white_space: Option, - pub ignore_trailing_white_space: Option, - pub null_value: Option, - pub empty_value: Option, - pub nan_value: Option, - pub positive_inf: Option, - pub negative_inf: Option, - pub date_format: Option, - pub timestamp_format: Option, - pub multi_line: Option, -} - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct ReadStepJsonLines { - pub schema: Option>, - pub date_format: Option, - pub encoding: Option, - pub multi_line: Option, - pub primitives_as_string: Option, - pub timestamp_format: Option, -} - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct ReadStepGeoJson { - pub schema: Option>, -} - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct ReadStepEsriShapefile { - pub schema: Option>, - pub sub_path: Option, -} - -//////////////////////////////////////////////////////////////////////////////// -// Transform -// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#transform-schema -//////////////////////////////////////////////////////////////////////////////// - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase", tag = "kind")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum Transform { - #[serde(rename_all = "camelCase")] - Sql(TransformSql), -} - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct TransformSql { - pub engine: String, - pub version: Option, - pub query: Option, - pub queries: Option>, - pub temporal_tables: Option>, -} - -//////////////////////////////////////////////////////////////////////////////// -// FetchStep -// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#fetchstep-schema -//////////////////////////////////////////////////////////////////////////////// - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase", tag = "kind")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum FetchStep { - #[serde(rename_all = "camelCase")] - Url(FetchStepUrl), - #[serde(rename_all = "camelCase")] - FilesGlob(FetchStepFilesGlob), -} - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct FetchStepUrl { - pub url: String, - pub event_time: Option, - pub cache: Option, -} - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct FetchStepFilesGlob { - pub path: String, - pub event_time: Option, - pub cache: Option, - pub order: Option, -} - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum SourceOrdering { - ByEventTime, - ByName, -} - -//////////////////////////////////////////////////////////////////////////////// -// PrepStep -// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#prepstep-schema -//////////////////////////////////////////////////////////////////////////////// - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase", tag = "kind")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum PrepStep { - #[serde(rename_all = "camelCase")] - Decompress(PrepStepDecompress), - #[serde(rename_all = "camelCase")] - Pipe(PrepStepPipe), -} - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct PrepStepDecompress { - pub format: CompressionFormat, - pub sub_path: Option, -} - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct PrepStepPipe { - pub command: Vec, -} - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum CompressionFormat { - Gzip, - Zip, -} - -//////////////////////////////////////////////////////////////////////////////// -// EventTimeSource -// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#eventtimesource-schema -//////////////////////////////////////////////////////////////////////////////// - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase", tag = "kind")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum EventTimeSource { - #[serde(rename_all = "camelCase")] - FromMetadata, - #[serde(rename_all = "camelCase")] - FromPath(EventTimeSourceFromPath), -} - -#[skip_serializing_none] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct EventTimeSourceFromPath { - pub pattern: String, - pub timestamp_format: Option, -} diff --git a/kamu-core/src/infra/serde/yaml/formats.rs b/kamu-core/src/infra/serde/yaml/formats.rs deleted file mode 100644 index 273a734ee3..0000000000 --- a/kamu-core/src/infra/serde/yaml/formats.rs +++ /dev/null @@ -1,51 +0,0 @@ -// TODO: Yep... all this to serde an Option in a slightly different -// format. See: https://github.com/serde-rs/serde/issues/723 - -pub mod datetime_rfc3339 { - use chrono::{DateTime, SecondsFormat, Utc}; - use serde::{Deserialize, Deserializer, Serializer}; - - pub fn serialize( - date: &DateTime, - serializer: S, - ) -> Result { - let s = date.to_rfc3339_opts(SecondsFormat::Millis, true); - serializer.serialize_str(&s) - } - - pub fn deserialize<'de, D: Deserializer<'de>>( - deserializer: D, - ) -> Result, D::Error> { - let s = String::deserialize(deserializer)?; - DateTime::parse_from_rfc3339(&s) - .map(|dt| dt.into()) - .map_err(serde::de::Error::custom) - } -} - -pub mod datetime_rfc3339_opt { - use chrono::{DateTime, SecondsFormat, Utc}; - use serde::{Deserialize, Deserializer, Serializer}; - - pub fn serialize( - option: &Option>, - serializer: S, - ) -> Result { - match option { - None => serializer.serialize_none(), - Some(date) => { - serializer.serialize_str(&date.to_rfc3339_opts(SecondsFormat::Millis, true)) - } - } - } - - pub fn deserialize<'de, D: Deserializer<'de>>( - deserializer: D, - ) -> Result>, D::Error> { - let s = String::deserialize(deserializer)?; - DateTime::parse_from_rfc3339(&s) - .map(|dt| dt.into()) - .map(Some) - .map_err(serde::de::Error::custom) - } -} diff --git a/kamu-core/src/infra/serde/yaml/mod.rs b/kamu-core/src/infra/serde/yaml/mod.rs deleted file mode 100644 index 5e4a6bf8de..0000000000 --- a/kamu-core/src/infra/serde/yaml/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub mod formats; - -mod dtos_odf; -pub use dtos_odf::*; - -mod dtos_extra; -pub use dtos_extra::*; diff --git a/kamu-core/src/infra/transform_service_impl.rs b/kamu-core/src/infra/transform_service_impl.rs index 8557d35d89..79d29d3f79 100644 --- a/kamu-core/src/infra/transform_service_impl.rs +++ b/kamu-core/src/infra/transform_service_impl.rs @@ -1,6 +1,6 @@ use crate::domain::*; -use crate::infra::serde::yaml::*; use crate::infra::*; +use opendatafabric::*; use slog::{info, o, Logger}; use std::cell::RefCell; @@ -57,7 +57,7 @@ impl TransformServiceImpl { mut meta_chain: Box, engine_factory: Arc>, ) -> Result { - let prev_hash = meta_chain.read_ref(&BlockRef::Head).unwrap(); + let prev_hash = meta_chain.read_ref(&BlockRef::Head); let engine = engine_factory .lock() diff --git a/kamu-core/src/infra/utils/docker_images.rs b/kamu-core/src/infra/utils/docker_images.rs index 45555b9a23..8574f473ad 100644 --- a/kamu-core/src/infra/utils/docker_images.rs +++ b/kamu-core/src/infra/utils/docker_images.rs @@ -1,8 +1,8 @@ -pub const SPARK: &str = "kamudata/engine-spark:0.8.2"; +pub const SPARK: &str = "kamudata/engine-spark:0.8.3"; // TODO: Replace with kamu image pub const SPARK_V3: &str = "bitnami/spark:3.0.0"; pub const LIVY: &str = SPARK; -pub const FLINK: &str = "kamudata/engine-flink:0.6.1"; +pub const FLINK: &str = "kamudata/engine-flink:0.6.2"; pub const JUPYTER: &str = "kamudata/jupyter-uber:0.0.1"; // Test Images diff --git a/kamu-core/tests/domain/mod.rs b/kamu-core/tests/domain/mod.rs deleted file mode 100644 index 06d67c4794..0000000000 --- a/kamu-core/tests/domain/mod.rs +++ /dev/null @@ -1 +0,0 @@ -mod test_dataset_id; diff --git a/kamu-core/tests/infra/engine/test_ingest_engine.rs b/kamu-core/tests/infra/engine/test_ingest_engine.rs index 459ed685a4..b5ee95d788 100644 --- a/kamu-core/tests/infra/engine/test_ingest_engine.rs +++ b/kamu-core/tests/infra/engine/test_ingest_engine.rs @@ -1,8 +1,8 @@ use indoc::indoc; use kamu::domain::*; -use kamu::infra::serde::yaml::*; use kamu::infra::*; use kamu_test::*; +use opendatafabric::*; use parquet::{ file::reader::{FileReader, SerializedFileReader}, diff --git a/kamu-core/tests/infra/engine/test_transform_engine.rs b/kamu-core/tests/infra/engine/test_transform_engine.rs index 0ae8a7bc5d..bc301bae06 100644 --- a/kamu-core/tests/infra/engine/test_transform_engine.rs +++ b/kamu-core/tests/infra/engine/test_transform_engine.rs @@ -1,8 +1,8 @@ use indoc::indoc; use kamu::domain::*; -use kamu::infra::serde::yaml::*; use kamu::infra::*; use kamu_test::*; +use opendatafabric::*; use parquet::{ file::reader::{FileReader, SerializedFileReader}, diff --git a/kamu-core/tests/infra/ingest/test_fetch.rs b/kamu-core/tests/infra/ingest/test_fetch.rs index 05d930a8c8..1ecb218cd4 100644 --- a/kamu-core/tests/infra/ingest/test_fetch.rs +++ b/kamu-core/tests/infra/ingest/test_fetch.rs @@ -2,8 +2,8 @@ use crate::utils::{FtpServer, HttpServer}; use indoc::indoc; use kamu::domain::*; use kamu::infra::ingest::*; -use kamu::infra::serde::yaml::*; use kamu_test::*; +use opendatafabric::*; use chrono::prelude::*; use chrono::Utc; diff --git a/kamu-core/tests/infra/ingest/test_prep.rs b/kamu-core/tests/infra/ingest/test_prep.rs index f3e079e9ef..798cd88d88 100644 --- a/kamu-core/tests/infra/ingest/test_prep.rs +++ b/kamu-core/tests/infra/ingest/test_prep.rs @@ -1,6 +1,6 @@ use indoc::indoc; use kamu::infra::ingest::*; -use kamu::infra::serde::yaml::*; +use opendatafabric::*; use chrono::Utc; use std::io::prelude::*; diff --git a/kamu-core/tests/infra/mod.rs b/kamu-core/tests/infra/mod.rs index 1bf41ff992..d1e61f320a 100644 --- a/kamu-core/tests/infra/mod.rs +++ b/kamu-core/tests/infra/mod.rs @@ -1,8 +1,8 @@ mod engine; mod ingest; -mod serde; mod test_metadata_chain_impl; mod test_metadata_repository_impl; mod test_pull_service_impl; mod test_resource_loader_impl; +mod test_serde_yaml; mod test_transform_service_impl; diff --git a/kamu-core/tests/infra/serde/mod.rs b/kamu-core/tests/infra/serde/mod.rs deleted file mode 100644 index 3ddee0fdae..0000000000 --- a/kamu-core/tests/infra/serde/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -mod test_flatbuffers; -mod test_yaml; diff --git a/kamu-core/tests/infra/serde/test_flatbuffers.rs b/kamu-core/tests/infra/serde/test_flatbuffers.rs deleted file mode 100644 index 686eb5d87d..0000000000 --- a/kamu-core/tests/infra/serde/test_flatbuffers.rs +++ /dev/null @@ -1,25 +0,0 @@ -/* -use kamu::domain::*; -use kamu::infra::serde::flatbuffers::*; -use std::io; - - -#[test] -fn to_and_from() -> Result<()> { - let block = MetadataBlock { - block_hash: "hash".to_string(), - prev_block_hash: "prev_hash".to_string(), - }; - - let ser = FlatbuffersSerializer; - let mut cursor = io::Cursor::new(Vec::new()); - ser.serialize(&block, &mut cursor)?; - //println!("{}", hex::encode(&buf)); - - let de = FlatbuffersDeserializer; - let block_prime = de.deserialize(cursor.get_ref())?; - assert_eq!(block, block_prime); - - Ok(()) -} -*/ diff --git a/kamu-core/tests/infra/serde/test_yaml.rs b/kamu-core/tests/infra/serde/test_yaml.rs deleted file mode 100644 index c6a2b907c6..0000000000 --- a/kamu-core/tests/infra/serde/test_yaml.rs +++ /dev/null @@ -1,388 +0,0 @@ -use chrono::prelude::*; -use indoc::indoc; -use std::convert::TryFrom; - -use kamu::domain::*; -use kamu::infra::serde::yaml::*; - -#[test] -fn de_dataset_snapshot_root() { - let data = indoc!( - " - --- - apiVersion: 1 - kind: DatasetSnapshot - content: - id: kamu.test - source: - kind: root - fetch: - kind: url - url: ftp://kamu.dev/test.zip - cache: - kind: forever - prepare: - - kind: decompress - format: zip - subPath: data_*.csv - read: - kind: csv - header: true - preprocess: - kind: sql - engine: spark - query: > - SELECT * FROM input - merge: - kind: snapshot - primaryKey: - - id - vocab: - eventTimeColumn: date" - ); - - let actual: Manifest = serde_yaml::from_str(data).unwrap(); - - let expected = Manifest { - api_version: 1, - kind: "DatasetSnapshot".to_owned(), - content: DatasetSnapshot { - id: DatasetIDBuf::try_from("kamu.test").unwrap(), - source: DatasetSource::Root(DatasetSourceRoot { - fetch: FetchStep::Url(FetchStepUrl { - url: "ftp://kamu.dev/test.zip".to_owned(), - event_time: None, - cache: Some(SourceCaching::Forever), - }), - prepare: Some(vec![PrepStep::Decompress(PrepStepDecompress { - format: CompressionFormat::Zip, - sub_path: Some("data_*.csv".to_owned()), - })]), - read: ReadStep::Csv(ReadStepCsv { - schema: None, - separator: None, - encoding: None, - quote: None, - escape: None, - comment: None, - header: Some(true), - enforce_schema: None, - infer_schema: None, - ignore_leading_white_space: None, - ignore_trailing_white_space: None, - null_value: None, - empty_value: None, - nan_value: None, - positive_inf: None, - negative_inf: None, - date_format: None, - timestamp_format: None, - multi_line: None, - }), - preprocess: Some(Transform::Sql(TransformSql { - engine: "spark".to_owned(), - version: None, - query: Some("SELECT * FROM input\n".to_owned()), - queries: None, - temporal_tables: None, - })), - merge: MergeStrategy::Snapshot(MergeStrategySnapshot { - primary_key: vec!["id".to_owned()], - compare_columns: None, - observation_column: None, - obsv_added: None, - obsv_changed: None, - obsv_removed: None, - }), - }), - vocab: Some(DatasetVocabulary { - system_time_column: None, - event_time_column: Some("date".to_owned()), - }), - }, - }; - - assert_eq!(expected, actual); -} - -#[test] -fn de_dataset_snapshot_derivative() { - let data = indoc!( - " - --- - apiVersion: 1 - kind: DatasetSnapshot - content: - id: com.naturalearthdata.admin0 - source: - kind: derivative - inputs: - - com.naturalearthdata.10m.admin0 - - com.naturalearthdata.50m.admin0 - transform: - kind: sql - engine: spark - query: SOME_SQL" - ); - - let actual: Manifest = serde_yaml::from_str(data).unwrap(); - - let expected = Manifest { - api_version: 1, - kind: "DatasetSnapshot".to_owned(), - content: DatasetSnapshot { - id: DatasetIDBuf::try_from("com.naturalearthdata.admin0").unwrap(), - source: DatasetSource::Derivative(DatasetSourceDerivative { - inputs: vec![ - DatasetIDBuf::try_from("com.naturalearthdata.10m.admin0").unwrap(), - DatasetIDBuf::try_from("com.naturalearthdata.50m.admin0").unwrap(), - ], - transform: Transform::Sql(TransformSql { - engine: "spark".to_owned(), - version: None, - query: Some("SOME_SQL".to_owned()), - queries: None, - temporal_tables: None, - }), - }), - vocab: None, - }, - }; - - assert_eq!(expected, actual); - assert_eq!(data, serde_yaml::to_string(&expected).unwrap()); -} - -#[test] -fn de_metadata_block() { - let data = indoc!( - " - --- - apiVersion: 1 - kind: MetadataBlock - content: - blockHash: ddeeaaddbbeeff - prevBlockHash: ffeebbddaaeedd - systemTime: \"2020-01-01T12:00:00.000Z\" - outputSlice: - hash: ffaabb - interval: \"[2020-01-01T12:00:00.000Z, 2020-01-01T12:00:00.000Z]\" - numRecords: 10 - outputWatermark: \"2020-01-01T12:00:00.000Z\" - inputSlices: - - hash: aa - interval: \"(-inf, 2020-01-01T12:00:00.000Z]\" - numRecords: 10 - - hash: zz - interval: () - numRecords: 0 - source: - kind: derivative - inputs: - - input1 - - input2 - transform: - kind: sql - engine: spark - query: SELECT * FROM input1 UNION ALL SELECT * FROM input2" - ); - - let actual: Manifest = serde_yaml::from_str(data).unwrap(); - - let expected = Manifest { - api_version: 1, - kind: "MetadataBlock".to_owned(), - content: MetadataBlock { - block_hash: "ddeeaaddbbeeff".to_owned(), - prev_block_hash: "ffeebbddaaeedd".to_owned(), - system_time: Utc.ymd(2020, 1, 1).and_hms(12, 0, 0), - source: Some(DatasetSource::Derivative(DatasetSourceDerivative { - inputs: vec![ - DatasetIDBuf::try_from("input1").unwrap(), - DatasetIDBuf::try_from("input2").unwrap(), - ], - transform: Transform::Sql(TransformSql { - engine: "spark".to_owned(), - version: None, - query: Some("SELECT * FROM input1 UNION ALL SELECT * FROM input2".to_owned()), - queries: None, - temporal_tables: None, - }), - })), - output_slice: Some(DataSlice { - hash: "ffaabb".to_owned(), - interval: TimeInterval::singleton(Utc.ymd(2020, 1, 1).and_hms(12, 0, 0)), - num_records: 10, - }), - output_watermark: Some(Utc.ymd(2020, 1, 1).and_hms(12, 0, 0)), - input_slices: Some(vec![ - DataSlice { - hash: "aa".to_owned(), - interval: TimeInterval::unbounded_closed_right( - Utc.ymd(2020, 1, 1).and_hms(12, 0, 0), - ), - num_records: 10, - }, - DataSlice { - hash: "zz".to_owned(), - interval: TimeInterval::empty(), - num_records: 0, - }, - ]), - }, - }; - - assert_eq!(expected, actual); - assert_eq!(data, serde_yaml::to_string(&expected).unwrap()); -} - -#[test] -fn serde_dataset_summary() { - let data = indoc!( - " - --- - apiVersion: 1 - kind: DatasetSummary - content: - id: foo.bar - kind: root - dependencies: - - foo - - bar - lastPulled: '2020-01-01T12:00:00.000Z' - dataSize: 1024 - numRecords: 100 - vocab: {}" - ); - - let actual: Manifest = serde_yaml::from_str(data).unwrap(); - - let expected = Manifest { - api_version: 1, - kind: "DatasetSummary".to_owned(), - content: DatasetSummary { - id: DatasetIDBuf::try_from("foo.bar").unwrap(), - kind: DatasetKind::Root, - dependencies: vec![ - DatasetIDBuf::try_from("foo").unwrap(), - DatasetIDBuf::try_from("bar").unwrap(), - ], - last_pulled: Some(Utc.ymd(2020, 1, 1).and_hms(12, 0, 0)), - data_size: 1024, - num_records: 100, - vocab: DatasetVocabulary::default(), - }, - }; - - assert_eq!(expected, actual); - - assert_eq!( - serde_yaml::to_string(&actual).unwrap(), - indoc!( - " - --- - apiVersion: 1 - kind: DatasetSummary - content: - id: foo.bar - kind: root - dependencies: - - foo - - bar - lastPulled: \"2020-01-01T12:00:00.000Z\" - numRecords: 100 - dataSize: 1024 - vocab: {}" - ) - ); -} - -#[test] -fn serde_fetch_step_files_glob() { - let data = indoc!( - " - --- - kind: filesGlob - path: /opt/x/*.txt - cache: - kind: forever - order: byName" - ); - - let actual: FetchStep = serde_yaml::from_str(data).unwrap(); - - let expected = FetchStep::FilesGlob(FetchStepFilesGlob { - path: "/opt/x/*.txt".to_owned(), - event_time: None, - cache: Some(SourceCaching::Forever), - order: Some(SourceOrdering::ByName), - }); - - assert_eq!(expected, actual); - - assert_eq!( - serde_yaml::to_string(&actual).unwrap(), - indoc!( - " - --- - kind: filesGlob - path: /opt/x/*.txt - cache: - kind: forever - order: byName" - ) - ); -} - -#[test] -fn serde_transform() { - let data = indoc!( - " - --- - kind: sql - engine: flink - temporalTables: - - id: foo - primaryKey: - - id - queries: - - alias: bar - query: > - SELECT * FROM foo" - ); - - let actual: Transform = serde_yaml::from_str(data).unwrap(); - - let expected = Transform::Sql(TransformSql { - engine: "flink".to_owned(), - version: None, - query: None, - temporal_tables: Some(vec![TemporalTable { - id: "foo".to_owned(), - primary_key: vec!["id".to_owned()], - }]), - queries: Some(vec![SqlQueryStep { - alias: Some("bar".to_owned()), - query: "SELECT * FROM foo".to_owned(), - }]), - }); - - assert_eq!(expected, actual); - - assert_eq!( - serde_yaml::to_string(&actual).unwrap(), - indoc!( - " - --- - kind: sql - engine: flink - queries: - - alias: bar - query: SELECT * FROM foo - temporalTables: - - id: foo - primaryKey: - - id" - ) - ); -} diff --git a/kamu-core/tests/infra/test_metadata_chain_impl.rs b/kamu-core/tests/infra/test_metadata_chain_impl.rs index a3581f5ab0..d88cbb74c0 100644 --- a/kamu-core/tests/infra/test_metadata_chain_impl.rs +++ b/kamu-core/tests/infra/test_metadata_chain_impl.rs @@ -4,23 +4,6 @@ use kamu_test::*; use chrono::{TimeZone, Utc}; -#[test] -fn test_block_hashing() { - assert_eq!( - MetadataChainImpl::block_hash(&MetadataFactory::metadata_block().build()), - "a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a" - ); - assert_eq!( - MetadataChainImpl::block_hash( - &MetadataFactory::metadata_block() - .prev("a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a") - .build() - ), - "42012470e20e90036b3098da71e1056ce0e561031bc41fa47faa1d1269f93a2e" - ); - // TODO: other fields -} - #[test] fn test_create_new_chain() { let tmp_dir = tempfile::tempdir().unwrap(); @@ -34,7 +17,7 @@ fn test_create_new_chain() { } #[test] -fn test_create_new_chain_err() { +fn test_create_new_chain_error_dir_already_exists() { let tmp_dir = tempfile::tempdir().unwrap(); let block = MetadataFactory::metadata_block().build(); @@ -51,18 +34,19 @@ fn test_append_and_iter_blocks() { let mut block1 = MetadataFactory::metadata_block() .system_time(Utc.ymd(2000, 1, 1).and_hms(12, 0, 0)) .build(); + let (mut chain, block1_hash) = MetadataChainImpl::create(&chain_dir, block1.clone()).unwrap(); + block1.block_hash = block1_hash; + let mut block2 = MetadataFactory::metadata_block() .system_time(Utc.ymd(2000, 1, 2).and_hms(12, 0, 0)) + .prev(&block1.block_hash) .build(); + block2.block_hash = chain.append(block2.clone()); + let mut block3 = MetadataFactory::metadata_block() .system_time(Utc.ymd(2000, 1, 3).and_hms(12, 0, 0)) + .prev(&block2.block_hash) .build(); - - let (mut chain, hash) = MetadataChainImpl::create(&chain_dir, block1.clone()).unwrap(); - block1.block_hash = hash; - block2.prev_block_hash = block1.block_hash.clone(); - block2.block_hash = chain.append(block2.clone()); - block3.prev_block_hash = block2.block_hash.clone(); block3.block_hash = chain.append(block3.clone()); let mut block_iter = chain.iter_blocks(); diff --git a/kamu-core/tests/infra/test_metadata_repository_impl.rs b/kamu-core/tests/infra/test_metadata_repository_impl.rs index b2c95bbcc6..c416688d33 100644 --- a/kamu-core/tests/infra/test_metadata_repository_impl.rs +++ b/kamu-core/tests/infra/test_metadata_repository_impl.rs @@ -1,6 +1,7 @@ use kamu::domain::*; use kamu::infra::*; use kamu_test::*; +use opendatafabric::*; #[test] fn test_delete_dataset() { diff --git a/kamu-core/tests/infra/test_pull_service_impl.rs b/kamu-core/tests/infra/test_pull_service_impl.rs index 9f4c21b249..90297cd5b1 100644 --- a/kamu-core/tests/infra/test_pull_service_impl.rs +++ b/kamu-core/tests/infra/test_pull_service_impl.rs @@ -1,6 +1,7 @@ use kamu::domain::*; use kamu::infra::*; use kamu_test::*; +use opendatafabric::*; use itertools::Itertools; use std::cell::RefCell; diff --git a/kamu-core/tests/infra/test_resource_loader_impl.rs b/kamu-core/tests/infra/test_resource_loader_impl.rs index ba856de2bb..4d792cdba0 100644 --- a/kamu-core/tests/infra/test_resource_loader_impl.rs +++ b/kamu-core/tests/infra/test_resource_loader_impl.rs @@ -1,23 +1,19 @@ use crate::utils::HttpServer; use kamu::domain::{ResourceError, ResourceLoader}; -use kamu::infra::serde::yaml::*; use kamu::infra::ResourceLoaderImpl; use kamu_test::*; +use opendatafabric::serde::yaml::*; +use opendatafabric::*; use std::path::Path; fn create_test_snapshot(path: &Path) -> DatasetSnapshot { let snapshot = MetadataFactory::dataset_snapshot().id("test").build(); - let manifest = Manifest { - api_version: 1, - kind: "DatasetSnapshot".to_owned(), - content: snapshot, - }; - - let file = std::fs::File::create(path).unwrap(); - serde_yaml::to_writer(file, &manifest).unwrap(); - - manifest.content + let buffer = YamlDatasetSnapshotSerializer + .write_manifest(&snapshot) + .unwrap(); + std::fs::write(path, &buffer).unwrap(); + snapshot } #[test] @@ -64,8 +60,7 @@ fn test_load_from_http_url() { let http_server = HttpServer::new(tempdir.path()); let url = url::Url::parse(&format!( "http://{}:{}/test.yaml", - http_server.address, - http_server.host_port, + http_server.address, http_server.host_port, )) .unwrap(); @@ -85,8 +80,7 @@ fn test_load_from_http_url_404() { let http_server = HttpServer::new(tempdir.path()); let url = url::Url::parse(&format!( "http://{}:{}/test.yaml", - http_server.address, - http_server.host_port, + http_server.address, http_server.host_port, )) .unwrap(); diff --git a/kamu-core/tests/infra/test_serde_yaml.rs b/kamu-core/tests/infra/test_serde_yaml.rs new file mode 100644 index 0000000000..83e16f489e --- /dev/null +++ b/kamu-core/tests/infra/test_serde_yaml.rs @@ -0,0 +1,48 @@ +use kamu::infra::{DatasetKind, DatasetSummary, Manifest}; +use opendatafabric::*; + +use chrono::prelude::*; +use indoc::indoc; +use std::convert::TryFrom; + +#[test] +fn serde_dataset_summary() { + let data = indoc!( + " + --- + apiVersion: 1 + kind: DatasetSummary + content: + id: foo.bar + kind: root + dependencies: + - foo + - bar + lastPulled: \"2020-01-01T12:00:00Z\" + numRecords: 100 + dataSize: 1024 + vocab: {}" + ); + + let actual: Manifest = serde_yaml::from_str(data).unwrap(); + + let expected = Manifest { + api_version: 1, + kind: "DatasetSummary".to_owned(), + content: DatasetSummary { + id: DatasetIDBuf::try_from("foo.bar").unwrap(), + kind: DatasetKind::Root, + dependencies: vec![ + DatasetIDBuf::try_from("foo").unwrap(), + DatasetIDBuf::try_from("bar").unwrap(), + ], + last_pulled: Some(Utc.ymd(2020, 1, 1).and_hms(12, 0, 0)), + data_size: 1024, + num_records: 100, + vocab: DatasetVocabulary::default(), + }, + }; + + assert_eq!(expected, actual); + assert_eq!(serde_yaml::to_string(&actual).unwrap(), data); +} diff --git a/kamu-core/tests/infra/test_transform_service_impl.rs b/kamu-core/tests/infra/test_transform_service_impl.rs index 4749bcc87b..e02fededed 100644 --- a/kamu-core/tests/infra/test_transform_service_impl.rs +++ b/kamu-core/tests/infra/test_transform_service_impl.rs @@ -1,7 +1,7 @@ use kamu::domain::*; -use kamu::infra::serde::yaml::*; use kamu::infra::*; use kamu_test::*; +use opendatafabric::*; use chrono::{DateTime, TimeZone, Utc}; use std::cell::RefCell; @@ -60,7 +60,7 @@ fn append_data_block( MetadataFactory::metadata_block() .prev(&chain.read_ref(&BlockRef::Head).unwrap()) .output_slice(DataSlice { - hash: "12345".to_owned(), + hash: Sha3_256::zero(), num_records: 100, interval: TimeInterval::singleton(Utc.ymd(2020, 1, 1).and_hms(12, 0, 0)), }) @@ -97,8 +97,6 @@ fn test_get_next_operation() { let foo_system_time = append_data_block(&metadata_repo, foo); - println!("{:?}", transform_svc.get_next_operation(bar)); - assert!(matches!( transform_svc.get_next_operation(bar).unwrap(), Some(ExecuteQueryRequest { source, input_slices, .. }) diff --git a/kamu-core/tests/mod.rs b/kamu-core/tests/mod.rs index e86db9b1ee..e144c2f299 100644 --- a/kamu-core/tests/mod.rs +++ b/kamu-core/tests/mod.rs @@ -1,5 +1,4 @@ #![feature(backtrace)] -mod domain; mod infra; pub mod utils; diff --git a/opendatafabric/Cargo.toml b/opendatafabric/Cargo.toml index e3641b2d37..63dcccc4ed 100644 --- a/opendatafabric/Cargo.toml +++ b/opendatafabric/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "opendatafabric" -version = "0.31.0" +version = "0.32.0" authors = ["Sergii Mikhtoniuk "] edition = "2018" diff --git a/opendatafabric/src/defaults.rs b/opendatafabric/src/defaults.rs new file mode 100644 index 0000000000..457fdb7289 --- /dev/null +++ b/opendatafabric/src/defaults.rs @@ -0,0 +1,10 @@ +use super::DatasetVocabulary; + +impl Default for DatasetVocabulary { + fn default() -> Self { + DatasetVocabulary { + system_time_column: None, + event_time_column: None, + } + } +} diff --git a/opendatafabric/src/lib.rs b/opendatafabric/src/lib.rs index 4f6d69f7f0..d2a3500333 100644 --- a/opendatafabric/src/lib.rs +++ b/opendatafabric/src/lib.rs @@ -12,12 +12,11 @@ pub use time_interval::*; mod sha; pub use sha::*; -mod manifest; -pub use manifest::*; - mod dtos; pub use dtos::*; pub mod dynamic; pub mod serde; + +mod defaults; diff --git a/opendatafabric/src/manifest.rs b/opendatafabric/src/manifest.rs deleted file mode 100644 index 07ca0fe21a..0000000000 --- a/opendatafabric/src/manifest.rs +++ /dev/null @@ -1,6 +0,0 @@ -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Manifest { - pub api_version: i32, - pub kind: String, - pub content: T, -} diff --git a/opendatafabric/src/serde/serdes.rs b/opendatafabric/src/serde/serdes.rs index 3665706a69..0f6140d7f6 100644 --- a/opendatafabric/src/serde/serdes.rs +++ b/opendatafabric/src/serde/serdes.rs @@ -83,7 +83,7 @@ impl Buffer { T: Default + Copy, { if self.head < space_left || self.buf.len() - self.tail < space_right { - let nlen = self.buf.len() + space_left + space_right; + let nlen = self.tail - self.head + space_left + space_right; let ntail = nlen - space_right; let mut nbuf = vec![T::default(); nlen]; nbuf[space_left..nlen - space_right].copy_from_slice(&self.buf[self.head..self.tail]); @@ -107,6 +107,12 @@ impl std::ops::Deref for Buffer { } } +impl std::convert::AsRef<[T]> for Buffer { + fn as_ref(&self) -> &[T] { + &self.buf + } +} + impl std::ops::DerefMut for Buffer { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.buf[self.head..self.tail] diff --git a/opendatafabric/src/serde/yaml/formats.rs b/opendatafabric/src/serde/yaml/formats.rs index 273a734ee3..66570d0431 100644 --- a/opendatafabric/src/serde/yaml/formats.rs +++ b/opendatafabric/src/serde/yaml/formats.rs @@ -9,7 +9,7 @@ pub mod datetime_rfc3339 { date: &DateTime, serializer: S, ) -> Result { - let s = date.to_rfc3339_opts(SecondsFormat::Millis, true); + let s = date.to_rfc3339_opts(SecondsFormat::AutoSi, true); serializer.serialize_str(&s) } @@ -24,8 +24,8 @@ pub mod datetime_rfc3339 { } pub mod datetime_rfc3339_opt { - use chrono::{DateTime, SecondsFormat, Utc}; - use serde::{Deserialize, Deserializer, Serializer}; + use chrono::{DateTime, Utc}; + use serde::{Deserializer, Serializer}; pub fn serialize( option: &Option>, @@ -33,19 +33,13 @@ pub mod datetime_rfc3339_opt { ) -> Result { match option { None => serializer.serialize_none(), - Some(date) => { - serializer.serialize_str(&date.to_rfc3339_opts(SecondsFormat::Millis, true)) - } + Some(date) => super::datetime_rfc3339::serialize(&date, serializer), } } pub fn deserialize<'de, D: Deserializer<'de>>( deserializer: D, ) -> Result>, D::Error> { - let s = String::deserialize(deserializer)?; - DateTime::parse_from_rfc3339(&s) - .map(|dt| dt.into()) - .map(Some) - .map_err(serde::de::Error::custom) + super::datetime_rfc3339::deserialize(deserializer).map(Some) } } diff --git a/opendatafabric/src/sha.rs b/opendatafabric/src/sha.rs index 413dfc68b0..3be67c8ac8 100644 --- a/opendatafabric/src/sha.rs +++ b/opendatafabric/src/sha.rs @@ -4,6 +4,8 @@ use std::convert::TryFrom; use std::fmt; use std::ops; +/////////////////////////////////////////////////////////////////////////////// + #[derive(Clone, Copy, PartialEq, Eq)] pub struct Sha3_256([u8; 32]); @@ -18,6 +20,12 @@ impl Sha3_256 { Self([0; Self::LENGTH]) } + pub fn from_str(s: &str) -> Result { + let mut slice: [u8; 32] = [0; 32]; + hex::decode_to_slice(s, &mut slice)?; + Ok(Self(slice)) + } + pub fn is_zero(&self) -> bool { for b in &self.0 { if *b != 0 { @@ -30,8 +38,18 @@ impl Sha3_256 { pub fn as_array(&self) -> &[u8; 32] { &self.0 } + + pub fn to_string(&self) -> String { + hex::encode(&self.0) + } + + pub fn short(&self) -> Sha3_256Short { + Sha3_256Short(self, 8) + } } +/////////////////////////////////////////////////////////////////////////////// + impl ops::Deref for Sha3_256 { type Target = [u8]; @@ -44,9 +62,7 @@ impl TryFrom<&str> for Sha3_256 { type Error = hex::FromHexError; fn try_from(value: &str) -> Result { - let mut slice: [u8; 32] = [0; 32]; - hex::decode_to_slice(value, &mut slice)?; - Ok(Sha3_256(slice)) + Self::from_str(value) } } @@ -91,3 +107,15 @@ impl<'de> Deserialize<'de> for Sha3_256 { deserializer.deserialize_str(Sha3_256Visitor) } } + +/////////////////////////////////////////////////////////////////////////////// + +#[derive(Clone, Copy)] +pub struct Sha3_256Short<'a>(&'a Sha3_256, usize); + +impl fmt::Display for Sha3_256Short<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s = hex::encode(&self.0.as_array()[..self.1 / 2]); + write!(f, "{}", s) + } +} diff --git a/opendatafabric/src/time_interval.rs b/opendatafabric/src/time_interval.rs index 6db2564bc7..cba547ebcf 100644 --- a/opendatafabric/src/time_interval.rs +++ b/opendatafabric/src/time_interval.rs @@ -152,7 +152,7 @@ impl Eq for TimeInterval {} impl fmt::Display for TimeInterval { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt_elem(v: &Element) -> String { - v.to_rfc3339_opts(SecondsFormat::Millis, true) + v.to_rfc3339_opts(SecondsFormat::AutoSi, true) } match self.0 { diff --git a/opendatafabric/tests/mod.rs b/opendatafabric/tests/mod.rs index 818ffa6fe0..aeaf96d8e7 100644 --- a/opendatafabric/tests/mod.rs +++ b/opendatafabric/tests/mod.rs @@ -1,3 +1,5 @@ +mod test_dataset_id; mod test_dynamic; +mod test_serde_buffer; mod test_serde_flatbuffers; mod test_serde_yaml; diff --git a/kamu-core/tests/domain/test_dataset_id.rs b/opendatafabric/tests/test_dataset_id.rs similarity index 98% rename from kamu-core/tests/domain/test_dataset_id.rs rename to opendatafabric/tests/test_dataset_id.rs index af7747c30e..e73b17ba36 100644 --- a/kamu-core/tests/domain/test_dataset_id.rs +++ b/opendatafabric/tests/test_dataset_id.rs @@ -1,4 +1,5 @@ -use kamu::domain::{DatasetID, DatasetIDBuf}; +use opendatafabric::*; + use std::convert::TryFrom; #[test] diff --git a/opendatafabric/tests/test_serde_buffer.rs b/opendatafabric/tests/test_serde_buffer.rs new file mode 100644 index 0000000000..6cf53d6e18 --- /dev/null +++ b/opendatafabric/tests/test_serde_buffer.rs @@ -0,0 +1,22 @@ +use opendatafabric::serde::Buffer; + +#[test] +fn ensure_capacity_realloc() { + let mut buf = Buffer::new(1, 4, vec![0, 3, 4, 5, 0]); + assert_eq!(buf.len(), 3); + assert_eq!(buf.inner(), [0, 3, 4, 5, 0]); + assert_eq!(&buf as &[i32], [3, 4, 5]); + + buf.ensure_capacity(2, 2); + assert_eq!(buf.len(), 3); + assert_eq!(buf.inner(), [0, 0, 3, 4, 5, 0, 0]); + + buf.set_head(0); + buf.set_tail(7); + assert_eq!(buf.len(), 7); + buf[0] = 1; + buf[1] = 2; + buf[5] = 6; + buf[6] = 7; + assert_eq!(buf.inner(), [1, 2, 3, 4, 5, 6, 7]); +} diff --git a/opendatafabric/tests/test_serde_yaml.rs b/opendatafabric/tests/test_serde_yaml.rs index b6d51dc233..f46ec8f538 100644 --- a/opendatafabric/tests/test_serde_yaml.rs +++ b/opendatafabric/tests/test_serde_yaml.rs @@ -170,15 +170,15 @@ fn serde_metadata_block() { content: blockHash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a prevBlockHash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b - systemTime: \"2020-01-01T12:00:00.000Z\" + systemTime: \"2020-01-01T12:00:00Z\" outputSlice: hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a - interval: \"[2020-01-01T12:00:00.000Z, 2020-01-01T12:00:00.000Z]\" + interval: \"[2020-01-01T12:00:00Z, 2020-01-01T12:00:00Z]\" numRecords: 10 - outputWatermark: \"2020-01-01T12:00:00.000Z\" + outputWatermark: \"2020-01-01T12:00:00Z\" inputSlices: - hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a - interval: \"(-inf, 2020-01-01T12:00:00.000Z]\" + interval: \"(-inf, 2020-01-01T12:00:00Z]\" numRecords: 10 - hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b interval: () @@ -247,24 +247,24 @@ fn serde_metadata_block() { } #[test] -fn ser_metadata_block_hashes() { +fn serde_metadata_block_hashes() { let expected = indoc!( " --- apiVersion: 1 kind: MetadataBlock content: - blockHash: efd99d69eba17c25d16bd58f296786f26234f63404a098339e646204cc9e1eeb + blockHash: decbb0476956d47e98ad2c30698d6282490ea6ee19f99139b2b6396a3589da9f prevBlockHash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b - systemTime: \"2020-01-01T12:00:00.000Z\" + systemTime: \"2020-01-01T12:00:00.123456789Z\" outputSlice: hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a - interval: \"[2020-01-01T12:00:00.000Z, 2020-01-01T12:00:00.000Z]\" + interval: \"[2020-01-01T12:00:00Z, 2020-01-01T12:00:00Z]\" numRecords: 10 - outputWatermark: \"2020-01-01T12:00:00.000Z\" + outputWatermark: \"2020-01-01T12:00:00Z\" inputSlices: - hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a - interval: \"(-inf, 2020-01-01T12:00:00.000Z]\" + interval: \"(-inf, 2020-01-01T12:00:00Z]\" numRecords: 10 - hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b interval: () @@ -283,7 +283,7 @@ fn ser_metadata_block_hashes() { let block = MetadataBlock { block_hash: Sha3_256::new([0; 32]), prev_block_hash: Some(Sha3_256::new([0x0b; 32])), - system_time: Utc.ymd(2020, 1, 1).and_hms(12, 0, 0), + system_time: Utc.ymd(2020, 1, 1).and_hms_nano(12, 0, 0, 123456789), source: Some(DatasetSource::Derivative(DatasetSourceDerivative { inputs: vec![ DatasetIDBuf::try_from("input1").unwrap(), @@ -321,6 +321,17 @@ fn ser_metadata_block_hashes() { let (_, actual) = YamlMetadataBlockSerializer.write_manifest(&block).unwrap(); assert_eq!(expected, std::str::from_utf8(&actual).unwrap()); + + let actual_block = YamlMetadataBlockDeserializer + .read_manifest(&actual) + .unwrap(); + assert_eq!( + block, + MetadataBlock { + block_hash: Sha3_256::zero(), + ..actual_block + } + ); } #[test] diff --git a/rust-toolchain b/rust-toolchain index b18a3f3f75..0ca96be9ae 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -nightly-2020-06-03 +nightly-2020-10-31