diff --git a/.gitignore b/.gitignore index fa002609ab..48535fb40b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /target +/opendatafabric/target /kamu-core/target /kamu-core-test/target /kamu-cli/target diff --git a/Cargo.lock b/Cargo.lock index a40af83188..48e6bdc286 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -582,6 +582,7 @@ dependencies = [ "flate2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", "fs_extra 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hex 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "indoc 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "intervals-general 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -823,6 +824,23 @@ name = "number_prefix" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "opendatafabric" +version = "0.31.0" +dependencies = [ + "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "chrono 0.4.19 (registry+https://github.com/rust-lang/crates.io-index)", + "flatbuffers 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "hex 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "indoc 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "intervals-general 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.116 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_with 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_yaml 0.8.13 (registry+https://github.com/rust-lang/crates.io-index)", + "thiserror 1.0.21 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "openssl-probe" version = "0.1.2" diff --git a/Cargo.toml b/Cargo.toml index 6ccffdce21..e65b1e7e95 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = [ + "opendatafabric", "kamu-core", "kamu-core-test", "kamu-cli", diff --git a/opendatafabric/Cargo.toml b/opendatafabric/Cargo.toml new file mode 100644 index 0000000000..e3641b2d37 --- /dev/null +++ b/opendatafabric/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "opendatafabric" +version = "0.31.0" +authors = ["Sergii Mikhtoniuk "] +edition = "2018" + +[dependencies] +# Domain +chrono = { version = "*", features = ["serde"] } +intervals-general = "*" +rust-crypto = "*" # Data and metadata hashing + +# Serialization +byteorder = "*" +flatbuffers = "*" +hex = "*" +serde = { version = "*", features = ["derive"] } +serde_with = "*" +serde_yaml = "*" + +# Utils +indoc = "*" # Compile-time unindent +thiserror = "*" # Structured error derivations diff --git a/opendatafabric/flatbuffers/odf.fbs b/opendatafabric/flatbuffers/odf.fbs new file mode 100644 index 0000000000..29f4bb2510 --- /dev/null +++ b/opendatafabric/flatbuffers/odf.fbs @@ -0,0 +1,309 @@ +//////////////////////////////////////////////////////////////////////////////// +// WARNING: This file is auto-generated from Open Data Fabric Schemas +// See: http://opendatafabric.org/ +//////////////////////////////////////////////////////////////////////////////// + +struct Option_bool { value: bool; } +struct Option_int64 { value: int64; } + +struct Timestamp { + year: int32; + ordinal: uint16; + seconds_from_midnight: uint32; + nanoseconds: uint32; +} + +enum TimeIntervalType: uint8 { + Closed, + Open, + LeftHalfOpen, + RightHalfOpen, + UnboundedClosedRight, + UnboundedOpenRight, + UnboundedClosedLeft, + UnboundedOpenLeft, + Singleton, + Unbounded, + Empty, +} + +struct TimeInterval { + type: TimeIntervalType; + left: Timestamp; + right: Timestamp; +} + +//////////////////////////////////////////////////////////////////////////////// +// EventTimeSource +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#eventtimesource-schema +//////////////////////////////////////////////////////////////////////////////// + +table EventTimeSourceFromMetadata { +} + +table EventTimeSourceFromPath { + pattern: string; + timestamp_format: string; +} + +union EventTimeSource { + EventTimeSourceFromMetadata, + EventTimeSourceFromPath, +} + +//////////////////////////////////////////////////////////////////////////////// +// SourceCaching +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#sourcecaching-schema +//////////////////////////////////////////////////////////////////////////////// + +table SourceCachingForever { +} + +union SourceCaching { + SourceCachingForever, +} + +//////////////////////////////////////////////////////////////////////////////// +// FetchStep +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#fetchstep-schema +//////////////////////////////////////////////////////////////////////////////// + +table FetchStepUrl { + url: string; + event_time: EventTimeSource; + cache: SourceCaching; +} + +enum SourceOrdering: int32 { + ByEventTime, + ByName, +} + +struct Option_SourceOrdering { value: SourceOrdering; } + +table FetchStepFilesGlob { + path: string; + event_time: EventTimeSource; + cache: SourceCaching; + order: Option_SourceOrdering; +} + +union FetchStep { + FetchStepUrl, + FetchStepFilesGlob, +} + +//////////////////////////////////////////////////////////////////////////////// +// PrepStep +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#prepstep-schema +//////////////////////////////////////////////////////////////////////////////// + +enum CompressionFormat: int32 { + Gzip, + Zip, +} + +struct Option_CompressionFormat { value: CompressionFormat; } + +table PrepStepDecompress { + format: Option_CompressionFormat; + sub_path: string; +} + +table PrepStepPipe { + command: [string]; +} + +union PrepStep { + PrepStepDecompress, + PrepStepPipe, +} + +//////////////////////////////////////////////////////////////////////////////// +// ReadStep +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#readstep-schema +//////////////////////////////////////////////////////////////////////////////// + +table ReadStepCsv { + schema: [string]; + separator: string; + encoding: string; + quote: string; + escape: string; + comment: string; + header: Option_bool; + enforce_schema: Option_bool; + infer_schema: Option_bool; + ignore_leading_white_space: Option_bool; + ignore_trailing_white_space: Option_bool; + null_value: string; + empty_value: string; + nan_value: string; + positive_inf: string; + negative_inf: string; + date_format: string; + timestamp_format: string; + multi_line: Option_bool; +} + +table ReadStepJsonLines { + schema: [string]; + date_format: string; + encoding: string; + multi_line: Option_bool; + primitives_as_string: Option_bool; + timestamp_format: string; +} + +table ReadStepGeoJson { + schema: [string]; +} + +table ReadStepEsriShapefile { + schema: [string]; + sub_path: string; +} + +union ReadStep { + ReadStepCsv, + ReadStepJsonLines, + ReadStepGeoJson, + ReadStepEsriShapefile, +} + +//////////////////////////////////////////////////////////////////////////////// +// Transform +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#transform-schema +//////////////////////////////////////////////////////////////////////////////// + +table TransformSql { + engine: string; + version: string; + query: string; + queries: [SqlQueryStep]; + temporal_tables: [TemporalTable]; +} + +union Transform { + TransformSql, +} + +//////////////////////////////////////////////////////////////////////////////// +// MergeStrategy +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#mergestrategy-schema +//////////////////////////////////////////////////////////////////////////////// + +table MergeStrategyAppend { +} + +table MergeStrategyLedger { + primary_key: [string]; +} + +table MergeStrategySnapshot { + primary_key: [string]; + compare_columns: [string]; + observation_column: string; + obsv_added: string; + obsv_changed: string; + obsv_removed: string; +} + +union MergeStrategy { + MergeStrategyAppend, + MergeStrategyLedger, + MergeStrategySnapshot, +} + +//////////////////////////////////////////////////////////////////////////////// +// DatasetSource +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#datasetsource-schema +//////////////////////////////////////////////////////////////////////////////// + +table PrepStepWrapper { value: PrepStep; } + +table DatasetSourceRoot { + fetch: FetchStep; + prepare: [PrepStepWrapper]; + read: ReadStep; + preprocess: Transform; + merge: MergeStrategy; +} + +table DatasetSourceDerivative { + inputs: [string]; + transform: Transform; +} + +union DatasetSource { + DatasetSourceRoot, + DatasetSourceDerivative, +} + +//////////////////////////////////////////////////////////////////////////////// +// DatasetVocabulary +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#datasetvocabulary-schema +//////////////////////////////////////////////////////////////////////////////// + +table DatasetVocabulary { + system_time_column: string; + event_time_column: string; +} + +//////////////////////////////////////////////////////////////////////////////// +// TemporalTable +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#temporaltable-schema +//////////////////////////////////////////////////////////////////////////////// + +table TemporalTable { + id: string; + primary_key: [string]; +} + +//////////////////////////////////////////////////////////////////////////////// +// DatasetSnapshot +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#datasetsnapshot-schema +//////////////////////////////////////////////////////////////////////////////// + +table DatasetSnapshot { + id: string; + source: DatasetSource; + vocab: DatasetVocabulary; +} + +//////////////////////////////////////////////////////////////////////////////// +// DataSlice +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#dataslice-schema +//////////////////////////////////////////////////////////////////////////////// + +table DataSlice { + hash: [ubyte]; + interval: TimeInterval; + num_records: Option_int64; +} + +//////////////////////////////////////////////////////////////////////////////// +// SqlQueryStep +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#sqlquerystep-schema +//////////////////////////////////////////////////////////////////////////////// + +table SqlQueryStep { + alias: string; + query: string; +} + +//////////////////////////////////////////////////////////////////////////////// +// MetadataBlock +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#metadatablock-schema +//////////////////////////////////////////////////////////////////////////////// + +table MetadataBlock { + block_hash: [ubyte]; + prev_block_hash: [ubyte]; + system_time: Timestamp; + output_slice: DataSlice; + output_watermark: Timestamp; + input_slices: [DataSlice]; + source: DatasetSource; +} + diff --git a/opendatafabric/src/dataset_id.rs b/opendatafabric/src/dataset_id.rs new file mode 100644 index 0000000000..30b9e53d6d --- /dev/null +++ b/opendatafabric/src/dataset_id.rs @@ -0,0 +1,265 @@ +use std::borrow; +use std::cmp; +use std::convert::{AsRef, TryFrom}; +use std::fmt; +use std::ops; + +use super::grammar::DatasetIDGrammar; + +//////////////////////////////////////////////////////////////////////////////// +// DatasetID (reference type) +//////////////////////////////////////////////////////////////////////////////// + +#[derive(Debug)] +pub struct DatasetID(str); + +impl DatasetID { + pub fn new_unchecked + ?Sized>(s: &S) -> &DatasetID { + unsafe { &*(s.as_ref() as *const str as *const DatasetID) } + } + + pub fn try_from + ?Sized>(s: &S) -> Result<&DatasetID, InvalidDatasetID> { + match DatasetIDGrammar::match_dataset_id(s.as_ref()) { + Some((_, "")) => Ok(DatasetID::new_unchecked(s)), + _ => Err(InvalidDatasetID { + invalid_id: String::from(s.as_ref()), + }), + } + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl ops::Deref for DatasetID { + type Target = str; + + fn deref(&self) -> &str { + &self.0 + } +} + +impl AsRef for DatasetID { + fn as_ref(&self) -> &str { + &self.0 + } +} + +impl AsRef for DatasetID { + fn as_ref(&self) -> &std::path::Path { + self.0.as_ref() + } +} + +impl ToOwned for DatasetID { + type Owned = DatasetIDBuf; + + fn to_owned(&self) -> DatasetIDBuf { + DatasetIDBuf::from(self) + } +} + +impl cmp::PartialEq for DatasetID { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } +} + +impl cmp::PartialEq for DatasetID { + fn eq(&self, other: &str) -> bool { + &self.0 == other + } +} + +impl cmp::Eq for DatasetID {} + +impl cmp::Ord for DatasetID { + fn cmp(&self, other: &Self) -> cmp::Ordering { + self.0.cmp(&other.0) + } +} + +impl cmp::PartialOrd for DatasetID { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl fmt::Display for DatasetID { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", &self.0) + } +} + +impl std::hash::Hash for DatasetID { + fn hash(&self, state: &mut H) { + self.0.hash(state) + } +} + +//////////////////////////////////////////////////////////////////////////////// +// DatasetIDBuf (buffer type) +//////////////////////////////////////////////////////////////////////////////// + +#[derive(Debug, Clone)] +pub struct DatasetIDBuf(String); + +impl DatasetIDBuf { + pub fn new() -> Self { + Self(String::new()) + } +} + +impl Default for DatasetIDBuf { + fn default() -> Self { + Self::new() + } +} + +impl From<&DatasetID> for DatasetIDBuf { + fn from(id: &DatasetID) -> Self { + Self(String::from(id as &str)) + } +} + +// TODO: Why TryFrom not enough? +impl std::str::FromStr for DatasetIDBuf { + type Err = InvalidDatasetID; + fn from_str(s: &str) -> Result { + Self::try_from(s) + } +} + +// TODO: Replace with AsRef matcher +// See: https://github.com/rust-lang/rust/issues/50133 +impl TryFrom<&str> for DatasetIDBuf { + type Error = InvalidDatasetID; + fn try_from(s: &str) -> Result { + let id = DatasetID::try_from(s)?; + Ok(Self::from(id)) + } +} +impl TryFrom<&std::ffi::OsString> for DatasetIDBuf { + type Error = InvalidDatasetID; + fn try_from(s: &std::ffi::OsString) -> Result { + Self::try_from(s.to_str().unwrap()) + } +} + +impl ops::Deref for DatasetIDBuf { + type Target = DatasetID; + + fn deref(&self) -> &DatasetID { + DatasetID::new_unchecked(&self.0) + } +} + +impl AsRef for DatasetIDBuf { + fn as_ref(&self) -> &str { + self.0.as_ref() + } +} + +impl AsRef for DatasetIDBuf { + fn as_ref(&self) -> &std::path::Path { + self.0.as_ref() + } +} + +impl AsRef for DatasetIDBuf { + fn as_ref(&self) -> &DatasetID { + DatasetID::new_unchecked(&self.0) + } +} + +impl borrow::Borrow for DatasetIDBuf { + fn borrow(&self) -> &DatasetID { + self + } +} + +impl cmp::PartialEq for DatasetIDBuf { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } +} + +impl cmp::PartialEq for DatasetIDBuf { + fn eq(&self, other: &DatasetID) -> bool { + self.0 == other.0 + } +} + +impl cmp::PartialEq for DatasetIDBuf { + fn eq(&self, other: &str) -> bool { + &self.0 == other + } +} + +impl cmp::Eq for DatasetIDBuf {} + +impl cmp::Ord for DatasetIDBuf { + fn cmp(&self, other: &Self) -> cmp::Ordering { + self.0.cmp(&other.0) + } +} + +impl cmp::PartialOrd for DatasetIDBuf { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl fmt::Display for DatasetIDBuf { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", &self.0) + } +} + +impl std::hash::Hash for DatasetIDBuf { + fn hash(&self, state: &mut H) { + self.0.hash(state) + } +} + +impl serde::Serialize for DatasetIDBuf { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(self) + } +} + +impl<'de> serde::Deserialize<'de> for DatasetIDBuf { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_string(DatasetIDBufSerdeVisitor) + } +} + +struct DatasetIDBufSerdeVisitor; + +impl<'de> serde::de::Visitor<'de> for DatasetIDBufSerdeVisitor { + type Value = DatasetIDBuf; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a DatasetID string") + } + + fn visit_str(self, v: &str) -> Result { + DatasetIDBuf::try_from(v).map_err(serde::de::Error::custom) + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Errors +//////////////////////////////////////////////////////////////////////////////// + +#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)] +pub struct InvalidDatasetID { + invalid_id: String, +} + +impl fmt::Display for InvalidDatasetID { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "Invalid DatasetID: {}", self.invalid_id) + } +} diff --git a/opendatafabric/src/dtos.rs b/opendatafabric/src/dtos.rs new file mode 100644 index 0000000000..d33e42cb38 --- /dev/null +++ b/opendatafabric/src/dtos.rs @@ -0,0 +1,297 @@ +//////////////////////////////////////////////////////////////////////////////// +// WARNING: This file is auto-generated from Open Data Fabric Schemas +// See: http://opendatafabric.org/ +//////////////////////////////////////////////////////////////////////////////// + +use super::{DatasetIDBuf, TimeInterval, Sha3_256}; +use chrono::{DateTime, Utc}; + +//////////////////////////////////////////////////////////////////////////////// +// DatasetSource +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#datasetsource-schema +//////////////////////////////////////////////////////////////////////////////// + +#[derive(Clone, PartialEq, Eq, Debug)] +pub enum DatasetSource { + Root(DatasetSourceRoot), + Derivative(DatasetSourceDerivative), +} + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct DatasetSourceRoot { + pub fetch: FetchStep, + pub prepare: Option>, + pub read: ReadStep, + pub preprocess: Option, + pub merge: MergeStrategy, +} + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct DatasetSourceDerivative { + pub inputs: Vec, + pub transform: Transform, +} + +//////////////////////////////////////////////////////////////////////////////// +// DatasetVocabulary +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#datasetvocabulary-schema +//////////////////////////////////////////////////////////////////////////////// + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct DatasetVocabulary { + pub system_time_column: Option, + pub event_time_column: Option, +} + +//////////////////////////////////////////////////////////////////////////////// +// SourceCaching +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#sourcecaching-schema +//////////////////////////////////////////////////////////////////////////////// + +#[derive(Clone, PartialEq, Eq, Debug)] +pub enum SourceCaching { + Forever, +} + +//////////////////////////////////////////////////////////////////////////////// +// TemporalTable +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#temporaltable-schema +//////////////////////////////////////////////////////////////////////////////// + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct TemporalTable { + pub id: String, + pub primary_key: Vec, +} + +//////////////////////////////////////////////////////////////////////////////// +// DatasetSnapshot +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#datasetsnapshot-schema +//////////////////////////////////////////////////////////////////////////////// + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct DatasetSnapshot { + pub id: DatasetIDBuf, + pub source: DatasetSource, + pub vocab: Option, +} + +//////////////////////////////////////////////////////////////////////////////// +// DataSlice +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#dataslice-schema +//////////////////////////////////////////////////////////////////////////////// + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct DataSlice { + pub hash: Sha3_256, + pub interval: TimeInterval, + pub num_records: i64, +} + +//////////////////////////////////////////////////////////////////////////////// +// SqlQueryStep +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#sqlquerystep-schema +//////////////////////////////////////////////////////////////////////////////// + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct SqlQueryStep { + pub alias: Option, + pub query: String, +} + +//////////////////////////////////////////////////////////////////////////////// +// MergeStrategy +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#mergestrategy-schema +//////////////////////////////////////////////////////////////////////////////// + +#[derive(Clone, PartialEq, Eq, Debug)] +pub enum MergeStrategy { + Append, + Ledger(MergeStrategyLedger), + Snapshot(MergeStrategySnapshot), +} + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct MergeStrategyLedger { + pub primary_key: Vec, +} + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct MergeStrategySnapshot { + pub primary_key: Vec, + pub compare_columns: Option>, + pub observation_column: Option, + pub obsv_added: Option, + pub obsv_changed: Option, + pub obsv_removed: Option, +} + +//////////////////////////////////////////////////////////////////////////////// +// MetadataBlock +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#metadatablock-schema +//////////////////////////////////////////////////////////////////////////////// + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct MetadataBlock { + pub block_hash: Sha3_256, + pub prev_block_hash: Option, + pub system_time: DateTime, + pub output_slice: Option, + pub output_watermark: Option>, + pub input_slices: Option>, + pub source: Option, +} + +//////////////////////////////////////////////////////////////////////////////// +// ReadStep +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#readstep-schema +//////////////////////////////////////////////////////////////////////////////// + +#[derive(Clone, PartialEq, Eq, Debug)] +pub enum ReadStep { + Csv(ReadStepCsv), + JsonLines(ReadStepJsonLines), + GeoJson(ReadStepGeoJson), + EsriShapefile(ReadStepEsriShapefile), +} + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct ReadStepCsv { + pub schema: Option>, + pub separator: Option, + pub encoding: Option, + pub quote: Option, + pub escape: Option, + pub comment: Option, + pub header: Option, + pub enforce_schema: Option, + pub infer_schema: Option, + pub ignore_leading_white_space: Option, + pub ignore_trailing_white_space: Option, + pub null_value: Option, + pub empty_value: Option, + pub nan_value: Option, + pub positive_inf: Option, + pub negative_inf: Option, + pub date_format: Option, + pub timestamp_format: Option, + pub multi_line: Option, +} + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct ReadStepJsonLines { + pub schema: Option>, + pub date_format: Option, + pub encoding: Option, + pub multi_line: Option, + pub primitives_as_string: Option, + pub timestamp_format: Option, +} + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct ReadStepGeoJson { + pub schema: Option>, +} + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct ReadStepEsriShapefile { + pub schema: Option>, + pub sub_path: Option, +} + +//////////////////////////////////////////////////////////////////////////////// +// Transform +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#transform-schema +//////////////////////////////////////////////////////////////////////////////// + +#[derive(Clone, PartialEq, Eq, Debug)] +pub enum Transform { + Sql(TransformSql), +} + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct TransformSql { + pub engine: String, + pub version: Option, + pub query: Option, + pub queries: Option>, + pub temporal_tables: Option>, +} + +//////////////////////////////////////////////////////////////////////////////// +// FetchStep +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#fetchstep-schema +//////////////////////////////////////////////////////////////////////////////// + +#[derive(Clone, PartialEq, Eq, Debug)] +pub enum FetchStep { + Url(FetchStepUrl), + FilesGlob(FetchStepFilesGlob), +} + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct FetchStepUrl { + pub url: String, + pub event_time: Option, + pub cache: Option, +} + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct FetchStepFilesGlob { + pub path: String, + pub event_time: Option, + pub cache: Option, + pub order: Option, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum SourceOrdering { + ByEventTime, + ByName, +} + +//////////////////////////////////////////////////////////////////////////////// +// PrepStep +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#prepstep-schema +//////////////////////////////////////////////////////////////////////////////// + +#[derive(Clone, PartialEq, Eq, Debug)] +pub enum PrepStep { + Decompress(PrepStepDecompress), + Pipe(PrepStepPipe), +} + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct PrepStepDecompress { + pub format: CompressionFormat, + pub sub_path: Option, +} + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct PrepStepPipe { + pub command: Vec, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum CompressionFormat { + Gzip, + Zip, +} + +//////////////////////////////////////////////////////////////////////////////// +// EventTimeSource +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#eventtimesource-schema +//////////////////////////////////////////////////////////////////////////////// + +#[derive(Clone, PartialEq, Eq, Debug)] +pub enum EventTimeSource { + FromMetadata, + FromPath(EventTimeSourceFromPath), +} + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct EventTimeSourceFromPath { + pub pattern: String, + pub timestamp_format: Option, +} + diff --git a/opendatafabric/src/dynamic.rs b/opendatafabric/src/dynamic.rs new file mode 100644 index 0000000000..eb4e2e7cb4 --- /dev/null +++ b/opendatafabric/src/dynamic.rs @@ -0,0 +1,978 @@ +//////////////////////////////////////////////////////////////////////////////// +// WARNING: This file is auto-generated from Open Data Fabric Schemas +// See: http://opendatafabric.org/ +//////////////////////////////////////////////////////////////////////////////// + +use super::{CompressionFormat, DatasetID, Sha3_256, SourceOrdering, TimeInterval}; +use chrono::{DateTime, Utc}; + +//////////////////////////////////////////////////////////////////////////////// +// DatasetSource +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#datasetsource-schema +//////////////////////////////////////////////////////////////////////////////// + +pub enum DatasetSource<'a> { + Root(&'a dyn DatasetSourceRoot), + Derivative(&'a dyn DatasetSourceDerivative), +} + +impl<'a> From<&'a super::DatasetSource> for DatasetSource<'a> { + fn from(other: &'a super::DatasetSource) -> Self { + match other { + super::DatasetSource::Root(v) => DatasetSource::Root(v), + super::DatasetSource::Derivative(v) => DatasetSource::Derivative(v), + } + } +} + +impl Into for DatasetSource<'_> { + fn into(self) -> super::DatasetSource { + match self { + DatasetSource::Root(v) => super::DatasetSource::Root(v.into()), + DatasetSource::Derivative(v) => super::DatasetSource::Derivative(v.into()), + } + } +} + +pub trait DatasetSourceRoot { + fn fetch(&self) -> FetchStep; + fn prepare(&self) -> Option + '_>>; + fn read(&self) -> ReadStep; + fn preprocess(&self) -> Option; + fn merge(&self) -> MergeStrategy; +} + +pub trait DatasetSourceDerivative { + fn inputs(&self) -> Box + '_>; + fn transform(&self) -> Transform; +} + +impl DatasetSourceRoot for super::DatasetSourceRoot { + fn fetch(&self) -> FetchStep { + (&self.fetch).into() + } + fn prepare(&self) -> Option + '_>> { + self.prepare + .as_ref() + .map(|v| -> Box + '_> { + Box::new(v.iter().map(|i| -> PrepStep { i.into() })) + }) + } + fn read(&self) -> ReadStep { + (&self.read).into() + } + fn preprocess(&self) -> Option { + self.preprocess.as_ref().map(|v| -> Transform { v.into() }) + } + fn merge(&self) -> MergeStrategy { + (&self.merge).into() + } +} + +impl DatasetSourceDerivative for super::DatasetSourceDerivative { + fn inputs(&self) -> Box + '_> { + Box::new(self.inputs.iter().map(|i| -> &DatasetID { i.as_ref() })) + } + fn transform(&self) -> Transform { + (&self.transform).into() + } +} + +impl Into for &dyn DatasetSourceRoot { + fn into(self) -> super::DatasetSourceRoot { + super::DatasetSourceRoot { + fetch: self.fetch().into(), + prepare: self.prepare().map(|v| v.map(|i| i.into()).collect()), + read: self.read().into(), + preprocess: self.preprocess().map(|v| v.into()), + merge: self.merge().into(), + } + } +} + +impl Into for &dyn DatasetSourceDerivative { + fn into(self) -> super::DatasetSourceDerivative { + super::DatasetSourceDerivative { + inputs: self.inputs().map(|i| i.to_owned()).collect(), + transform: self.transform().into(), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// DatasetVocabulary +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#datasetvocabulary-schema +//////////////////////////////////////////////////////////////////////////////// + +pub trait DatasetVocabulary { + fn system_time_column(&self) -> Option<&str>; + fn event_time_column(&self) -> Option<&str>; +} + +impl DatasetVocabulary for super::DatasetVocabulary { + fn system_time_column(&self) -> Option<&str> { + self.system_time_column + .as_ref() + .map(|v| -> &str { v.as_ref() }) + } + fn event_time_column(&self) -> Option<&str> { + self.event_time_column + .as_ref() + .map(|v| -> &str { v.as_ref() }) + } +} + +impl Into for &dyn DatasetVocabulary { + fn into(self) -> super::DatasetVocabulary { + super::DatasetVocabulary { + system_time_column: self.system_time_column().map(|v| v.to_owned()), + event_time_column: self.event_time_column().map(|v| v.to_owned()), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// SourceCaching +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#sourcecaching-schema +//////////////////////////////////////////////////////////////////////////////// + +pub enum SourceCaching<'a> { + Forever, + _Phantom(std::marker::PhantomData<&'a ()>), +} + +impl<'a> From<&'a super::SourceCaching> for SourceCaching<'a> { + fn from(other: &'a super::SourceCaching) -> Self { + match other { + super::SourceCaching::Forever => SourceCaching::Forever, + } + } +} + +impl Into for SourceCaching<'_> { + fn into(self) -> super::SourceCaching { + match self { + SourceCaching::Forever => super::SourceCaching::Forever, + SourceCaching::_Phantom(_) => panic!(), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// TemporalTable +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#temporaltable-schema +//////////////////////////////////////////////////////////////////////////////// + +pub trait TemporalTable { + fn id(&self) -> &str; + fn primary_key(&self) -> Box + '_>; +} + +impl TemporalTable for super::TemporalTable { + fn id(&self) -> &str { + self.id.as_ref() + } + fn primary_key(&self) -> Box + '_> { + Box::new(self.primary_key.iter().map(|i| -> &str { i.as_ref() })) + } +} + +impl Into for &dyn TemporalTable { + fn into(self) -> super::TemporalTable { + super::TemporalTable { + id: self.id().to_owned(), + primary_key: self.primary_key().map(|i| i.to_owned()).collect(), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// DatasetSnapshot +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#datasetsnapshot-schema +//////////////////////////////////////////////////////////////////////////////// + +pub trait DatasetSnapshot { + fn id(&self) -> &DatasetID; + fn source(&self) -> DatasetSource; + fn vocab(&self) -> Option<&dyn DatasetVocabulary>; +} + +impl DatasetSnapshot for super::DatasetSnapshot { + fn id(&self) -> &DatasetID { + self.id.as_ref() + } + fn source(&self) -> DatasetSource { + (&self.source).into() + } + fn vocab(&self) -> Option<&dyn DatasetVocabulary> { + self.vocab.as_ref().map(|v| -> &dyn DatasetVocabulary { v }) + } +} + +impl Into for &dyn DatasetSnapshot { + fn into(self) -> super::DatasetSnapshot { + super::DatasetSnapshot { + id: self.id().to_owned(), + source: self.source().into(), + vocab: self.vocab().map(|v| v.into()), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// DataSlice +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#dataslice-schema +//////////////////////////////////////////////////////////////////////////////// + +pub trait DataSlice { + fn hash(&self) -> &Sha3_256; + fn interval(&self) -> TimeInterval; + fn num_records(&self) -> i64; +} + +impl DataSlice for super::DataSlice { + fn hash(&self) -> &Sha3_256 { + &self.hash + } + fn interval(&self) -> TimeInterval { + self.interval + } + fn num_records(&self) -> i64 { + self.num_records + } +} + +impl Into for &dyn DataSlice { + fn into(self) -> super::DataSlice { + super::DataSlice { + hash: *self.hash(), + interval: self.interval(), + num_records: self.num_records(), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// SqlQueryStep +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#sqlquerystep-schema +//////////////////////////////////////////////////////////////////////////////// + +pub trait SqlQueryStep { + fn alias(&self) -> Option<&str>; + fn query(&self) -> &str; +} + +impl SqlQueryStep for super::SqlQueryStep { + fn alias(&self) -> Option<&str> { + self.alias.as_ref().map(|v| -> &str { v.as_ref() }) + } + fn query(&self) -> &str { + self.query.as_ref() + } +} + +impl Into for &dyn SqlQueryStep { + fn into(self) -> super::SqlQueryStep { + super::SqlQueryStep { + alias: self.alias().map(|v| v.to_owned()), + query: self.query().to_owned(), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// MergeStrategy +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#mergestrategy-schema +//////////////////////////////////////////////////////////////////////////////// + +pub enum MergeStrategy<'a> { + Append, + Ledger(&'a dyn MergeStrategyLedger), + Snapshot(&'a dyn MergeStrategySnapshot), +} + +impl<'a> From<&'a super::MergeStrategy> for MergeStrategy<'a> { + fn from(other: &'a super::MergeStrategy) -> Self { + match other { + super::MergeStrategy::Append => MergeStrategy::Append, + super::MergeStrategy::Ledger(v) => MergeStrategy::Ledger(v), + super::MergeStrategy::Snapshot(v) => MergeStrategy::Snapshot(v), + } + } +} + +impl Into for MergeStrategy<'_> { + fn into(self) -> super::MergeStrategy { + match self { + MergeStrategy::Append => super::MergeStrategy::Append, + MergeStrategy::Ledger(v) => super::MergeStrategy::Ledger(v.into()), + MergeStrategy::Snapshot(v) => super::MergeStrategy::Snapshot(v.into()), + } + } +} + +pub trait MergeStrategyLedger { + fn primary_key(&self) -> Box + '_>; +} + +pub trait MergeStrategySnapshot { + fn primary_key(&self) -> Box + '_>; + fn compare_columns(&self) -> Option + '_>>; + fn observation_column(&self) -> Option<&str>; + fn obsv_added(&self) -> Option<&str>; + fn obsv_changed(&self) -> Option<&str>; + fn obsv_removed(&self) -> Option<&str>; +} + +impl MergeStrategyLedger for super::MergeStrategyLedger { + fn primary_key(&self) -> Box + '_> { + Box::new(self.primary_key.iter().map(|i| -> &str { i.as_ref() })) + } +} + +impl MergeStrategySnapshot for super::MergeStrategySnapshot { + fn primary_key(&self) -> Box + '_> { + Box::new(self.primary_key.iter().map(|i| -> &str { i.as_ref() })) + } + fn compare_columns(&self) -> Option + '_>> { + self.compare_columns + .as_ref() + .map(|v| -> Box + '_> { + Box::new(v.iter().map(|i| -> &str { i.as_ref() })) + }) + } + fn observation_column(&self) -> Option<&str> { + self.observation_column + .as_ref() + .map(|v| -> &str { v.as_ref() }) + } + fn obsv_added(&self) -> Option<&str> { + self.obsv_added.as_ref().map(|v| -> &str { v.as_ref() }) + } + fn obsv_changed(&self) -> Option<&str> { + self.obsv_changed.as_ref().map(|v| -> &str { v.as_ref() }) + } + fn obsv_removed(&self) -> Option<&str> { + self.obsv_removed.as_ref().map(|v| -> &str { v.as_ref() }) + } +} + +impl Into for &dyn MergeStrategyLedger { + fn into(self) -> super::MergeStrategyLedger { + super::MergeStrategyLedger { + primary_key: self.primary_key().map(|i| i.to_owned()).collect(), + } + } +} + +impl Into for &dyn MergeStrategySnapshot { + fn into(self) -> super::MergeStrategySnapshot { + super::MergeStrategySnapshot { + primary_key: self.primary_key().map(|i| i.to_owned()).collect(), + compare_columns: self + .compare_columns() + .map(|v| v.map(|i| i.to_owned()).collect()), + observation_column: self.observation_column().map(|v| v.to_owned()), + obsv_added: self.obsv_added().map(|v| v.to_owned()), + obsv_changed: self.obsv_changed().map(|v| v.to_owned()), + obsv_removed: self.obsv_removed().map(|v| v.to_owned()), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// MetadataBlock +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#metadatablock-schema +//////////////////////////////////////////////////////////////////////////////// + +pub trait MetadataBlock { + fn block_hash(&self) -> &Sha3_256; + fn prev_block_hash(&self) -> Option<&Sha3_256>; + fn system_time(&self) -> DateTime; + fn output_slice(&self) -> Option<&dyn DataSlice>; + fn output_watermark(&self) -> Option>; + fn input_slices(&self) -> Option + '_>>; + fn source(&self) -> Option; +} + +impl MetadataBlock for super::MetadataBlock { + fn block_hash(&self) -> &Sha3_256 { + &self.block_hash + } + fn prev_block_hash(&self) -> Option<&Sha3_256> { + self.prev_block_hash.as_ref().map(|v| -> &Sha3_256 { v }) + } + fn system_time(&self) -> DateTime { + self.system_time + } + fn output_slice(&self) -> Option<&dyn DataSlice> { + self.output_slice.as_ref().map(|v| -> &dyn DataSlice { v }) + } + fn output_watermark(&self) -> Option> { + self.output_watermark + .as_ref() + .map(|v| -> DateTime { *v }) + } + fn input_slices(&self) -> Option + '_>> { + self.input_slices + .as_ref() + .map(|v| -> Box + '_> { + Box::new(v.iter().map(|i| -> &dyn DataSlice { i })) + }) + } + fn source(&self) -> Option { + self.source.as_ref().map(|v| -> DatasetSource { v.into() }) + } +} + +impl Into for &dyn MetadataBlock { + fn into(self) -> super::MetadataBlock { + super::MetadataBlock { + block_hash: *self.block_hash(), + prev_block_hash: self.prev_block_hash().map(|v| *v), + system_time: self.system_time(), + output_slice: self.output_slice().map(|v| v.into()), + output_watermark: self.output_watermark().map(|v| v), + input_slices: self.input_slices().map(|v| v.map(|i| i.into()).collect()), + source: self.source().map(|v| v.into()), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// ReadStep +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#readstep-schema +//////////////////////////////////////////////////////////////////////////////// + +pub enum ReadStep<'a> { + Csv(&'a dyn ReadStepCsv), + JsonLines(&'a dyn ReadStepJsonLines), + GeoJson(&'a dyn ReadStepGeoJson), + EsriShapefile(&'a dyn ReadStepEsriShapefile), +} + +impl<'a> From<&'a super::ReadStep> for ReadStep<'a> { + fn from(other: &'a super::ReadStep) -> Self { + match other { + super::ReadStep::Csv(v) => ReadStep::Csv(v), + super::ReadStep::JsonLines(v) => ReadStep::JsonLines(v), + super::ReadStep::GeoJson(v) => ReadStep::GeoJson(v), + super::ReadStep::EsriShapefile(v) => ReadStep::EsriShapefile(v), + } + } +} + +impl Into for ReadStep<'_> { + fn into(self) -> super::ReadStep { + match self { + ReadStep::Csv(v) => super::ReadStep::Csv(v.into()), + ReadStep::JsonLines(v) => super::ReadStep::JsonLines(v.into()), + ReadStep::GeoJson(v) => super::ReadStep::GeoJson(v.into()), + ReadStep::EsriShapefile(v) => super::ReadStep::EsriShapefile(v.into()), + } + } +} + +pub trait ReadStepCsv { + fn schema(&self) -> Option + '_>>; + fn separator(&self) -> Option<&str>; + fn encoding(&self) -> Option<&str>; + fn quote(&self) -> Option<&str>; + fn escape(&self) -> Option<&str>; + fn comment(&self) -> Option<&str>; + fn header(&self) -> Option; + fn enforce_schema(&self) -> Option; + fn infer_schema(&self) -> Option; + fn ignore_leading_white_space(&self) -> Option; + fn ignore_trailing_white_space(&self) -> Option; + fn null_value(&self) -> Option<&str>; + fn empty_value(&self) -> Option<&str>; + fn nan_value(&self) -> Option<&str>; + fn positive_inf(&self) -> Option<&str>; + fn negative_inf(&self) -> Option<&str>; + fn date_format(&self) -> Option<&str>; + fn timestamp_format(&self) -> Option<&str>; + fn multi_line(&self) -> Option; +} + +pub trait ReadStepJsonLines { + fn schema(&self) -> Option + '_>>; + fn date_format(&self) -> Option<&str>; + fn encoding(&self) -> Option<&str>; + fn multi_line(&self) -> Option; + fn primitives_as_string(&self) -> Option; + fn timestamp_format(&self) -> Option<&str>; +} + +pub trait ReadStepGeoJson { + fn schema(&self) -> Option + '_>>; +} + +pub trait ReadStepEsriShapefile { + fn schema(&self) -> Option + '_>>; + fn sub_path(&self) -> Option<&str>; +} + +impl ReadStepCsv for super::ReadStepCsv { + fn schema(&self) -> Option + '_>> { + self.schema + .as_ref() + .map(|v| -> Box + '_> { + Box::new(v.iter().map(|i| -> &str { i.as_ref() })) + }) + } + fn separator(&self) -> Option<&str> { + self.separator.as_ref().map(|v| -> &str { v.as_ref() }) + } + fn encoding(&self) -> Option<&str> { + self.encoding.as_ref().map(|v| -> &str { v.as_ref() }) + } + fn quote(&self) -> Option<&str> { + self.quote.as_ref().map(|v| -> &str { v.as_ref() }) + } + fn escape(&self) -> Option<&str> { + self.escape.as_ref().map(|v| -> &str { v.as_ref() }) + } + fn comment(&self) -> Option<&str> { + self.comment.as_ref().map(|v| -> &str { v.as_ref() }) + } + fn header(&self) -> Option { + self.header.as_ref().map(|v| -> bool { *v }) + } + fn enforce_schema(&self) -> Option { + self.enforce_schema.as_ref().map(|v| -> bool { *v }) + } + fn infer_schema(&self) -> Option { + self.infer_schema.as_ref().map(|v| -> bool { *v }) + } + fn ignore_leading_white_space(&self) -> Option { + self.ignore_leading_white_space + .as_ref() + .map(|v| -> bool { *v }) + } + fn ignore_trailing_white_space(&self) -> Option { + self.ignore_trailing_white_space + .as_ref() + .map(|v| -> bool { *v }) + } + fn null_value(&self) -> Option<&str> { + self.null_value.as_ref().map(|v| -> &str { v.as_ref() }) + } + fn empty_value(&self) -> Option<&str> { + self.empty_value.as_ref().map(|v| -> &str { v.as_ref() }) + } + fn nan_value(&self) -> Option<&str> { + self.nan_value.as_ref().map(|v| -> &str { v.as_ref() }) + } + fn positive_inf(&self) -> Option<&str> { + self.positive_inf.as_ref().map(|v| -> &str { v.as_ref() }) + } + fn negative_inf(&self) -> Option<&str> { + self.negative_inf.as_ref().map(|v| -> &str { v.as_ref() }) + } + fn date_format(&self) -> Option<&str> { + self.date_format.as_ref().map(|v| -> &str { v.as_ref() }) + } + fn timestamp_format(&self) -> Option<&str> { + self.timestamp_format + .as_ref() + .map(|v| -> &str { v.as_ref() }) + } + fn multi_line(&self) -> Option { + self.multi_line.as_ref().map(|v| -> bool { *v }) + } +} + +impl ReadStepJsonLines for super::ReadStepJsonLines { + fn schema(&self) -> Option + '_>> { + self.schema + .as_ref() + .map(|v| -> Box + '_> { + Box::new(v.iter().map(|i| -> &str { i.as_ref() })) + }) + } + fn date_format(&self) -> Option<&str> { + self.date_format.as_ref().map(|v| -> &str { v.as_ref() }) + } + fn encoding(&self) -> Option<&str> { + self.encoding.as_ref().map(|v| -> &str { v.as_ref() }) + } + fn multi_line(&self) -> Option { + self.multi_line.as_ref().map(|v| -> bool { *v }) + } + fn primitives_as_string(&self) -> Option { + self.primitives_as_string.as_ref().map(|v| -> bool { *v }) + } + fn timestamp_format(&self) -> Option<&str> { + self.timestamp_format + .as_ref() + .map(|v| -> &str { v.as_ref() }) + } +} + +impl ReadStepGeoJson for super::ReadStepGeoJson { + fn schema(&self) -> Option + '_>> { + self.schema + .as_ref() + .map(|v| -> Box + '_> { + Box::new(v.iter().map(|i| -> &str { i.as_ref() })) + }) + } +} + +impl ReadStepEsriShapefile for super::ReadStepEsriShapefile { + fn schema(&self) -> Option + '_>> { + self.schema + .as_ref() + .map(|v| -> Box + '_> { + Box::new(v.iter().map(|i| -> &str { i.as_ref() })) + }) + } + fn sub_path(&self) -> Option<&str> { + self.sub_path.as_ref().map(|v| -> &str { v.as_ref() }) + } +} + +impl Into for &dyn ReadStepCsv { + fn into(self) -> super::ReadStepCsv { + super::ReadStepCsv { + schema: self.schema().map(|v| v.map(|i| i.to_owned()).collect()), + separator: self.separator().map(|v| v.to_owned()), + encoding: self.encoding().map(|v| v.to_owned()), + quote: self.quote().map(|v| v.to_owned()), + escape: self.escape().map(|v| v.to_owned()), + comment: self.comment().map(|v| v.to_owned()), + header: self.header().map(|v| v), + enforce_schema: self.enforce_schema().map(|v| v), + infer_schema: self.infer_schema().map(|v| v), + ignore_leading_white_space: self.ignore_leading_white_space().map(|v| v), + ignore_trailing_white_space: self.ignore_trailing_white_space().map(|v| v), + null_value: self.null_value().map(|v| v.to_owned()), + empty_value: self.empty_value().map(|v| v.to_owned()), + nan_value: self.nan_value().map(|v| v.to_owned()), + positive_inf: self.positive_inf().map(|v| v.to_owned()), + negative_inf: self.negative_inf().map(|v| v.to_owned()), + date_format: self.date_format().map(|v| v.to_owned()), + timestamp_format: self.timestamp_format().map(|v| v.to_owned()), + multi_line: self.multi_line().map(|v| v), + } + } +} + +impl Into for &dyn ReadStepJsonLines { + fn into(self) -> super::ReadStepJsonLines { + super::ReadStepJsonLines { + schema: self.schema().map(|v| v.map(|i| i.to_owned()).collect()), + date_format: self.date_format().map(|v| v.to_owned()), + encoding: self.encoding().map(|v| v.to_owned()), + multi_line: self.multi_line().map(|v| v), + primitives_as_string: self.primitives_as_string().map(|v| v), + timestamp_format: self.timestamp_format().map(|v| v.to_owned()), + } + } +} + +impl Into for &dyn ReadStepGeoJson { + fn into(self) -> super::ReadStepGeoJson { + super::ReadStepGeoJson { + schema: self.schema().map(|v| v.map(|i| i.to_owned()).collect()), + } + } +} + +impl Into for &dyn ReadStepEsriShapefile { + fn into(self) -> super::ReadStepEsriShapefile { + super::ReadStepEsriShapefile { + schema: self.schema().map(|v| v.map(|i| i.to_owned()).collect()), + sub_path: self.sub_path().map(|v| v.to_owned()), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Transform +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#transform-schema +//////////////////////////////////////////////////////////////////////////////// + +pub enum Transform<'a> { + Sql(&'a dyn TransformSql), +} + +impl<'a> From<&'a super::Transform> for Transform<'a> { + fn from(other: &'a super::Transform) -> Self { + match other { + super::Transform::Sql(v) => Transform::Sql(v), + } + } +} + +impl Into for Transform<'_> { + fn into(self) -> super::Transform { + match self { + Transform::Sql(v) => super::Transform::Sql(v.into()), + } + } +} + +pub trait TransformSql { + fn engine(&self) -> &str; + fn version(&self) -> Option<&str>; + fn query(&self) -> Option<&str>; + fn queries(&self) -> Option + '_>>; + fn temporal_tables(&self) -> Option + '_>>; +} + +impl TransformSql for super::TransformSql { + fn engine(&self) -> &str { + self.engine.as_ref() + } + fn version(&self) -> Option<&str> { + self.version.as_ref().map(|v| -> &str { v.as_ref() }) + } + fn query(&self) -> Option<&str> { + self.query.as_ref().map(|v| -> &str { v.as_ref() }) + } + fn queries(&self) -> Option + '_>> { + self.queries + .as_ref() + .map(|v| -> Box + '_> { + Box::new(v.iter().map(|i| -> &dyn SqlQueryStep { i })) + }) + } + fn temporal_tables(&self) -> Option + '_>> { + self.temporal_tables.as_ref().map( + |v| -> Box + '_> { + Box::new(v.iter().map(|i| -> &dyn TemporalTable { i })) + }, + ) + } +} + +impl Into for &dyn TransformSql { + fn into(self) -> super::TransformSql { + super::TransformSql { + engine: self.engine().to_owned(), + version: self.version().map(|v| v.to_owned()), + query: self.query().map(|v| v.to_owned()), + queries: self.queries().map(|v| v.map(|i| i.into()).collect()), + temporal_tables: self + .temporal_tables() + .map(|v| v.map(|i| i.into()).collect()), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// FetchStep +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#fetchstep-schema +//////////////////////////////////////////////////////////////////////////////// + +pub enum FetchStep<'a> { + Url(&'a dyn FetchStepUrl), + FilesGlob(&'a dyn FetchStepFilesGlob), +} + +impl<'a> From<&'a super::FetchStep> for FetchStep<'a> { + fn from(other: &'a super::FetchStep) -> Self { + match other { + super::FetchStep::Url(v) => FetchStep::Url(v), + super::FetchStep::FilesGlob(v) => FetchStep::FilesGlob(v), + } + } +} + +impl Into for FetchStep<'_> { + fn into(self) -> super::FetchStep { + match self { + FetchStep::Url(v) => super::FetchStep::Url(v.into()), + FetchStep::FilesGlob(v) => super::FetchStep::FilesGlob(v.into()), + } + } +} + +pub trait FetchStepUrl { + fn url(&self) -> &str; + fn event_time(&self) -> Option; + fn cache(&self) -> Option; +} + +pub trait FetchStepFilesGlob { + fn path(&self) -> &str; + fn event_time(&self) -> Option; + fn cache(&self) -> Option; + fn order(&self) -> Option; +} + +impl FetchStepUrl for super::FetchStepUrl { + fn url(&self) -> &str { + self.url.as_ref() + } + fn event_time(&self) -> Option { + self.event_time + .as_ref() + .map(|v| -> EventTimeSource { v.into() }) + } + fn cache(&self) -> Option { + self.cache.as_ref().map(|v| -> SourceCaching { v.into() }) + } +} + +impl FetchStepFilesGlob for super::FetchStepFilesGlob { + fn path(&self) -> &str { + self.path.as_ref() + } + fn event_time(&self) -> Option { + self.event_time + .as_ref() + .map(|v| -> EventTimeSource { v.into() }) + } + fn cache(&self) -> Option { + self.cache.as_ref().map(|v| -> SourceCaching { v.into() }) + } + fn order(&self) -> Option { + self.order.as_ref().map(|v| -> SourceOrdering { *v }) + } +} + +impl Into for &dyn FetchStepUrl { + fn into(self) -> super::FetchStepUrl { + super::FetchStepUrl { + url: self.url().to_owned(), + event_time: self.event_time().map(|v| v.into()), + cache: self.cache().map(|v| v.into()), + } + } +} + +impl Into for &dyn FetchStepFilesGlob { + fn into(self) -> super::FetchStepFilesGlob { + super::FetchStepFilesGlob { + path: self.path().to_owned(), + event_time: self.event_time().map(|v| v.into()), + cache: self.cache().map(|v| v.into()), + order: self.order().map(|v| v.into()), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// PrepStep +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#prepstep-schema +//////////////////////////////////////////////////////////////////////////////// + +pub enum PrepStep<'a> { + Decompress(&'a dyn PrepStepDecompress), + Pipe(&'a dyn PrepStepPipe), +} + +impl<'a> From<&'a super::PrepStep> for PrepStep<'a> { + fn from(other: &'a super::PrepStep) -> Self { + match other { + super::PrepStep::Decompress(v) => PrepStep::Decompress(v), + super::PrepStep::Pipe(v) => PrepStep::Pipe(v), + } + } +} + +impl Into for PrepStep<'_> { + fn into(self) -> super::PrepStep { + match self { + PrepStep::Decompress(v) => super::PrepStep::Decompress(v.into()), + PrepStep::Pipe(v) => super::PrepStep::Pipe(v.into()), + } + } +} + +pub trait PrepStepDecompress { + fn format(&self) -> CompressionFormat; + fn sub_path(&self) -> Option<&str>; +} + +pub trait PrepStepPipe { + fn command(&self) -> Box + '_>; +} + +impl PrepStepDecompress for super::PrepStepDecompress { + fn format(&self) -> CompressionFormat { + self.format + } + fn sub_path(&self) -> Option<&str> { + self.sub_path.as_ref().map(|v| -> &str { v.as_ref() }) + } +} + +impl PrepStepPipe for super::PrepStepPipe { + fn command(&self) -> Box + '_> { + Box::new(self.command.iter().map(|i| -> &str { i.as_ref() })) + } +} + +impl Into for &dyn PrepStepDecompress { + fn into(self) -> super::PrepStepDecompress { + super::PrepStepDecompress { + format: self.format().into(), + sub_path: self.sub_path().map(|v| v.to_owned()), + } + } +} + +impl Into for &dyn PrepStepPipe { + fn into(self) -> super::PrepStepPipe { + super::PrepStepPipe { + command: self.command().map(|i| i.to_owned()).collect(), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// EventTimeSource +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#eventtimesource-schema +//////////////////////////////////////////////////////////////////////////////// + +pub enum EventTimeSource<'a> { + FromMetadata, + FromPath(&'a dyn EventTimeSourceFromPath), +} + +impl<'a> From<&'a super::EventTimeSource> for EventTimeSource<'a> { + fn from(other: &'a super::EventTimeSource) -> Self { + match other { + super::EventTimeSource::FromMetadata => EventTimeSource::FromMetadata, + super::EventTimeSource::FromPath(v) => EventTimeSource::FromPath(v), + } + } +} + +impl Into for EventTimeSource<'_> { + fn into(self) -> super::EventTimeSource { + match self { + EventTimeSource::FromMetadata => super::EventTimeSource::FromMetadata, + EventTimeSource::FromPath(v) => super::EventTimeSource::FromPath(v.into()), + } + } +} + +pub trait EventTimeSourceFromPath { + fn pattern(&self) -> &str; + fn timestamp_format(&self) -> Option<&str>; +} + +impl EventTimeSourceFromPath for super::EventTimeSourceFromPath { + fn pattern(&self) -> &str { + self.pattern.as_ref() + } + fn timestamp_format(&self) -> Option<&str> { + self.timestamp_format + .as_ref() + .map(|v| -> &str { v.as_ref() }) + } +} + +impl Into for &dyn EventTimeSourceFromPath { + fn into(self) -> super::EventTimeSourceFromPath { + super::EventTimeSourceFromPath { + pattern: self.pattern().to_owned(), + timestamp_format: self.timestamp_format().map(|v| v.to_owned()), + } + } +} diff --git a/opendatafabric/src/grammar.rs b/opendatafabric/src/grammar.rs new file mode 100644 index 0000000000..8640d0e4d2 --- /dev/null +++ b/opendatafabric/src/grammar.rs @@ -0,0 +1,62 @@ +pub struct DatasetIDGrammar; + +impl DatasetIDGrammar { + fn match_zero_or_many( + s: &str, + matcher: impl Fn(&str) -> Option<(&str, &str)>, + ) -> Option<(&str, &str)> { + let mut len = 0; + let mut tail = s; + loop { + match matcher(tail) { + Some((head, ntail)) => { + len += head.len(); + tail = ntail; + } + _ => break Some((&s[0..len], &s[len..s.len()])), + } + } + } + + fn match_char(s: &str, c: char) -> Option<(&str, &str)> { + if s.len() > 0 && s.as_bytes()[0] == (c as u8) { + Some((&s[0..1], &s[1..s.len()])) + } else { + None + } + } + + fn match_alphanums(s: &str) -> Option<(&str, &str)> { + let alnums = s.bytes().take_while(|b| b.is_ascii_alphanumeric()).count(); + + if alnums == 0 { + None + } else { + Some((&s[0..alnums], &s[alnums..s.len()])) + } + } + + fn match_subdomain(s: &str) -> Option<(&str, &str)> { + let (h, t) = Self::match_alphanums(s)?; + + let (hh, tt) = Self::match_zero_or_many(t, |s| { + let (_, t) = Self::match_char(s, '-')?; + let (h, tt) = Self::match_alphanums(t)?; + Some((&s[0..h.len() + 1], tt)) + })?; + + Some((&s[0..h.len() + hh.len()], tt)) + } + + pub fn match_dataset_id(s: &str) -> Option<(&str, &str)> { + let (h, t) = Self::match_subdomain(s)?; + + let (hh, tt) = Self::match_zero_or_many(t, |s| { + let (_, t) = Self::match_char(s, '.')?; + let (h, tt) = Self::match_subdomain(t)?; + Some((&s[0..h.len() + 1], tt)) + })?; + + Some((&s[0..h.len() + hh.len()], tt)) + } +} diff --git a/opendatafabric/src/lib.rs b/opendatafabric/src/lib.rs new file mode 100644 index 0000000000..4f6d69f7f0 --- /dev/null +++ b/opendatafabric/src/lib.rs @@ -0,0 +1,23 @@ +#![feature(backtrace)] + +mod grammar; +pub use grammar::*; + +mod dataset_id; +pub use dataset_id::*; + +mod time_interval; +pub use time_interval::*; + +mod sha; +pub use sha::*; + +mod manifest; +pub use manifest::*; + +mod dtos; +pub use dtos::*; + +pub mod dynamic; + +pub mod serde; diff --git a/opendatafabric/src/manifest.rs b/opendatafabric/src/manifest.rs new file mode 100644 index 0000000000..07ca0fe21a --- /dev/null +++ b/opendatafabric/src/manifest.rs @@ -0,0 +1,6 @@ +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Manifest { + pub api_version: i32, + pub kind: String, + pub content: T, +} diff --git a/opendatafabric/src/serde/flatbuffers/convertors.rs b/opendatafabric/src/serde/flatbuffers/convertors.rs new file mode 100644 index 0000000000..09e6445f16 --- /dev/null +++ b/opendatafabric/src/serde/flatbuffers/convertors.rs @@ -0,0 +1,1252 @@ +//////////////////////////////////////////////////////////////////////////////// +// WARNING: This file is auto-generated from Open Data Fabric Schemas +// See: http://opendatafabric.org/ +//////////////////////////////////////////////////////////////////////////////// + +use super::odf_generated as fb; +use crate as odf; +use ::flatbuffers::{FlatBufferBuilder, Table, UnionWIPOffset, WIPOffset}; +use chrono::prelude::*; +use std::convert::{TryFrom, TryInto}; + +pub trait FlatbuffersSerializable<'fb> { + type OffsetT; + fn serialize(&self, fb: &mut FlatBufferBuilder<'fb>) -> Self::OffsetT; +} + +pub trait FlatbuffersDeserializable { + fn deserialize(fb: T) -> Self; +} + +trait FlatbuffersEnumSerializable<'fb, E> { + fn serialize(&self, fb: &mut FlatBufferBuilder<'fb>) -> (E, Option>); +} + +trait FlatbuffersEnumDeserializable<'fb, E> { + fn deserialize(table: Option>, t: E) -> Option + where + Self: Sized; +} + +//////////////////////////////////////////////////////////////////////////////// +// DatasetSource +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#datasetsource-schema +//////////////////////////////////////////////////////////////////////////////// + +impl<'fb> FlatbuffersEnumSerializable<'fb, fb::DatasetSource> for odf::DatasetSource { + fn serialize( + &self, + fb: &mut FlatBufferBuilder<'fb>, + ) -> (fb::DatasetSource, Option>) { + match self { + odf::DatasetSource::Root(v) => ( + fb::DatasetSource::DatasetSourceRoot, + Some(v.serialize(fb).as_union_value()), + ), + odf::DatasetSource::Derivative(v) => ( + fb::DatasetSource::DatasetSourceDerivative, + Some(v.serialize(fb).as_union_value()), + ), + } + } +} + +impl<'fb> FlatbuffersEnumDeserializable<'fb, fb::DatasetSource> for odf::DatasetSource { + fn deserialize(table: Option>, t: fb::DatasetSource) -> Option { + match t { + fb::DatasetSource::NONE => None, + fb::DatasetSource::DatasetSourceRoot => Some(odf::DatasetSource::Root( + odf::DatasetSourceRoot::deserialize(fb::DatasetSourceRoot::init_from_table( + table.unwrap(), + )), + )), + fb::DatasetSource::DatasetSourceDerivative => Some(odf::DatasetSource::Derivative( + odf::DatasetSourceDerivative::deserialize( + fb::DatasetSourceDerivative::init_from_table(table.unwrap()), + ), + )), + } + } +} + +impl<'fb> FlatbuffersSerializable<'fb> for odf::DatasetSourceRoot { + type OffsetT = WIPOffset>; + + fn serialize(&self, fb: &mut FlatBufferBuilder<'fb>) -> Self::OffsetT { + let fetch_offset = { self.fetch.serialize(fb) }; + let prepare_offset = self.prepare.as_ref().map(|v| { + let offsets: Vec<_> = v + .iter() + .map(|i| { + let (value_type, value_offset) = i.serialize(fb); + let mut builder = fb::PrepStepWrapperBuilder::new(fb); + builder.add_value_type(value_type); + value_offset.map(|off| builder.add_value(off)); + builder.finish() + }) + .collect(); + fb.create_vector(&offsets) + }); + let read_offset = { self.read.serialize(fb) }; + let preprocess_offset = self.preprocess.as_ref().map(|v| v.serialize(fb)); + let merge_offset = { self.merge.serialize(fb) }; + let mut builder = fb::DatasetSourceRootBuilder::new(fb); + builder.add_fetch_type(fetch_offset.0); + fetch_offset.1.map(|off| builder.add_fetch(off)); + prepare_offset.map(|off| builder.add_prepare(off)); + builder.add_read_type(read_offset.0); + read_offset.1.map(|off| builder.add_read(off)); + preprocess_offset.map(|off| { + builder.add_preprocess_type(off.0); + off.1.map(|v| builder.add_preprocess(v)); + }); + builder.add_merge_type(merge_offset.0); + merge_offset.1.map(|off| builder.add_merge(off)); + builder.finish() + } +} + +impl<'fb> FlatbuffersDeserializable> for odf::DatasetSourceRoot { + fn deserialize(proxy: fb::DatasetSourceRoot<'fb>) -> Self { + odf::DatasetSourceRoot { + fetch: odf::FetchStep::deserialize(proxy.fetch(), proxy.fetch_type()).unwrap(), + prepare: proxy.prepare().map(|v| { + v.iter() + .map(|i| odf::PrepStep::deserialize(i.value(), i.value_type()).unwrap()) + .collect() + }), + read: odf::ReadStep::deserialize(proxy.read(), proxy.read_type()).unwrap(), + preprocess: odf::Transform::deserialize(proxy.preprocess(), proxy.preprocess_type()), + merge: odf::MergeStrategy::deserialize(proxy.merge(), proxy.merge_type()).unwrap(), + } + } +} + +impl<'fb> FlatbuffersSerializable<'fb> for odf::DatasetSourceDerivative { + type OffsetT = WIPOffset>; + + fn serialize(&self, fb: &mut FlatBufferBuilder<'fb>) -> Self::OffsetT { + let inputs_offset = { + let offsets: Vec<_> = self.inputs.iter().map(|i| fb.create_string(&i)).collect(); + fb.create_vector(&offsets) + }; + let transform_offset = { self.transform.serialize(fb) }; + let mut builder = fb::DatasetSourceDerivativeBuilder::new(fb); + builder.add_inputs(inputs_offset); + builder.add_transform_type(transform_offset.0); + transform_offset.1.map(|off| builder.add_transform(off)); + builder.finish() + } +} + +impl<'fb> FlatbuffersDeserializable> + for odf::DatasetSourceDerivative +{ + fn deserialize(proxy: fb::DatasetSourceDerivative<'fb>) -> Self { + use std::str::FromStr; + + odf::DatasetSourceDerivative { + inputs: proxy + .inputs() + .map(|v| { + v.iter() + .map(|i| odf::DatasetIDBuf::from_str(i).unwrap()) + .collect() + }) + .unwrap(), + transform: odf::Transform::deserialize(proxy.transform(), proxy.transform_type()) + .unwrap(), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// DatasetVocabulary +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#datasetvocabulary-schema +//////////////////////////////////////////////////////////////////////////////// + +impl<'fb> FlatbuffersSerializable<'fb> for odf::DatasetVocabulary { + type OffsetT = WIPOffset>; + + fn serialize(&self, fb: &mut FlatBufferBuilder<'fb>) -> Self::OffsetT { + let system_time_column_offset = self + .system_time_column + .as_ref() + .map(|v| fb.create_string(&v)); + let event_time_column_offset = self + .event_time_column + .as_ref() + .map(|v| fb.create_string(&v)); + let mut builder = fb::DatasetVocabularyBuilder::new(fb); + system_time_column_offset.map(|off| builder.add_system_time_column(off)); + event_time_column_offset.map(|off| builder.add_event_time_column(off)); + builder.finish() + } +} + +impl<'fb> FlatbuffersDeserializable> for odf::DatasetVocabulary { + fn deserialize(proxy: fb::DatasetVocabulary<'fb>) -> Self { + odf::DatasetVocabulary { + system_time_column: proxy.system_time_column().map(|v| v.to_owned()), + event_time_column: proxy.event_time_column().map(|v| v.to_owned()), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// SourceCaching +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#sourcecaching-schema +//////////////////////////////////////////////////////////////////////////////// + +impl<'fb> FlatbuffersEnumSerializable<'fb, fb::SourceCaching> for odf::SourceCaching { + fn serialize( + &self, + _fb: &mut FlatBufferBuilder<'fb>, + ) -> (fb::SourceCaching, Option>) { + match self { + odf::SourceCaching::Forever => (fb::SourceCaching::SourceCachingForever, None), + } + } +} + +impl<'fb> FlatbuffersEnumDeserializable<'fb, fb::SourceCaching> for odf::SourceCaching { + fn deserialize(_table: Option>, t: fb::SourceCaching) -> Option { + match t { + fb::SourceCaching::NONE => None, + fb::SourceCaching::SourceCachingForever => Some(odf::SourceCaching::Forever), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// TemporalTable +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#temporaltable-schema +//////////////////////////////////////////////////////////////////////////////// + +impl<'fb> FlatbuffersSerializable<'fb> for odf::TemporalTable { + type OffsetT = WIPOffset>; + + fn serialize(&self, fb: &mut FlatBufferBuilder<'fb>) -> Self::OffsetT { + let id_offset = { fb.create_string(&self.id) }; + let primary_key_offset = { + let offsets: Vec<_> = self + .primary_key + .iter() + .map(|i| fb.create_string(&i)) + .collect(); + fb.create_vector(&offsets) + }; + let mut builder = fb::TemporalTableBuilder::new(fb); + builder.add_id(id_offset); + builder.add_primary_key(primary_key_offset); + builder.finish() + } +} + +impl<'fb> FlatbuffersDeserializable> for odf::TemporalTable { + fn deserialize(proxy: fb::TemporalTable<'fb>) -> Self { + odf::TemporalTable { + id: proxy.id().map(|v| v.to_owned()).unwrap(), + primary_key: proxy + .primary_key() + .map(|v| v.iter().map(|i| i.to_owned()).collect()) + .unwrap(), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// DatasetSnapshot +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#datasetsnapshot-schema +//////////////////////////////////////////////////////////////////////////////// + +impl<'fb> FlatbuffersSerializable<'fb> for odf::DatasetSnapshot { + type OffsetT = WIPOffset>; + + fn serialize(&self, fb: &mut FlatBufferBuilder<'fb>) -> Self::OffsetT { + let id_offset = { fb.create_string(&self.id) }; + let source_offset = { self.source.serialize(fb) }; + let vocab_offset = self.vocab.as_ref().map(|v| v.serialize(fb)); + let mut builder = fb::DatasetSnapshotBuilder::new(fb); + builder.add_id(id_offset); + builder.add_source_type(source_offset.0); + source_offset.1.map(|off| builder.add_source(off)); + vocab_offset.map(|off| builder.add_vocab(off)); + builder.finish() + } +} + +impl<'fb> FlatbuffersDeserializable> for odf::DatasetSnapshot { + fn deserialize(proxy: fb::DatasetSnapshot<'fb>) -> Self { + odf::DatasetSnapshot { + id: proxy + .id() + .map(|v| odf::DatasetIDBuf::try_from(v).unwrap()) + .unwrap(), + source: odf::DatasetSource::deserialize(proxy.source(), proxy.source_type()).unwrap(), + vocab: proxy + .vocab() + .map(|v| odf::DatasetVocabulary::deserialize(v)), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// DataSlice +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#dataslice-schema +//////////////////////////////////////////////////////////////////////////////// + +impl<'fb> FlatbuffersSerializable<'fb> for odf::DataSlice { + type OffsetT = WIPOffset>; + + fn serialize(&self, fb: &mut FlatBufferBuilder<'fb>) -> Self::OffsetT { + let hash_offset = { fb.create_vector(&self.hash) }; + let mut builder = fb::DataSliceBuilder::new(fb); + builder.add_hash(hash_offset); + let interval = interval_to_fb(&self.interval); + builder.add_interval(&interval); + let num_records = fb::Option_int64::new(self.num_records); + builder.add_num_records(&num_records); + builder.finish() + } +} + +impl<'fb> FlatbuffersDeserializable> for odf::DataSlice { + fn deserialize(proxy: fb::DataSlice<'fb>) -> Self { + odf::DataSlice { + hash: proxy + .hash() + .map(|v| odf::Sha3_256::new(v.try_into().unwrap())) + .unwrap(), + interval: proxy.interval().map(|v| fb_to_interval(v)).unwrap(), + num_records: proxy.num_records().map(|v| v.value()).unwrap(), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// SqlQueryStep +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#sqlquerystep-schema +//////////////////////////////////////////////////////////////////////////////// + +impl<'fb> FlatbuffersSerializable<'fb> for odf::SqlQueryStep { + type OffsetT = WIPOffset>; + + fn serialize(&self, fb: &mut FlatBufferBuilder<'fb>) -> Self::OffsetT { + let alias_offset = self.alias.as_ref().map(|v| fb.create_string(&v)); + let query_offset = { fb.create_string(&self.query) }; + let mut builder = fb::SqlQueryStepBuilder::new(fb); + alias_offset.map(|off| builder.add_alias(off)); + builder.add_query(query_offset); + builder.finish() + } +} + +impl<'fb> FlatbuffersDeserializable> for odf::SqlQueryStep { + fn deserialize(proxy: fb::SqlQueryStep<'fb>) -> Self { + odf::SqlQueryStep { + alias: proxy.alias().map(|v| v.to_owned()), + query: proxy.query().map(|v| v.to_owned()).unwrap(), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// MergeStrategy +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#mergestrategy-schema +//////////////////////////////////////////////////////////////////////////////// + +impl<'fb> FlatbuffersEnumSerializable<'fb, fb::MergeStrategy> for odf::MergeStrategy { + fn serialize( + &self, + fb: &mut FlatBufferBuilder<'fb>, + ) -> (fb::MergeStrategy, Option>) { + match self { + odf::MergeStrategy::Append => (fb::MergeStrategy::MergeStrategyAppend, None), + odf::MergeStrategy::Ledger(v) => ( + fb::MergeStrategy::MergeStrategyLedger, + Some(v.serialize(fb).as_union_value()), + ), + odf::MergeStrategy::Snapshot(v) => ( + fb::MergeStrategy::MergeStrategySnapshot, + Some(v.serialize(fb).as_union_value()), + ), + } + } +} + +impl<'fb> FlatbuffersEnumDeserializable<'fb, fb::MergeStrategy> for odf::MergeStrategy { + fn deserialize(table: Option>, t: fb::MergeStrategy) -> Option { + match t { + fb::MergeStrategy::NONE => None, + fb::MergeStrategy::MergeStrategyAppend => Some(odf::MergeStrategy::Append), + fb::MergeStrategy::MergeStrategyLedger => Some(odf::MergeStrategy::Ledger( + odf::MergeStrategyLedger::deserialize(fb::MergeStrategyLedger::init_from_table( + table.unwrap(), + )), + )), + fb::MergeStrategy::MergeStrategySnapshot => Some(odf::MergeStrategy::Snapshot( + odf::MergeStrategySnapshot::deserialize( + fb::MergeStrategySnapshot::init_from_table(table.unwrap()), + ), + )), + } + } +} + +impl<'fb> FlatbuffersSerializable<'fb> for odf::MergeStrategyLedger { + type OffsetT = WIPOffset>; + + fn serialize(&self, fb: &mut FlatBufferBuilder<'fb>) -> Self::OffsetT { + let primary_key_offset = { + let offsets: Vec<_> = self + .primary_key + .iter() + .map(|i| fb.create_string(&i)) + .collect(); + fb.create_vector(&offsets) + }; + let mut builder = fb::MergeStrategyLedgerBuilder::new(fb); + builder.add_primary_key(primary_key_offset); + builder.finish() + } +} + +impl<'fb> FlatbuffersDeserializable> for odf::MergeStrategyLedger { + fn deserialize(proxy: fb::MergeStrategyLedger<'fb>) -> Self { + odf::MergeStrategyLedger { + primary_key: proxy + .primary_key() + .map(|v| v.iter().map(|i| i.to_owned()).collect()) + .unwrap(), + } + } +} + +impl<'fb> FlatbuffersSerializable<'fb> for odf::MergeStrategySnapshot { + type OffsetT = WIPOffset>; + + fn serialize(&self, fb: &mut FlatBufferBuilder<'fb>) -> Self::OffsetT { + let primary_key_offset = { + let offsets: Vec<_> = self + .primary_key + .iter() + .map(|i| fb.create_string(&i)) + .collect(); + fb.create_vector(&offsets) + }; + let compare_columns_offset = self.compare_columns.as_ref().map(|v| { + let offsets: Vec<_> = v.iter().map(|i| fb.create_string(&i)).collect(); + fb.create_vector(&offsets) + }); + let observation_column_offset = self + .observation_column + .as_ref() + .map(|v| fb.create_string(&v)); + let obsv_added_offset = self.obsv_added.as_ref().map(|v| fb.create_string(&v)); + let obsv_changed_offset = self.obsv_changed.as_ref().map(|v| fb.create_string(&v)); + let obsv_removed_offset = self.obsv_removed.as_ref().map(|v| fb.create_string(&v)); + let mut builder = fb::MergeStrategySnapshotBuilder::new(fb); + builder.add_primary_key(primary_key_offset); + compare_columns_offset.map(|off| builder.add_compare_columns(off)); + observation_column_offset.map(|off| builder.add_observation_column(off)); + obsv_added_offset.map(|off| builder.add_obsv_added(off)); + obsv_changed_offset.map(|off| builder.add_obsv_changed(off)); + obsv_removed_offset.map(|off| builder.add_obsv_removed(off)); + builder.finish() + } +} + +impl<'fb> FlatbuffersDeserializable> for odf::MergeStrategySnapshot { + fn deserialize(proxy: fb::MergeStrategySnapshot<'fb>) -> Self { + odf::MergeStrategySnapshot { + primary_key: proxy + .primary_key() + .map(|v| v.iter().map(|i| i.to_owned()).collect()) + .unwrap(), + compare_columns: proxy + .compare_columns() + .map(|v| v.iter().map(|i| i.to_owned()).collect()), + observation_column: proxy.observation_column().map(|v| v.to_owned()), + obsv_added: proxy.obsv_added().map(|v| v.to_owned()), + obsv_changed: proxy.obsv_changed().map(|v| v.to_owned()), + obsv_removed: proxy.obsv_removed().map(|v| v.to_owned()), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// MetadataBlock +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#metadatablock-schema +//////////////////////////////////////////////////////////////////////////////// + +impl<'fb> FlatbuffersSerializable<'fb> for odf::MetadataBlock { + type OffsetT = WIPOffset>; + + fn serialize(&self, fb: &mut FlatBufferBuilder<'fb>) -> Self::OffsetT { + let block_hash_offset = { fb.create_vector(&self.block_hash) }; + let prev_block_hash_offset = self.prev_block_hash.as_ref().map(|v| fb.create_vector(&v)); + let output_slice_offset = self.output_slice.as_ref().map(|v| v.serialize(fb)); + let input_slices_offset = self.input_slices.as_ref().map(|v| { + let offsets: Vec<_> = v.iter().map(|i| i.serialize(fb)).collect(); + fb.create_vector(&offsets) + }); + let source_offset = self.source.as_ref().map(|v| v.serialize(fb)); + let mut builder = fb::MetadataBlockBuilder::new(fb); + builder.add_block_hash(block_hash_offset); + prev_block_hash_offset.map(|off| builder.add_prev_block_hash(off)); + let system_time = datetime_to_fb(&self.system_time); + builder.add_system_time(&system_time); + output_slice_offset.map(|off| builder.add_output_slice(off)); + let output_watermark = self.output_watermark.map(|t| datetime_to_fb(&t)); + self.output_watermark + .as_ref() + .map(|_| builder.add_output_watermark(output_watermark.as_ref().unwrap())); + input_slices_offset.map(|off| builder.add_input_slices(off)); + source_offset.map(|off| { + builder.add_source_type(off.0); + off.1.map(|v| builder.add_source(v)); + }); + builder.finish() + } +} + +impl<'fb> FlatbuffersDeserializable> for odf::MetadataBlock { + fn deserialize(proxy: fb::MetadataBlock<'fb>) -> Self { + odf::MetadataBlock { + block_hash: proxy + .block_hash() + .map(|v| odf::Sha3_256::new(v.try_into().unwrap())) + .unwrap(), + prev_block_hash: proxy + .prev_block_hash() + .map(|v| odf::Sha3_256::new(v.try_into().unwrap())), + system_time: proxy.system_time().map(|v| fb_to_datetime(v)).unwrap(), + output_slice: proxy.output_slice().map(|v| odf::DataSlice::deserialize(v)), + output_watermark: proxy.output_watermark().map(|v| fb_to_datetime(v)), + input_slices: proxy + .input_slices() + .map(|v| v.iter().map(|i| odf::DataSlice::deserialize(i)).collect()), + source: odf::DatasetSource::deserialize(proxy.source(), proxy.source_type()), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// ReadStep +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#readstep-schema +//////////////////////////////////////////////////////////////////////////////// + +impl<'fb> FlatbuffersEnumSerializable<'fb, fb::ReadStep> for odf::ReadStep { + fn serialize( + &self, + fb: &mut FlatBufferBuilder<'fb>, + ) -> (fb::ReadStep, Option>) { + match self { + odf::ReadStep::Csv(v) => ( + fb::ReadStep::ReadStepCsv, + Some(v.serialize(fb).as_union_value()), + ), + odf::ReadStep::JsonLines(v) => ( + fb::ReadStep::ReadStepJsonLines, + Some(v.serialize(fb).as_union_value()), + ), + odf::ReadStep::GeoJson(v) => ( + fb::ReadStep::ReadStepGeoJson, + Some(v.serialize(fb).as_union_value()), + ), + odf::ReadStep::EsriShapefile(v) => ( + fb::ReadStep::ReadStepEsriShapefile, + Some(v.serialize(fb).as_union_value()), + ), + } + } +} + +impl<'fb> FlatbuffersEnumDeserializable<'fb, fb::ReadStep> for odf::ReadStep { + fn deserialize(table: Option>, t: fb::ReadStep) -> Option { + match t { + fb::ReadStep::NONE => None, + fb::ReadStep::ReadStepCsv => Some(odf::ReadStep::Csv(odf::ReadStepCsv::deserialize( + fb::ReadStepCsv::init_from_table(table.unwrap()), + ))), + fb::ReadStep::ReadStepJsonLines => Some(odf::ReadStep::JsonLines( + odf::ReadStepJsonLines::deserialize(fb::ReadStepJsonLines::init_from_table( + table.unwrap(), + )), + )), + fb::ReadStep::ReadStepGeoJson => { + Some(odf::ReadStep::GeoJson(odf::ReadStepGeoJson::deserialize( + fb::ReadStepGeoJson::init_from_table(table.unwrap()), + ))) + } + fb::ReadStep::ReadStepEsriShapefile => Some(odf::ReadStep::EsriShapefile( + odf::ReadStepEsriShapefile::deserialize( + fb::ReadStepEsriShapefile::init_from_table(table.unwrap()), + ), + )), + } + } +} + +impl<'fb> FlatbuffersSerializable<'fb> for odf::ReadStepCsv { + type OffsetT = WIPOffset>; + + fn serialize(&self, fb: &mut FlatBufferBuilder<'fb>) -> Self::OffsetT { + let schema_offset = self.schema.as_ref().map(|v| { + let offsets: Vec<_> = v.iter().map(|i| fb.create_string(&i)).collect(); + fb.create_vector(&offsets) + }); + let separator_offset = self.separator.as_ref().map(|v| fb.create_string(&v)); + let encoding_offset = self.encoding.as_ref().map(|v| fb.create_string(&v)); + let quote_offset = self.quote.as_ref().map(|v| fb.create_string(&v)); + let escape_offset = self.escape.as_ref().map(|v| fb.create_string(&v)); + let comment_offset = self.comment.as_ref().map(|v| fb.create_string(&v)); + let null_value_offset = self.null_value.as_ref().map(|v| fb.create_string(&v)); + let empty_value_offset = self.empty_value.as_ref().map(|v| fb.create_string(&v)); + let nan_value_offset = self.nan_value.as_ref().map(|v| fb.create_string(&v)); + let positive_inf_offset = self.positive_inf.as_ref().map(|v| fb.create_string(&v)); + let negative_inf_offset = self.negative_inf.as_ref().map(|v| fb.create_string(&v)); + let date_format_offset = self.date_format.as_ref().map(|v| fb.create_string(&v)); + let timestamp_format_offset = self.timestamp_format.as_ref().map(|v| fb.create_string(&v)); + let mut builder = fb::ReadStepCsvBuilder::new(fb); + schema_offset.map(|off| builder.add_schema(off)); + separator_offset.map(|off| builder.add_separator(off)); + encoding_offset.map(|off| builder.add_encoding(off)); + quote_offset.map(|off| builder.add_quote(off)); + escape_offset.map(|off| builder.add_escape(off)); + comment_offset.map(|off| builder.add_comment(off)); + let header = self.header.map(|v| fb::Option_bool::new(v)); + self.header + .as_ref() + .map(|_| builder.add_header(header.as_ref().unwrap())); + let enforce_schema = self.enforce_schema.map(|v| fb::Option_bool::new(v)); + self.enforce_schema + .as_ref() + .map(|_| builder.add_enforce_schema(enforce_schema.as_ref().unwrap())); + let infer_schema = self.infer_schema.map(|v| fb::Option_bool::new(v)); + self.infer_schema + .as_ref() + .map(|_| builder.add_infer_schema(infer_schema.as_ref().unwrap())); + let ignore_leading_white_space = self + .ignore_leading_white_space + .map(|v| fb::Option_bool::new(v)); + self.ignore_leading_white_space.as_ref().map(|_| { + builder.add_ignore_leading_white_space(ignore_leading_white_space.as_ref().unwrap()) + }); + let ignore_trailing_white_space = self + .ignore_trailing_white_space + .map(|v| fb::Option_bool::new(v)); + self.ignore_trailing_white_space.as_ref().map(|_| { + builder.add_ignore_trailing_white_space(ignore_trailing_white_space.as_ref().unwrap()) + }); + null_value_offset.map(|off| builder.add_null_value(off)); + empty_value_offset.map(|off| builder.add_empty_value(off)); + nan_value_offset.map(|off| builder.add_nan_value(off)); + positive_inf_offset.map(|off| builder.add_positive_inf(off)); + negative_inf_offset.map(|off| builder.add_negative_inf(off)); + date_format_offset.map(|off| builder.add_date_format(off)); + timestamp_format_offset.map(|off| builder.add_timestamp_format(off)); + let multi_line = self.multi_line.map(|v| fb::Option_bool::new(v)); + self.multi_line + .as_ref() + .map(|_| builder.add_multi_line(multi_line.as_ref().unwrap())); + builder.finish() + } +} + +impl<'fb> FlatbuffersDeserializable> for odf::ReadStepCsv { + fn deserialize(proxy: fb::ReadStepCsv<'fb>) -> Self { + odf::ReadStepCsv { + schema: proxy + .schema() + .map(|v| v.iter().map(|i| i.to_owned()).collect()), + separator: proxy.separator().map(|v| v.to_owned()), + encoding: proxy.encoding().map(|v| v.to_owned()), + quote: proxy.quote().map(|v| v.to_owned()), + escape: proxy.escape().map(|v| v.to_owned()), + comment: proxy.comment().map(|v| v.to_owned()), + header: proxy.header().map(|v| v.value()), + enforce_schema: proxy.enforce_schema().map(|v| v.value()), + infer_schema: proxy.infer_schema().map(|v| v.value()), + ignore_leading_white_space: proxy.ignore_leading_white_space().map(|v| v.value()), + ignore_trailing_white_space: proxy.ignore_trailing_white_space().map(|v| v.value()), + null_value: proxy.null_value().map(|v| v.to_owned()), + empty_value: proxy.empty_value().map(|v| v.to_owned()), + nan_value: proxy.nan_value().map(|v| v.to_owned()), + positive_inf: proxy.positive_inf().map(|v| v.to_owned()), + negative_inf: proxy.negative_inf().map(|v| v.to_owned()), + date_format: proxy.date_format().map(|v| v.to_owned()), + timestamp_format: proxy.timestamp_format().map(|v| v.to_owned()), + multi_line: proxy.multi_line().map(|v| v.value()), + } + } +} + +impl<'fb> FlatbuffersSerializable<'fb> for odf::ReadStepJsonLines { + type OffsetT = WIPOffset>; + + fn serialize(&self, fb: &mut FlatBufferBuilder<'fb>) -> Self::OffsetT { + let schema_offset = self.schema.as_ref().map(|v| { + let offsets: Vec<_> = v.iter().map(|i| fb.create_string(&i)).collect(); + fb.create_vector(&offsets) + }); + let date_format_offset = self.date_format.as_ref().map(|v| fb.create_string(&v)); + let encoding_offset = self.encoding.as_ref().map(|v| fb.create_string(&v)); + let timestamp_format_offset = self.timestamp_format.as_ref().map(|v| fb.create_string(&v)); + let mut builder = fb::ReadStepJsonLinesBuilder::new(fb); + schema_offset.map(|off| builder.add_schema(off)); + date_format_offset.map(|off| builder.add_date_format(off)); + encoding_offset.map(|off| builder.add_encoding(off)); + let multi_line = self.multi_line.map(|v| fb::Option_bool::new(v)); + self.multi_line + .as_ref() + .map(|_| builder.add_multi_line(multi_line.as_ref().unwrap())); + let primitives_as_string = self.primitives_as_string.map(|v| fb::Option_bool::new(v)); + self.primitives_as_string + .as_ref() + .map(|_| builder.add_primitives_as_string(primitives_as_string.as_ref().unwrap())); + timestamp_format_offset.map(|off| builder.add_timestamp_format(off)); + builder.finish() + } +} + +impl<'fb> FlatbuffersDeserializable> for odf::ReadStepJsonLines { + fn deserialize(proxy: fb::ReadStepJsonLines<'fb>) -> Self { + odf::ReadStepJsonLines { + schema: proxy + .schema() + .map(|v| v.iter().map(|i| i.to_owned()).collect()), + date_format: proxy.date_format().map(|v| v.to_owned()), + encoding: proxy.encoding().map(|v| v.to_owned()), + multi_line: proxy.multi_line().map(|v| v.value()), + primitives_as_string: proxy.primitives_as_string().map(|v| v.value()), + timestamp_format: proxy.timestamp_format().map(|v| v.to_owned()), + } + } +} + +impl<'fb> FlatbuffersSerializable<'fb> for odf::ReadStepGeoJson { + type OffsetT = WIPOffset>; + + fn serialize(&self, fb: &mut FlatBufferBuilder<'fb>) -> Self::OffsetT { + let schema_offset = self.schema.as_ref().map(|v| { + let offsets: Vec<_> = v.iter().map(|i| fb.create_string(&i)).collect(); + fb.create_vector(&offsets) + }); + let mut builder = fb::ReadStepGeoJsonBuilder::new(fb); + schema_offset.map(|off| builder.add_schema(off)); + builder.finish() + } +} + +impl<'fb> FlatbuffersDeserializable> for odf::ReadStepGeoJson { + fn deserialize(proxy: fb::ReadStepGeoJson<'fb>) -> Self { + odf::ReadStepGeoJson { + schema: proxy + .schema() + .map(|v| v.iter().map(|i| i.to_owned()).collect()), + } + } +} + +impl<'fb> FlatbuffersSerializable<'fb> for odf::ReadStepEsriShapefile { + type OffsetT = WIPOffset>; + + fn serialize(&self, fb: &mut FlatBufferBuilder<'fb>) -> Self::OffsetT { + let schema_offset = self.schema.as_ref().map(|v| { + let offsets: Vec<_> = v.iter().map(|i| fb.create_string(&i)).collect(); + fb.create_vector(&offsets) + }); + let sub_path_offset = self.sub_path.as_ref().map(|v| fb.create_string(&v)); + let mut builder = fb::ReadStepEsriShapefileBuilder::new(fb); + schema_offset.map(|off| builder.add_schema(off)); + sub_path_offset.map(|off| builder.add_sub_path(off)); + builder.finish() + } +} + +impl<'fb> FlatbuffersDeserializable> for odf::ReadStepEsriShapefile { + fn deserialize(proxy: fb::ReadStepEsriShapefile<'fb>) -> Self { + odf::ReadStepEsriShapefile { + schema: proxy + .schema() + .map(|v| v.iter().map(|i| i.to_owned()).collect()), + sub_path: proxy.sub_path().map(|v| v.to_owned()), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Transform +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#transform-schema +//////////////////////////////////////////////////////////////////////////////// + +impl<'fb> FlatbuffersEnumSerializable<'fb, fb::Transform> for odf::Transform { + fn serialize( + &self, + fb: &mut FlatBufferBuilder<'fb>, + ) -> (fb::Transform, Option>) { + match self { + odf::Transform::Sql(v) => ( + fb::Transform::TransformSql, + Some(v.serialize(fb).as_union_value()), + ), + } + } +} + +impl<'fb> FlatbuffersEnumDeserializable<'fb, fb::Transform> for odf::Transform { + fn deserialize(table: Option>, t: fb::Transform) -> Option { + match t { + fb::Transform::NONE => None, + fb::Transform::TransformSql => Some(odf::Transform::Sql( + odf::TransformSql::deserialize(fb::TransformSql::init_from_table(table.unwrap())), + )), + } + } +} + +impl<'fb> FlatbuffersSerializable<'fb> for odf::TransformSql { + type OffsetT = WIPOffset>; + + fn serialize(&self, fb: &mut FlatBufferBuilder<'fb>) -> Self::OffsetT { + let engine_offset = { fb.create_string(&self.engine) }; + let version_offset = self.version.as_ref().map(|v| fb.create_string(&v)); + let query_offset = self.query.as_ref().map(|v| fb.create_string(&v)); + let queries_offset = self.queries.as_ref().map(|v| { + let offsets: Vec<_> = v.iter().map(|i| i.serialize(fb)).collect(); + fb.create_vector(&offsets) + }); + let temporal_tables_offset = self.temporal_tables.as_ref().map(|v| { + let offsets: Vec<_> = v.iter().map(|i| i.serialize(fb)).collect(); + fb.create_vector(&offsets) + }); + let mut builder = fb::TransformSqlBuilder::new(fb); + builder.add_engine(engine_offset); + version_offset.map(|off| builder.add_version(off)); + query_offset.map(|off| builder.add_query(off)); + queries_offset.map(|off| builder.add_queries(off)); + temporal_tables_offset.map(|off| builder.add_temporal_tables(off)); + builder.finish() + } +} + +impl<'fb> FlatbuffersDeserializable> for odf::TransformSql { + fn deserialize(proxy: fb::TransformSql<'fb>) -> Self { + odf::TransformSql { + engine: proxy.engine().map(|v| v.to_owned()).unwrap(), + version: proxy.version().map(|v| v.to_owned()), + query: proxy.query().map(|v| v.to_owned()), + queries: proxy.queries().map(|v| { + v.iter() + .map(|i| odf::SqlQueryStep::deserialize(i)) + .collect() + }), + temporal_tables: proxy.temporal_tables().map(|v| { + v.iter() + .map(|i| odf::TemporalTable::deserialize(i)) + .collect() + }), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// FetchStep +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#fetchstep-schema +//////////////////////////////////////////////////////////////////////////////// + +impl<'fb> FlatbuffersEnumSerializable<'fb, fb::FetchStep> for odf::FetchStep { + fn serialize( + &self, + fb: &mut FlatBufferBuilder<'fb>, + ) -> (fb::FetchStep, Option>) { + match self { + odf::FetchStep::Url(v) => ( + fb::FetchStep::FetchStepUrl, + Some(v.serialize(fb).as_union_value()), + ), + odf::FetchStep::FilesGlob(v) => ( + fb::FetchStep::FetchStepFilesGlob, + Some(v.serialize(fb).as_union_value()), + ), + } + } +} + +impl<'fb> FlatbuffersEnumDeserializable<'fb, fb::FetchStep> for odf::FetchStep { + fn deserialize(table: Option>, t: fb::FetchStep) -> Option { + match t { + fb::FetchStep::NONE => panic!("Property is missing"), + fb::FetchStep::FetchStepUrl => Some(odf::FetchStep::Url( + odf::FetchStepUrl::deserialize(fb::FetchStepUrl::init_from_table(table.unwrap())), + )), + fb::FetchStep::FetchStepFilesGlob => Some(odf::FetchStep::FilesGlob( + odf::FetchStepFilesGlob::deserialize(fb::FetchStepFilesGlob::init_from_table( + table.unwrap(), + )), + )), + } + } +} + +impl<'fb> FlatbuffersSerializable<'fb> for odf::FetchStepUrl { + type OffsetT = WIPOffset>; + + fn serialize(&self, fb: &mut FlatBufferBuilder<'fb>) -> Self::OffsetT { + let url_offset = { fb.create_string(&self.url) }; + let event_time_offset = self.event_time.as_ref().map(|v| v.serialize(fb)); + let cache_offset = self.cache.as_ref().map(|v| v.serialize(fb)); + let mut builder = fb::FetchStepUrlBuilder::new(fb); + builder.add_url(url_offset); + event_time_offset.map(|off| { + builder.add_event_time_type(off.0); + off.1.map(|v| builder.add_event_time(v)); + }); + cache_offset.map(|off| { + builder.add_cache_type(off.0); + off.1.map(|v| builder.add_cache(v)); + }); + builder.finish() + } +} + +impl<'fb> FlatbuffersDeserializable> for odf::FetchStepUrl { + fn deserialize(proxy: fb::FetchStepUrl<'fb>) -> Self { + odf::FetchStepUrl { + url: proxy.url().map(|v| v.to_owned()).unwrap(), + event_time: odf::EventTimeSource::deserialize( + proxy.event_time(), + proxy.event_time_type(), + ), + cache: odf::SourceCaching::deserialize(proxy.cache(), proxy.cache_type()), + } + } +} + +impl<'fb> FlatbuffersSerializable<'fb> for odf::FetchStepFilesGlob { + type OffsetT = WIPOffset>; + + fn serialize(&self, fb: &mut FlatBufferBuilder<'fb>) -> Self::OffsetT { + let path_offset = { fb.create_string(&self.path) }; + let event_time_offset = self.event_time.as_ref().map(|v| v.serialize(fb)); + let cache_offset = self.cache.as_ref().map(|v| v.serialize(fb)); + let mut builder = fb::FetchStepFilesGlobBuilder::new(fb); + builder.add_path(path_offset); + event_time_offset.map(|off| { + builder.add_event_time_type(off.0); + off.1.map(|v| builder.add_event_time(v)); + }); + cache_offset.map(|off| { + builder.add_cache_type(off.0); + off.1.map(|v| builder.add_cache(v)); + }); + let order = self.order.map(|v| { + fb::Option_SourceOrdering::new(match v { + odf::SourceOrdering::ByEventTime => fb::SourceOrdering::ByEventTime, + odf::SourceOrdering::ByName => fb::SourceOrdering::ByName, + }) + }); + self.order + .as_ref() + .map(|_| builder.add_order(order.as_ref().unwrap())); + builder.finish() + } +} + +impl<'fb> FlatbuffersDeserializable> for odf::FetchStepFilesGlob { + fn deserialize(proxy: fb::FetchStepFilesGlob<'fb>) -> Self { + odf::FetchStepFilesGlob { + path: proxy.path().map(|v| v.to_owned()).unwrap(), + event_time: odf::EventTimeSource::deserialize( + proxy.event_time(), + proxy.event_time_type(), + ), + cache: odf::SourceCaching::deserialize(proxy.cache(), proxy.cache_type()), + order: proxy.order().map(|v| match v.value() { + fb::SourceOrdering::ByEventTime => odf::SourceOrdering::ByEventTime, + fb::SourceOrdering::ByName => odf::SourceOrdering::ByName, + }), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// PrepStep +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#prepstep-schema +//////////////////////////////////////////////////////////////////////////////// + +impl<'fb> FlatbuffersEnumSerializable<'fb, fb::PrepStep> for odf::PrepStep { + fn serialize( + &self, + fb: &mut FlatBufferBuilder<'fb>, + ) -> (fb::PrepStep, Option>) { + match self { + odf::PrepStep::Decompress(v) => ( + fb::PrepStep::PrepStepDecompress, + Some(v.serialize(fb).as_union_value()), + ), + odf::PrepStep::Pipe(v) => ( + fb::PrepStep::PrepStepPipe, + Some(v.serialize(fb).as_union_value()), + ), + } + } +} + +impl<'fb> FlatbuffersEnumDeserializable<'fb, fb::PrepStep> for odf::PrepStep { + fn deserialize(table: Option>, t: fb::PrepStep) -> Option { + match t { + fb::PrepStep::NONE => None, + fb::PrepStep::PrepStepDecompress => Some(odf::PrepStep::Decompress( + odf::PrepStepDecompress::deserialize(fb::PrepStepDecompress::init_from_table( + table.unwrap(), + )), + )), + fb::PrepStep::PrepStepPipe => Some(odf::PrepStep::Pipe( + odf::PrepStepPipe::deserialize(fb::PrepStepPipe::init_from_table(table.unwrap())), + )), + } + } +} + +impl<'fb> FlatbuffersSerializable<'fb> for odf::PrepStepDecompress { + type OffsetT = WIPOffset>; + + fn serialize(&self, fb: &mut FlatBufferBuilder<'fb>) -> Self::OffsetT { + let sub_path_offset = self.sub_path.as_ref().map(|v| fb.create_string(&v)); + let mut builder = fb::PrepStepDecompressBuilder::new(fb); + let format = fb::Option_CompressionFormat::new(match self.format { + odf::CompressionFormat::Gzip => fb::CompressionFormat::Gzip, + odf::CompressionFormat::Zip => fb::CompressionFormat::Zip, + }); + builder.add_format(&format); + sub_path_offset.map(|off| builder.add_sub_path(off)); + builder.finish() + } +} + +impl<'fb> FlatbuffersDeserializable> for odf::PrepStepDecompress { + fn deserialize(proxy: fb::PrepStepDecompress<'fb>) -> Self { + odf::PrepStepDecompress { + format: proxy + .format() + .map(|v| match v.value() { + fb::CompressionFormat::Gzip => odf::CompressionFormat::Gzip, + fb::CompressionFormat::Zip => odf::CompressionFormat::Zip, + }) + .unwrap(), + sub_path: proxy.sub_path().map(|v| v.to_owned()), + } + } +} + +impl<'fb> FlatbuffersSerializable<'fb> for odf::PrepStepPipe { + type OffsetT = WIPOffset>; + + fn serialize(&self, fb: &mut FlatBufferBuilder<'fb>) -> Self::OffsetT { + let command_offset = { + let offsets: Vec<_> = self.command.iter().map(|i| fb.create_string(&i)).collect(); + fb.create_vector(&offsets) + }; + let mut builder = fb::PrepStepPipeBuilder::new(fb); + builder.add_command(command_offset); + builder.finish() + } +} + +impl<'fb> FlatbuffersDeserializable> for odf::PrepStepPipe { + fn deserialize(proxy: fb::PrepStepPipe<'fb>) -> Self { + odf::PrepStepPipe { + command: proxy + .command() + .map(|v| v.iter().map(|i| i.to_owned()).collect()) + .unwrap(), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// EventTimeSource +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#eventtimesource-schema +//////////////////////////////////////////////////////////////////////////////// + +impl<'fb> FlatbuffersEnumSerializable<'fb, fb::EventTimeSource> for odf::EventTimeSource { + fn serialize( + &self, + fb: &mut FlatBufferBuilder<'fb>, + ) -> (fb::EventTimeSource, Option>) { + match self { + odf::EventTimeSource::FromMetadata => { + (fb::EventTimeSource::EventTimeSourceFromMetadata, None) + } + odf::EventTimeSource::FromPath(v) => ( + fb::EventTimeSource::EventTimeSourceFromPath, + Some(v.serialize(fb).as_union_value()), + ), + } + } +} + +impl<'fb> FlatbuffersEnumDeserializable<'fb, fb::EventTimeSource> for odf::EventTimeSource { + fn deserialize(table: Option>, t: fb::EventTimeSource) -> Option { + match t { + fb::EventTimeSource::NONE => None, + fb::EventTimeSource::EventTimeSourceFromMetadata => { + Some(odf::EventTimeSource::FromMetadata) + } + fb::EventTimeSource::EventTimeSourceFromPath => Some(odf::EventTimeSource::FromPath( + odf::EventTimeSourceFromPath::deserialize( + fb::EventTimeSourceFromPath::init_from_table(table.unwrap()), + ), + )), + } + } +} + +impl<'fb> FlatbuffersSerializable<'fb> for odf::EventTimeSourceFromPath { + type OffsetT = WIPOffset>; + + fn serialize(&self, fb: &mut FlatBufferBuilder<'fb>) -> Self::OffsetT { + let pattern_offset = { fb.create_string(&self.pattern) }; + let timestamp_format_offset = self.timestamp_format.as_ref().map(|v| fb.create_string(&v)); + let mut builder = fb::EventTimeSourceFromPathBuilder::new(fb); + builder.add_pattern(pattern_offset); + timestamp_format_offset.map(|off| builder.add_timestamp_format(off)); + builder.finish() + } +} + +impl<'fb> FlatbuffersDeserializable> + for odf::EventTimeSourceFromPath +{ + fn deserialize(proxy: fb::EventTimeSourceFromPath<'fb>) -> Self { + odf::EventTimeSourceFromPath { + pattern: proxy.pattern().map(|v| v.to_owned()).unwrap(), + timestamp_format: proxy.timestamp_format().map(|v| v.to_owned()), + } + } +} + +/////////////////////////////////////////////////////////////////////////////// +// Helpers +/////////////////////////////////////////////////////////////////////////////// + +fn datetime_to_fb(dt: &DateTime) -> fb::Timestamp { + fb::Timestamp::new( + dt.year(), + dt.ordinal() as u16, + dt.naive_utc().num_seconds_from_midnight(), + dt.naive_utc().nanosecond(), + ) +} + +fn fb_to_datetime(dt: &fb::Timestamp) -> DateTime { + Utc.yo(dt.year(), dt.ordinal() as u32) + .and_time( + NaiveTime::from_num_seconds_from_midnight_opt( + dt.seconds_from_midnight(), + dt.nanoseconds(), + ) + .unwrap(), + ) + .unwrap() +} + +fn interval_to_fb(iv: &odf::TimeInterval) -> fb::TimeInterval { + use intervals_general::interval::Interval; + match iv.0 { + Interval::Closed { bound_pair: p } => fb::TimeInterval::new( + fb::TimeIntervalType::Closed, + &datetime_to_fb(p.left()), + &datetime_to_fb(p.right()), + ), + Interval::Open { bound_pair: p } => fb::TimeInterval::new( + fb::TimeIntervalType::Open, + &datetime_to_fb(p.left()), + &datetime_to_fb(p.right()), + ), + Interval::LeftHalfOpen { bound_pair: p } => fb::TimeInterval::new( + fb::TimeIntervalType::LeftHalfOpen, + &datetime_to_fb(p.left()), + &datetime_to_fb(p.right()), + ), + Interval::RightHalfOpen { bound_pair: p } => fb::TimeInterval::new( + fb::TimeIntervalType::RightHalfOpen, + &datetime_to_fb(p.left()), + &datetime_to_fb(p.right()), + ), + Interval::UnboundedClosedRight { right } => fb::TimeInterval::new( + fb::TimeIntervalType::UnboundedClosedRight, + &datetime_to_fb(&Utc.yo(0, 1).and_hms(0, 0, 0)), + &datetime_to_fb(&right), + ), + Interval::UnboundedOpenRight { right } => fb::TimeInterval::new( + fb::TimeIntervalType::UnboundedOpenRight, + &datetime_to_fb(&Utc.yo(0, 1).and_hms(0, 0, 0)), + &datetime_to_fb(&right), + ), + Interval::UnboundedClosedLeft { left } => fb::TimeInterval::new( + fb::TimeIntervalType::UnboundedClosedLeft, + &datetime_to_fb(&left), + &datetime_to_fb(&Utc.yo(0, 1).and_hms(0, 0, 0)), + ), + Interval::UnboundedOpenLeft { left } => fb::TimeInterval::new( + fb::TimeIntervalType::UnboundedOpenLeft, + &datetime_to_fb(&left), + &datetime_to_fb(&Utc.yo(0, 1).and_hms(0, 0, 0)), + ), + Interval::Singleton { at } => fb::TimeInterval::new( + fb::TimeIntervalType::Singleton, + &datetime_to_fb(&at), + &datetime_to_fb(&Utc.yo(0, 1).and_hms(0, 0, 0)), + ), + Interval::Unbounded => fb::TimeInterval::new( + fb::TimeIntervalType::Unbounded, + &datetime_to_fb(&Utc.yo(0, 1).and_hms(0, 0, 0)), + &datetime_to_fb(&Utc.yo(0, 1).and_hms(0, 0, 0)), + ), + Interval::Empty => fb::TimeInterval::new( + fb::TimeIntervalType::Empty, + &datetime_to_fb(&Utc.yo(0, 1).and_hms(0, 0, 0)), + &datetime_to_fb(&Utc.yo(0, 1).and_hms(0, 0, 0)), + ), + } +} + +fn fb_to_interval(iv: &fb::TimeInterval) -> odf::TimeInterval { + match iv.type_() { + fb::TimeIntervalType::Closed => { + odf::TimeInterval::closed(fb_to_datetime(iv.left()), fb_to_datetime(iv.right())) + .unwrap() + } + fb::TimeIntervalType::Open => { + odf::TimeInterval::open(fb_to_datetime(iv.left()), fb_to_datetime(iv.right())).unwrap() + } + fb::TimeIntervalType::LeftHalfOpen => { + odf::TimeInterval::left_half_open(fb_to_datetime(iv.left()), fb_to_datetime(iv.right())) + .unwrap() + } + fb::TimeIntervalType::RightHalfOpen => odf::TimeInterval::right_half_open( + fb_to_datetime(iv.left()), + fb_to_datetime(iv.right()), + ) + .unwrap(), + fb::TimeIntervalType::UnboundedClosedRight => { + odf::TimeInterval::unbounded_closed_right(fb_to_datetime(iv.right())) + } + fb::TimeIntervalType::UnboundedOpenRight => { + odf::TimeInterval::unbounded_open_right(fb_to_datetime(iv.right())) + } + fb::TimeIntervalType::UnboundedClosedLeft => { + odf::TimeInterval::unbounded_closed_left(fb_to_datetime(iv.left())) + } + fb::TimeIntervalType::UnboundedOpenLeft => { + odf::TimeInterval::unbounded_open_left(fb_to_datetime(iv.left())) + } + fb::TimeIntervalType::Singleton => odf::TimeInterval::singleton(fb_to_datetime(iv.left())), + fb::TimeIntervalType::Unbounded => odf::TimeInterval::unbounded(), + fb::TimeIntervalType::Empty => odf::TimeInterval::empty(), + } +} diff --git a/opendatafabric/src/serde/flatbuffers/mod.rs b/opendatafabric/src/serde/flatbuffers/mod.rs new file mode 100644 index 0000000000..2ad9cac5ac --- /dev/null +++ b/opendatafabric/src/serde/flatbuffers/mod.rs @@ -0,0 +1,7 @@ +mod convertors; + +#[allow(dead_code, unused_imports)] +mod odf_generated; + +mod serdes; +pub use serdes::*; diff --git a/opendatafabric/src/serde/flatbuffers/odf_generated.rs b/opendatafabric/src/serde/flatbuffers/odf_generated.rs new file mode 100644 index 0000000000..da139cedd2 --- /dev/null +++ b/opendatafabric/src/serde/flatbuffers/odf_generated.rs @@ -0,0 +1,3982 @@ +// automatically generated by the FlatBuffers compiler, do not modify + + + +use std::mem; +use std::cmp::Ordering; + +extern crate flatbuffers; +use self::flatbuffers::EndianScalar; + +#[allow(non_camel_case_types)] +#[repr(u8)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub enum TimeIntervalType { + Closed = 0, + Open = 1, + LeftHalfOpen = 2, + RightHalfOpen = 3, + UnboundedClosedRight = 4, + UnboundedOpenRight = 5, + UnboundedClosedLeft = 6, + UnboundedOpenLeft = 7, + Singleton = 8, + Unbounded = 9, + Empty = 10, + +} + +pub const ENUM_MIN_TIME_INTERVAL_TYPE: u8 = 0; +pub const ENUM_MAX_TIME_INTERVAL_TYPE: u8 = 10; + +impl<'a> flatbuffers::Follow<'a> for TimeIntervalType { + type Inner = Self; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + flatbuffers::read_scalar_at::(buf, loc) + } +} + +impl flatbuffers::EndianScalar for TimeIntervalType { + #[inline] + fn to_little_endian(self) -> Self { + let n = u8::to_le(self as u8); + let p = &n as *const u8 as *const TimeIntervalType; + unsafe { *p } + } + #[inline] + fn from_little_endian(self) -> Self { + let n = u8::from_le(self as u8); + let p = &n as *const u8 as *const TimeIntervalType; + unsafe { *p } + } +} + +impl flatbuffers::Push for TimeIntervalType { + type Output = TimeIntervalType; + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + flatbuffers::emplace_scalar::(dst, *self); + } +} + +#[allow(non_camel_case_types)] +pub const ENUM_VALUES_TIME_INTERVAL_TYPE:[TimeIntervalType; 11] = [ + TimeIntervalType::Closed, + TimeIntervalType::Open, + TimeIntervalType::LeftHalfOpen, + TimeIntervalType::RightHalfOpen, + TimeIntervalType::UnboundedClosedRight, + TimeIntervalType::UnboundedOpenRight, + TimeIntervalType::UnboundedClosedLeft, + TimeIntervalType::UnboundedOpenLeft, + TimeIntervalType::Singleton, + TimeIntervalType::Unbounded, + TimeIntervalType::Empty +]; + +#[allow(non_camel_case_types)] +pub const ENUM_NAMES_TIME_INTERVAL_TYPE:[&'static str; 11] = [ + "Closed", + "Open", + "LeftHalfOpen", + "RightHalfOpen", + "UnboundedClosedRight", + "UnboundedOpenRight", + "UnboundedClosedLeft", + "UnboundedOpenLeft", + "Singleton", + "Unbounded", + "Empty" +]; + +pub fn enum_name_time_interval_type(e: TimeIntervalType) -> &'static str { + let index = e as u8; + ENUM_NAMES_TIME_INTERVAL_TYPE[index as usize] +} + +#[allow(non_camel_case_types)] +#[repr(u8)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub enum EventTimeSource { + NONE = 0, + EventTimeSourceFromMetadata = 1, + EventTimeSourceFromPath = 2, + +} + +pub const ENUM_MIN_EVENT_TIME_SOURCE: u8 = 0; +pub const ENUM_MAX_EVENT_TIME_SOURCE: u8 = 2; + +impl<'a> flatbuffers::Follow<'a> for EventTimeSource { + type Inner = Self; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + flatbuffers::read_scalar_at::(buf, loc) + } +} + +impl flatbuffers::EndianScalar for EventTimeSource { + #[inline] + fn to_little_endian(self) -> Self { + let n = u8::to_le(self as u8); + let p = &n as *const u8 as *const EventTimeSource; + unsafe { *p } + } + #[inline] + fn from_little_endian(self) -> Self { + let n = u8::from_le(self as u8); + let p = &n as *const u8 as *const EventTimeSource; + unsafe { *p } + } +} + +impl flatbuffers::Push for EventTimeSource { + type Output = EventTimeSource; + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + flatbuffers::emplace_scalar::(dst, *self); + } +} + +#[allow(non_camel_case_types)] +pub const ENUM_VALUES_EVENT_TIME_SOURCE:[EventTimeSource; 3] = [ + EventTimeSource::NONE, + EventTimeSource::EventTimeSourceFromMetadata, + EventTimeSource::EventTimeSourceFromPath +]; + +#[allow(non_camel_case_types)] +pub const ENUM_NAMES_EVENT_TIME_SOURCE:[&'static str; 3] = [ + "NONE", + "EventTimeSourceFromMetadata", + "EventTimeSourceFromPath" +]; + +pub fn enum_name_event_time_source(e: EventTimeSource) -> &'static str { + let index = e as u8; + ENUM_NAMES_EVENT_TIME_SOURCE[index as usize] +} + +pub struct EventTimeSourceUnionTableOffset {} +#[allow(non_camel_case_types)] +#[repr(u8)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub enum SourceCaching { + NONE = 0, + SourceCachingForever = 1, + +} + +pub const ENUM_MIN_SOURCE_CACHING: u8 = 0; +pub const ENUM_MAX_SOURCE_CACHING: u8 = 1; + +impl<'a> flatbuffers::Follow<'a> for SourceCaching { + type Inner = Self; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + flatbuffers::read_scalar_at::(buf, loc) + } +} + +impl flatbuffers::EndianScalar for SourceCaching { + #[inline] + fn to_little_endian(self) -> Self { + let n = u8::to_le(self as u8); + let p = &n as *const u8 as *const SourceCaching; + unsafe { *p } + } + #[inline] + fn from_little_endian(self) -> Self { + let n = u8::from_le(self as u8); + let p = &n as *const u8 as *const SourceCaching; + unsafe { *p } + } +} + +impl flatbuffers::Push for SourceCaching { + type Output = SourceCaching; + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + flatbuffers::emplace_scalar::(dst, *self); + } +} + +#[allow(non_camel_case_types)] +pub const ENUM_VALUES_SOURCE_CACHING:[SourceCaching; 2] = [ + SourceCaching::NONE, + SourceCaching::SourceCachingForever +]; + +#[allow(non_camel_case_types)] +pub const ENUM_NAMES_SOURCE_CACHING:[&'static str; 2] = [ + "NONE", + "SourceCachingForever" +]; + +pub fn enum_name_source_caching(e: SourceCaching) -> &'static str { + let index = e as u8; + ENUM_NAMES_SOURCE_CACHING[index as usize] +} + +pub struct SourceCachingUnionTableOffset {} +#[allow(non_camel_case_types)] +#[repr(i32)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub enum SourceOrdering { + ByEventTime = 0, + ByName = 1, + +} + +pub const ENUM_MIN_SOURCE_ORDERING: i32 = 0; +pub const ENUM_MAX_SOURCE_ORDERING: i32 = 1; + +impl<'a> flatbuffers::Follow<'a> for SourceOrdering { + type Inner = Self; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + flatbuffers::read_scalar_at::(buf, loc) + } +} + +impl flatbuffers::EndianScalar for SourceOrdering { + #[inline] + fn to_little_endian(self) -> Self { + let n = i32::to_le(self as i32); + let p = &n as *const i32 as *const SourceOrdering; + unsafe { *p } + } + #[inline] + fn from_little_endian(self) -> Self { + let n = i32::from_le(self as i32); + let p = &n as *const i32 as *const SourceOrdering; + unsafe { *p } + } +} + +impl flatbuffers::Push for SourceOrdering { + type Output = SourceOrdering; + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + flatbuffers::emplace_scalar::(dst, *self); + } +} + +#[allow(non_camel_case_types)] +pub const ENUM_VALUES_SOURCE_ORDERING:[SourceOrdering; 2] = [ + SourceOrdering::ByEventTime, + SourceOrdering::ByName +]; + +#[allow(non_camel_case_types)] +pub const ENUM_NAMES_SOURCE_ORDERING:[&'static str; 2] = [ + "ByEventTime", + "ByName" +]; + +pub fn enum_name_source_ordering(e: SourceOrdering) -> &'static str { + let index = e as i32; + ENUM_NAMES_SOURCE_ORDERING[index as usize] +} + +#[allow(non_camel_case_types)] +#[repr(u8)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub enum FetchStep { + NONE = 0, + FetchStepUrl = 1, + FetchStepFilesGlob = 2, + +} + +pub const ENUM_MIN_FETCH_STEP: u8 = 0; +pub const ENUM_MAX_FETCH_STEP: u8 = 2; + +impl<'a> flatbuffers::Follow<'a> for FetchStep { + type Inner = Self; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + flatbuffers::read_scalar_at::(buf, loc) + } +} + +impl flatbuffers::EndianScalar for FetchStep { + #[inline] + fn to_little_endian(self) -> Self { + let n = u8::to_le(self as u8); + let p = &n as *const u8 as *const FetchStep; + unsafe { *p } + } + #[inline] + fn from_little_endian(self) -> Self { + let n = u8::from_le(self as u8); + let p = &n as *const u8 as *const FetchStep; + unsafe { *p } + } +} + +impl flatbuffers::Push for FetchStep { + type Output = FetchStep; + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + flatbuffers::emplace_scalar::(dst, *self); + } +} + +#[allow(non_camel_case_types)] +pub const ENUM_VALUES_FETCH_STEP:[FetchStep; 3] = [ + FetchStep::NONE, + FetchStep::FetchStepUrl, + FetchStep::FetchStepFilesGlob +]; + +#[allow(non_camel_case_types)] +pub const ENUM_NAMES_FETCH_STEP:[&'static str; 3] = [ + "NONE", + "FetchStepUrl", + "FetchStepFilesGlob" +]; + +pub fn enum_name_fetch_step(e: FetchStep) -> &'static str { + let index = e as u8; + ENUM_NAMES_FETCH_STEP[index as usize] +} + +pub struct FetchStepUnionTableOffset {} +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +#[allow(non_camel_case_types)] +#[repr(i32)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub enum CompressionFormat { + Gzip = 0, + Zip = 1, + +} + +pub const ENUM_MIN_COMPRESSION_FORMAT: i32 = 0; +pub const ENUM_MAX_COMPRESSION_FORMAT: i32 = 1; + +impl<'a> flatbuffers::Follow<'a> for CompressionFormat { + type Inner = Self; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + flatbuffers::read_scalar_at::(buf, loc) + } +} + +impl flatbuffers::EndianScalar for CompressionFormat { + #[inline] + fn to_little_endian(self) -> Self { + let n = i32::to_le(self as i32); + let p = &n as *const i32 as *const CompressionFormat; + unsafe { *p } + } + #[inline] + fn from_little_endian(self) -> Self { + let n = i32::from_le(self as i32); + let p = &n as *const i32 as *const CompressionFormat; + unsafe { *p } + } +} + +impl flatbuffers::Push for CompressionFormat { + type Output = CompressionFormat; + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + flatbuffers::emplace_scalar::(dst, *self); + } +} + +#[allow(non_camel_case_types)] +pub const ENUM_VALUES_COMPRESSION_FORMAT:[CompressionFormat; 2] = [ + CompressionFormat::Gzip, + CompressionFormat::Zip +]; + +#[allow(non_camel_case_types)] +pub const ENUM_NAMES_COMPRESSION_FORMAT:[&'static str; 2] = [ + "Gzip", + "Zip" +]; + +pub fn enum_name_compression_format(e: CompressionFormat) -> &'static str { + let index = e as i32; + ENUM_NAMES_COMPRESSION_FORMAT[index as usize] +} + +#[allow(non_camel_case_types)] +#[repr(u8)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub enum PrepStep { + NONE = 0, + PrepStepDecompress = 1, + PrepStepPipe = 2, + +} + +pub const ENUM_MIN_PREP_STEP: u8 = 0; +pub const ENUM_MAX_PREP_STEP: u8 = 2; + +impl<'a> flatbuffers::Follow<'a> for PrepStep { + type Inner = Self; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + flatbuffers::read_scalar_at::(buf, loc) + } +} + +impl flatbuffers::EndianScalar for PrepStep { + #[inline] + fn to_little_endian(self) -> Self { + let n = u8::to_le(self as u8); + let p = &n as *const u8 as *const PrepStep; + unsafe { *p } + } + #[inline] + fn from_little_endian(self) -> Self { + let n = u8::from_le(self as u8); + let p = &n as *const u8 as *const PrepStep; + unsafe { *p } + } +} + +impl flatbuffers::Push for PrepStep { + type Output = PrepStep; + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + flatbuffers::emplace_scalar::(dst, *self); + } +} + +#[allow(non_camel_case_types)] +pub const ENUM_VALUES_PREP_STEP:[PrepStep; 3] = [ + PrepStep::NONE, + PrepStep::PrepStepDecompress, + PrepStep::PrepStepPipe +]; + +#[allow(non_camel_case_types)] +pub const ENUM_NAMES_PREP_STEP:[&'static str; 3] = [ + "NONE", + "PrepStepDecompress", + "PrepStepPipe" +]; + +pub fn enum_name_prep_step(e: PrepStep) -> &'static str { + let index = e as u8; + ENUM_NAMES_PREP_STEP[index as usize] +} + +pub struct PrepStepUnionTableOffset {} +#[allow(non_camel_case_types)] +#[repr(u8)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub enum ReadStep { + NONE = 0, + ReadStepCsv = 1, + ReadStepJsonLines = 2, + ReadStepGeoJson = 3, + ReadStepEsriShapefile = 4, + +} + +pub const ENUM_MIN_READ_STEP: u8 = 0; +pub const ENUM_MAX_READ_STEP: u8 = 4; + +impl<'a> flatbuffers::Follow<'a> for ReadStep { + type Inner = Self; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + flatbuffers::read_scalar_at::(buf, loc) + } +} + +impl flatbuffers::EndianScalar for ReadStep { + #[inline] + fn to_little_endian(self) -> Self { + let n = u8::to_le(self as u8); + let p = &n as *const u8 as *const ReadStep; + unsafe { *p } + } + #[inline] + fn from_little_endian(self) -> Self { + let n = u8::from_le(self as u8); + let p = &n as *const u8 as *const ReadStep; + unsafe { *p } + } +} + +impl flatbuffers::Push for ReadStep { + type Output = ReadStep; + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + flatbuffers::emplace_scalar::(dst, *self); + } +} + +#[allow(non_camel_case_types)] +pub const ENUM_VALUES_READ_STEP:[ReadStep; 5] = [ + ReadStep::NONE, + ReadStep::ReadStepCsv, + ReadStep::ReadStepJsonLines, + ReadStep::ReadStepGeoJson, + ReadStep::ReadStepEsriShapefile +]; + +#[allow(non_camel_case_types)] +pub const ENUM_NAMES_READ_STEP:[&'static str; 5] = [ + "NONE", + "ReadStepCsv", + "ReadStepJsonLines", + "ReadStepGeoJson", + "ReadStepEsriShapefile" +]; + +pub fn enum_name_read_step(e: ReadStep) -> &'static str { + let index = e as u8; + ENUM_NAMES_READ_STEP[index as usize] +} + +pub struct ReadStepUnionTableOffset {} +#[allow(non_camel_case_types)] +#[repr(u8)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub enum Transform { + NONE = 0, + TransformSql = 1, + +} + +pub const ENUM_MIN_TRANSFORM: u8 = 0; +pub const ENUM_MAX_TRANSFORM: u8 = 1; + +impl<'a> flatbuffers::Follow<'a> for Transform { + type Inner = Self; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + flatbuffers::read_scalar_at::(buf, loc) + } +} + +impl flatbuffers::EndianScalar for Transform { + #[inline] + fn to_little_endian(self) -> Self { + let n = u8::to_le(self as u8); + let p = &n as *const u8 as *const Transform; + unsafe { *p } + } + #[inline] + fn from_little_endian(self) -> Self { + let n = u8::from_le(self as u8); + let p = &n as *const u8 as *const Transform; + unsafe { *p } + } +} + +impl flatbuffers::Push for Transform { + type Output = Transform; + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + flatbuffers::emplace_scalar::(dst, *self); + } +} + +#[allow(non_camel_case_types)] +pub const ENUM_VALUES_TRANSFORM:[Transform; 2] = [ + Transform::NONE, + Transform::TransformSql +]; + +#[allow(non_camel_case_types)] +pub const ENUM_NAMES_TRANSFORM:[&'static str; 2] = [ + "NONE", + "TransformSql" +]; + +pub fn enum_name_transform(e: Transform) -> &'static str { + let index = e as u8; + ENUM_NAMES_TRANSFORM[index as usize] +} + +pub struct TransformUnionTableOffset {} +#[allow(non_camel_case_types)] +#[repr(u8)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub enum MergeStrategy { + NONE = 0, + MergeStrategyAppend = 1, + MergeStrategyLedger = 2, + MergeStrategySnapshot = 3, + +} + +pub const ENUM_MIN_MERGE_STRATEGY: u8 = 0; +pub const ENUM_MAX_MERGE_STRATEGY: u8 = 3; + +impl<'a> flatbuffers::Follow<'a> for MergeStrategy { + type Inner = Self; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + flatbuffers::read_scalar_at::(buf, loc) + } +} + +impl flatbuffers::EndianScalar for MergeStrategy { + #[inline] + fn to_little_endian(self) -> Self { + let n = u8::to_le(self as u8); + let p = &n as *const u8 as *const MergeStrategy; + unsafe { *p } + } + #[inline] + fn from_little_endian(self) -> Self { + let n = u8::from_le(self as u8); + let p = &n as *const u8 as *const MergeStrategy; + unsafe { *p } + } +} + +impl flatbuffers::Push for MergeStrategy { + type Output = MergeStrategy; + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + flatbuffers::emplace_scalar::(dst, *self); + } +} + +#[allow(non_camel_case_types)] +pub const ENUM_VALUES_MERGE_STRATEGY:[MergeStrategy; 4] = [ + MergeStrategy::NONE, + MergeStrategy::MergeStrategyAppend, + MergeStrategy::MergeStrategyLedger, + MergeStrategy::MergeStrategySnapshot +]; + +#[allow(non_camel_case_types)] +pub const ENUM_NAMES_MERGE_STRATEGY:[&'static str; 4] = [ + "NONE", + "MergeStrategyAppend", + "MergeStrategyLedger", + "MergeStrategySnapshot" +]; + +pub fn enum_name_merge_strategy(e: MergeStrategy) -> &'static str { + let index = e as u8; + ENUM_NAMES_MERGE_STRATEGY[index as usize] +} + +pub struct MergeStrategyUnionTableOffset {} +#[allow(non_camel_case_types)] +#[repr(u8)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub enum DatasetSource { + NONE = 0, + DatasetSourceRoot = 1, + DatasetSourceDerivative = 2, + +} + +pub const ENUM_MIN_DATASET_SOURCE: u8 = 0; +pub const ENUM_MAX_DATASET_SOURCE: u8 = 2; + +impl<'a> flatbuffers::Follow<'a> for DatasetSource { + type Inner = Self; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + flatbuffers::read_scalar_at::(buf, loc) + } +} + +impl flatbuffers::EndianScalar for DatasetSource { + #[inline] + fn to_little_endian(self) -> Self { + let n = u8::to_le(self as u8); + let p = &n as *const u8 as *const DatasetSource; + unsafe { *p } + } + #[inline] + fn from_little_endian(self) -> Self { + let n = u8::from_le(self as u8); + let p = &n as *const u8 as *const DatasetSource; + unsafe { *p } + } +} + +impl flatbuffers::Push for DatasetSource { + type Output = DatasetSource; + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + flatbuffers::emplace_scalar::(dst, *self); + } +} + +#[allow(non_camel_case_types)] +pub const ENUM_VALUES_DATASET_SOURCE:[DatasetSource; 3] = [ + DatasetSource::NONE, + DatasetSource::DatasetSourceRoot, + DatasetSource::DatasetSourceDerivative +]; + +#[allow(non_camel_case_types)] +pub const ENUM_NAMES_DATASET_SOURCE:[&'static str; 3] = [ + "NONE", + "DatasetSourceRoot", + "DatasetSourceDerivative" +]; + +pub fn enum_name_dataset_source(e: DatasetSource) -> &'static str { + let index = e as u8; + ENUM_NAMES_DATASET_SOURCE[index as usize] +} + +pub struct DatasetSourceUnionTableOffset {} +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +// struct Option_bool, aligned to 1 +#[repr(C, align(1))] +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Option_bool { + value_: bool, +} // pub struct Option_bool +impl flatbuffers::SafeSliceAccess for Option_bool {} +impl<'a> flatbuffers::Follow<'a> for Option_bool { + type Inner = &'a Option_bool; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + <&'a Option_bool>::follow(buf, loc) + } +} +impl<'a> flatbuffers::Follow<'a> for &'a Option_bool { + type Inner = &'a Option_bool; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + flatbuffers::follow_cast_ref::(buf, loc) + } +} +impl<'b> flatbuffers::Push for Option_bool { + type Output = Option_bool; + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + let src = unsafe { + ::std::slice::from_raw_parts(self as *const Option_bool as *const u8, Self::size()) + }; + dst.copy_from_slice(src); + } +} +impl<'b> flatbuffers::Push for &'b Option_bool { + type Output = Option_bool; + + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + let src = unsafe { + ::std::slice::from_raw_parts(*self as *const Option_bool as *const u8, Self::size()) + }; + dst.copy_from_slice(src); + } +} + + +impl Option_bool { + pub fn new<'a>(_value: bool) -> Self { + Option_bool { + value_: _value.to_little_endian(), + + } + } + pub fn value<'a>(&'a self) -> bool { + self.value_.from_little_endian() + } +} + +// struct Option_int64, aligned to 8 +#[repr(C, align(8))] +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Option_int64 { + value_: i64, +} // pub struct Option_int64 +impl flatbuffers::SafeSliceAccess for Option_int64 {} +impl<'a> flatbuffers::Follow<'a> for Option_int64 { + type Inner = &'a Option_int64; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + <&'a Option_int64>::follow(buf, loc) + } +} +impl<'a> flatbuffers::Follow<'a> for &'a Option_int64 { + type Inner = &'a Option_int64; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + flatbuffers::follow_cast_ref::(buf, loc) + } +} +impl<'b> flatbuffers::Push for Option_int64 { + type Output = Option_int64; + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + let src = unsafe { + ::std::slice::from_raw_parts(self as *const Option_int64 as *const u8, Self::size()) + }; + dst.copy_from_slice(src); + } +} +impl<'b> flatbuffers::Push for &'b Option_int64 { + type Output = Option_int64; + + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + let src = unsafe { + ::std::slice::from_raw_parts(*self as *const Option_int64 as *const u8, Self::size()) + }; + dst.copy_from_slice(src); + } +} + + +impl Option_int64 { + pub fn new<'a>(_value: i64) -> Self { + Option_int64 { + value_: _value.to_little_endian(), + + } + } + pub fn value<'a>(&'a self) -> i64 { + self.value_.from_little_endian() + } +} + +// struct Timestamp, aligned to 4 +#[repr(C, align(4))] +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Timestamp { + year_: i32, + ordinal_: u16, + padding0__: u16, + seconds_from_midnight_: u32, + nanoseconds_: u32, +} // pub struct Timestamp +impl flatbuffers::SafeSliceAccess for Timestamp {} +impl<'a> flatbuffers::Follow<'a> for Timestamp { + type Inner = &'a Timestamp; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + <&'a Timestamp>::follow(buf, loc) + } +} +impl<'a> flatbuffers::Follow<'a> for &'a Timestamp { + type Inner = &'a Timestamp; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + flatbuffers::follow_cast_ref::(buf, loc) + } +} +impl<'b> flatbuffers::Push for Timestamp { + type Output = Timestamp; + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + let src = unsafe { + ::std::slice::from_raw_parts(self as *const Timestamp as *const u8, Self::size()) + }; + dst.copy_from_slice(src); + } +} +impl<'b> flatbuffers::Push for &'b Timestamp { + type Output = Timestamp; + + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + let src = unsafe { + ::std::slice::from_raw_parts(*self as *const Timestamp as *const u8, Self::size()) + }; + dst.copy_from_slice(src); + } +} + + +impl Timestamp { + pub fn new<'a>(_year: i32, _ordinal: u16, _seconds_from_midnight: u32, _nanoseconds: u32) -> Self { + Timestamp { + year_: _year.to_little_endian(), + ordinal_: _ordinal.to_little_endian(), + seconds_from_midnight_: _seconds_from_midnight.to_little_endian(), + nanoseconds_: _nanoseconds.to_little_endian(), + + padding0__: 0, + } + } + pub fn year<'a>(&'a self) -> i32 { + self.year_.from_little_endian() + } + pub fn ordinal<'a>(&'a self) -> u16 { + self.ordinal_.from_little_endian() + } + pub fn seconds_from_midnight<'a>(&'a self) -> u32 { + self.seconds_from_midnight_.from_little_endian() + } + pub fn nanoseconds<'a>(&'a self) -> u32 { + self.nanoseconds_.from_little_endian() + } +} + +// struct TimeInterval, aligned to 4 +#[repr(C, align(4))] +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct TimeInterval { + type__: TimeIntervalType, + padding0__: u8, padding1__: u16, + left_: Timestamp, + right_: Timestamp, +} // pub struct TimeInterval +impl flatbuffers::SafeSliceAccess for TimeInterval {} +impl<'a> flatbuffers::Follow<'a> for TimeInterval { + type Inner = &'a TimeInterval; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + <&'a TimeInterval>::follow(buf, loc) + } +} +impl<'a> flatbuffers::Follow<'a> for &'a TimeInterval { + type Inner = &'a TimeInterval; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + flatbuffers::follow_cast_ref::(buf, loc) + } +} +impl<'b> flatbuffers::Push for TimeInterval { + type Output = TimeInterval; + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + let src = unsafe { + ::std::slice::from_raw_parts(self as *const TimeInterval as *const u8, Self::size()) + }; + dst.copy_from_slice(src); + } +} +impl<'b> flatbuffers::Push for &'b TimeInterval { + type Output = TimeInterval; + + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + let src = unsafe { + ::std::slice::from_raw_parts(*self as *const TimeInterval as *const u8, Self::size()) + }; + dst.copy_from_slice(src); + } +} + + +impl TimeInterval { + pub fn new<'a>(_type_: TimeIntervalType, _left: &'a Timestamp, _right: &'a Timestamp) -> Self { + TimeInterval { + type__: _type_.to_little_endian(), + left_: *_left, + right_: *_right, + + padding0__: 0,padding1__: 0, + } + } + pub fn type_<'a>(&'a self) -> TimeIntervalType { + self.type__.from_little_endian() + } + pub fn left<'a>(&'a self) -> &'a Timestamp { + &self.left_ + } + pub fn right<'a>(&'a self) -> &'a Timestamp { + &self.right_ + } +} + +// struct Option_SourceOrdering, aligned to 4 +#[repr(C, align(4))] +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Option_SourceOrdering { + value_: SourceOrdering, +} // pub struct Option_SourceOrdering +impl flatbuffers::SafeSliceAccess for Option_SourceOrdering {} +impl<'a> flatbuffers::Follow<'a> for Option_SourceOrdering { + type Inner = &'a Option_SourceOrdering; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + <&'a Option_SourceOrdering>::follow(buf, loc) + } +} +impl<'a> flatbuffers::Follow<'a> for &'a Option_SourceOrdering { + type Inner = &'a Option_SourceOrdering; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + flatbuffers::follow_cast_ref::(buf, loc) + } +} +impl<'b> flatbuffers::Push for Option_SourceOrdering { + type Output = Option_SourceOrdering; + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + let src = unsafe { + ::std::slice::from_raw_parts(self as *const Option_SourceOrdering as *const u8, Self::size()) + }; + dst.copy_from_slice(src); + } +} +impl<'b> flatbuffers::Push for &'b Option_SourceOrdering { + type Output = Option_SourceOrdering; + + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + let src = unsafe { + ::std::slice::from_raw_parts(*self as *const Option_SourceOrdering as *const u8, Self::size()) + }; + dst.copy_from_slice(src); + } +} + + +impl Option_SourceOrdering { + pub fn new<'a>(_value: SourceOrdering) -> Self { + Option_SourceOrdering { + value_: _value.to_little_endian(), + + } + } + pub fn value<'a>(&'a self) -> SourceOrdering { + self.value_.from_little_endian() + } +} + +// struct Option_CompressionFormat, aligned to 4 +#[repr(C, align(4))] +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Option_CompressionFormat { + value_: CompressionFormat, +} // pub struct Option_CompressionFormat +impl flatbuffers::SafeSliceAccess for Option_CompressionFormat {} +impl<'a> flatbuffers::Follow<'a> for Option_CompressionFormat { + type Inner = &'a Option_CompressionFormat; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + <&'a Option_CompressionFormat>::follow(buf, loc) + } +} +impl<'a> flatbuffers::Follow<'a> for &'a Option_CompressionFormat { + type Inner = &'a Option_CompressionFormat; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + flatbuffers::follow_cast_ref::(buf, loc) + } +} +impl<'b> flatbuffers::Push for Option_CompressionFormat { + type Output = Option_CompressionFormat; + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + let src = unsafe { + ::std::slice::from_raw_parts(self as *const Option_CompressionFormat as *const u8, Self::size()) + }; + dst.copy_from_slice(src); + } +} +impl<'b> flatbuffers::Push for &'b Option_CompressionFormat { + type Output = Option_CompressionFormat; + + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + let src = unsafe { + ::std::slice::from_raw_parts(*self as *const Option_CompressionFormat as *const u8, Self::size()) + }; + dst.copy_from_slice(src); + } +} + + +impl Option_CompressionFormat { + pub fn new<'a>(_value: CompressionFormat) -> Self { + Option_CompressionFormat { + value_: _value.to_little_endian(), + + } + } + pub fn value<'a>(&'a self) -> CompressionFormat { + self.value_.from_little_endian() + } +} + +pub enum EventTimeSourceFromMetadataOffset {} +#[derive(Copy, Clone, Debug, PartialEq)] + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +pub struct EventTimeSourceFromMetadata<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for EventTimeSourceFromMetadata<'a> { + type Inner = EventTimeSourceFromMetadata<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf: buf, loc: loc }, + } + } +} + +impl<'a> EventTimeSourceFromMetadata<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + EventTimeSourceFromMetadata { + _tab: table, + } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + _args: &'args EventTimeSourceFromMetadataArgs) -> flatbuffers::WIPOffset> { + let mut builder = EventTimeSourceFromMetadataBuilder::new(_fbb); + builder.finish() + } + +} + +pub struct EventTimeSourceFromMetadataArgs { +} +impl<'a> Default for EventTimeSourceFromMetadataArgs { + #[inline] + fn default() -> Self { + EventTimeSourceFromMetadataArgs { + } + } +} +pub struct EventTimeSourceFromMetadataBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b> EventTimeSourceFromMetadataBuilder<'a, 'b> { + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> EventTimeSourceFromMetadataBuilder<'a, 'b> { + let start = _fbb.start_table(); + EventTimeSourceFromMetadataBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} + +pub enum EventTimeSourceFromPathOffset {} +#[derive(Copy, Clone, Debug, PartialEq)] + +pub struct EventTimeSourceFromPath<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for EventTimeSourceFromPath<'a> { + type Inner = EventTimeSourceFromPath<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf: buf, loc: loc }, + } + } +} + +impl<'a> EventTimeSourceFromPath<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + EventTimeSourceFromPath { + _tab: table, + } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args EventTimeSourceFromPathArgs<'args>) -> flatbuffers::WIPOffset> { + let mut builder = EventTimeSourceFromPathBuilder::new(_fbb); + if let Some(x) = args.timestamp_format { builder.add_timestamp_format(x); } + if let Some(x) = args.pattern { builder.add_pattern(x); } + builder.finish() + } + + pub const VT_PATTERN: flatbuffers::VOffsetT = 4; + pub const VT_TIMESTAMP_FORMAT: flatbuffers::VOffsetT = 6; + + #[inline] + pub fn pattern(&self) -> Option<&'a str> { + self._tab.get::>(EventTimeSourceFromPath::VT_PATTERN, None) + } + #[inline] + pub fn timestamp_format(&self) -> Option<&'a str> { + self._tab.get::>(EventTimeSourceFromPath::VT_TIMESTAMP_FORMAT, None) + } +} + +pub struct EventTimeSourceFromPathArgs<'a> { + pub pattern: Option>, + pub timestamp_format: Option>, +} +impl<'a> Default for EventTimeSourceFromPathArgs<'a> { + #[inline] + fn default() -> Self { + EventTimeSourceFromPathArgs { + pattern: None, + timestamp_format: None, + } + } +} +pub struct EventTimeSourceFromPathBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b> EventTimeSourceFromPathBuilder<'a, 'b> { + #[inline] + pub fn add_pattern(&mut self, pattern: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(EventTimeSourceFromPath::VT_PATTERN, pattern); + } + #[inline] + pub fn add_timestamp_format(&mut self, timestamp_format: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(EventTimeSourceFromPath::VT_TIMESTAMP_FORMAT, timestamp_format); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> EventTimeSourceFromPathBuilder<'a, 'b> { + let start = _fbb.start_table(); + EventTimeSourceFromPathBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} + +pub enum SourceCachingForeverOffset {} +#[derive(Copy, Clone, Debug, PartialEq)] + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +pub struct SourceCachingForever<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for SourceCachingForever<'a> { + type Inner = SourceCachingForever<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf: buf, loc: loc }, + } + } +} + +impl<'a> SourceCachingForever<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + SourceCachingForever { + _tab: table, + } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + _args: &'args SourceCachingForeverArgs) -> flatbuffers::WIPOffset> { + let mut builder = SourceCachingForeverBuilder::new(_fbb); + builder.finish() + } + +} + +pub struct SourceCachingForeverArgs { +} +impl<'a> Default for SourceCachingForeverArgs { + #[inline] + fn default() -> Self { + SourceCachingForeverArgs { + } + } +} +pub struct SourceCachingForeverBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b> SourceCachingForeverBuilder<'a, 'b> { + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> SourceCachingForeverBuilder<'a, 'b> { + let start = _fbb.start_table(); + SourceCachingForeverBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} + +pub enum FetchStepUrlOffset {} +#[derive(Copy, Clone, Debug, PartialEq)] + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +pub struct FetchStepUrl<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for FetchStepUrl<'a> { + type Inner = FetchStepUrl<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf: buf, loc: loc }, + } + } +} + +impl<'a> FetchStepUrl<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + FetchStepUrl { + _tab: table, + } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args FetchStepUrlArgs<'args>) -> flatbuffers::WIPOffset> { + let mut builder = FetchStepUrlBuilder::new(_fbb); + if let Some(x) = args.cache { builder.add_cache(x); } + if let Some(x) = args.event_time { builder.add_event_time(x); } + if let Some(x) = args.url { builder.add_url(x); } + builder.add_cache_type(args.cache_type); + builder.add_event_time_type(args.event_time_type); + builder.finish() + } + + pub const VT_URL: flatbuffers::VOffsetT = 4; + pub const VT_EVENT_TIME_TYPE: flatbuffers::VOffsetT = 6; + pub const VT_EVENT_TIME: flatbuffers::VOffsetT = 8; + pub const VT_CACHE_TYPE: flatbuffers::VOffsetT = 10; + pub const VT_CACHE: flatbuffers::VOffsetT = 12; + + #[inline] + pub fn url(&self) -> Option<&'a str> { + self._tab.get::>(FetchStepUrl::VT_URL, None) + } + #[inline] + pub fn event_time_type(&self) -> EventTimeSource { + self._tab.get::(FetchStepUrl::VT_EVENT_TIME_TYPE, Some(EventTimeSource::NONE)).unwrap() + } + #[inline] + pub fn event_time(&self) -> Option> { + self._tab.get::>>(FetchStepUrl::VT_EVENT_TIME, None) + } + #[inline] + pub fn cache_type(&self) -> SourceCaching { + self._tab.get::(FetchStepUrl::VT_CACHE_TYPE, Some(SourceCaching::NONE)).unwrap() + } + #[inline] + pub fn cache(&self) -> Option> { + self._tab.get::>>(FetchStepUrl::VT_CACHE, None) + } + #[inline] + #[allow(non_snake_case)] + pub fn event_time_as_event_time_source_from_metadata(&self) -> Option> { + if self.event_time_type() == EventTimeSource::EventTimeSourceFromMetadata { + self.event_time().map(|u| EventTimeSourceFromMetadata::init_from_table(u)) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn event_time_as_event_time_source_from_path(&self) -> Option> { + if self.event_time_type() == EventTimeSource::EventTimeSourceFromPath { + self.event_time().map(|u| EventTimeSourceFromPath::init_from_table(u)) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn cache_as_source_caching_forever(&self) -> Option> { + if self.cache_type() == SourceCaching::SourceCachingForever { + self.cache().map(|u| SourceCachingForever::init_from_table(u)) + } else { + None + } + } + +} + +pub struct FetchStepUrlArgs<'a> { + pub url: Option>, + pub event_time_type: EventTimeSource, + pub event_time: Option>, + pub cache_type: SourceCaching, + pub cache: Option>, +} +impl<'a> Default for FetchStepUrlArgs<'a> { + #[inline] + fn default() -> Self { + FetchStepUrlArgs { + url: None, + event_time_type: EventTimeSource::NONE, + event_time: None, + cache_type: SourceCaching::NONE, + cache: None, + } + } +} +pub struct FetchStepUrlBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b> FetchStepUrlBuilder<'a, 'b> { + #[inline] + pub fn add_url(&mut self, url: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(FetchStepUrl::VT_URL, url); + } + #[inline] + pub fn add_event_time_type(&mut self, event_time_type: EventTimeSource) { + self.fbb_.push_slot::(FetchStepUrl::VT_EVENT_TIME_TYPE, event_time_type, EventTimeSource::NONE); + } + #[inline] + pub fn add_event_time(&mut self, event_time: flatbuffers::WIPOffset) { + self.fbb_.push_slot_always::>(FetchStepUrl::VT_EVENT_TIME, event_time); + } + #[inline] + pub fn add_cache_type(&mut self, cache_type: SourceCaching) { + self.fbb_.push_slot::(FetchStepUrl::VT_CACHE_TYPE, cache_type, SourceCaching::NONE); + } + #[inline] + pub fn add_cache(&mut self, cache: flatbuffers::WIPOffset) { + self.fbb_.push_slot_always::>(FetchStepUrl::VT_CACHE, cache); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> FetchStepUrlBuilder<'a, 'b> { + let start = _fbb.start_table(); + FetchStepUrlBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} + +pub enum FetchStepFilesGlobOffset {} +#[derive(Copy, Clone, Debug, PartialEq)] + +pub struct FetchStepFilesGlob<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for FetchStepFilesGlob<'a> { + type Inner = FetchStepFilesGlob<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf: buf, loc: loc }, + } + } +} + +impl<'a> FetchStepFilesGlob<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + FetchStepFilesGlob { + _tab: table, + } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args FetchStepFilesGlobArgs<'args>) -> flatbuffers::WIPOffset> { + let mut builder = FetchStepFilesGlobBuilder::new(_fbb); + if let Some(x) = args.order { builder.add_order(x); } + if let Some(x) = args.cache { builder.add_cache(x); } + if let Some(x) = args.event_time { builder.add_event_time(x); } + if let Some(x) = args.path { builder.add_path(x); } + builder.add_cache_type(args.cache_type); + builder.add_event_time_type(args.event_time_type); + builder.finish() + } + + pub const VT_PATH: flatbuffers::VOffsetT = 4; + pub const VT_EVENT_TIME_TYPE: flatbuffers::VOffsetT = 6; + pub const VT_EVENT_TIME: flatbuffers::VOffsetT = 8; + pub const VT_CACHE_TYPE: flatbuffers::VOffsetT = 10; + pub const VT_CACHE: flatbuffers::VOffsetT = 12; + pub const VT_ORDER: flatbuffers::VOffsetT = 14; + + #[inline] + pub fn path(&self) -> Option<&'a str> { + self._tab.get::>(FetchStepFilesGlob::VT_PATH, None) + } + #[inline] + pub fn event_time_type(&self) -> EventTimeSource { + self._tab.get::(FetchStepFilesGlob::VT_EVENT_TIME_TYPE, Some(EventTimeSource::NONE)).unwrap() + } + #[inline] + pub fn event_time(&self) -> Option> { + self._tab.get::>>(FetchStepFilesGlob::VT_EVENT_TIME, None) + } + #[inline] + pub fn cache_type(&self) -> SourceCaching { + self._tab.get::(FetchStepFilesGlob::VT_CACHE_TYPE, Some(SourceCaching::NONE)).unwrap() + } + #[inline] + pub fn cache(&self) -> Option> { + self._tab.get::>>(FetchStepFilesGlob::VT_CACHE, None) + } + #[inline] + pub fn order(&self) -> Option<&'a Option_SourceOrdering> { + self._tab.get::(FetchStepFilesGlob::VT_ORDER, None) + } + #[inline] + #[allow(non_snake_case)] + pub fn event_time_as_event_time_source_from_metadata(&self) -> Option> { + if self.event_time_type() == EventTimeSource::EventTimeSourceFromMetadata { + self.event_time().map(|u| EventTimeSourceFromMetadata::init_from_table(u)) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn event_time_as_event_time_source_from_path(&self) -> Option> { + if self.event_time_type() == EventTimeSource::EventTimeSourceFromPath { + self.event_time().map(|u| EventTimeSourceFromPath::init_from_table(u)) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn cache_as_source_caching_forever(&self) -> Option> { + if self.cache_type() == SourceCaching::SourceCachingForever { + self.cache().map(|u| SourceCachingForever::init_from_table(u)) + } else { + None + } + } + +} + +pub struct FetchStepFilesGlobArgs<'a> { + pub path: Option>, + pub event_time_type: EventTimeSource, + pub event_time: Option>, + pub cache_type: SourceCaching, + pub cache: Option>, + pub order: Option<&'a Option_SourceOrdering>, +} +impl<'a> Default for FetchStepFilesGlobArgs<'a> { + #[inline] + fn default() -> Self { + FetchStepFilesGlobArgs { + path: None, + event_time_type: EventTimeSource::NONE, + event_time: None, + cache_type: SourceCaching::NONE, + cache: None, + order: None, + } + } +} +pub struct FetchStepFilesGlobBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b> FetchStepFilesGlobBuilder<'a, 'b> { + #[inline] + pub fn add_path(&mut self, path: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(FetchStepFilesGlob::VT_PATH, path); + } + #[inline] + pub fn add_event_time_type(&mut self, event_time_type: EventTimeSource) { + self.fbb_.push_slot::(FetchStepFilesGlob::VT_EVENT_TIME_TYPE, event_time_type, EventTimeSource::NONE); + } + #[inline] + pub fn add_event_time(&mut self, event_time: flatbuffers::WIPOffset) { + self.fbb_.push_slot_always::>(FetchStepFilesGlob::VT_EVENT_TIME, event_time); + } + #[inline] + pub fn add_cache_type(&mut self, cache_type: SourceCaching) { + self.fbb_.push_slot::(FetchStepFilesGlob::VT_CACHE_TYPE, cache_type, SourceCaching::NONE); + } + #[inline] + pub fn add_cache(&mut self, cache: flatbuffers::WIPOffset) { + self.fbb_.push_slot_always::>(FetchStepFilesGlob::VT_CACHE, cache); + } + #[inline] + pub fn add_order(&mut self, order: &'b Option_SourceOrdering) { + self.fbb_.push_slot_always::<&Option_SourceOrdering>(FetchStepFilesGlob::VT_ORDER, order); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> FetchStepFilesGlobBuilder<'a, 'b> { + let start = _fbb.start_table(); + FetchStepFilesGlobBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} + +pub enum PrepStepDecompressOffset {} +#[derive(Copy, Clone, Debug, PartialEq)] + +pub struct PrepStepDecompress<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for PrepStepDecompress<'a> { + type Inner = PrepStepDecompress<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf: buf, loc: loc }, + } + } +} + +impl<'a> PrepStepDecompress<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + PrepStepDecompress { + _tab: table, + } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args PrepStepDecompressArgs<'args>) -> flatbuffers::WIPOffset> { + let mut builder = PrepStepDecompressBuilder::new(_fbb); + if let Some(x) = args.sub_path { builder.add_sub_path(x); } + if let Some(x) = args.format { builder.add_format(x); } + builder.finish() + } + + pub const VT_FORMAT: flatbuffers::VOffsetT = 4; + pub const VT_SUB_PATH: flatbuffers::VOffsetT = 6; + + #[inline] + pub fn format(&self) -> Option<&'a Option_CompressionFormat> { + self._tab.get::(PrepStepDecompress::VT_FORMAT, None) + } + #[inline] + pub fn sub_path(&self) -> Option<&'a str> { + self._tab.get::>(PrepStepDecompress::VT_SUB_PATH, None) + } +} + +pub struct PrepStepDecompressArgs<'a> { + pub format: Option<&'a Option_CompressionFormat>, + pub sub_path: Option>, +} +impl<'a> Default for PrepStepDecompressArgs<'a> { + #[inline] + fn default() -> Self { + PrepStepDecompressArgs { + format: None, + sub_path: None, + } + } +} +pub struct PrepStepDecompressBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b> PrepStepDecompressBuilder<'a, 'b> { + #[inline] + pub fn add_format(&mut self, format: &'b Option_CompressionFormat) { + self.fbb_.push_slot_always::<&Option_CompressionFormat>(PrepStepDecompress::VT_FORMAT, format); + } + #[inline] + pub fn add_sub_path(&mut self, sub_path: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(PrepStepDecompress::VT_SUB_PATH, sub_path); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> PrepStepDecompressBuilder<'a, 'b> { + let start = _fbb.start_table(); + PrepStepDecompressBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} + +pub enum PrepStepPipeOffset {} +#[derive(Copy, Clone, Debug, PartialEq)] + +pub struct PrepStepPipe<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for PrepStepPipe<'a> { + type Inner = PrepStepPipe<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf: buf, loc: loc }, + } + } +} + +impl<'a> PrepStepPipe<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + PrepStepPipe { + _tab: table, + } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args PrepStepPipeArgs<'args>) -> flatbuffers::WIPOffset> { + let mut builder = PrepStepPipeBuilder::new(_fbb); + if let Some(x) = args.command { builder.add_command(x); } + builder.finish() + } + + pub const VT_COMMAND: flatbuffers::VOffsetT = 4; + + #[inline] + pub fn command(&self) -> Option>> { + self._tab.get::>>>(PrepStepPipe::VT_COMMAND, None) + } +} + +pub struct PrepStepPipeArgs<'a> { + pub command: Option>>>, +} +impl<'a> Default for PrepStepPipeArgs<'a> { + #[inline] + fn default() -> Self { + PrepStepPipeArgs { + command: None, + } + } +} +pub struct PrepStepPipeBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b> PrepStepPipeBuilder<'a, 'b> { + #[inline] + pub fn add_command(&mut self, command: flatbuffers::WIPOffset>>) { + self.fbb_.push_slot_always::>(PrepStepPipe::VT_COMMAND, command); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> PrepStepPipeBuilder<'a, 'b> { + let start = _fbb.start_table(); + PrepStepPipeBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} + +pub enum ReadStepCsvOffset {} +#[derive(Copy, Clone, Debug, PartialEq)] + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +pub struct ReadStepCsv<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for ReadStepCsv<'a> { + type Inner = ReadStepCsv<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf: buf, loc: loc }, + } + } +} + +impl<'a> ReadStepCsv<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + ReadStepCsv { + _tab: table, + } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args ReadStepCsvArgs<'args>) -> flatbuffers::WIPOffset> { + let mut builder = ReadStepCsvBuilder::new(_fbb); + if let Some(x) = args.multi_line { builder.add_multi_line(x); } + if let Some(x) = args.timestamp_format { builder.add_timestamp_format(x); } + if let Some(x) = args.date_format { builder.add_date_format(x); } + if let Some(x) = args.negative_inf { builder.add_negative_inf(x); } + if let Some(x) = args.positive_inf { builder.add_positive_inf(x); } + if let Some(x) = args.nan_value { builder.add_nan_value(x); } + if let Some(x) = args.empty_value { builder.add_empty_value(x); } + if let Some(x) = args.null_value { builder.add_null_value(x); } + if let Some(x) = args.ignore_trailing_white_space { builder.add_ignore_trailing_white_space(x); } + if let Some(x) = args.ignore_leading_white_space { builder.add_ignore_leading_white_space(x); } + if let Some(x) = args.infer_schema { builder.add_infer_schema(x); } + if let Some(x) = args.enforce_schema { builder.add_enforce_schema(x); } + if let Some(x) = args.header { builder.add_header(x); } + if let Some(x) = args.comment { builder.add_comment(x); } + if let Some(x) = args.escape { builder.add_escape(x); } + if let Some(x) = args.quote { builder.add_quote(x); } + if let Some(x) = args.encoding { builder.add_encoding(x); } + if let Some(x) = args.separator { builder.add_separator(x); } + if let Some(x) = args.schema { builder.add_schema(x); } + builder.finish() + } + + pub const VT_SCHEMA: flatbuffers::VOffsetT = 4; + pub const VT_SEPARATOR: flatbuffers::VOffsetT = 6; + pub const VT_ENCODING: flatbuffers::VOffsetT = 8; + pub const VT_QUOTE: flatbuffers::VOffsetT = 10; + pub const VT_ESCAPE: flatbuffers::VOffsetT = 12; + pub const VT_COMMENT: flatbuffers::VOffsetT = 14; + pub const VT_HEADER: flatbuffers::VOffsetT = 16; + pub const VT_ENFORCE_SCHEMA: flatbuffers::VOffsetT = 18; + pub const VT_INFER_SCHEMA: flatbuffers::VOffsetT = 20; + pub const VT_IGNORE_LEADING_WHITE_SPACE: flatbuffers::VOffsetT = 22; + pub const VT_IGNORE_TRAILING_WHITE_SPACE: flatbuffers::VOffsetT = 24; + pub const VT_NULL_VALUE: flatbuffers::VOffsetT = 26; + pub const VT_EMPTY_VALUE: flatbuffers::VOffsetT = 28; + pub const VT_NAN_VALUE: flatbuffers::VOffsetT = 30; + pub const VT_POSITIVE_INF: flatbuffers::VOffsetT = 32; + pub const VT_NEGATIVE_INF: flatbuffers::VOffsetT = 34; + pub const VT_DATE_FORMAT: flatbuffers::VOffsetT = 36; + pub const VT_TIMESTAMP_FORMAT: flatbuffers::VOffsetT = 38; + pub const VT_MULTI_LINE: flatbuffers::VOffsetT = 40; + + #[inline] + pub fn schema(&self) -> Option>> { + self._tab.get::>>>(ReadStepCsv::VT_SCHEMA, None) + } + #[inline] + pub fn separator(&self) -> Option<&'a str> { + self._tab.get::>(ReadStepCsv::VT_SEPARATOR, None) + } + #[inline] + pub fn encoding(&self) -> Option<&'a str> { + self._tab.get::>(ReadStepCsv::VT_ENCODING, None) + } + #[inline] + pub fn quote(&self) -> Option<&'a str> { + self._tab.get::>(ReadStepCsv::VT_QUOTE, None) + } + #[inline] + pub fn escape(&self) -> Option<&'a str> { + self._tab.get::>(ReadStepCsv::VT_ESCAPE, None) + } + #[inline] + pub fn comment(&self) -> Option<&'a str> { + self._tab.get::>(ReadStepCsv::VT_COMMENT, None) + } + #[inline] + pub fn header(&self) -> Option<&'a Option_bool> { + self._tab.get::(ReadStepCsv::VT_HEADER, None) + } + #[inline] + pub fn enforce_schema(&self) -> Option<&'a Option_bool> { + self._tab.get::(ReadStepCsv::VT_ENFORCE_SCHEMA, None) + } + #[inline] + pub fn infer_schema(&self) -> Option<&'a Option_bool> { + self._tab.get::(ReadStepCsv::VT_INFER_SCHEMA, None) + } + #[inline] + pub fn ignore_leading_white_space(&self) -> Option<&'a Option_bool> { + self._tab.get::(ReadStepCsv::VT_IGNORE_LEADING_WHITE_SPACE, None) + } + #[inline] + pub fn ignore_trailing_white_space(&self) -> Option<&'a Option_bool> { + self._tab.get::(ReadStepCsv::VT_IGNORE_TRAILING_WHITE_SPACE, None) + } + #[inline] + pub fn null_value(&self) -> Option<&'a str> { + self._tab.get::>(ReadStepCsv::VT_NULL_VALUE, None) + } + #[inline] + pub fn empty_value(&self) -> Option<&'a str> { + self._tab.get::>(ReadStepCsv::VT_EMPTY_VALUE, None) + } + #[inline] + pub fn nan_value(&self) -> Option<&'a str> { + self._tab.get::>(ReadStepCsv::VT_NAN_VALUE, None) + } + #[inline] + pub fn positive_inf(&self) -> Option<&'a str> { + self._tab.get::>(ReadStepCsv::VT_POSITIVE_INF, None) + } + #[inline] + pub fn negative_inf(&self) -> Option<&'a str> { + self._tab.get::>(ReadStepCsv::VT_NEGATIVE_INF, None) + } + #[inline] + pub fn date_format(&self) -> Option<&'a str> { + self._tab.get::>(ReadStepCsv::VT_DATE_FORMAT, None) + } + #[inline] + pub fn timestamp_format(&self) -> Option<&'a str> { + self._tab.get::>(ReadStepCsv::VT_TIMESTAMP_FORMAT, None) + } + #[inline] + pub fn multi_line(&self) -> Option<&'a Option_bool> { + self._tab.get::(ReadStepCsv::VT_MULTI_LINE, None) + } +} + +pub struct ReadStepCsvArgs<'a> { + pub schema: Option>>>, + pub separator: Option>, + pub encoding: Option>, + pub quote: Option>, + pub escape: Option>, + pub comment: Option>, + pub header: Option<&'a Option_bool>, + pub enforce_schema: Option<&'a Option_bool>, + pub infer_schema: Option<&'a Option_bool>, + pub ignore_leading_white_space: Option<&'a Option_bool>, + pub ignore_trailing_white_space: Option<&'a Option_bool>, + pub null_value: Option>, + pub empty_value: Option>, + pub nan_value: Option>, + pub positive_inf: Option>, + pub negative_inf: Option>, + pub date_format: Option>, + pub timestamp_format: Option>, + pub multi_line: Option<&'a Option_bool>, +} +impl<'a> Default for ReadStepCsvArgs<'a> { + #[inline] + fn default() -> Self { + ReadStepCsvArgs { + schema: None, + separator: None, + encoding: None, + quote: None, + escape: None, + comment: None, + header: None, + enforce_schema: None, + infer_schema: None, + ignore_leading_white_space: None, + ignore_trailing_white_space: None, + null_value: None, + empty_value: None, + nan_value: None, + positive_inf: None, + negative_inf: None, + date_format: None, + timestamp_format: None, + multi_line: None, + } + } +} +pub struct ReadStepCsvBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b> ReadStepCsvBuilder<'a, 'b> { + #[inline] + pub fn add_schema(&mut self, schema: flatbuffers::WIPOffset>>) { + self.fbb_.push_slot_always::>(ReadStepCsv::VT_SCHEMA, schema); + } + #[inline] + pub fn add_separator(&mut self, separator: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(ReadStepCsv::VT_SEPARATOR, separator); + } + #[inline] + pub fn add_encoding(&mut self, encoding: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(ReadStepCsv::VT_ENCODING, encoding); + } + #[inline] + pub fn add_quote(&mut self, quote: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(ReadStepCsv::VT_QUOTE, quote); + } + #[inline] + pub fn add_escape(&mut self, escape: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(ReadStepCsv::VT_ESCAPE, escape); + } + #[inline] + pub fn add_comment(&mut self, comment: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(ReadStepCsv::VT_COMMENT, comment); + } + #[inline] + pub fn add_header(&mut self, header: &'b Option_bool) { + self.fbb_.push_slot_always::<&Option_bool>(ReadStepCsv::VT_HEADER, header); + } + #[inline] + pub fn add_enforce_schema(&mut self, enforce_schema: &'b Option_bool) { + self.fbb_.push_slot_always::<&Option_bool>(ReadStepCsv::VT_ENFORCE_SCHEMA, enforce_schema); + } + #[inline] + pub fn add_infer_schema(&mut self, infer_schema: &'b Option_bool) { + self.fbb_.push_slot_always::<&Option_bool>(ReadStepCsv::VT_INFER_SCHEMA, infer_schema); + } + #[inline] + pub fn add_ignore_leading_white_space(&mut self, ignore_leading_white_space: &'b Option_bool) { + self.fbb_.push_slot_always::<&Option_bool>(ReadStepCsv::VT_IGNORE_LEADING_WHITE_SPACE, ignore_leading_white_space); + } + #[inline] + pub fn add_ignore_trailing_white_space(&mut self, ignore_trailing_white_space: &'b Option_bool) { + self.fbb_.push_slot_always::<&Option_bool>(ReadStepCsv::VT_IGNORE_TRAILING_WHITE_SPACE, ignore_trailing_white_space); + } + #[inline] + pub fn add_null_value(&mut self, null_value: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(ReadStepCsv::VT_NULL_VALUE, null_value); + } + #[inline] + pub fn add_empty_value(&mut self, empty_value: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(ReadStepCsv::VT_EMPTY_VALUE, empty_value); + } + #[inline] + pub fn add_nan_value(&mut self, nan_value: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(ReadStepCsv::VT_NAN_VALUE, nan_value); + } + #[inline] + pub fn add_positive_inf(&mut self, positive_inf: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(ReadStepCsv::VT_POSITIVE_INF, positive_inf); + } + #[inline] + pub fn add_negative_inf(&mut self, negative_inf: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(ReadStepCsv::VT_NEGATIVE_INF, negative_inf); + } + #[inline] + pub fn add_date_format(&mut self, date_format: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(ReadStepCsv::VT_DATE_FORMAT, date_format); + } + #[inline] + pub fn add_timestamp_format(&mut self, timestamp_format: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(ReadStepCsv::VT_TIMESTAMP_FORMAT, timestamp_format); + } + #[inline] + pub fn add_multi_line(&mut self, multi_line: &'b Option_bool) { + self.fbb_.push_slot_always::<&Option_bool>(ReadStepCsv::VT_MULTI_LINE, multi_line); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> ReadStepCsvBuilder<'a, 'b> { + let start = _fbb.start_table(); + ReadStepCsvBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} + +pub enum ReadStepJsonLinesOffset {} +#[derive(Copy, Clone, Debug, PartialEq)] + +pub struct ReadStepJsonLines<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for ReadStepJsonLines<'a> { + type Inner = ReadStepJsonLines<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf: buf, loc: loc }, + } + } +} + +impl<'a> ReadStepJsonLines<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + ReadStepJsonLines { + _tab: table, + } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args ReadStepJsonLinesArgs<'args>) -> flatbuffers::WIPOffset> { + let mut builder = ReadStepJsonLinesBuilder::new(_fbb); + if let Some(x) = args.timestamp_format { builder.add_timestamp_format(x); } + if let Some(x) = args.primitives_as_string { builder.add_primitives_as_string(x); } + if let Some(x) = args.multi_line { builder.add_multi_line(x); } + if let Some(x) = args.encoding { builder.add_encoding(x); } + if let Some(x) = args.date_format { builder.add_date_format(x); } + if let Some(x) = args.schema { builder.add_schema(x); } + builder.finish() + } + + pub const VT_SCHEMA: flatbuffers::VOffsetT = 4; + pub const VT_DATE_FORMAT: flatbuffers::VOffsetT = 6; + pub const VT_ENCODING: flatbuffers::VOffsetT = 8; + pub const VT_MULTI_LINE: flatbuffers::VOffsetT = 10; + pub const VT_PRIMITIVES_AS_STRING: flatbuffers::VOffsetT = 12; + pub const VT_TIMESTAMP_FORMAT: flatbuffers::VOffsetT = 14; + + #[inline] + pub fn schema(&self) -> Option>> { + self._tab.get::>>>(ReadStepJsonLines::VT_SCHEMA, None) + } + #[inline] + pub fn date_format(&self) -> Option<&'a str> { + self._tab.get::>(ReadStepJsonLines::VT_DATE_FORMAT, None) + } + #[inline] + pub fn encoding(&self) -> Option<&'a str> { + self._tab.get::>(ReadStepJsonLines::VT_ENCODING, None) + } + #[inline] + pub fn multi_line(&self) -> Option<&'a Option_bool> { + self._tab.get::(ReadStepJsonLines::VT_MULTI_LINE, None) + } + #[inline] + pub fn primitives_as_string(&self) -> Option<&'a Option_bool> { + self._tab.get::(ReadStepJsonLines::VT_PRIMITIVES_AS_STRING, None) + } + #[inline] + pub fn timestamp_format(&self) -> Option<&'a str> { + self._tab.get::>(ReadStepJsonLines::VT_TIMESTAMP_FORMAT, None) + } +} + +pub struct ReadStepJsonLinesArgs<'a> { + pub schema: Option>>>, + pub date_format: Option>, + pub encoding: Option>, + pub multi_line: Option<&'a Option_bool>, + pub primitives_as_string: Option<&'a Option_bool>, + pub timestamp_format: Option>, +} +impl<'a> Default for ReadStepJsonLinesArgs<'a> { + #[inline] + fn default() -> Self { + ReadStepJsonLinesArgs { + schema: None, + date_format: None, + encoding: None, + multi_line: None, + primitives_as_string: None, + timestamp_format: None, + } + } +} +pub struct ReadStepJsonLinesBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b> ReadStepJsonLinesBuilder<'a, 'b> { + #[inline] + pub fn add_schema(&mut self, schema: flatbuffers::WIPOffset>>) { + self.fbb_.push_slot_always::>(ReadStepJsonLines::VT_SCHEMA, schema); + } + #[inline] + pub fn add_date_format(&mut self, date_format: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(ReadStepJsonLines::VT_DATE_FORMAT, date_format); + } + #[inline] + pub fn add_encoding(&mut self, encoding: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(ReadStepJsonLines::VT_ENCODING, encoding); + } + #[inline] + pub fn add_multi_line(&mut self, multi_line: &'b Option_bool) { + self.fbb_.push_slot_always::<&Option_bool>(ReadStepJsonLines::VT_MULTI_LINE, multi_line); + } + #[inline] + pub fn add_primitives_as_string(&mut self, primitives_as_string: &'b Option_bool) { + self.fbb_.push_slot_always::<&Option_bool>(ReadStepJsonLines::VT_PRIMITIVES_AS_STRING, primitives_as_string); + } + #[inline] + pub fn add_timestamp_format(&mut self, timestamp_format: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(ReadStepJsonLines::VT_TIMESTAMP_FORMAT, timestamp_format); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> ReadStepJsonLinesBuilder<'a, 'b> { + let start = _fbb.start_table(); + ReadStepJsonLinesBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} + +pub enum ReadStepGeoJsonOffset {} +#[derive(Copy, Clone, Debug, PartialEq)] + +pub struct ReadStepGeoJson<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for ReadStepGeoJson<'a> { + type Inner = ReadStepGeoJson<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf: buf, loc: loc }, + } + } +} + +impl<'a> ReadStepGeoJson<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + ReadStepGeoJson { + _tab: table, + } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args ReadStepGeoJsonArgs<'args>) -> flatbuffers::WIPOffset> { + let mut builder = ReadStepGeoJsonBuilder::new(_fbb); + if let Some(x) = args.schema { builder.add_schema(x); } + builder.finish() + } + + pub const VT_SCHEMA: flatbuffers::VOffsetT = 4; + + #[inline] + pub fn schema(&self) -> Option>> { + self._tab.get::>>>(ReadStepGeoJson::VT_SCHEMA, None) + } +} + +pub struct ReadStepGeoJsonArgs<'a> { + pub schema: Option>>>, +} +impl<'a> Default for ReadStepGeoJsonArgs<'a> { + #[inline] + fn default() -> Self { + ReadStepGeoJsonArgs { + schema: None, + } + } +} +pub struct ReadStepGeoJsonBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b> ReadStepGeoJsonBuilder<'a, 'b> { + #[inline] + pub fn add_schema(&mut self, schema: flatbuffers::WIPOffset>>) { + self.fbb_.push_slot_always::>(ReadStepGeoJson::VT_SCHEMA, schema); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> ReadStepGeoJsonBuilder<'a, 'b> { + let start = _fbb.start_table(); + ReadStepGeoJsonBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} + +pub enum ReadStepEsriShapefileOffset {} +#[derive(Copy, Clone, Debug, PartialEq)] + +pub struct ReadStepEsriShapefile<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for ReadStepEsriShapefile<'a> { + type Inner = ReadStepEsriShapefile<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf: buf, loc: loc }, + } + } +} + +impl<'a> ReadStepEsriShapefile<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + ReadStepEsriShapefile { + _tab: table, + } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args ReadStepEsriShapefileArgs<'args>) -> flatbuffers::WIPOffset> { + let mut builder = ReadStepEsriShapefileBuilder::new(_fbb); + if let Some(x) = args.sub_path { builder.add_sub_path(x); } + if let Some(x) = args.schema { builder.add_schema(x); } + builder.finish() + } + + pub const VT_SCHEMA: flatbuffers::VOffsetT = 4; + pub const VT_SUB_PATH: flatbuffers::VOffsetT = 6; + + #[inline] + pub fn schema(&self) -> Option>> { + self._tab.get::>>>(ReadStepEsriShapefile::VT_SCHEMA, None) + } + #[inline] + pub fn sub_path(&self) -> Option<&'a str> { + self._tab.get::>(ReadStepEsriShapefile::VT_SUB_PATH, None) + } +} + +pub struct ReadStepEsriShapefileArgs<'a> { + pub schema: Option>>>, + pub sub_path: Option>, +} +impl<'a> Default for ReadStepEsriShapefileArgs<'a> { + #[inline] + fn default() -> Self { + ReadStepEsriShapefileArgs { + schema: None, + sub_path: None, + } + } +} +pub struct ReadStepEsriShapefileBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b> ReadStepEsriShapefileBuilder<'a, 'b> { + #[inline] + pub fn add_schema(&mut self, schema: flatbuffers::WIPOffset>>) { + self.fbb_.push_slot_always::>(ReadStepEsriShapefile::VT_SCHEMA, schema); + } + #[inline] + pub fn add_sub_path(&mut self, sub_path: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(ReadStepEsriShapefile::VT_SUB_PATH, sub_path); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> ReadStepEsriShapefileBuilder<'a, 'b> { + let start = _fbb.start_table(); + ReadStepEsriShapefileBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} + +pub enum TransformSqlOffset {} +#[derive(Copy, Clone, Debug, PartialEq)] + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +pub struct TransformSql<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for TransformSql<'a> { + type Inner = TransformSql<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf: buf, loc: loc }, + } + } +} + +impl<'a> TransformSql<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + TransformSql { + _tab: table, + } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args TransformSqlArgs<'args>) -> flatbuffers::WIPOffset> { + let mut builder = TransformSqlBuilder::new(_fbb); + if let Some(x) = args.temporal_tables { builder.add_temporal_tables(x); } + if let Some(x) = args.queries { builder.add_queries(x); } + if let Some(x) = args.query { builder.add_query(x); } + if let Some(x) = args.version { builder.add_version(x); } + if let Some(x) = args.engine { builder.add_engine(x); } + builder.finish() + } + + pub const VT_ENGINE: flatbuffers::VOffsetT = 4; + pub const VT_VERSION: flatbuffers::VOffsetT = 6; + pub const VT_QUERY: flatbuffers::VOffsetT = 8; + pub const VT_QUERIES: flatbuffers::VOffsetT = 10; + pub const VT_TEMPORAL_TABLES: flatbuffers::VOffsetT = 12; + + #[inline] + pub fn engine(&self) -> Option<&'a str> { + self._tab.get::>(TransformSql::VT_ENGINE, None) + } + #[inline] + pub fn version(&self) -> Option<&'a str> { + self._tab.get::>(TransformSql::VT_VERSION, None) + } + #[inline] + pub fn query(&self) -> Option<&'a str> { + self._tab.get::>(TransformSql::VT_QUERY, None) + } + #[inline] + pub fn queries(&self) -> Option>>> { + self._tab.get::>>>>(TransformSql::VT_QUERIES, None) + } + #[inline] + pub fn temporal_tables(&self) -> Option>>> { + self._tab.get::>>>>(TransformSql::VT_TEMPORAL_TABLES, None) + } +} + +pub struct TransformSqlArgs<'a> { + pub engine: Option>, + pub version: Option>, + pub query: Option>, + pub queries: Option>>>>, + pub temporal_tables: Option>>>>, +} +impl<'a> Default for TransformSqlArgs<'a> { + #[inline] + fn default() -> Self { + TransformSqlArgs { + engine: None, + version: None, + query: None, + queries: None, + temporal_tables: None, + } + } +} +pub struct TransformSqlBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b> TransformSqlBuilder<'a, 'b> { + #[inline] + pub fn add_engine(&mut self, engine: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(TransformSql::VT_ENGINE, engine); + } + #[inline] + pub fn add_version(&mut self, version: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(TransformSql::VT_VERSION, version); + } + #[inline] + pub fn add_query(&mut self, query: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(TransformSql::VT_QUERY, query); + } + #[inline] + pub fn add_queries(&mut self, queries: flatbuffers::WIPOffset>>>) { + self.fbb_.push_slot_always::>(TransformSql::VT_QUERIES, queries); + } + #[inline] + pub fn add_temporal_tables(&mut self, temporal_tables: flatbuffers::WIPOffset>>>) { + self.fbb_.push_slot_always::>(TransformSql::VT_TEMPORAL_TABLES, temporal_tables); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> TransformSqlBuilder<'a, 'b> { + let start = _fbb.start_table(); + TransformSqlBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} + +pub enum MergeStrategyAppendOffset {} +#[derive(Copy, Clone, Debug, PartialEq)] + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +pub struct MergeStrategyAppend<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for MergeStrategyAppend<'a> { + type Inner = MergeStrategyAppend<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf: buf, loc: loc }, + } + } +} + +impl<'a> MergeStrategyAppend<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + MergeStrategyAppend { + _tab: table, + } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + _args: &'args MergeStrategyAppendArgs) -> flatbuffers::WIPOffset> { + let mut builder = MergeStrategyAppendBuilder::new(_fbb); + builder.finish() + } + +} + +pub struct MergeStrategyAppendArgs { +} +impl<'a> Default for MergeStrategyAppendArgs { + #[inline] + fn default() -> Self { + MergeStrategyAppendArgs { + } + } +} +pub struct MergeStrategyAppendBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b> MergeStrategyAppendBuilder<'a, 'b> { + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> MergeStrategyAppendBuilder<'a, 'b> { + let start = _fbb.start_table(); + MergeStrategyAppendBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} + +pub enum MergeStrategyLedgerOffset {} +#[derive(Copy, Clone, Debug, PartialEq)] + +pub struct MergeStrategyLedger<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for MergeStrategyLedger<'a> { + type Inner = MergeStrategyLedger<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf: buf, loc: loc }, + } + } +} + +impl<'a> MergeStrategyLedger<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + MergeStrategyLedger { + _tab: table, + } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args MergeStrategyLedgerArgs<'args>) -> flatbuffers::WIPOffset> { + let mut builder = MergeStrategyLedgerBuilder::new(_fbb); + if let Some(x) = args.primary_key { builder.add_primary_key(x); } + builder.finish() + } + + pub const VT_PRIMARY_KEY: flatbuffers::VOffsetT = 4; + + #[inline] + pub fn primary_key(&self) -> Option>> { + self._tab.get::>>>(MergeStrategyLedger::VT_PRIMARY_KEY, None) + } +} + +pub struct MergeStrategyLedgerArgs<'a> { + pub primary_key: Option>>>, +} +impl<'a> Default for MergeStrategyLedgerArgs<'a> { + #[inline] + fn default() -> Self { + MergeStrategyLedgerArgs { + primary_key: None, + } + } +} +pub struct MergeStrategyLedgerBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b> MergeStrategyLedgerBuilder<'a, 'b> { + #[inline] + pub fn add_primary_key(&mut self, primary_key: flatbuffers::WIPOffset>>) { + self.fbb_.push_slot_always::>(MergeStrategyLedger::VT_PRIMARY_KEY, primary_key); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> MergeStrategyLedgerBuilder<'a, 'b> { + let start = _fbb.start_table(); + MergeStrategyLedgerBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} + +pub enum MergeStrategySnapshotOffset {} +#[derive(Copy, Clone, Debug, PartialEq)] + +pub struct MergeStrategySnapshot<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for MergeStrategySnapshot<'a> { + type Inner = MergeStrategySnapshot<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf: buf, loc: loc }, + } + } +} + +impl<'a> MergeStrategySnapshot<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + MergeStrategySnapshot { + _tab: table, + } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args MergeStrategySnapshotArgs<'args>) -> flatbuffers::WIPOffset> { + let mut builder = MergeStrategySnapshotBuilder::new(_fbb); + if let Some(x) = args.obsv_removed { builder.add_obsv_removed(x); } + if let Some(x) = args.obsv_changed { builder.add_obsv_changed(x); } + if let Some(x) = args.obsv_added { builder.add_obsv_added(x); } + if let Some(x) = args.observation_column { builder.add_observation_column(x); } + if let Some(x) = args.compare_columns { builder.add_compare_columns(x); } + if let Some(x) = args.primary_key { builder.add_primary_key(x); } + builder.finish() + } + + pub const VT_PRIMARY_KEY: flatbuffers::VOffsetT = 4; + pub const VT_COMPARE_COLUMNS: flatbuffers::VOffsetT = 6; + pub const VT_OBSERVATION_COLUMN: flatbuffers::VOffsetT = 8; + pub const VT_OBSV_ADDED: flatbuffers::VOffsetT = 10; + pub const VT_OBSV_CHANGED: flatbuffers::VOffsetT = 12; + pub const VT_OBSV_REMOVED: flatbuffers::VOffsetT = 14; + + #[inline] + pub fn primary_key(&self) -> Option>> { + self._tab.get::>>>(MergeStrategySnapshot::VT_PRIMARY_KEY, None) + } + #[inline] + pub fn compare_columns(&self) -> Option>> { + self._tab.get::>>>(MergeStrategySnapshot::VT_COMPARE_COLUMNS, None) + } + #[inline] + pub fn observation_column(&self) -> Option<&'a str> { + self._tab.get::>(MergeStrategySnapshot::VT_OBSERVATION_COLUMN, None) + } + #[inline] + pub fn obsv_added(&self) -> Option<&'a str> { + self._tab.get::>(MergeStrategySnapshot::VT_OBSV_ADDED, None) + } + #[inline] + pub fn obsv_changed(&self) -> Option<&'a str> { + self._tab.get::>(MergeStrategySnapshot::VT_OBSV_CHANGED, None) + } + #[inline] + pub fn obsv_removed(&self) -> Option<&'a str> { + self._tab.get::>(MergeStrategySnapshot::VT_OBSV_REMOVED, None) + } +} + +pub struct MergeStrategySnapshotArgs<'a> { + pub primary_key: Option>>>, + pub compare_columns: Option>>>, + pub observation_column: Option>, + pub obsv_added: Option>, + pub obsv_changed: Option>, + pub obsv_removed: Option>, +} +impl<'a> Default for MergeStrategySnapshotArgs<'a> { + #[inline] + fn default() -> Self { + MergeStrategySnapshotArgs { + primary_key: None, + compare_columns: None, + observation_column: None, + obsv_added: None, + obsv_changed: None, + obsv_removed: None, + } + } +} +pub struct MergeStrategySnapshotBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b> MergeStrategySnapshotBuilder<'a, 'b> { + #[inline] + pub fn add_primary_key(&mut self, primary_key: flatbuffers::WIPOffset>>) { + self.fbb_.push_slot_always::>(MergeStrategySnapshot::VT_PRIMARY_KEY, primary_key); + } + #[inline] + pub fn add_compare_columns(&mut self, compare_columns: flatbuffers::WIPOffset>>) { + self.fbb_.push_slot_always::>(MergeStrategySnapshot::VT_COMPARE_COLUMNS, compare_columns); + } + #[inline] + pub fn add_observation_column(&mut self, observation_column: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(MergeStrategySnapshot::VT_OBSERVATION_COLUMN, observation_column); + } + #[inline] + pub fn add_obsv_added(&mut self, obsv_added: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(MergeStrategySnapshot::VT_OBSV_ADDED, obsv_added); + } + #[inline] + pub fn add_obsv_changed(&mut self, obsv_changed: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(MergeStrategySnapshot::VT_OBSV_CHANGED, obsv_changed); + } + #[inline] + pub fn add_obsv_removed(&mut self, obsv_removed: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(MergeStrategySnapshot::VT_OBSV_REMOVED, obsv_removed); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> MergeStrategySnapshotBuilder<'a, 'b> { + let start = _fbb.start_table(); + MergeStrategySnapshotBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} + +pub enum PrepStepWrapperOffset {} +#[derive(Copy, Clone, Debug, PartialEq)] + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +pub struct PrepStepWrapper<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for PrepStepWrapper<'a> { + type Inner = PrepStepWrapper<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf: buf, loc: loc }, + } + } +} + +impl<'a> PrepStepWrapper<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + PrepStepWrapper { + _tab: table, + } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args PrepStepWrapperArgs) -> flatbuffers::WIPOffset> { + let mut builder = PrepStepWrapperBuilder::new(_fbb); + if let Some(x) = args.value { builder.add_value(x); } + builder.add_value_type(args.value_type); + builder.finish() + } + + pub const VT_VALUE_TYPE: flatbuffers::VOffsetT = 4; + pub const VT_VALUE: flatbuffers::VOffsetT = 6; + + #[inline] + pub fn value_type(&self) -> PrepStep { + self._tab.get::(PrepStepWrapper::VT_VALUE_TYPE, Some(PrepStep::NONE)).unwrap() + } + #[inline] + pub fn value(&self) -> Option> { + self._tab.get::>>(PrepStepWrapper::VT_VALUE, None) + } + #[inline] + #[allow(non_snake_case)] + pub fn value_as_prep_step_decompress(&self) -> Option> { + if self.value_type() == PrepStep::PrepStepDecompress { + self.value().map(|u| PrepStepDecompress::init_from_table(u)) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn value_as_prep_step_pipe(&self) -> Option> { + if self.value_type() == PrepStep::PrepStepPipe { + self.value().map(|u| PrepStepPipe::init_from_table(u)) + } else { + None + } + } + +} + +pub struct PrepStepWrapperArgs { + pub value_type: PrepStep, + pub value: Option>, +} +impl<'a> Default for PrepStepWrapperArgs { + #[inline] + fn default() -> Self { + PrepStepWrapperArgs { + value_type: PrepStep::NONE, + value: None, + } + } +} +pub struct PrepStepWrapperBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b> PrepStepWrapperBuilder<'a, 'b> { + #[inline] + pub fn add_value_type(&mut self, value_type: PrepStep) { + self.fbb_.push_slot::(PrepStepWrapper::VT_VALUE_TYPE, value_type, PrepStep::NONE); + } + #[inline] + pub fn add_value(&mut self, value: flatbuffers::WIPOffset) { + self.fbb_.push_slot_always::>(PrepStepWrapper::VT_VALUE, value); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> PrepStepWrapperBuilder<'a, 'b> { + let start = _fbb.start_table(); + PrepStepWrapperBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} + +pub enum DatasetSourceRootOffset {} +#[derive(Copy, Clone, Debug, PartialEq)] + +pub struct DatasetSourceRoot<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for DatasetSourceRoot<'a> { + type Inner = DatasetSourceRoot<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf: buf, loc: loc }, + } + } +} + +impl<'a> DatasetSourceRoot<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + DatasetSourceRoot { + _tab: table, + } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args DatasetSourceRootArgs<'args>) -> flatbuffers::WIPOffset> { + let mut builder = DatasetSourceRootBuilder::new(_fbb); + if let Some(x) = args.merge { builder.add_merge(x); } + if let Some(x) = args.preprocess { builder.add_preprocess(x); } + if let Some(x) = args.read { builder.add_read(x); } + if let Some(x) = args.prepare { builder.add_prepare(x); } + if let Some(x) = args.fetch { builder.add_fetch(x); } + builder.add_merge_type(args.merge_type); + builder.add_preprocess_type(args.preprocess_type); + builder.add_read_type(args.read_type); + builder.add_fetch_type(args.fetch_type); + builder.finish() + } + + pub const VT_FETCH_TYPE: flatbuffers::VOffsetT = 4; + pub const VT_FETCH: flatbuffers::VOffsetT = 6; + pub const VT_PREPARE: flatbuffers::VOffsetT = 8; + pub const VT_READ_TYPE: flatbuffers::VOffsetT = 10; + pub const VT_READ: flatbuffers::VOffsetT = 12; + pub const VT_PREPROCESS_TYPE: flatbuffers::VOffsetT = 14; + pub const VT_PREPROCESS: flatbuffers::VOffsetT = 16; + pub const VT_MERGE_TYPE: flatbuffers::VOffsetT = 18; + pub const VT_MERGE: flatbuffers::VOffsetT = 20; + + #[inline] + pub fn fetch_type(&self) -> FetchStep { + self._tab.get::(DatasetSourceRoot::VT_FETCH_TYPE, Some(FetchStep::NONE)).unwrap() + } + #[inline] + pub fn fetch(&self) -> Option> { + self._tab.get::>>(DatasetSourceRoot::VT_FETCH, None) + } + #[inline] + pub fn prepare(&self) -> Option>>> { + self._tab.get::>>>>(DatasetSourceRoot::VT_PREPARE, None) + } + #[inline] + pub fn read_type(&self) -> ReadStep { + self._tab.get::(DatasetSourceRoot::VT_READ_TYPE, Some(ReadStep::NONE)).unwrap() + } + #[inline] + pub fn read(&self) -> Option> { + self._tab.get::>>(DatasetSourceRoot::VT_READ, None) + } + #[inline] + pub fn preprocess_type(&self) -> Transform { + self._tab.get::(DatasetSourceRoot::VT_PREPROCESS_TYPE, Some(Transform::NONE)).unwrap() + } + #[inline] + pub fn preprocess(&self) -> Option> { + self._tab.get::>>(DatasetSourceRoot::VT_PREPROCESS, None) + } + #[inline] + pub fn merge_type(&self) -> MergeStrategy { + self._tab.get::(DatasetSourceRoot::VT_MERGE_TYPE, Some(MergeStrategy::NONE)).unwrap() + } + #[inline] + pub fn merge(&self) -> Option> { + self._tab.get::>>(DatasetSourceRoot::VT_MERGE, None) + } + #[inline] + #[allow(non_snake_case)] + pub fn fetch_as_fetch_step_url(&self) -> Option> { + if self.fetch_type() == FetchStep::FetchStepUrl { + self.fetch().map(|u| FetchStepUrl::init_from_table(u)) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn fetch_as_fetch_step_files_glob(&self) -> Option> { + if self.fetch_type() == FetchStep::FetchStepFilesGlob { + self.fetch().map(|u| FetchStepFilesGlob::init_from_table(u)) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn read_as_read_step_csv(&self) -> Option> { + if self.read_type() == ReadStep::ReadStepCsv { + self.read().map(|u| ReadStepCsv::init_from_table(u)) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn read_as_read_step_json_lines(&self) -> Option> { + if self.read_type() == ReadStep::ReadStepJsonLines { + self.read().map(|u| ReadStepJsonLines::init_from_table(u)) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn read_as_read_step_geo_json(&self) -> Option> { + if self.read_type() == ReadStep::ReadStepGeoJson { + self.read().map(|u| ReadStepGeoJson::init_from_table(u)) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn read_as_read_step_esri_shapefile(&self) -> Option> { + if self.read_type() == ReadStep::ReadStepEsriShapefile { + self.read().map(|u| ReadStepEsriShapefile::init_from_table(u)) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn preprocess_as_transform_sql(&self) -> Option> { + if self.preprocess_type() == Transform::TransformSql { + self.preprocess().map(|u| TransformSql::init_from_table(u)) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn merge_as_merge_strategy_append(&self) -> Option> { + if self.merge_type() == MergeStrategy::MergeStrategyAppend { + self.merge().map(|u| MergeStrategyAppend::init_from_table(u)) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn merge_as_merge_strategy_ledger(&self) -> Option> { + if self.merge_type() == MergeStrategy::MergeStrategyLedger { + self.merge().map(|u| MergeStrategyLedger::init_from_table(u)) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn merge_as_merge_strategy_snapshot(&self) -> Option> { + if self.merge_type() == MergeStrategy::MergeStrategySnapshot { + self.merge().map(|u| MergeStrategySnapshot::init_from_table(u)) + } else { + None + } + } + +} + +pub struct DatasetSourceRootArgs<'a> { + pub fetch_type: FetchStep, + pub fetch: Option>, + pub prepare: Option>>>>, + pub read_type: ReadStep, + pub read: Option>, + pub preprocess_type: Transform, + pub preprocess: Option>, + pub merge_type: MergeStrategy, + pub merge: Option>, +} +impl<'a> Default for DatasetSourceRootArgs<'a> { + #[inline] + fn default() -> Self { + DatasetSourceRootArgs { + fetch_type: FetchStep::NONE, + fetch: None, + prepare: None, + read_type: ReadStep::NONE, + read: None, + preprocess_type: Transform::NONE, + preprocess: None, + merge_type: MergeStrategy::NONE, + merge: None, + } + } +} +pub struct DatasetSourceRootBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b> DatasetSourceRootBuilder<'a, 'b> { + #[inline] + pub fn add_fetch_type(&mut self, fetch_type: FetchStep) { + self.fbb_.push_slot::(DatasetSourceRoot::VT_FETCH_TYPE, fetch_type, FetchStep::NONE); + } + #[inline] + pub fn add_fetch(&mut self, fetch: flatbuffers::WIPOffset) { + self.fbb_.push_slot_always::>(DatasetSourceRoot::VT_FETCH, fetch); + } + #[inline] + pub fn add_prepare(&mut self, prepare: flatbuffers::WIPOffset>>>) { + self.fbb_.push_slot_always::>(DatasetSourceRoot::VT_PREPARE, prepare); + } + #[inline] + pub fn add_read_type(&mut self, read_type: ReadStep) { + self.fbb_.push_slot::(DatasetSourceRoot::VT_READ_TYPE, read_type, ReadStep::NONE); + } + #[inline] + pub fn add_read(&mut self, read: flatbuffers::WIPOffset) { + self.fbb_.push_slot_always::>(DatasetSourceRoot::VT_READ, read); + } + #[inline] + pub fn add_preprocess_type(&mut self, preprocess_type: Transform) { + self.fbb_.push_slot::(DatasetSourceRoot::VT_PREPROCESS_TYPE, preprocess_type, Transform::NONE); + } + #[inline] + pub fn add_preprocess(&mut self, preprocess: flatbuffers::WIPOffset) { + self.fbb_.push_slot_always::>(DatasetSourceRoot::VT_PREPROCESS, preprocess); + } + #[inline] + pub fn add_merge_type(&mut self, merge_type: MergeStrategy) { + self.fbb_.push_slot::(DatasetSourceRoot::VT_MERGE_TYPE, merge_type, MergeStrategy::NONE); + } + #[inline] + pub fn add_merge(&mut self, merge: flatbuffers::WIPOffset) { + self.fbb_.push_slot_always::>(DatasetSourceRoot::VT_MERGE, merge); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> DatasetSourceRootBuilder<'a, 'b> { + let start = _fbb.start_table(); + DatasetSourceRootBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} + +pub enum DatasetSourceDerivativeOffset {} +#[derive(Copy, Clone, Debug, PartialEq)] + +pub struct DatasetSourceDerivative<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for DatasetSourceDerivative<'a> { + type Inner = DatasetSourceDerivative<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf: buf, loc: loc }, + } + } +} + +impl<'a> DatasetSourceDerivative<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + DatasetSourceDerivative { + _tab: table, + } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args DatasetSourceDerivativeArgs<'args>) -> flatbuffers::WIPOffset> { + let mut builder = DatasetSourceDerivativeBuilder::new(_fbb); + if let Some(x) = args.transform { builder.add_transform(x); } + if let Some(x) = args.inputs { builder.add_inputs(x); } + builder.add_transform_type(args.transform_type); + builder.finish() + } + + pub const VT_INPUTS: flatbuffers::VOffsetT = 4; + pub const VT_TRANSFORM_TYPE: flatbuffers::VOffsetT = 6; + pub const VT_TRANSFORM: flatbuffers::VOffsetT = 8; + + #[inline] + pub fn inputs(&self) -> Option>> { + self._tab.get::>>>(DatasetSourceDerivative::VT_INPUTS, None) + } + #[inline] + pub fn transform_type(&self) -> Transform { + self._tab.get::(DatasetSourceDerivative::VT_TRANSFORM_TYPE, Some(Transform::NONE)).unwrap() + } + #[inline] + pub fn transform(&self) -> Option> { + self._tab.get::>>(DatasetSourceDerivative::VT_TRANSFORM, None) + } + #[inline] + #[allow(non_snake_case)] + pub fn transform_as_transform_sql(&self) -> Option> { + if self.transform_type() == Transform::TransformSql { + self.transform().map(|u| TransformSql::init_from_table(u)) + } else { + None + } + } + +} + +pub struct DatasetSourceDerivativeArgs<'a> { + pub inputs: Option>>>, + pub transform_type: Transform, + pub transform: Option>, +} +impl<'a> Default for DatasetSourceDerivativeArgs<'a> { + #[inline] + fn default() -> Self { + DatasetSourceDerivativeArgs { + inputs: None, + transform_type: Transform::NONE, + transform: None, + } + } +} +pub struct DatasetSourceDerivativeBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b> DatasetSourceDerivativeBuilder<'a, 'b> { + #[inline] + pub fn add_inputs(&mut self, inputs: flatbuffers::WIPOffset>>) { + self.fbb_.push_slot_always::>(DatasetSourceDerivative::VT_INPUTS, inputs); + } + #[inline] + pub fn add_transform_type(&mut self, transform_type: Transform) { + self.fbb_.push_slot::(DatasetSourceDerivative::VT_TRANSFORM_TYPE, transform_type, Transform::NONE); + } + #[inline] + pub fn add_transform(&mut self, transform: flatbuffers::WIPOffset) { + self.fbb_.push_slot_always::>(DatasetSourceDerivative::VT_TRANSFORM, transform); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> DatasetSourceDerivativeBuilder<'a, 'b> { + let start = _fbb.start_table(); + DatasetSourceDerivativeBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} + +pub enum DatasetVocabularyOffset {} +#[derive(Copy, Clone, Debug, PartialEq)] + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +pub struct DatasetVocabulary<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for DatasetVocabulary<'a> { + type Inner = DatasetVocabulary<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf: buf, loc: loc }, + } + } +} + +impl<'a> DatasetVocabulary<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + DatasetVocabulary { + _tab: table, + } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args DatasetVocabularyArgs<'args>) -> flatbuffers::WIPOffset> { + let mut builder = DatasetVocabularyBuilder::new(_fbb); + if let Some(x) = args.event_time_column { builder.add_event_time_column(x); } + if let Some(x) = args.system_time_column { builder.add_system_time_column(x); } + builder.finish() + } + + pub const VT_SYSTEM_TIME_COLUMN: flatbuffers::VOffsetT = 4; + pub const VT_EVENT_TIME_COLUMN: flatbuffers::VOffsetT = 6; + + #[inline] + pub fn system_time_column(&self) -> Option<&'a str> { + self._tab.get::>(DatasetVocabulary::VT_SYSTEM_TIME_COLUMN, None) + } + #[inline] + pub fn event_time_column(&self) -> Option<&'a str> { + self._tab.get::>(DatasetVocabulary::VT_EVENT_TIME_COLUMN, None) + } +} + +pub struct DatasetVocabularyArgs<'a> { + pub system_time_column: Option>, + pub event_time_column: Option>, +} +impl<'a> Default for DatasetVocabularyArgs<'a> { + #[inline] + fn default() -> Self { + DatasetVocabularyArgs { + system_time_column: None, + event_time_column: None, + } + } +} +pub struct DatasetVocabularyBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b> DatasetVocabularyBuilder<'a, 'b> { + #[inline] + pub fn add_system_time_column(&mut self, system_time_column: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(DatasetVocabulary::VT_SYSTEM_TIME_COLUMN, system_time_column); + } + #[inline] + pub fn add_event_time_column(&mut self, event_time_column: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(DatasetVocabulary::VT_EVENT_TIME_COLUMN, event_time_column); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> DatasetVocabularyBuilder<'a, 'b> { + let start = _fbb.start_table(); + DatasetVocabularyBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} + +pub enum TemporalTableOffset {} +#[derive(Copy, Clone, Debug, PartialEq)] + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +pub struct TemporalTable<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for TemporalTable<'a> { + type Inner = TemporalTable<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf: buf, loc: loc }, + } + } +} + +impl<'a> TemporalTable<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + TemporalTable { + _tab: table, + } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args TemporalTableArgs<'args>) -> flatbuffers::WIPOffset> { + let mut builder = TemporalTableBuilder::new(_fbb); + if let Some(x) = args.primary_key { builder.add_primary_key(x); } + if let Some(x) = args.id { builder.add_id(x); } + builder.finish() + } + + pub const VT_ID: flatbuffers::VOffsetT = 4; + pub const VT_PRIMARY_KEY: flatbuffers::VOffsetT = 6; + + #[inline] + pub fn id(&self) -> Option<&'a str> { + self._tab.get::>(TemporalTable::VT_ID, None) + } + #[inline] + pub fn primary_key(&self) -> Option>> { + self._tab.get::>>>(TemporalTable::VT_PRIMARY_KEY, None) + } +} + +pub struct TemporalTableArgs<'a> { + pub id: Option>, + pub primary_key: Option>>>, +} +impl<'a> Default for TemporalTableArgs<'a> { + #[inline] + fn default() -> Self { + TemporalTableArgs { + id: None, + primary_key: None, + } + } +} +pub struct TemporalTableBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b> TemporalTableBuilder<'a, 'b> { + #[inline] + pub fn add_id(&mut self, id: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(TemporalTable::VT_ID, id); + } + #[inline] + pub fn add_primary_key(&mut self, primary_key: flatbuffers::WIPOffset>>) { + self.fbb_.push_slot_always::>(TemporalTable::VT_PRIMARY_KEY, primary_key); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> TemporalTableBuilder<'a, 'b> { + let start = _fbb.start_table(); + TemporalTableBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} + +pub enum DatasetSnapshotOffset {} +#[derive(Copy, Clone, Debug, PartialEq)] + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +pub struct DatasetSnapshot<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for DatasetSnapshot<'a> { + type Inner = DatasetSnapshot<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf: buf, loc: loc }, + } + } +} + +impl<'a> DatasetSnapshot<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + DatasetSnapshot { + _tab: table, + } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args DatasetSnapshotArgs<'args>) -> flatbuffers::WIPOffset> { + let mut builder = DatasetSnapshotBuilder::new(_fbb); + if let Some(x) = args.vocab { builder.add_vocab(x); } + if let Some(x) = args.source { builder.add_source(x); } + if let Some(x) = args.id { builder.add_id(x); } + builder.add_source_type(args.source_type); + builder.finish() + } + + pub const VT_ID: flatbuffers::VOffsetT = 4; + pub const VT_SOURCE_TYPE: flatbuffers::VOffsetT = 6; + pub const VT_SOURCE: flatbuffers::VOffsetT = 8; + pub const VT_VOCAB: flatbuffers::VOffsetT = 10; + + #[inline] + pub fn id(&self) -> Option<&'a str> { + self._tab.get::>(DatasetSnapshot::VT_ID, None) + } + #[inline] + pub fn source_type(&self) -> DatasetSource { + self._tab.get::(DatasetSnapshot::VT_SOURCE_TYPE, Some(DatasetSource::NONE)).unwrap() + } + #[inline] + pub fn source(&self) -> Option> { + self._tab.get::>>(DatasetSnapshot::VT_SOURCE, None) + } + #[inline] + pub fn vocab(&self) -> Option> { + self._tab.get::>>(DatasetSnapshot::VT_VOCAB, None) + } + #[inline] + #[allow(non_snake_case)] + pub fn source_as_dataset_source_root(&self) -> Option> { + if self.source_type() == DatasetSource::DatasetSourceRoot { + self.source().map(|u| DatasetSourceRoot::init_from_table(u)) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn source_as_dataset_source_derivative(&self) -> Option> { + if self.source_type() == DatasetSource::DatasetSourceDerivative { + self.source().map(|u| DatasetSourceDerivative::init_from_table(u)) + } else { + None + } + } + +} + +pub struct DatasetSnapshotArgs<'a> { + pub id: Option>, + pub source_type: DatasetSource, + pub source: Option>, + pub vocab: Option>>, +} +impl<'a> Default for DatasetSnapshotArgs<'a> { + #[inline] + fn default() -> Self { + DatasetSnapshotArgs { + id: None, + source_type: DatasetSource::NONE, + source: None, + vocab: None, + } + } +} +pub struct DatasetSnapshotBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b> DatasetSnapshotBuilder<'a, 'b> { + #[inline] + pub fn add_id(&mut self, id: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(DatasetSnapshot::VT_ID, id); + } + #[inline] + pub fn add_source_type(&mut self, source_type: DatasetSource) { + self.fbb_.push_slot::(DatasetSnapshot::VT_SOURCE_TYPE, source_type, DatasetSource::NONE); + } + #[inline] + pub fn add_source(&mut self, source: flatbuffers::WIPOffset) { + self.fbb_.push_slot_always::>(DatasetSnapshot::VT_SOURCE, source); + } + #[inline] + pub fn add_vocab(&mut self, vocab: flatbuffers::WIPOffset>) { + self.fbb_.push_slot_always::>(DatasetSnapshot::VT_VOCAB, vocab); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> DatasetSnapshotBuilder<'a, 'b> { + let start = _fbb.start_table(); + DatasetSnapshotBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} + +pub enum DataSliceOffset {} +#[derive(Copy, Clone, Debug, PartialEq)] + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +pub struct DataSlice<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for DataSlice<'a> { + type Inner = DataSlice<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf: buf, loc: loc }, + } + } +} + +impl<'a> DataSlice<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + DataSlice { + _tab: table, + } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args DataSliceArgs<'args>) -> flatbuffers::WIPOffset> { + let mut builder = DataSliceBuilder::new(_fbb); + if let Some(x) = args.num_records { builder.add_num_records(x); } + if let Some(x) = args.interval { builder.add_interval(x); } + if let Some(x) = args.hash { builder.add_hash(x); } + builder.finish() + } + + pub const VT_HASH: flatbuffers::VOffsetT = 4; + pub const VT_INTERVAL: flatbuffers::VOffsetT = 6; + pub const VT_NUM_RECORDS: flatbuffers::VOffsetT = 8; + + #[inline] + pub fn hash(&self) -> Option<&'a [u8]> { + self._tab.get::>>(DataSlice::VT_HASH, None).map(|v| v.safe_slice()) + } + #[inline] + pub fn interval(&self) -> Option<&'a TimeInterval> { + self._tab.get::(DataSlice::VT_INTERVAL, None) + } + #[inline] + pub fn num_records(&self) -> Option<&'a Option_int64> { + self._tab.get::(DataSlice::VT_NUM_RECORDS, None) + } +} + +pub struct DataSliceArgs<'a> { + pub hash: Option>>, + pub interval: Option<&'a TimeInterval>, + pub num_records: Option<&'a Option_int64>, +} +impl<'a> Default for DataSliceArgs<'a> { + #[inline] + fn default() -> Self { + DataSliceArgs { + hash: None, + interval: None, + num_records: None, + } + } +} +pub struct DataSliceBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b> DataSliceBuilder<'a, 'b> { + #[inline] + pub fn add_hash(&mut self, hash: flatbuffers::WIPOffset>) { + self.fbb_.push_slot_always::>(DataSlice::VT_HASH, hash); + } + #[inline] + pub fn add_interval(&mut self, interval: &'b TimeInterval) { + self.fbb_.push_slot_always::<&TimeInterval>(DataSlice::VT_INTERVAL, interval); + } + #[inline] + pub fn add_num_records(&mut self, num_records: &'b Option_int64) { + self.fbb_.push_slot_always::<&Option_int64>(DataSlice::VT_NUM_RECORDS, num_records); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> DataSliceBuilder<'a, 'b> { + let start = _fbb.start_table(); + DataSliceBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} + +pub enum SqlQueryStepOffset {} +#[derive(Copy, Clone, Debug, PartialEq)] + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +pub struct SqlQueryStep<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for SqlQueryStep<'a> { + type Inner = SqlQueryStep<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf: buf, loc: loc }, + } + } +} + +impl<'a> SqlQueryStep<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + SqlQueryStep { + _tab: table, + } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args SqlQueryStepArgs<'args>) -> flatbuffers::WIPOffset> { + let mut builder = SqlQueryStepBuilder::new(_fbb); + if let Some(x) = args.query { builder.add_query(x); } + if let Some(x) = args.alias { builder.add_alias(x); } + builder.finish() + } + + pub const VT_ALIAS: flatbuffers::VOffsetT = 4; + pub const VT_QUERY: flatbuffers::VOffsetT = 6; + + #[inline] + pub fn alias(&self) -> Option<&'a str> { + self._tab.get::>(SqlQueryStep::VT_ALIAS, None) + } + #[inline] + pub fn query(&self) -> Option<&'a str> { + self._tab.get::>(SqlQueryStep::VT_QUERY, None) + } +} + +pub struct SqlQueryStepArgs<'a> { + pub alias: Option>, + pub query: Option>, +} +impl<'a> Default for SqlQueryStepArgs<'a> { + #[inline] + fn default() -> Self { + SqlQueryStepArgs { + alias: None, + query: None, + } + } +} +pub struct SqlQueryStepBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b> SqlQueryStepBuilder<'a, 'b> { + #[inline] + pub fn add_alias(&mut self, alias: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(SqlQueryStep::VT_ALIAS, alias); + } + #[inline] + pub fn add_query(&mut self, query: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>(SqlQueryStep::VT_QUERY, query); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> SqlQueryStepBuilder<'a, 'b> { + let start = _fbb.start_table(); + SqlQueryStepBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} + +pub enum MetadataBlockOffset {} +#[derive(Copy, Clone, Debug, PartialEq)] + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +pub struct MetadataBlock<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for MetadataBlock<'a> { + type Inner = MetadataBlock<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf: buf, loc: loc }, + } + } +} + +impl<'a> MetadataBlock<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + MetadataBlock { + _tab: table, + } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args MetadataBlockArgs<'args>) -> flatbuffers::WIPOffset> { + let mut builder = MetadataBlockBuilder::new(_fbb); + if let Some(x) = args.source { builder.add_source(x); } + if let Some(x) = args.input_slices { builder.add_input_slices(x); } + if let Some(x) = args.output_watermark { builder.add_output_watermark(x); } + if let Some(x) = args.output_slice { builder.add_output_slice(x); } + if let Some(x) = args.system_time { builder.add_system_time(x); } + if let Some(x) = args.prev_block_hash { builder.add_prev_block_hash(x); } + if let Some(x) = args.block_hash { builder.add_block_hash(x); } + builder.add_source_type(args.source_type); + builder.finish() + } + + pub const VT_BLOCK_HASH: flatbuffers::VOffsetT = 4; + pub const VT_PREV_BLOCK_HASH: flatbuffers::VOffsetT = 6; + pub const VT_SYSTEM_TIME: flatbuffers::VOffsetT = 8; + pub const VT_OUTPUT_SLICE: flatbuffers::VOffsetT = 10; + pub const VT_OUTPUT_WATERMARK: flatbuffers::VOffsetT = 12; + pub const VT_INPUT_SLICES: flatbuffers::VOffsetT = 14; + pub const VT_SOURCE_TYPE: flatbuffers::VOffsetT = 16; + pub const VT_SOURCE: flatbuffers::VOffsetT = 18; + + #[inline] + pub fn block_hash(&self) -> Option<&'a [u8]> { + self._tab.get::>>(MetadataBlock::VT_BLOCK_HASH, None).map(|v| v.safe_slice()) + } + #[inline] + pub fn prev_block_hash(&self) -> Option<&'a [u8]> { + self._tab.get::>>(MetadataBlock::VT_PREV_BLOCK_HASH, None).map(|v| v.safe_slice()) + } + #[inline] + pub fn system_time(&self) -> Option<&'a Timestamp> { + self._tab.get::(MetadataBlock::VT_SYSTEM_TIME, None) + } + #[inline] + pub fn output_slice(&self) -> Option> { + self._tab.get::>>(MetadataBlock::VT_OUTPUT_SLICE, None) + } + #[inline] + pub fn output_watermark(&self) -> Option<&'a Timestamp> { + self._tab.get::(MetadataBlock::VT_OUTPUT_WATERMARK, None) + } + #[inline] + pub fn input_slices(&self) -> Option>>> { + self._tab.get::>>>>(MetadataBlock::VT_INPUT_SLICES, None) + } + #[inline] + pub fn source_type(&self) -> DatasetSource { + self._tab.get::(MetadataBlock::VT_SOURCE_TYPE, Some(DatasetSource::NONE)).unwrap() + } + #[inline] + pub fn source(&self) -> Option> { + self._tab.get::>>(MetadataBlock::VT_SOURCE, None) + } + #[inline] + #[allow(non_snake_case)] + pub fn source_as_dataset_source_root(&self) -> Option> { + if self.source_type() == DatasetSource::DatasetSourceRoot { + self.source().map(|u| DatasetSourceRoot::init_from_table(u)) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn source_as_dataset_source_derivative(&self) -> Option> { + if self.source_type() == DatasetSource::DatasetSourceDerivative { + self.source().map(|u| DatasetSourceDerivative::init_from_table(u)) + } else { + None + } + } + +} + +pub struct MetadataBlockArgs<'a> { + pub block_hash: Option>>, + pub prev_block_hash: Option>>, + pub system_time: Option<&'a Timestamp>, + pub output_slice: Option>>, + pub output_watermark: Option<&'a Timestamp>, + pub input_slices: Option>>>>, + pub source_type: DatasetSource, + pub source: Option>, +} +impl<'a> Default for MetadataBlockArgs<'a> { + #[inline] + fn default() -> Self { + MetadataBlockArgs { + block_hash: None, + prev_block_hash: None, + system_time: None, + output_slice: None, + output_watermark: None, + input_slices: None, + source_type: DatasetSource::NONE, + source: None, + } + } +} +pub struct MetadataBlockBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b> MetadataBlockBuilder<'a, 'b> { + #[inline] + pub fn add_block_hash(&mut self, block_hash: flatbuffers::WIPOffset>) { + self.fbb_.push_slot_always::>(MetadataBlock::VT_BLOCK_HASH, block_hash); + } + #[inline] + pub fn add_prev_block_hash(&mut self, prev_block_hash: flatbuffers::WIPOffset>) { + self.fbb_.push_slot_always::>(MetadataBlock::VT_PREV_BLOCK_HASH, prev_block_hash); + } + #[inline] + pub fn add_system_time(&mut self, system_time: &'b Timestamp) { + self.fbb_.push_slot_always::<&Timestamp>(MetadataBlock::VT_SYSTEM_TIME, system_time); + } + #[inline] + pub fn add_output_slice(&mut self, output_slice: flatbuffers::WIPOffset>) { + self.fbb_.push_slot_always::>(MetadataBlock::VT_OUTPUT_SLICE, output_slice); + } + #[inline] + pub fn add_output_watermark(&mut self, output_watermark: &'b Timestamp) { + self.fbb_.push_slot_always::<&Timestamp>(MetadataBlock::VT_OUTPUT_WATERMARK, output_watermark); + } + #[inline] + pub fn add_input_slices(&mut self, input_slices: flatbuffers::WIPOffset>>>) { + self.fbb_.push_slot_always::>(MetadataBlock::VT_INPUT_SLICES, input_slices); + } + #[inline] + pub fn add_source_type(&mut self, source_type: DatasetSource) { + self.fbb_.push_slot::(MetadataBlock::VT_SOURCE_TYPE, source_type, DatasetSource::NONE); + } + #[inline] + pub fn add_source(&mut self, source: flatbuffers::WIPOffset) { + self.fbb_.push_slot_always::>(MetadataBlock::VT_SOURCE, source); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> MetadataBlockBuilder<'a, 'b> { + let start = _fbb.start_table(); + MetadataBlockBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} + diff --git a/opendatafabric/src/serde/flatbuffers/serdes.rs b/opendatafabric/src/serde/flatbuffers/serdes.rs new file mode 100644 index 0000000000..a18b1addc2 --- /dev/null +++ b/opendatafabric/src/serde/flatbuffers/serdes.rs @@ -0,0 +1,183 @@ +use super::convertors::*; +pub use crate::serde::{Buffer, Error, MetadataBlockDeserializer, MetadataBlockSerializer}; +use crate::{MetadataBlock, Sha3_256}; +use crypto::digest::Digest; +use crypto::sha3::Sha3; +use std::convert::TryInto; +use thiserror::Error; + +/////////////////////////////////////////////////////////////////////////////// +// FlatbuffersSerializer +/////////////////////////////////////////////////////////////////////////////// + +pub struct FlatbuffersMetadataBlockSerializer; + +/////////////////////////////////////////////////////////////////////////////// + +impl FlatbuffersMetadataBlockSerializer { + pub fn serialize_metadata_block(&self, block: &MetadataBlock) -> Buffer { + let mut fb = flatbuffers::FlatBufferBuilder::new_with_capacity(0); + let offset = block.serialize(&mut fb); + fb.finish(offset, None); + let (buf, head) = fb.collapse(); + Buffer::new(head, buf.len(), buf) + } + + pub fn set_metadata_block_hash(&self, data: &mut [u8]) -> Result { + let hash_index = data.len() - Sha3_256::LENGTH; + + FlatbuffersMetadataBlockDeserializer::validate_hash_buffer_position(data)?; + + let mut digest = Sha3::sha3_256(); + digest.input(&data[..hash_index]); + digest.result(&mut data[hash_index..]); + + Ok(Sha3_256::new(data[hash_index..].try_into().unwrap())) + } + + pub fn prepend_manifest( + &self, + mut buffer: Buffer, + version: u16, + kind: &str, + ) -> Result, Error> { + use byteorder::{LittleEndian, WriteBytesExt}; + use std::io::Write; + + let kind_bytes = kind.as_bytes(); + let header_size = kind_bytes.len() + 2 + 2; + buffer.ensure_capacity(header_size, 0); + + let header_start = buffer.head() - header_size; + let header_end = buffer.head(); + let mut cursor = std::io::Cursor::new(&mut buffer.inner_mut()[header_start..header_end]); + cursor.write_u16::(version)?; + cursor.write_u16::(kind_bytes.len() as u16)?; + cursor.write(kind_bytes)?; + buffer.set_head(header_start); + Ok(buffer) + } +} + +/////////////////////////////////////////////////////////////////////////////// + +impl MetadataBlockSerializer for FlatbuffersMetadataBlockSerializer { + fn write_manifest(&self, block: &MetadataBlock) -> Result<(Sha3_256, Buffer), Error> { + let mut buffer = self.serialize_metadata_block(block); + + let block_hash = if block.block_hash.is_zero() { + self.set_metadata_block_hash(&mut buffer)? + } else { + FlatbuffersMetadataBlockDeserializer::validate_block_hash(&buffer)?; + block.block_hash + }; + + let buffer = self.prepend_manifest(buffer, 1, "MetadataBlock")?; + Ok((block_hash, buffer)) + } + + fn write_manifest_unchecked(&self, block: &MetadataBlock) -> Result, Error> { + let buffer = self.serialize_metadata_block(block); + self.prepend_manifest(buffer, 1, "MetadataBlock") + } +} + +/////////////////////////////////////////////////////////////////////////////// +// FlatbuffersMetadataBlockDeserializer +/////////////////////////////////////////////////////////////////////////////// + +pub struct FlatbuffersMetadataBlockDeserializer; + +/////////////////////////////////////////////////////////////////////////////// + +impl FlatbuffersMetadataBlockDeserializer { + pub fn read_manifest_header<'a>( + &self, + data: &'a [u8], + ) -> Result<(u16, &'a str, &'a [u8]), Error> { + use byteorder::{LittleEndian, ReadBytesExt}; + + let mut cursor = std::io::Cursor::new(data); + let version = cursor.read_u16::()?; + let kind_len = cursor.read_u16::()? as usize; + let kind = std::str::from_utf8(&data[4..4 + kind_len]).unwrap(); + Ok((version, kind, &data[4 + kind_len..])) + } + + fn validate_hash_buffer_position(data: &[u8]) -> Result<(), Error> { + let proxy = flatbuffers::get_root::(data); + let block_hash = proxy.block_hash().unwrap(); + + if block_hash.len() != Sha3_256::LENGTH { + return Err(Error::serde(MalformedFlatbufferError)); + } + + let p_buf_end: *const u8 = data.as_ptr().wrapping_add(data.len() - Sha3_256::LENGTH); + let p_hash: *const u8 = block_hash.as_ptr(); + if p_buf_end != p_hash { + return Err(Error::serde(MalformedFlatbufferError)); + } + + Ok(()) + } + + fn validate_block_hash(data: &[u8]) -> Result<(), Error> { + Self::validate_hash_buffer_position(data)?; + + let hash_index = data.len() - Sha3_256::LENGTH; + let block_hash = &data[hash_index..]; + let mut true_block_hash = [0u8; 32]; + + let mut digest = Sha3::sha3_256(); + digest.input(&data[..hash_index]); + digest.result(&mut true_block_hash); + + if &true_block_hash != block_hash { + return Err(Error::invalid_hash( + Sha3_256::new(block_hash.try_into().unwrap()), + Sha3_256::new(true_block_hash), + )); + } + + Ok(()) + } +} + +/////////////////////////////////////////////////////////////////////////////// + +impl MetadataBlockDeserializer for FlatbuffersMetadataBlockDeserializer { + fn validate_manifest(&self, data: &[u8]) -> Result<(), Error> { + let (_, _, tail) = self.read_manifest_header(data)?; + Self::validate_block_hash(tail) + } + + fn read_manifest(&self, data: &[u8]) -> Result { + let (version, kind, tail) = self.read_manifest_header(data)?; + + // TODO: Handle conversions? + assert_eq!(version, 1); + assert_eq!(kind, "MetadataBlock"); + + Self::validate_block_hash(tail)?; + + let proxy = flatbuffers::get_root::(tail); + Ok(MetadataBlock::deserialize(proxy)) + } + + fn read_manifest_unchecked(&self, data: &[u8]) -> Result { + let (version, kind, tail) = self.read_manifest_header(data)?; + + // TODO: Handle conversions? + assert_eq!(version, 1); + assert_eq!(kind, "MetadataBlock"); + + let proxy = flatbuffers::get_root::(tail); + Ok(MetadataBlock::deserialize(proxy)) + } +} + +/////////////////////////////////////////////////////////////////////////////// + +#[derive(Error, Debug)] +#[error("Malformet flatbuffers data")] +pub struct MalformedFlatbufferError; diff --git a/opendatafabric/src/serde/mod.rs b/opendatafabric/src/serde/mod.rs new file mode 100644 index 0000000000..15fd8ffa08 --- /dev/null +++ b/opendatafabric/src/serde/mod.rs @@ -0,0 +1,5 @@ +mod serdes; +pub use serdes::*; + +pub mod flatbuffers; +pub mod yaml; diff --git a/opendatafabric/src/serde/serdes.rs b/opendatafabric/src/serde/serdes.rs new file mode 100644 index 0000000000..3665706a69 --- /dev/null +++ b/opendatafabric/src/serde/serdes.rs @@ -0,0 +1,161 @@ +use crate::{DatasetSnapshot, MetadataBlock, Sha3_256}; +use std::backtrace::Backtrace; +use thiserror::Error; + +/////////////////////////////////////////////////////////////////////////////// + +pub trait MetadataBlockSerializer { + fn write_manifest(&self, block: &MetadataBlock) -> Result<(Sha3_256, Buffer), Error>; + + fn write_manifest_unchecked(&self, block: &MetadataBlock) -> Result, Error>; +} + +/////////////////////////////////////////////////////////////////////////////// + +pub trait MetadataBlockDeserializer { + fn validate_manifest(&self, data: &[u8]) -> Result<(), Error>; + + fn read_manifest(&self, data: &[u8]) -> Result; + + fn read_manifest_unchecked(&self, data: &[u8]) -> Result; +} + +/////////////////////////////////////////////////////////////////////////////// + +pub trait DatasetSnapshotSerializer { + fn write_manifest(&self, snapshot: &DatasetSnapshot) -> Result, Error>; +} + +/////////////////////////////////////////////////////////////////////////////// + +pub trait DatasetSnapshotDeserializer { + fn read_manifest(&self, data: &[u8]) -> Result; +} + +/////////////////////////////////////////////////////////////////////////////// + +#[derive(Clone, Debug)] +pub struct Buffer { + buf: Vec, + head: usize, + tail: usize, +} + +impl Buffer { + pub fn new(head: usize, tail: usize, buf: Vec) -> Self { + Self { + buf: buf, + head: head, + tail: tail, + } + } + + pub fn inner(&self) -> &[T] { + &self.buf + } + + pub fn inner_mut(&mut self) -> &mut [T] { + &mut self.buf + } + + pub fn head(&self) -> usize { + self.head + } + + pub fn set_head(&mut self, head: usize) { + self.head = head; + } + + pub fn tail(&self) -> usize { + self.tail + } + + pub fn set_tail(&mut self, tail: usize) { + self.tail = tail; + } + + pub fn capacity(&self) -> usize { + self.buf.len() + } + + pub fn ensure_capacity(&mut self, space_left: usize, space_right: usize) + where + T: Default + Copy, + { + if self.head < space_left || self.buf.len() - self.tail < space_right { + let nlen = self.buf.len() + space_left + space_right; + let ntail = nlen - space_right; + let mut nbuf = vec![T::default(); nlen]; + nbuf[space_left..nlen - space_right].copy_from_slice(&self.buf[self.head..self.tail]); + + self.head = space_left; + self.tail = ntail; + self.buf = nbuf; + } + } + + pub fn collapse(self) -> (Vec, usize, usize) { + (self.buf, self.head, self.tail) + } +} + +impl std::ops::Deref for Buffer { + type Target = [T]; + + fn deref(&self) -> &Self::Target { + &self.buf[self.head..self.tail] + } +} + +impl std::ops::DerefMut for Buffer { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.buf[self.head..self.tail] + } +} + +/////////////////////////////////////////////////////////////////////////////// + +type BoxedError = Box; + +#[derive(Error, Debug)] +pub enum Error { + #[error("Invalid hash {actual} expected {expected}")] + InvalidHash { + actual: Sha3_256, + expected: Sha3_256, + }, + #[error("IO error: {source}")] + IoError { + #[from] + source: std::io::Error, + backtrace: Backtrace, + }, + #[error("Serde error: {source}")] + SerdeError { + source: BoxedError, + backtrace: Backtrace, + }, +} + +impl Error { + pub fn invalid_hash(actual: Sha3_256, expected: Sha3_256) -> Self { + Self::InvalidHash { + actual: actual, + expected: expected, + } + } + + pub fn io_error(e: std::io::Error) -> Self { + Self::IoError { + source: e, + backtrace: Backtrace::capture(), + } + } + + pub fn serde(e: impl std::error::Error + Send + Sync + 'static) -> Self { + Self::SerdeError { + source: e.into(), + backtrace: Backtrace::capture(), + } + } +} diff --git a/opendatafabric/src/serde/yaml/formats.rs b/opendatafabric/src/serde/yaml/formats.rs new file mode 100644 index 0000000000..273a734ee3 --- /dev/null +++ b/opendatafabric/src/serde/yaml/formats.rs @@ -0,0 +1,51 @@ +// TODO: Yep... all this to serde an Option in a slightly different +// format. See: https://github.com/serde-rs/serde/issues/723 + +pub mod datetime_rfc3339 { + use chrono::{DateTime, SecondsFormat, Utc}; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize( + date: &DateTime, + serializer: S, + ) -> Result { + let s = date.to_rfc3339_opts(SecondsFormat::Millis, true); + serializer.serialize_str(&s) + } + + pub fn deserialize<'de, D: Deserializer<'de>>( + deserializer: D, + ) -> Result, D::Error> { + let s = String::deserialize(deserializer)?; + DateTime::parse_from_rfc3339(&s) + .map(|dt| dt.into()) + .map_err(serde::de::Error::custom) + } +} + +pub mod datetime_rfc3339_opt { + use chrono::{DateTime, SecondsFormat, Utc}; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize( + option: &Option>, + serializer: S, + ) -> Result { + match option { + None => serializer.serialize_none(), + Some(date) => { + serializer.serialize_str(&date.to_rfc3339_opts(SecondsFormat::Millis, true)) + } + } + } + + pub fn deserialize<'de, D: Deserializer<'de>>( + deserializer: D, + ) -> Result>, D::Error> { + let s = String::deserialize(deserializer)?; + DateTime::parse_from_rfc3339(&s) + .map(|dt| dt.into()) + .map(Some) + .map_err(serde::de::Error::custom) + } +} diff --git a/opendatafabric/src/serde/yaml/generated.rs b/opendatafabric/src/serde/yaml/generated.rs new file mode 100644 index 0000000000..ee7e8b40ac --- /dev/null +++ b/opendatafabric/src/serde/yaml/generated.rs @@ -0,0 +1,1189 @@ +//////////////////////////////////////////////////////////////////////////////// +// WARNING: This file is auto-generated from Open Data Fabric Schemas +// See: http://opendatafabric.org/ +//////////////////////////////////////////////////////////////////////////////// + +use super::formats::{datetime_rfc3339, datetime_rfc3339_opt}; +use crate::*; +use ::chrono::{DateTime, Utc}; +use ::serde::{Deserialize, Deserializer, Serialize, Serializer}; +use ::serde_with::serde_as; +use ::serde_with::skip_serializing_none; + +//////////////////////////////////////////////////////////////////////////////// +// DatasetSource +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#datasetsource-schema +//////////////////////////////////////////////////////////////////////////////// + +#[serde(remote = "DatasetSource")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase", tag = "kind")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum DatasetSourceDef { + #[serde(rename_all = "camelCase")] + Root(#[serde_as(as = "DatasetSourceRootDef")] DatasetSourceRoot), + #[serde(rename_all = "camelCase")] + Derivative(#[serde_as(as = "DatasetSourceDerivativeDef")] DatasetSourceDerivative), +} + +impl serde_with::SerializeAs for DatasetSourceDef { + fn serialize_as(source: &DatasetSource, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "DatasetSourceDef")] &'a DatasetSource); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, DatasetSource> for DatasetSourceDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "DatasetSourceDef")] DatasetSource); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +impl serde_with::SerializeAs for DatasetSourceRootDef { + fn serialize_as(source: &DatasetSourceRoot, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "DatasetSourceRootDef")] &'a DatasetSourceRoot); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, DatasetSourceRoot> for DatasetSourceRootDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "DatasetSourceRootDef")] DatasetSourceRoot); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +impl serde_with::SerializeAs for DatasetSourceDerivativeDef { + fn serialize_as(source: &DatasetSourceDerivative, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "DatasetSourceDerivativeDef")] &'a DatasetSourceDerivative); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, DatasetSourceDerivative> for DatasetSourceDerivativeDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "DatasetSourceDerivativeDef")] DatasetSourceDerivative); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +#[serde(remote = "DatasetSourceRoot")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DatasetSourceRootDef { + #[serde_as(as = "FetchStepDef")] + pub fetch: FetchStep, + #[serde_as(as = "Option>")] + #[serde(default)] + pub prepare: Option>, + #[serde_as(as = "ReadStepDef")] + pub read: ReadStep, + #[serde_as(as = "Option")] + #[serde(default)] + pub preprocess: Option, + #[serde_as(as = "MergeStrategyDef")] + pub merge: MergeStrategy, +} + +#[serde(remote = "DatasetSourceDerivative")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DatasetSourceDerivativeDef { + pub inputs: Vec, + #[serde_as(as = "TransformDef")] + pub transform: Transform, +} + +//////////////////////////////////////////////////////////////////////////////// +// DatasetVocabulary +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#datasetvocabulary-schema +//////////////////////////////////////////////////////////////////////////////// + +#[serde(remote = "DatasetVocabulary")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DatasetVocabularyDef { + pub system_time_column: Option, + pub event_time_column: Option, +} + +impl serde_with::SerializeAs for DatasetVocabularyDef { + fn serialize_as(source: &DatasetVocabulary, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "DatasetVocabularyDef")] &'a DatasetVocabulary); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, DatasetVocabulary> for DatasetVocabularyDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "DatasetVocabularyDef")] DatasetVocabulary); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +//////////////////////////////////////////////////////////////////////////////// +// SourceCaching +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#sourcecaching-schema +//////////////////////////////////////////////////////////////////////////////// + +#[serde(remote = "SourceCaching")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase", tag = "kind")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum SourceCachingDef { + #[serde(rename_all = "camelCase")] + Forever, +} + +impl serde_with::SerializeAs for SourceCachingDef { + fn serialize_as(source: &SourceCaching, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "SourceCachingDef")] &'a SourceCaching); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, SourceCaching> for SourceCachingDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "SourceCachingDef")] SourceCaching); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +//////////////////////////////////////////////////////////////////////////////// +// TemporalTable +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#temporaltable-schema +//////////////////////////////////////////////////////////////////////////////// + +#[serde(remote = "TemporalTable")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TemporalTableDef { + pub id: String, + pub primary_key: Vec, +} + +impl serde_with::SerializeAs for TemporalTableDef { + fn serialize_as(source: &TemporalTable, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "TemporalTableDef")] &'a TemporalTable); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, TemporalTable> for TemporalTableDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "TemporalTableDef")] TemporalTable); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +//////////////////////////////////////////////////////////////////////////////// +// DatasetSnapshot +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#datasetsnapshot-schema +//////////////////////////////////////////////////////////////////////////////// + +#[serde(remote = "DatasetSnapshot")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DatasetSnapshotDef { + pub id: DatasetIDBuf, + #[serde_as(as = "DatasetSourceDef")] + pub source: DatasetSource, + #[serde_as(as = "Option")] + #[serde(default)] + pub vocab: Option, +} + +impl serde_with::SerializeAs for DatasetSnapshotDef { + fn serialize_as(source: &DatasetSnapshot, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "DatasetSnapshotDef")] &'a DatasetSnapshot); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, DatasetSnapshot> for DatasetSnapshotDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "DatasetSnapshotDef")] DatasetSnapshot); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +//////////////////////////////////////////////////////////////////////////////// +// DataSlice +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#dataslice-schema +//////////////////////////////////////////////////////////////////////////////// + +#[serde(remote = "DataSlice")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DataSliceDef { + pub hash: Sha3_256, + pub interval: TimeInterval, + pub num_records: i64, +} + +impl serde_with::SerializeAs for DataSliceDef { + fn serialize_as(source: &DataSlice, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "DataSliceDef")] &'a DataSlice); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, DataSlice> for DataSliceDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "DataSliceDef")] DataSlice); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +//////////////////////////////////////////////////////////////////////////////// +// SqlQueryStep +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#sqlquerystep-schema +//////////////////////////////////////////////////////////////////////////////// + +#[serde(remote = "SqlQueryStep")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SqlQueryStepDef { + pub alias: Option, + pub query: String, +} + +impl serde_with::SerializeAs for SqlQueryStepDef { + fn serialize_as(source: &SqlQueryStep, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "SqlQueryStepDef")] &'a SqlQueryStep); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, SqlQueryStep> for SqlQueryStepDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "SqlQueryStepDef")] SqlQueryStep); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +//////////////////////////////////////////////////////////////////////////////// +// MergeStrategy +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#mergestrategy-schema +//////////////////////////////////////////////////////////////////////////////// + +#[serde(remote = "MergeStrategy")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase", tag = "kind")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum MergeStrategyDef { + #[serde(rename_all = "camelCase")] + Append, + #[serde(rename_all = "camelCase")] + Ledger(#[serde_as(as = "MergeStrategyLedgerDef")] MergeStrategyLedger), + #[serde(rename_all = "camelCase")] + Snapshot(#[serde_as(as = "MergeStrategySnapshotDef")] MergeStrategySnapshot), +} + +impl serde_with::SerializeAs for MergeStrategyDef { + fn serialize_as(source: &MergeStrategy, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "MergeStrategyDef")] &'a MergeStrategy); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, MergeStrategy> for MergeStrategyDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "MergeStrategyDef")] MergeStrategy); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +impl serde_with::SerializeAs for MergeStrategyLedgerDef { + fn serialize_as(source: &MergeStrategyLedger, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "MergeStrategyLedgerDef")] &'a MergeStrategyLedger); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, MergeStrategyLedger> for MergeStrategyLedgerDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "MergeStrategyLedgerDef")] MergeStrategyLedger); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +impl serde_with::SerializeAs for MergeStrategySnapshotDef { + fn serialize_as(source: &MergeStrategySnapshot, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "MergeStrategySnapshotDef")] &'a MergeStrategySnapshot); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, MergeStrategySnapshot> for MergeStrategySnapshotDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "MergeStrategySnapshotDef")] MergeStrategySnapshot); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +#[serde(remote = "MergeStrategyLedger")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MergeStrategyLedgerDef { + pub primary_key: Vec, +} + +#[serde(remote = "MergeStrategySnapshot")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MergeStrategySnapshotDef { + pub primary_key: Vec, + pub compare_columns: Option>, + pub observation_column: Option, + pub obsv_added: Option, + pub obsv_changed: Option, + pub obsv_removed: Option, +} + +//////////////////////////////////////////////////////////////////////////////// +// MetadataBlock +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#metadatablock-schema +//////////////////////////////////////////////////////////////////////////////// + +#[serde(remote = "MetadataBlock")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MetadataBlockDef { + pub block_hash: Sha3_256, + pub prev_block_hash: Option, + #[serde(with = "datetime_rfc3339")] + pub system_time: DateTime, + #[serde_as(as = "Option")] + #[serde(default)] + pub output_slice: Option, + #[serde(default, with = "datetime_rfc3339_opt")] + pub output_watermark: Option>, + #[serde_as(as = "Option>")] + #[serde(default)] + pub input_slices: Option>, + #[serde_as(as = "Option")] + #[serde(default)] + pub source: Option, +} + +impl serde_with::SerializeAs for MetadataBlockDef { + fn serialize_as(source: &MetadataBlock, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "MetadataBlockDef")] &'a MetadataBlock); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, MetadataBlock> for MetadataBlockDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "MetadataBlockDef")] MetadataBlock); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +//////////////////////////////////////////////////////////////////////////////// +// ReadStep +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#readstep-schema +//////////////////////////////////////////////////////////////////////////////// + +#[serde(remote = "ReadStep")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase", tag = "kind")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum ReadStepDef { + #[serde(rename_all = "camelCase")] + Csv(#[serde_as(as = "ReadStepCsvDef")] ReadStepCsv), + #[serde(rename_all = "camelCase")] + JsonLines(#[serde_as(as = "ReadStepJsonLinesDef")] ReadStepJsonLines), + #[serde(rename_all = "camelCase")] + GeoJson(#[serde_as(as = "ReadStepGeoJsonDef")] ReadStepGeoJson), + #[serde(rename_all = "camelCase")] + EsriShapefile(#[serde_as(as = "ReadStepEsriShapefileDef")] ReadStepEsriShapefile), +} + +impl serde_with::SerializeAs for ReadStepDef { + fn serialize_as(source: &ReadStep, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "ReadStepDef")] &'a ReadStep); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, ReadStep> for ReadStepDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "ReadStepDef")] ReadStep); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +impl serde_with::SerializeAs for ReadStepCsvDef { + fn serialize_as(source: &ReadStepCsv, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "ReadStepCsvDef")] &'a ReadStepCsv); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, ReadStepCsv> for ReadStepCsvDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "ReadStepCsvDef")] ReadStepCsv); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +impl serde_with::SerializeAs for ReadStepJsonLinesDef { + fn serialize_as(source: &ReadStepJsonLines, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "ReadStepJsonLinesDef")] &'a ReadStepJsonLines); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, ReadStepJsonLines> for ReadStepJsonLinesDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "ReadStepJsonLinesDef")] ReadStepJsonLines); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +impl serde_with::SerializeAs for ReadStepGeoJsonDef { + fn serialize_as(source: &ReadStepGeoJson, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "ReadStepGeoJsonDef")] &'a ReadStepGeoJson); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, ReadStepGeoJson> for ReadStepGeoJsonDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "ReadStepGeoJsonDef")] ReadStepGeoJson); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +impl serde_with::SerializeAs for ReadStepEsriShapefileDef { + fn serialize_as(source: &ReadStepEsriShapefile, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "ReadStepEsriShapefileDef")] &'a ReadStepEsriShapefile); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, ReadStepEsriShapefile> for ReadStepEsriShapefileDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "ReadStepEsriShapefileDef")] ReadStepEsriShapefile); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +#[serde(remote = "ReadStepCsv")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ReadStepCsvDef { + pub schema: Option>, + pub separator: Option, + pub encoding: Option, + pub quote: Option, + pub escape: Option, + pub comment: Option, + pub header: Option, + pub enforce_schema: Option, + pub infer_schema: Option, + pub ignore_leading_white_space: Option, + pub ignore_trailing_white_space: Option, + pub null_value: Option, + pub empty_value: Option, + pub nan_value: Option, + pub positive_inf: Option, + pub negative_inf: Option, + pub date_format: Option, + pub timestamp_format: Option, + pub multi_line: Option, +} + +#[serde(remote = "ReadStepJsonLines")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ReadStepJsonLinesDef { + pub schema: Option>, + pub date_format: Option, + pub encoding: Option, + pub multi_line: Option, + pub primitives_as_string: Option, + pub timestamp_format: Option, +} + +#[serde(remote = "ReadStepGeoJson")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ReadStepGeoJsonDef { + pub schema: Option>, +} + +#[serde(remote = "ReadStepEsriShapefile")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ReadStepEsriShapefileDef { + pub schema: Option>, + pub sub_path: Option, +} + +//////////////////////////////////////////////////////////////////////////////// +// Transform +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#transform-schema +//////////////////////////////////////////////////////////////////////////////// + +#[serde(remote = "Transform")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase", tag = "kind")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum TransformDef { + #[serde(rename_all = "camelCase")] + Sql(#[serde_as(as = "TransformSqlDef")] TransformSql), +} + +impl serde_with::SerializeAs for TransformDef { + fn serialize_as(source: &Transform, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "TransformDef")] &'a Transform); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, Transform> for TransformDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "TransformDef")] Transform); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +impl serde_with::SerializeAs for TransformSqlDef { + fn serialize_as(source: &TransformSql, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "TransformSqlDef")] &'a TransformSql); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, TransformSql> for TransformSqlDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "TransformSqlDef")] TransformSql); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +#[serde(remote = "TransformSql")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TransformSqlDef { + pub engine: String, + pub version: Option, + pub query: Option, + #[serde_as(as = "Option>")] + #[serde(default)] + pub queries: Option>, + #[serde_as(as = "Option>")] + #[serde(default)] + pub temporal_tables: Option>, +} + +//////////////////////////////////////////////////////////////////////////////// +// FetchStep +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#fetchstep-schema +//////////////////////////////////////////////////////////////////////////////// + +#[serde(remote = "FetchStep")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase", tag = "kind")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum FetchStepDef { + #[serde(rename_all = "camelCase")] + Url(#[serde_as(as = "FetchStepUrlDef")] FetchStepUrl), + #[serde(rename_all = "camelCase")] + FilesGlob(#[serde_as(as = "FetchStepFilesGlobDef")] FetchStepFilesGlob), +} + +impl serde_with::SerializeAs for FetchStepDef { + fn serialize_as(source: &FetchStep, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "FetchStepDef")] &'a FetchStep); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, FetchStep> for FetchStepDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "FetchStepDef")] FetchStep); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +impl serde_with::SerializeAs for FetchStepUrlDef { + fn serialize_as(source: &FetchStepUrl, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "FetchStepUrlDef")] &'a FetchStepUrl); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, FetchStepUrl> for FetchStepUrlDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "FetchStepUrlDef")] FetchStepUrl); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +impl serde_with::SerializeAs for FetchStepFilesGlobDef { + fn serialize_as(source: &FetchStepFilesGlob, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "FetchStepFilesGlobDef")] &'a FetchStepFilesGlob); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, FetchStepFilesGlob> for FetchStepFilesGlobDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "FetchStepFilesGlobDef")] FetchStepFilesGlob); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +#[serde(remote = "FetchStepUrl")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct FetchStepUrlDef { + pub url: String, + #[serde_as(as = "Option")] + #[serde(default)] + pub event_time: Option, + #[serde_as(as = "Option")] + #[serde(default)] + pub cache: Option, +} + +#[serde(remote = "FetchStepFilesGlob")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct FetchStepFilesGlobDef { + pub path: String, + #[serde_as(as = "Option")] + #[serde(default)] + pub event_time: Option, + #[serde_as(as = "Option")] + #[serde(default)] + pub cache: Option, + #[serde_as(as = "Option")] + #[serde(default)] + pub order: Option, +} + +#[serde(remote = "SourceOrdering")] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum SourceOrderingDef { + ByEventTime, + ByName, +} + +impl serde_with::SerializeAs for SourceOrderingDef { + fn serialize_as(source: &SourceOrdering, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "SourceOrderingDef")] &'a SourceOrdering); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, SourceOrdering> for SourceOrderingDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "SourceOrderingDef")] SourceOrdering); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +//////////////////////////////////////////////////////////////////////////////// +// PrepStep +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#prepstep-schema +//////////////////////////////////////////////////////////////////////////////// + +#[serde(remote = "PrepStep")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase", tag = "kind")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum PrepStepDef { + #[serde(rename_all = "camelCase")] + Decompress(#[serde_as(as = "PrepStepDecompressDef")] PrepStepDecompress), + #[serde(rename_all = "camelCase")] + Pipe(#[serde_as(as = "PrepStepPipeDef")] PrepStepPipe), +} + +impl serde_with::SerializeAs for PrepStepDef { + fn serialize_as(source: &PrepStep, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "PrepStepDef")] &'a PrepStep); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, PrepStep> for PrepStepDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "PrepStepDef")] PrepStep); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +impl serde_with::SerializeAs for PrepStepDecompressDef { + fn serialize_as(source: &PrepStepDecompress, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "PrepStepDecompressDef")] &'a PrepStepDecompress); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, PrepStepDecompress> for PrepStepDecompressDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "PrepStepDecompressDef")] PrepStepDecompress); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +impl serde_with::SerializeAs for PrepStepPipeDef { + fn serialize_as(source: &PrepStepPipe, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "PrepStepPipeDef")] &'a PrepStepPipe); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, PrepStepPipe> for PrepStepPipeDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "PrepStepPipeDef")] PrepStepPipe); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +#[serde(remote = "PrepStepDecompress")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PrepStepDecompressDef { + #[serde_as(as = "CompressionFormatDef")] + pub format: CompressionFormat, + pub sub_path: Option, +} + +#[serde(remote = "PrepStepPipe")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PrepStepPipeDef { + pub command: Vec, +} + +#[serde(remote = "CompressionFormat")] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum CompressionFormatDef { + Gzip, + Zip, +} + +impl serde_with::SerializeAs for CompressionFormatDef { + fn serialize_as(source: &CompressionFormat, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "CompressionFormatDef")] &'a CompressionFormat); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, CompressionFormat> for CompressionFormatDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "CompressionFormatDef")] CompressionFormat); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +//////////////////////////////////////////////////////////////////////////////// +// EventTimeSource +// https://github.com/kamu-data/open-data-fabric/blob/master/open-data-fabric.md#eventtimesource-schema +//////////////////////////////////////////////////////////////////////////////// + +#[serde(remote = "EventTimeSource")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase", tag = "kind")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum EventTimeSourceDef { + #[serde(rename_all = "camelCase")] + FromMetadata, + #[serde(rename_all = "camelCase")] + FromPath(#[serde_as(as = "EventTimeSourceFromPathDef")] EventTimeSourceFromPath), +} + +impl serde_with::SerializeAs for EventTimeSourceDef { + fn serialize_as(source: &EventTimeSource, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "EventTimeSourceDef")] &'a EventTimeSource); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, EventTimeSource> for EventTimeSourceDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "EventTimeSourceDef")] EventTimeSource); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +impl serde_with::SerializeAs for EventTimeSourceFromPathDef { + fn serialize_as(source: &EventTimeSourceFromPath, serializer: S) -> Result + where + S: Serializer, + { + #[derive(Serialize)] + struct Helper<'a>(#[serde(with = "EventTimeSourceFromPathDef")] &'a EventTimeSourceFromPath); + Helper(source).serialize(serializer) + } +} + +impl<'de> serde_with::DeserializeAs<'de, EventTimeSourceFromPath> for EventTimeSourceFromPathDef { + fn deserialize_as(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + #[derive(Deserialize)] + struct Helper(#[serde(with = "EventTimeSourceFromPathDef")] EventTimeSourceFromPath); + let helper = Helper::deserialize(deserializer)?; + let Helper(v) = helper; + Ok(v) + } +} + +#[serde(remote = "EventTimeSourceFromPath")] +#[serde_as] +#[skip_serializing_none] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct EventTimeSourceFromPathDef { + pub pattern: String, + pub timestamp_format: Option, +} diff --git a/opendatafabric/src/serde/yaml/mod.rs b/opendatafabric/src/serde/yaml/mod.rs new file mode 100644 index 0000000000..5b82d0623f --- /dev/null +++ b/opendatafabric/src/serde/yaml/mod.rs @@ -0,0 +1,5 @@ +pub mod formats; +pub mod generated; + +mod serdes; +pub use serdes::*; diff --git a/opendatafabric/src/serde/yaml/serdes.rs b/opendatafabric/src/serde/yaml/serdes.rs new file mode 100644 index 0000000000..f753091b5a --- /dev/null +++ b/opendatafabric/src/serde/yaml/serdes.rs @@ -0,0 +1,150 @@ +use super::generated::*; +use crate::serde::flatbuffers::{ + FlatbuffersMetadataBlockDeserializer, FlatbuffersMetadataBlockSerializer, +}; +pub use crate::serde::{ + Buffer, DatasetSnapshotDeserializer, DatasetSnapshotSerializer, Error, + MetadataBlockDeserializer, MetadataBlockSerializer, +}; +use crate::{DatasetSnapshot, MetadataBlock, Sha3_256}; +use ::serde::{Deserialize, Serialize}; + +/////////////////////////////////////////////////////////////////////////////// +// ManifestDef +/////////////////////////////////////////////////////////////////////////////// + +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[derive(Deserialize, Serialize)] +struct ManifestDefMetadataBlock { + pub api_version: i32, + pub kind: String, + #[serde(with = "MetadataBlockDef")] + pub content: MetadataBlock, +} + +#[serde(deny_unknown_fields, rename_all = "camelCase")] +#[derive(Deserialize, Serialize)] +struct ManifestDefDatasetSnapshot { + pub api_version: i32, + pub kind: String, + #[serde(with = "DatasetSnapshotDef")] + pub content: DatasetSnapshot, +} + +/////////////////////////////////////////////////////////////////////////////// +// YamlMetadataBlockSerializer +/////////////////////////////////////////////////////////////////////////////// + +pub struct YamlMetadataBlockSerializer; + +/////////////////////////////////////////////////////////////////////////////// + +impl YamlMetadataBlockSerializer {} + +/////////////////////////////////////////////////////////////////////////////// + +impl MetadataBlockSerializer for YamlMetadataBlockSerializer { + fn write_manifest(&self, block: &MetadataBlock) -> Result<(Sha3_256, Buffer), Error> { + let (block_hash, _) = FlatbuffersMetadataBlockSerializer.write_manifest(block)?; + let new_block = MetadataBlock { + block_hash: block_hash, + ..block.clone() + }; + + let manifest = ManifestDefMetadataBlock { + api_version: 1, + kind: "MetadataBlock".to_owned(), + content: new_block, + }; + + let buf = serde_yaml::to_vec(&manifest).map_err(|e| Error::serde(e))?; + Ok((block_hash, Buffer::new(0, buf.len(), buf))) + } + + fn write_manifest_unchecked(&self, block: &MetadataBlock) -> Result, Error> { + let manifest = ManifestDefMetadataBlock { + api_version: 1, + kind: "MetadataBlock".to_owned(), + content: block.clone(), + }; + + let buf = serde_yaml::to_vec(&manifest).map_err(|e| Error::serde(e))?; + Ok(Buffer::new(0, buf.len(), buf)) + } +} + +/////////////////////////////////////////////////////////////////////////////// +// YamlMetadataBlockDeserializer +/////////////////////////////////////////////////////////////////////////////// + +pub struct YamlMetadataBlockDeserializer; + +/////////////////////////////////////////////////////////////////////////////// + +impl YamlMetadataBlockDeserializer {} + +/////////////////////////////////////////////////////////////////////////////// + +impl MetadataBlockDeserializer for YamlMetadataBlockDeserializer { + fn validate_manifest(&self, data: &[u8]) -> Result<(), Error> { + self.read_manifest(data)?; + Ok(()) + } + + fn read_manifest(&self, data: &[u8]) -> Result { + // TODO: Will not work if conversion took place + let block = self.read_manifest_unchecked(data)?; + let fbdata = FlatbuffersMetadataBlockSerializer.write_manifest_unchecked(&block)?; + FlatbuffersMetadataBlockDeserializer.validate_manifest(&fbdata)?; + Ok(block) + } + + fn read_manifest_unchecked(&self, data: &[u8]) -> Result { + let manifest: ManifestDefMetadataBlock = + serde_yaml::from_slice(data).map_err(|e| Error::serde(e))?; + + // TODO: Handle conversions? + assert_eq!(manifest.api_version, 1); + assert_eq!(manifest.kind, "MetadataBlock"); + + Ok(manifest.content) + } +} + +/////////////////////////////////////////////////////////////////////////////// +// YamlDatasetSnapshotSerializer +/////////////////////////////////////////////////////////////////////////////// + +pub struct YamlDatasetSnapshotSerializer; + +impl DatasetSnapshotSerializer for YamlDatasetSnapshotSerializer { + fn write_manifest(&self, snapshot: &DatasetSnapshot) -> Result, Error> { + let manifest = ManifestDefDatasetSnapshot { + api_version: 1, + kind: "DatasetSnapshot".to_owned(), + content: snapshot.clone(), + }; + + let buf = serde_yaml::to_vec(&manifest).map_err(|e| Error::serde(e))?; + Ok(Buffer::new(0, buf.len(), buf)) + } +} + +/////////////////////////////////////////////////////////////////////////////// +// YamlDatasetSnapshotDeserializer +/////////////////////////////////////////////////////////////////////////////// + +pub struct YamlDatasetSnapshotDeserializer; + +impl DatasetSnapshotDeserializer for YamlDatasetSnapshotDeserializer { + fn read_manifest(&self, data: &[u8]) -> Result { + let manifest: ManifestDefDatasetSnapshot = + serde_yaml::from_slice(data).map_err(|e| Error::serde(e))?; + + // TODO: Handle conversions? + assert_eq!(manifest.api_version, 1); + assert_eq!(manifest.kind, "DatasetSnapshot"); + + Ok(manifest.content) + } +} diff --git a/opendatafabric/src/sha.rs b/opendatafabric/src/sha.rs new file mode 100644 index 0000000000..413dfc68b0 --- /dev/null +++ b/opendatafabric/src/sha.rs @@ -0,0 +1,93 @@ +use serde::de::{Deserialize, Deserializer, Error, Visitor}; +use serde::{Serialize, Serializer}; +use std::convert::TryFrom; +use std::fmt; +use std::ops; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct Sha3_256([u8; 32]); + +impl Sha3_256 { + pub const LENGTH: usize = 32; + + pub fn new(data: [u8; Self::LENGTH]) -> Self { + Self(data) + } + + pub fn zero() -> Self { + Self([0; Self::LENGTH]) + } + + pub fn is_zero(&self) -> bool { + for b in &self.0 { + if *b != 0 { + return false; + } + } + return true; + } + + pub fn as_array(&self) -> &[u8; 32] { + &self.0 + } +} + +impl ops::Deref for Sha3_256 { + type Target = [u8]; + + fn deref(&self) -> &[u8] { + &self.0 + } +} + +impl TryFrom<&str> for Sha3_256 { + type Error = hex::FromHexError; + + fn try_from(value: &str) -> Result { + let mut slice: [u8; 32] = [0; 32]; + hex::decode_to_slice(value, &mut slice)?; + Ok(Sha3_256(slice)) + } +} + +impl fmt::Debug for Sha3_256 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("Sha3_256") + .field(&hex::encode(&self.0)) + .finish() + } +} + +impl fmt::Display for Sha3_256 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s = hex::encode(&self.0); + write!(f, "{}", s) + } +} + +impl Serialize for Sha3_256 { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_str(self) + } +} + +struct Sha3_256Visitor; + +impl<'de> Visitor<'de> for Sha3_256Visitor { + type Value = Sha3_256; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a sha3-256 hex string") + } + + fn visit_str(self, v: &str) -> Result { + Sha3_256::try_from(v).map_err(serde::de::Error::custom) + } +} + +// This is the trait that informs Serde how to deserialize MyMap. +impl<'de> Deserialize<'de> for Sha3_256 { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_str(Sha3_256Visitor) + } +} diff --git a/opendatafabric/src/time_interval.rs b/opendatafabric/src/time_interval.rs new file mode 100644 index 0000000000..6db2564bc7 --- /dev/null +++ b/opendatafabric/src/time_interval.rs @@ -0,0 +1,270 @@ +use chrono::prelude::*; +use intervals_general::bound_pair::BoundPair; +use intervals_general::interval::Interval; + +use serde::de::{Deserialize, Deserializer, Error, Visitor}; +use serde::{Serialize, Serializer}; +use std::convert::TryFrom; +use std::fmt; + +type Element = DateTime; + +#[derive(Debug, Copy, Clone, PartialEq)] +pub struct TimeInterval(pub Interval); + +#[derive(Debug, Clone)] +pub struct InvalidTimeInterval; + +impl fmt::Display for InvalidTimeInterval { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "invalid time interval") + } +} + +impl TimeInterval { + pub fn empty() -> Self { + Self(Interval::Empty) + } + + pub fn closed(left: Element, right: Element) -> Result { + match BoundPair::new(left, right) { + None => Err(InvalidTimeInterval), + Some(p) => Ok(Self(Interval::Closed { bound_pair: p })), + } + } + + pub fn open(left: Element, right: Element) -> Result { + match BoundPair::new(left, right) { + None => Err(InvalidTimeInterval), + Some(p) => Ok(Self(Interval::Open { bound_pair: p })), + } + } + + pub fn left_half_open(left: Element, right: Element) -> Result { + match BoundPair::new(left, right) { + None => Err(InvalidTimeInterval), + Some(p) => Ok(Self(Interval::LeftHalfOpen { bound_pair: p })), + } + } + + pub fn right_half_open(left: Element, right: Element) -> Result { + match BoundPair::new(left, right) { + None => Err(InvalidTimeInterval), + Some(p) => Ok(Self(Interval::RightHalfOpen { bound_pair: p })), + } + } + + pub fn unbounded_closed_right(right: Element) -> Self { + Self(Interval::UnboundedClosedRight { right: right }) + } + + pub fn unbounded_open_right(right: Element) -> Self { + Self(Interval::UnboundedOpenRight { right: right }) + } + + pub fn unbounded_closed_left(left: Element) -> Self { + Self(Interval::UnboundedClosedLeft { left: left }) + } + + pub fn unbounded_open_left(left: Element) -> Self { + Self(Interval::UnboundedOpenLeft { left: left }) + } + + pub fn singleton(at: Element) -> Self { + Self(Interval::Singleton { at: at }) + } + + pub fn unbounded() -> Self { + Self(Interval::Unbounded) + } + + pub fn is_empty(&self) -> bool { + match self.0 { + Interval::Empty => true, + _ => false, + } + } + + // TODO: upstream + pub fn left_complement(&self) -> TimeInterval { + match self.0 { + Interval::Empty => Self(Interval::Unbounded), + Interval::Unbounded => Self(Interval::Empty), + Interval::Singleton { at } => Self::unbounded_open_right(at), + Interval::Open { bound_pair } => { + Self::unbounded_closed_right(bound_pair.left().clone()) + } + Interval::Closed { bound_pair } => { + Self::unbounded_open_right(bound_pair.left().clone()) + } + Interval::LeftHalfOpen { bound_pair } => { + Self::unbounded_closed_right(bound_pair.left().clone()) + } + Interval::RightHalfOpen { bound_pair } => { + Self::unbounded_open_right(bound_pair.left().clone()) + } + Interval::UnboundedOpenRight { .. } => Self(Interval::Empty), + Interval::UnboundedClosedRight { .. } => Self(Interval::Empty), + Interval::UnboundedOpenLeft { left } => Self::unbounded_closed_right(left.clone()), + Interval::UnboundedClosedLeft { left } => Self::unbounded_open_right(left.clone()), + } + } + + pub fn right_complement(&self) -> TimeInterval { + match self.0 { + Interval::Empty => Self(Interval::Unbounded), + Interval::Unbounded => Self(Interval::Empty), + Interval::Singleton { at } => Self::unbounded_open_left(at), + Interval::Open { bound_pair } => { + Self::unbounded_closed_left(bound_pair.right().clone()) + } + Interval::Closed { bound_pair } => { + Self::unbounded_open_left(bound_pair.right().clone()) + } + Interval::LeftHalfOpen { bound_pair } => { + Self::unbounded_open_left(bound_pair.right().clone()) + } + Interval::RightHalfOpen { bound_pair } => { + Self::unbounded_closed_left(bound_pair.right().clone()) + } + Interval::UnboundedOpenRight { right } => Self::unbounded_closed_left(right.clone()), + Interval::UnboundedClosedRight { right } => Self::unbounded_open_left(right.clone()), + Interval::UnboundedOpenLeft { .. } => Self(Interval::Empty), + Interval::UnboundedClosedLeft { .. } => Self(Interval::Empty), + } + } + + pub fn contains(&self, other: &TimeInterval) -> bool { + self.0.contains(&other.0) + } + + pub fn contains_point(&self, point: &Element) -> bool { + self.contains(&TimeInterval::singleton(point.clone())) + } + + pub fn intersect(&self, other: &TimeInterval) -> TimeInterval { + Self(self.0.intersect(&other.0)) + } +} + +impl Eq for TimeInterval {} + +impl fmt::Display for TimeInterval { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fn fmt_elem(v: &Element) -> String { + v.to_rfc3339_opts(SecondsFormat::Millis, true) + } + + match self.0 { + Interval::Unbounded => f.write_str("(-inf, inf)"), + Interval::Empty => f.write_str("()"), + Interval::Closed { bound_pair: p } => { + write!(f, "[{}, {}]", fmt_elem(p.left()), fmt_elem(p.right())) + } + Interval::Open { bound_pair: p } => { + write!(f, "({}, {})", fmt_elem(p.left()), fmt_elem(p.right())) + } + Interval::LeftHalfOpen { bound_pair: p } => { + write!(f, "({}, {}]", fmt_elem(p.left()), fmt_elem(p.right())) + } + Interval::RightHalfOpen { bound_pair: p } => { + write!(f, "[{}, {})", fmt_elem(p.left()), fmt_elem(p.right())) + } + Interval::UnboundedClosedRight { right } => write!(f, "(-inf, {}]", fmt_elem(&right)), + Interval::UnboundedOpenRight { right } => write!(f, "(-inf, {})", fmt_elem(&right)), + Interval::UnboundedClosedLeft { left } => write!(f, "[{}, inf)", fmt_elem(&left)), + Interval::UnboundedOpenLeft { left } => write!(f, "({}, inf)", fmt_elem(&left)), + Interval::Singleton { at } => write!(f, "[{}, {}]", fmt_elem(&at), fmt_elem(&at)), + } + } +} + +impl TryFrom<&str> for TimeInterval { + type Error = InvalidTimeInterval; + + fn try_from(value: &str) -> Result { + if value == "()" { + return Ok(TimeInterval::empty()); + } else if value.len() < 3 { + return Err(InvalidTimeInterval); + } + + fn parse_dt(s: &str) -> Result { + DateTime::parse_from_rfc3339(s.trim()) + .map(|dt| dt.into()) + .map_err(|_| InvalidTimeInterval) + } + + let lbound = &value[0..1]; + let rbound = &value[value.len() - 1..value.len()]; + let mut split = value[1..value.len() - 1].split(','); + + let left = match split.next() { + None => return Err(InvalidTimeInterval), + Some("-inf") => None, + Some(v) => match parse_dt(v) { + Ok(dt) => Some(dt), + _ => return Err(InvalidTimeInterval), + }, + }; + + let right = match split.next() { + None => return Err(InvalidTimeInterval), + Some("inf") => None, + Some(v) => match parse_dt(v) { + Ok(dt) => Some(dt), + _ => return Err(InvalidTimeInterval), + }, + }; + + if split.next() != None { + return Err(InvalidTimeInterval); + } + + match (lbound, left, right, rbound) { + ("(", None, None, ")") => Ok(TimeInterval::unbounded()), + ("(", None, Some(r), ")") => Ok(TimeInterval::unbounded_open_right(r)), + ("(", None, Some(r), "]") => Ok(TimeInterval::unbounded_closed_right(r)), + ("(", Some(l), None, ")") => Ok(TimeInterval::unbounded_open_left(l)), + ("[", Some(l), None, ")") => Ok(TimeInterval::unbounded_closed_left(l)), + ("(", Some(l), Some(r), ")") => TimeInterval::open(l, r), + ("(", Some(l), Some(r), "]") => TimeInterval::left_half_open(l, r), + ("[", Some(l), Some(r), ")") => TimeInterval::right_half_open(l, r), + ("[", Some(l), Some(r), "]") => { + if l == r { + Ok(TimeInterval::singleton(l)) + } else { + TimeInterval::closed(l, r) + } + } + _ => Err(InvalidTimeInterval), + } + } +} + +impl Serialize for TimeInterval { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_str(self) + } +} + +struct TimeIntervalVisitor; + +impl<'de> Visitor<'de> for TimeIntervalVisitor { + type Value = TimeInterval; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a time interval") + } + + fn visit_str(self, v: &str) -> Result { + TimeInterval::try_from(v).map_err(serde::de::Error::custom) + } +} + +// This is the trait that informs Serde how to deserialize MyMap. +impl<'de> Deserialize<'de> for TimeInterval { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_str(TimeIntervalVisitor) + } +} diff --git a/opendatafabric/tests/mod.rs b/opendatafabric/tests/mod.rs new file mode 100644 index 0000000000..818ffa6fe0 --- /dev/null +++ b/opendatafabric/tests/mod.rs @@ -0,0 +1,3 @@ +mod test_dynamic; +mod test_serde_flatbuffers; +mod test_serde_yaml; diff --git a/opendatafabric/tests/test_dynamic.rs b/opendatafabric/tests/test_dynamic.rs new file mode 100644 index 0000000000..3a25f05f0d --- /dev/null +++ b/opendatafabric/tests/test_dynamic.rs @@ -0,0 +1,74 @@ +use chrono::prelude::*; +use opendatafabric::*; +use std::convert::TryFrom; + +fn load() -> MetadataBlock { + MetadataBlock { + block_hash: Sha3_256::new([0xAA; 32]), + prev_block_hash: Some(Sha3_256::new([0xBB; 32])), + system_time: Utc.ymd(2020, 1, 1).and_hms(12, 0, 0), + source: Some(DatasetSource::Derivative(DatasetSourceDerivative { + inputs: vec![ + DatasetIDBuf::try_from("input1").unwrap(), + DatasetIDBuf::try_from("input2").unwrap(), + ], + transform: Transform::Sql(TransformSql { + engine: "spark".to_owned(), + version: None, + query: Some("SELECT * FROM input1 UNION ALL SELECT * FROM input2".to_owned()), + queries: None, + temporal_tables: None, + }), + })), + output_slice: Some(DataSlice { + hash: Sha3_256::new([0xA; 32]), + interval: TimeInterval::singleton(Utc.ymd(2020, 1, 1).and_hms(12, 0, 0)), + num_records: 10, + }), + output_watermark: Some(Utc.ymd(2020, 1, 1).and_hms(12, 0, 0)), + input_slices: Some(vec![ + DataSlice { + hash: Sha3_256::new([0xB; 32]), + interval: TimeInterval::unbounded_closed_right( + Utc.ymd(2020, 1, 1).and_hms(12, 0, 0), + ), + num_records: 10, + }, + DataSlice { + hash: Sha3_256::new([0xC; 32]), + interval: TimeInterval::empty(), + num_records: 0, + }, + ]), + } +} + +fn load_dynamic() -> Box { + Box::new(load()) +} + +#[test] +fn test_accessors() { + let block = load_dynamic(); + assert_eq!(block.block_hash(), &Sha3_256::new([0xAA; 32])); + assert_eq!(block.prev_block_hash().unwrap(), &Sha3_256::new([0xBB; 32])); + let source = match block.source().unwrap() { + dynamic::DatasetSource::Derivative(source) => source, + _ => panic!(), + }; + let inputs: Vec = source.inputs().map(|id| id.to_owned()).collect(); + assert_eq!( + inputs, + vec![ + DatasetIDBuf::try_from("input1").unwrap(), + DatasetIDBuf::try_from("input2").unwrap(), + ] + ); +} + +#[test] +fn test_conversion() { + let block_dyn = load_dynamic(); + let block_dto: MetadataBlock = block_dyn.as_ref().into(); + assert_eq!(block_dto, load()); +} diff --git a/opendatafabric/tests/test_serde_flatbuffers.rs b/opendatafabric/tests/test_serde_flatbuffers.rs new file mode 100644 index 0000000000..224e2cccff --- /dev/null +++ b/opendatafabric/tests/test_serde_flatbuffers.rs @@ -0,0 +1,189 @@ +use opendatafabric::serde::flatbuffers::*; +use opendatafabric::*; + +use chrono::prelude::*; +use std::convert::TryFrom; + +/////////////////////////////////////////////////////////////////////////////// + +fn get_block_root() -> MetadataBlock { + MetadataBlock { + block_hash: Sha3_256::zero(), + prev_block_hash: Some(Sha3_256::new([0x0b; 32])), + system_time: Utc.ymd(2020, 1, 1).and_hms(12, 0, 0), + input_slices: None, + output_slice: None, + output_watermark: None, + source: Some(DatasetSource::Root(DatasetSourceRoot { + fetch: FetchStep::FilesGlob(FetchStepFilesGlob { + path: "./*.csv".to_owned(), + event_time: Some(EventTimeSource::FromMetadata), + cache: Some(SourceCaching::Forever), + order: Some(SourceOrdering::ByName), + }), + prepare: Some(vec![PrepStep::Decompress(PrepStepDecompress { + format: CompressionFormat::Gzip, + sub_path: None, + })]), + read: ReadStep::GeoJson(ReadStepGeoJson { + schema: Some(vec!["a: INT".to_owned(), "b: INT".to_owned()]), + }), + preprocess: Some(Transform::Sql(TransformSql { + engine: "spark".to_owned(), + version: Some("1.0.0".to_owned()), + query: Some("SELECT * FROM input".to_owned()), + queries: None, + temporal_tables: None, + })), + merge: MergeStrategy::Ledger(MergeStrategyLedger { + primary_key: vec!["a".to_owned()], + }), + })), + } +} + +fn get_block_deriv() -> MetadataBlock { + MetadataBlock { + block_hash: Sha3_256::zero(), + prev_block_hash: Some(Sha3_256::new([0x0b; 32])), + system_time: Utc.ymd(2020, 1, 1).and_hms(12, 0, 0), + source: Some(DatasetSource::Derivative(DatasetSourceDerivative { + inputs: vec![ + DatasetIDBuf::try_from("input1").unwrap(), + DatasetIDBuf::try_from("input2").unwrap(), + ], + transform: Transform::Sql(TransformSql { + engine: "spark".to_owned(), + version: None, + query: Some("SELECT * FROM input1 UNION ALL SELECT * FROM input2".to_owned()), + queries: None, + temporal_tables: None, + }), + })), + output_slice: Some(DataSlice { + hash: Sha3_256::new([0x0a; 32]), + interval: TimeInterval::singleton(Utc.ymd(2020, 1, 1).and_hms(12, 0, 0)), + num_records: 10, + }), + output_watermark: Some(Utc.ymd(2020, 1, 1).and_hms(12, 0, 0)), + input_slices: Some(vec![ + DataSlice { + hash: Sha3_256::new([0x0a; 32]), + interval: TimeInterval::unbounded_closed_right( + Utc.ymd(2020, 1, 1).and_hms(12, 0, 0), + ), + num_records: 10, + }, + DataSlice { + hash: Sha3_256::new([0x0b; 32]), + interval: TimeInterval::empty(), + num_records: 0, + }, + ]), + } +} + +/////////////////////////////////////////////////////////////////////////////// + +#[test] +fn serde_metadata_block_root() { + let expected = get_block_root(); + + let buffer = FlatbuffersMetadataBlockSerializer + .write_manifest_unchecked(&expected) + .unwrap(); + let actual = FlatbuffersMetadataBlockDeserializer + .read_manifest_unchecked(&buffer) + .unwrap(); + assert_eq!(expected, actual); + + // Ensure produces same binary result + let buffer2 = FlatbuffersMetadataBlockSerializer + .write_manifest_unchecked(&actual) + .unwrap(); + assert_eq!(buffer.inner(), buffer2.inner()); +} + +#[test] +fn serde_metadata_block_deriv() { + let expected = get_block_deriv(); + + let buffer = FlatbuffersMetadataBlockSerializer + .write_manifest_unchecked(&expected) + .unwrap(); + let actual = FlatbuffersMetadataBlockDeserializer + .read_manifest_unchecked(&buffer) + .unwrap(); + assert_eq!(expected, actual); + + // Ensure produces same binary result + let buffer2 = FlatbuffersMetadataBlockSerializer + .write_manifest_unchecked(&actual) + .unwrap(); + assert_eq!(buffer.inner(), buffer2.inner()); +} + +#[test] +fn serializer_hashes_are_stable_root() { + let block = get_block_root(); + + let (block_hash, _) = FlatbuffersMetadataBlockSerializer + .write_manifest(&block) + .unwrap(); + + assert_eq!( + block_hash, + Sha3_256::try_from("7d5c8d887e706bb2fada712e4bb892794c611c5b4347c195582d88704079afb4") + .unwrap() + ); +} + +#[test] +fn serializer_hashes_are_stable_deriv() { + let block = get_block_deriv(); + + let (block_hash, _) = FlatbuffersMetadataBlockSerializer + .write_manifest(&block) + .unwrap(); + + assert_eq!( + block_hash, + Sha3_256::try_from("efd99d69eba17c25d16bd58f296786f26234f63404a098339e646204cc9e1eeb") + .unwrap() + ); +} + +#[test] +fn serializer_rejects_incorrect_hashes() { + let invalid = MetadataBlock { + block_hash: Sha3_256::new([0xab; 32]), + ..get_block_root() + }; + + assert!(matches!( + FlatbuffersMetadataBlockSerializer.write_manifest(&invalid), + Err(opendatafabric::serde::Error::InvalidHash { .. }) + )); +} + +#[test] +fn deserializer_rejects_incorrect_hashes() { + let invalid = MetadataBlock { + block_hash: Sha3_256::new([0xab; 32]), + ..get_block_root() + }; + + let buf = FlatbuffersMetadataBlockSerializer + .write_manifest_unchecked(&invalid) + .unwrap(); + + assert!(matches!( + FlatbuffersMetadataBlockDeserializer.validate_manifest(&buf), + Err(opendatafabric::serde::Error::InvalidHash { .. }) + )); + + assert!(matches!( + FlatbuffersMetadataBlockDeserializer.read_manifest(&buf), + Err(opendatafabric::serde::Error::InvalidHash { .. }) + )); +} diff --git a/opendatafabric/tests/test_serde_yaml.rs b/opendatafabric/tests/test_serde_yaml.rs new file mode 100644 index 0000000000..b6d51dc233 --- /dev/null +++ b/opendatafabric/tests/test_serde_yaml.rs @@ -0,0 +1,420 @@ +use ::serde::{Deserialize, Serialize}; +use chrono::prelude::*; +use indoc::indoc; +use opendatafabric::serde::yaml::generated::*; +use opendatafabric::serde::yaml::*; +use opendatafabric::*; +use std::convert::TryFrom; + +#[test] +fn serde_dataset_snapshot_root() { + let data = indoc!( + " + --- + apiVersion: 1 + kind: DatasetSnapshot + content: + id: kamu.test + source: + kind: root + fetch: + kind: url + url: \"ftp://kamu.dev/test.zip\" + cache: + kind: forever + prepare: + - kind: decompress + format: zip + subPath: data_*.csv + read: + kind: csv + header: true + preprocess: + kind: sql + engine: spark + query: SELECT * FROM input + merge: + kind: snapshot + primaryKey: + - id + vocab: + eventTimeColumn: date" + ); + + let expected = DatasetSnapshot { + id: DatasetIDBuf::try_from("kamu.test").unwrap(), + source: DatasetSource::Root(DatasetSourceRoot { + fetch: FetchStep::Url(FetchStepUrl { + url: "ftp://kamu.dev/test.zip".to_owned(), + event_time: None, + cache: Some(SourceCaching::Forever), + }), + prepare: Some(vec![PrepStep::Decompress(PrepStepDecompress { + format: CompressionFormat::Zip, + sub_path: Some("data_*.csv".to_owned()), + })]), + read: ReadStep::Csv(ReadStepCsv { + schema: None, + separator: None, + encoding: None, + quote: None, + escape: None, + comment: None, + header: Some(true), + enforce_schema: None, + infer_schema: None, + ignore_leading_white_space: None, + ignore_trailing_white_space: None, + null_value: None, + empty_value: None, + nan_value: None, + positive_inf: None, + negative_inf: None, + date_format: None, + timestamp_format: None, + multi_line: None, + }), + preprocess: Some(Transform::Sql(TransformSql { + engine: "spark".to_owned(), + version: None, + query: Some("SELECT * FROM input".to_owned()), + queries: None, + temporal_tables: None, + })), + merge: MergeStrategy::Snapshot(MergeStrategySnapshot { + primary_key: vec!["id".to_owned()], + compare_columns: None, + observation_column: None, + obsv_added: None, + obsv_changed: None, + obsv_removed: None, + }), + }), + vocab: Some(DatasetVocabulary { + system_time_column: None, + event_time_column: Some("date".to_owned()), + }), + }; + + let actual = YamlDatasetSnapshotDeserializer + .read_manifest(data.as_bytes()) + .unwrap(); + + assert_eq!(expected, actual); + + let data2 = YamlDatasetSnapshotSerializer + .write_manifest(&actual) + .unwrap(); + + assert_eq!(data, std::str::from_utf8(&data2).unwrap()); +} + +#[test] +fn serde_dataset_snapshot_derivative() { + let data = indoc!( + " + --- + apiVersion: 1 + kind: DatasetSnapshot + content: + id: com.naturalearthdata.admin0 + source: + kind: derivative + inputs: + - com.naturalearthdata.10m.admin0 + - com.naturalearthdata.50m.admin0 + transform: + kind: sql + engine: spark + query: SOME_SQL" + ); + + let expected = DatasetSnapshot { + id: DatasetIDBuf::try_from("com.naturalearthdata.admin0").unwrap(), + source: DatasetSource::Derivative(DatasetSourceDerivative { + inputs: vec![ + DatasetIDBuf::try_from("com.naturalearthdata.10m.admin0").unwrap(), + DatasetIDBuf::try_from("com.naturalearthdata.50m.admin0").unwrap(), + ], + transform: Transform::Sql(TransformSql { + engine: "spark".to_owned(), + version: None, + query: Some("SOME_SQL".to_owned()), + queries: None, + temporal_tables: None, + }), + }), + vocab: None, + }; + + let actual = YamlDatasetSnapshotDeserializer + .read_manifest(data.as_bytes()) + .unwrap(); + + assert_eq!(expected, actual); + + let data2 = YamlDatasetSnapshotSerializer + .write_manifest(&actual) + .unwrap(); + + assert_eq!(data, std::str::from_utf8(&data2).unwrap()); +} + +#[test] +fn serde_metadata_block() { + let data = indoc!( + " + --- + apiVersion: 1 + kind: MetadataBlock + content: + blockHash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a + prevBlockHash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b + systemTime: \"2020-01-01T12:00:00.000Z\" + outputSlice: + hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a + interval: \"[2020-01-01T12:00:00.000Z, 2020-01-01T12:00:00.000Z]\" + numRecords: 10 + outputWatermark: \"2020-01-01T12:00:00.000Z\" + inputSlices: + - hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a + interval: \"(-inf, 2020-01-01T12:00:00.000Z]\" + numRecords: 10 + - hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b + interval: () + numRecords: 0 + source: + kind: derivative + inputs: + - input1 + - input2 + transform: + kind: sql + engine: spark + query: SELECT * FROM input1 UNION ALL SELECT * FROM input2" + ); + + let expected = MetadataBlock { + block_hash: Sha3_256::new([0x0a; 32]), + prev_block_hash: Some(Sha3_256::new([0x0b; 32])), + system_time: Utc.ymd(2020, 1, 1).and_hms(12, 0, 0), + source: Some(DatasetSource::Derivative(DatasetSourceDerivative { + inputs: vec![ + DatasetIDBuf::try_from("input1").unwrap(), + DatasetIDBuf::try_from("input2").unwrap(), + ], + transform: Transform::Sql(TransformSql { + engine: "spark".to_owned(), + version: None, + query: Some("SELECT * FROM input1 UNION ALL SELECT * FROM input2".to_owned()), + queries: None, + temporal_tables: None, + }), + })), + output_slice: Some(DataSlice { + hash: Sha3_256::new([0x0a; 32]), + interval: TimeInterval::singleton(Utc.ymd(2020, 1, 1).and_hms(12, 0, 0)), + num_records: 10, + }), + output_watermark: Some(Utc.ymd(2020, 1, 1).and_hms(12, 0, 0)), + input_slices: Some(vec![ + DataSlice { + hash: Sha3_256::new([0x0a; 32]), + interval: TimeInterval::unbounded_closed_right( + Utc.ymd(2020, 1, 1).and_hms(12, 0, 0), + ), + num_records: 10, + }, + DataSlice { + hash: Sha3_256::new([0x0b; 32]), + interval: TimeInterval::empty(), + num_records: 0, + }, + ]), + }; + + let actual = YamlMetadataBlockDeserializer + .read_manifest_unchecked(data.as_bytes()) + .unwrap(); + + assert_eq!(expected, actual); + + let data2 = YamlMetadataBlockSerializer + .write_manifest_unchecked(&actual) + .unwrap(); + + assert_eq!(data, std::str::from_utf8(&data2).unwrap()); +} + +#[test] +fn ser_metadata_block_hashes() { + let expected = indoc!( + " + --- + apiVersion: 1 + kind: MetadataBlock + content: + blockHash: efd99d69eba17c25d16bd58f296786f26234f63404a098339e646204cc9e1eeb + prevBlockHash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b + systemTime: \"2020-01-01T12:00:00.000Z\" + outputSlice: + hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a + interval: \"[2020-01-01T12:00:00.000Z, 2020-01-01T12:00:00.000Z]\" + numRecords: 10 + outputWatermark: \"2020-01-01T12:00:00.000Z\" + inputSlices: + - hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a + interval: \"(-inf, 2020-01-01T12:00:00.000Z]\" + numRecords: 10 + - hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b + interval: () + numRecords: 0 + source: + kind: derivative + inputs: + - input1 + - input2 + transform: + kind: sql + engine: spark + query: SELECT * FROM input1 UNION ALL SELECT * FROM input2" + ); + + let block = MetadataBlock { + block_hash: Sha3_256::new([0; 32]), + prev_block_hash: Some(Sha3_256::new([0x0b; 32])), + system_time: Utc.ymd(2020, 1, 1).and_hms(12, 0, 0), + source: Some(DatasetSource::Derivative(DatasetSourceDerivative { + inputs: vec![ + DatasetIDBuf::try_from("input1").unwrap(), + DatasetIDBuf::try_from("input2").unwrap(), + ], + transform: Transform::Sql(TransformSql { + engine: "spark".to_owned(), + version: None, + query: Some("SELECT * FROM input1 UNION ALL SELECT * FROM input2".to_owned()), + queries: None, + temporal_tables: None, + }), + })), + output_slice: Some(DataSlice { + hash: Sha3_256::new([0x0a; 32]), + interval: TimeInterval::singleton(Utc.ymd(2020, 1, 1).and_hms(12, 0, 0)), + num_records: 10, + }), + output_watermark: Some(Utc.ymd(2020, 1, 1).and_hms(12, 0, 0)), + input_slices: Some(vec![ + DataSlice { + hash: Sha3_256::new([0x0a; 32]), + interval: TimeInterval::unbounded_closed_right( + Utc.ymd(2020, 1, 1).and_hms(12, 0, 0), + ), + num_records: 10, + }, + DataSlice { + hash: Sha3_256::new([0x0b; 32]), + interval: TimeInterval::empty(), + num_records: 0, + }, + ]), + }; + + let (_, actual) = YamlMetadataBlockSerializer.write_manifest(&block).unwrap(); + assert_eq!(expected, std::str::from_utf8(&actual).unwrap()); +} + +#[test] +fn serde_fetch_step_files_glob() { + let data = indoc!( + " + --- + kind: filesGlob + path: /opt/x/*.txt + cache: + kind: forever + order: byName" + ); + + #[derive(Serialize, Deserialize)] + struct Helper(#[serde(with = "FetchStepDef")] FetchStep); + let hlp: Helper = serde_yaml::from_str(data).unwrap(); + let actual = hlp.0; + + let expected = FetchStep::FilesGlob(FetchStepFilesGlob { + path: "/opt/x/*.txt".to_owned(), + event_time: None, + cache: Some(SourceCaching::Forever), + order: Some(SourceOrdering::ByName), + }); + + assert_eq!(expected, actual); + + assert_eq!( + serde_yaml::to_string(&Helper(actual)).unwrap(), + indoc!( + " + --- + kind: filesGlob + path: /opt/x/*.txt + cache: + kind: forever + order: byName" + ) + ); +} + +#[test] +fn serde_transform() { + let data = indoc!( + " + --- + kind: sql + engine: flink + temporalTables: + - id: foo + primaryKey: + - id + queries: + - alias: bar + query: > + SELECT * FROM foo" + ); + + #[derive(Serialize, Deserialize)] + struct Helper(#[serde(with = "TransformDef")] Transform); + let hlp: Helper = serde_yaml::from_str(data).unwrap(); + let actual = hlp.0; + + let expected = Transform::Sql(TransformSql { + engine: "flink".to_owned(), + version: None, + query: None, + temporal_tables: Some(vec![TemporalTable { + id: "foo".to_owned(), + primary_key: vec!["id".to_owned()], + }]), + queries: Some(vec![SqlQueryStep { + alias: Some("bar".to_owned()), + query: "SELECT * FROM foo".to_owned(), + }]), + }); + + assert_eq!(expected, actual); + + assert_eq!( + serde_yaml::to_string(&Helper(actual)).unwrap(), + indoc!( + " + --- + kind: sql + engine: flink + queries: + - alias: bar + query: SELECT * FROM foo + temporalTables: + - id: foo + primaryKey: + - id" + ) + ); +}