From 9bd2c4d46c2b2fb174448bb90cf3ee0cda78c798 Mon Sep 17 00:00:00 2001 From: Dima Pristupa Date: Mon, 18 Nov 2024 12:02:27 +0200 Subject: [PATCH] Fix `kamu push` (multi-tenant) (#962) * E2E, test_smart_push_pull_sequence: tenantness-matrix * E2E, test_smart_push_force_pull_force: tenantness-matrix * E2E, test_smart_push_no_alias_pull_no_alias: tenantness-matrix * E2E, test_smart_pull_as: tenantness-matrix * E2E, test_smart_push_all_pull_all: tenantness-matrix * E2E, test_smart_push_recursive_pull_recursive: tenantness-matrix * E2E, test_smart_pull_set_watermark: tenantness-matrix * E2E, test_smart_pull_reset_derivative: tenantness-matrix * E2E, test_smart_push_visibility: tenantness-matrix * E2E, test_s3_push_smart_pull: tenantness-matrix * E2E, test_pull_derivative: tenantness-matrix * E2E, test_smart_push_to_registered_repo_smart_pull: tenantness-matrix * E2E, test_smart_push_no_alias_pull_no_alias: fix alias checks * E2E, test_smart_pull_as_st: fix * E2E, test_smart_push_visibility_st: fix * E2E, test_smart_transfer_protocol: use odf import alias * E2E: Reorganization * E2E, test_smart_transfer_protocol: add "smart_" prefixes * E2E, test_smart_transfer_protocol: activate to mysql/postgres/sqlite * E2E, test_pull_command: st/mt * E2E, test_search_multi_user: continue use st * E2E, test_login_command: st/mt * E2E, test_search_command: st/mt * E2E, KamuCliPuppetExt: use odf type alias * E2E, use pub mod commands * E2E, postgres/tests: parity with inmem * E2E, sqlite/tests: parity with inmem * WsSmartTransferProtocolClient: use static AppendDatasetMetadataBatchUseCase * kamu push: enable transaction for push * WsSmartTransferProtocolClient::pull_protocol_client_flow(): return transaction * kamu {pull,push}: remove global transaction * CHANGELOG.md: update * Release (patch): 0.207.2 * Revert patch and use global transaction only for kamu push as well --- CHANGELOG.md | 11 + Cargo.lock | 147 ++-- Cargo.toml | 138 ++-- LICENSE.txt | 2 +- resources/openapi-mt.json | 2 +- resources/openapi.json | 2 +- src/app/cli/src/cli_commands.rs | 1 + src/e2e/app/cli/common-macros/src/lib.rs | 2 + .../tests/tests/commands/test_add_command.rs | 10 +- .../tests/commands/test_compact_command.rs | 6 +- .../tests/commands/test_complete_command.rs | 6 +- .../tests/commands/test_config_command.rs | 8 +- .../tests/commands/test_delete_command.rs | 6 +- .../tests/commands/test_ingest_command.rs | 11 +- .../tests/tests/commands/test_init_command.rs | 10 +- .../tests/commands/test_inspect_command.rs | 6 +- .../tests/tests/commands/test_log_command.rs | 2 +- .../tests/commands/test_login_command.rs | 35 +- .../tests/tests/commands/test_new_command.rs | 4 +- .../tests/tests/commands/test_pull_command.rs | 65 +- .../tests/commands/test_rename_command.rs | 2 +- .../tests/tests/commands/test_repo_command.rs | 4 +- .../tests/commands/test_reset_command.rs | 2 +- .../tests/commands/test_search_command.rs | 66 +- .../tests/tests/commands/test_sql_command.rs | 6 +- .../test_system_api_server_gql_query.rs | 2 +- .../commands/test_system_diagnose_command.rs | 2 +- .../tests/commands/test_system_gc_command.rs | 5 +- .../test_system_generate_token_command.rs | 2 +- .../commands/test_system_info_command.rs | 2 +- .../tests/tests/commands/test_tail_command.rs | 2 +- .../tests/commands/test_verify_command.rs | 6 +- .../tests/test_smart_transfer_protocol.rs | 275 ++++++- .../test_system_api_server_gql_query.rs | 2 +- .../test_system_generate_token_command.rs | 2 +- .../tests/test_smart_transfer_protocol.rs | 360 ++++++++- .../tests/tests/commands/test_add_command.rs | 10 +- .../tests/commands/test_compact_command.rs | 6 +- .../tests/commands/test_delete_command.rs | 6 +- .../tests/commands/test_ingest_command.rs | 11 +- .../tests/tests/commands/test_init_command.rs | 6 +- .../tests/commands/test_inspect_command.rs | 6 +- .../tests/tests/commands/test_log_command.rs | 2 +- .../tests/commands/test_login_command.rs | 35 +- .../tests/tests/commands/test_new_command.rs | 4 +- .../tests/tests/commands/test_pull_command.rs | 69 +- .../tests/commands/test_rename_command.rs | 2 +- .../tests/tests/commands/test_repo_command.rs | 4 +- .../tests/commands/test_reset_command.rs | 2 +- .../tests/commands/test_search_command.rs | 66 +- .../tests/tests/commands/test_sql_command.rs | 6 +- .../test_system_api_server_gql_query.rs | 2 +- .../commands/test_system_diagnose_command.rs | 2 +- .../tests/commands/test_system_gc_command.rs | 2 +- .../test_system_generate_token_command.rs | 2 +- .../commands/test_system_info_command.rs | 2 +- .../tests/tests/commands/test_tail_command.rs | 2 +- .../tests/commands/test_verify_command.rs | 6 +- .../tests/test_smart_transfer_protocol.rs | 275 ++++++- src/e2e/app/cli/repo-tests/Cargo.toml | 1 + .../src/commands/test_login_command.rs | 77 +- .../src/commands/test_pull_command.rs | 189 ++--- .../src/commands/test_search_command.rs | 65 +- src/e2e/app/cli/repo-tests/src/lib.rs | 3 +- .../src/test_smart_transfer_protocol.rs | 705 ++++++++---------- .../tests/tests/commands/test_add_command.rs | 10 +- .../tests/commands/test_compact_command.rs | 6 +- .../tests/commands/test_delete_command.rs | 6 +- .../tests/commands/test_ingest_command.rs | 11 +- .../tests/tests/commands/test_init_command.rs | 23 +- .../tests/commands/test_inspect_command.rs | 6 +- .../tests/tests/commands/test_log_command.rs | 2 +- .../tests/commands/test_login_command.rs | 35 +- .../tests/tests/commands/test_new_command.rs | 4 +- .../tests/tests/commands/test_pull_command.rs | 69 +- .../tests/commands/test_rename_command.rs | 2 +- .../tests/tests/commands/test_repo_command.rs | 4 +- .../tests/commands/test_reset_command.rs | 2 +- .../tests/commands/test_search_command.rs | 66 +- .../tests/tests/commands/test_sql_command.rs | 6 +- .../test_system_api_server_gql_query.rs | 2 +- .../commands/test_system_diagnose_command.rs | 2 +- .../tests/commands/test_system_gc_command.rs | 5 +- .../test_system_generate_token_command.rs | 2 +- .../commands/test_system_info_command.rs | 2 +- .../tests/tests/commands/test_tail_command.rs | 2 +- .../tests/commands/test_verify_command.rs | 6 +- .../tests/test_smart_transfer_protocol.rs | 275 ++++++- .../kamu-cli-puppet/src/kamu_cli_puppet.rs | 8 +- .../src/kamu_cli_puppet_ext.rs | 47 +- 90 files changed, 2433 insertions(+), 952 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44bae1e7ca..ca04f05947 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,17 @@ Recommendation: for ease of reading, use the following order: - Fixed --> +## [0.207.2] - 2024-11-15 +### Fixed +- E2E: revision of st/mt tests: + - In cases where temporary workspaces are created, + test variants for both single-tenant and multi-tenant have been added + - New combinations activated + - Certain duplicate tests have been removed + - Some of the tests related to `kamu pull` only have been moved to the appropriate module + - Activated missing tests for databases +- `kamu push`: crash in multi-tenant mode + ## [0.207.1] - 2024-11-14 ### Fixed - `kamu pull`: crash in multi-tenant mode diff --git a/Cargo.lock b/Cargo.lock index fa4b34adb7..fa4bb306c5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1314,7 +1314,7 @@ dependencies = [ [[package]] name = "async-utils" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", ] @@ -2472,7 +2472,7 @@ checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" [[package]] name = "container-runtime" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "cfg-if", @@ -2896,7 +2896,7 @@ checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" [[package]] name = "database-common" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "aws-config", @@ -2920,7 +2920,7 @@ dependencies = [ [[package]] name = "database-common-macros" -version = "0.207.1" +version = "0.207.2" dependencies = [ "quote", "syn 2.0.87", @@ -3764,7 +3764,7 @@ dependencies = [ [[package]] name = "enum-variants" -version = "0.207.1" +version = "0.207.2" [[package]] name = "env_filter" @@ -3833,7 +3833,7 @@ dependencies = [ [[package]] name = "event-sourcing" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-stream", "async-trait", @@ -3849,7 +3849,7 @@ dependencies = [ [[package]] name = "event-sourcing-macros" -version = "0.207.1" +version = "0.207.2" dependencies = [ "quote", "syn 2.0.87", @@ -4534,7 +4534,7 @@ dependencies = [ [[package]] name = "http-common" -version = "0.207.1" +version = "0.207.2" dependencies = [ "axum", "http 1.1.0", @@ -4940,7 +4940,7 @@ checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" [[package]] name = "init-on-startup" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "database-common", @@ -4983,7 +4983,7 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "internal-error" -version = "0.207.1" +version = "0.207.2" dependencies = [ "thiserror", ] @@ -5131,7 +5131,7 @@ dependencies = [ [[package]] name = "kamu" -version = "0.207.1" +version = "0.207.2" dependencies = [ "alloy", "async-recursion", @@ -5221,7 +5221,7 @@ dependencies = [ [[package]] name = "kamu-accounts" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "base32", @@ -5247,7 +5247,7 @@ dependencies = [ [[package]] name = "kamu-accounts-inmem" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "chrono", @@ -5268,7 +5268,7 @@ dependencies = [ [[package]] name = "kamu-accounts-mysql" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "chrono", @@ -5289,7 +5289,7 @@ dependencies = [ [[package]] name = "kamu-accounts-postgres" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "chrono", @@ -5310,7 +5310,7 @@ dependencies = [ [[package]] name = "kamu-accounts-repo-tests" -version = "0.207.1" +version = "0.207.2" dependencies = [ "argon2", "chrono", @@ -5326,7 +5326,7 @@ dependencies = [ [[package]] name = "kamu-accounts-services" -version = "0.207.1" +version = "0.207.2" dependencies = [ "argon2", "async-trait", @@ -5353,7 +5353,7 @@ dependencies = [ [[package]] name = "kamu-accounts-sqlite" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "chrono", @@ -5374,7 +5374,7 @@ dependencies = [ [[package]] name = "kamu-adapter-auth-oso" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "dill", @@ -5396,7 +5396,7 @@ dependencies = [ [[package]] name = "kamu-adapter-flight-sql" -version = "0.207.1" +version = "0.207.2" dependencies = [ "arrow-flight", "async-trait", @@ -5419,7 +5419,7 @@ dependencies = [ [[package]] name = "kamu-adapter-graphql" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-graphql", "async-trait", @@ -5471,7 +5471,7 @@ dependencies = [ [[package]] name = "kamu-adapter-http" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "aws-sdk-s3", @@ -5540,7 +5540,7 @@ dependencies = [ [[package]] name = "kamu-adapter-oauth" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "chrono", @@ -5559,7 +5559,7 @@ dependencies = [ [[package]] name = "kamu-adapter-odata" -version = "0.207.1" +version = "0.207.2" dependencies = [ "axum", "chrono", @@ -5597,7 +5597,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "internal-error", @@ -5609,7 +5609,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-inmem" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "database-common-macros", @@ -5623,7 +5623,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-postgres" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "database-common", @@ -5640,7 +5640,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-repo-tests" -version = "0.207.1" +version = "0.207.2" dependencies = [ "dill", "kamu-auth-rebac", @@ -5649,7 +5649,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-services" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "dill", @@ -5668,7 +5668,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-sqlite" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "database-common", @@ -5685,7 +5685,7 @@ dependencies = [ [[package]] name = "kamu-cli" -version = "0.207.1" +version = "0.207.2" dependencies = [ "arrow-flight", "async-graphql", @@ -5809,7 +5809,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-common" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "chrono", @@ -5839,7 +5839,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-common-macros" -version = "0.207.1" +version = "0.207.2" dependencies = [ "quote", "syn 2.0.87", @@ -5847,7 +5847,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-inmem" -version = "0.207.1" +version = "0.207.2" dependencies = [ "indoc 2.0.5", "kamu-cli-e2e-common", @@ -5860,7 +5860,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-mysql" -version = "0.207.1" +version = "0.207.2" dependencies = [ "indoc 2.0.5", "kamu-cli-e2e-common", @@ -5874,7 +5874,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-postgres" -version = "0.207.1" +version = "0.207.2" dependencies = [ "indoc 2.0.5", "kamu-cli-e2e-common", @@ -5888,7 +5888,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-repo-tests" -version = "0.207.1" +version = "0.207.2" dependencies = [ "chrono", "http-common", @@ -5902,6 +5902,7 @@ dependencies = [ "kamu-cli-puppet", "kamu-flow-system", "opendatafabric", + "paste", "pretty_assertions", "reqwest", "serde_json", @@ -5912,7 +5913,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-sqlite" -version = "0.207.1" +version = "0.207.2" dependencies = [ "indoc 2.0.5", "kamu-cli-e2e-common", @@ -5926,7 +5927,7 @@ dependencies = [ [[package]] name = "kamu-cli-puppet" -version = "0.207.1" +version = "0.207.2" dependencies = [ "assert_cmd", "async-trait", @@ -5945,7 +5946,7 @@ dependencies = [ [[package]] name = "kamu-core" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-stream", "async-trait", @@ -5976,7 +5977,7 @@ dependencies = [ [[package]] name = "kamu-data-utils" -version = "0.207.1" +version = "0.207.2" dependencies = [ "arrow", "arrow-digest", @@ -6001,7 +6002,7 @@ dependencies = [ [[package]] name = "kamu-datafusion-cli" -version = "0.207.1" +version = "0.207.2" dependencies = [ "arrow", "async-trait", @@ -6025,7 +6026,7 @@ dependencies = [ [[package]] name = "kamu-datasets" -version = "0.207.1" +version = "0.207.2" dependencies = [ "aes-gcm", "async-trait", @@ -6045,7 +6046,7 @@ dependencies = [ [[package]] name = "kamu-datasets-inmem" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "chrono", @@ -6068,7 +6069,7 @@ dependencies = [ [[package]] name = "kamu-datasets-postgres" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "chrono", @@ -6091,7 +6092,7 @@ dependencies = [ [[package]] name = "kamu-datasets-repo-tests" -version = "0.207.1" +version = "0.207.2" dependencies = [ "chrono", "database-common", @@ -6105,7 +6106,7 @@ dependencies = [ [[package]] name = "kamu-datasets-services" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "chrono", @@ -6136,7 +6137,7 @@ dependencies = [ [[package]] name = "kamu-datasets-sqlite" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "chrono", @@ -6159,7 +6160,7 @@ dependencies = [ [[package]] name = "kamu-flow-system" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "chrono", @@ -6188,7 +6189,7 @@ dependencies = [ [[package]] name = "kamu-flow-system-inmem" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-stream", "async-trait", @@ -6218,7 +6219,7 @@ dependencies = [ [[package]] name = "kamu-flow-system-postgres" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-stream", "async-trait", @@ -6243,7 +6244,7 @@ dependencies = [ [[package]] name = "kamu-flow-system-repo-tests" -version = "0.207.1" +version = "0.207.2" dependencies = [ "chrono", "database-common", @@ -6256,7 +6257,7 @@ dependencies = [ [[package]] name = "kamu-flow-system-services" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-stream", "async-trait", @@ -6300,7 +6301,7 @@ dependencies = [ [[package]] name = "kamu-flow-system-sqlite" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-stream", "async-trait", @@ -6325,7 +6326,7 @@ dependencies = [ [[package]] name = "kamu-ingest-datafusion" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "chrono", @@ -6361,7 +6362,7 @@ dependencies = [ [[package]] name = "kamu-messaging-outbox-inmem" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "chrono", @@ -6380,7 +6381,7 @@ dependencies = [ [[package]] name = "kamu-messaging-outbox-postgres" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-stream", "async-trait", @@ -6403,7 +6404,7 @@ dependencies = [ [[package]] name = "kamu-messaging-outbox-repo-tests" -version = "0.207.1" +version = "0.207.2" dependencies = [ "chrono", "database-common", @@ -6417,7 +6418,7 @@ dependencies = [ [[package]] name = "kamu-messaging-outbox-sqlite" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-stream", "async-trait", @@ -6439,7 +6440,7 @@ dependencies = [ [[package]] name = "kamu-repo-tools" -version = "0.207.1" +version = "0.207.2" dependencies = [ "chrono", "clap", @@ -6454,7 +6455,7 @@ dependencies = [ [[package]] name = "kamu-task-system" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "chrono", @@ -6472,7 +6473,7 @@ dependencies = [ [[package]] name = "kamu-task-system-inmem" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "chrono", @@ -6491,7 +6492,7 @@ dependencies = [ [[package]] name = "kamu-task-system-postgres" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-stream", "async-trait", @@ -6514,7 +6515,7 @@ dependencies = [ [[package]] name = "kamu-task-system-repo-tests" -version = "0.207.1" +version = "0.207.2" dependencies = [ "chrono", "database-common", @@ -6526,7 +6527,7 @@ dependencies = [ [[package]] name = "kamu-task-system-services" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-stream", "async-trait", @@ -6554,7 +6555,7 @@ dependencies = [ [[package]] name = "kamu-task-system-sqlite" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-stream", "async-trait", @@ -6963,7 +6964,7 @@ dependencies = [ [[package]] name = "messaging-outbox" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "chrono", @@ -7104,7 +7105,7 @@ dependencies = [ [[package]] name = "multiformats" -version = "0.207.1" +version = "0.207.2" dependencies = [ "base64 0.22.1", "bs58", @@ -7429,7 +7430,7 @@ dependencies = [ [[package]] name = "observability" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "axum", @@ -7487,7 +7488,7 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "opendatafabric" -version = "0.207.1" +version = "0.207.2" dependencies = [ "arrow", "base64 0.22.1", @@ -8448,7 +8449,7 @@ dependencies = [ [[package]] name = "random-names" -version = "0.207.1" +version = "0.207.2" dependencies = [ "rand", ] @@ -10029,7 +10030,7 @@ dependencies = [ [[package]] name = "time-source" -version = "0.207.1" +version = "0.207.2" dependencies = [ "async-trait", "chrono", @@ -10437,7 +10438,7 @@ dependencies = [ [[package]] name = "tracing-perfetto" -version = "0.207.1" +version = "0.207.2" dependencies = [ "conv", "serde", diff --git a/Cargo.toml b/Cargo.toml index 4e82b39901..0ab06cee80 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,95 +92,95 @@ resolver = "2" [workspace.dependencies] # Apps -kamu-cli = { version = "0.207.1", path = "src/app/cli", default-features = false } +kamu-cli = { version = "0.207.2", path = "src/app/cli", default-features = false } # Utils -async-utils = { version = "0.207.1", path = "src/utils/async-utils", default-features = false } -container-runtime = { version = "0.207.1", path = "src/utils/container-runtime", default-features = false } -database-common = { version = "0.207.1", path = "src/utils/database-common", default-features = false } -database-common-macros = { version = "0.207.1", path = "src/utils/database-common-macros", default-features = false } -enum-variants = { version = "0.207.1", path = "src/utils/enum-variants", default-features = false } -event-sourcing = { version = "0.207.1", path = "src/utils/event-sourcing", default-features = false } -event-sourcing-macros = { version = "0.207.1", path = "src/utils/event-sourcing-macros", default-features = false } -http-common = { version = "0.207.1", path = "src/utils/http-common", default-features = false } -init-on-startup = { version = "0.207.1", path = "src/utils/init-on-startup", default-features = false } -internal-error = { version = "0.207.1", path = "src/utils/internal-error", default-features = false } -kamu-cli-puppet = { version = "0.207.1", path = "src/utils/kamu-cli-puppet", default-features = false } -kamu-data-utils = { version = "0.207.1", path = "src/utils/data-utils", default-features = false } -kamu-datafusion-cli = { version = "0.207.1", path = "src/utils/datafusion-cli", default-features = false } -messaging-outbox = { version = "0.207.1", path = "src/utils/messaging-outbox", default-features = false } -multiformats = { version = "0.207.1", path = "src/utils/multiformats", default-features = false } -observability = { version = "0.207.1", path = "src/utils/observability", default-features = false } -random-names = { version = "0.207.1", path = "src/utils/random-names", default-features = false } -time-source = { version = "0.207.1", path = "src/utils/time-source", default-features = false } -tracing-perfetto = { version = "0.207.1", path = "src/utils/tracing-perfetto", default-features = false } +async-utils = { version = "0.207.2", path = "src/utils/async-utils", default-features = false } +container-runtime = { version = "0.207.2", path = "src/utils/container-runtime", default-features = false } +database-common = { version = "0.207.2", path = "src/utils/database-common", default-features = false } +database-common-macros = { version = "0.207.2", path = "src/utils/database-common-macros", default-features = false } +enum-variants = { version = "0.207.2", path = "src/utils/enum-variants", default-features = false } +event-sourcing = { version = "0.207.2", path = "src/utils/event-sourcing", default-features = false } +event-sourcing-macros = { version = "0.207.2", path = "src/utils/event-sourcing-macros", default-features = false } +http-common = { version = "0.207.2", path = "src/utils/http-common", default-features = false } +init-on-startup = { version = "0.207.2", path = "src/utils/init-on-startup", default-features = false } +internal-error = { version = "0.207.2", path = "src/utils/internal-error", default-features = false } +kamu-cli-puppet = { version = "0.207.2", path = "src/utils/kamu-cli-puppet", default-features = false } +kamu-data-utils = { version = "0.207.2", path = "src/utils/data-utils", default-features = false } +kamu-datafusion-cli = { version = "0.207.2", path = "src/utils/datafusion-cli", default-features = false } +messaging-outbox = { version = "0.207.2", path = "src/utils/messaging-outbox", default-features = false } +multiformats = { version = "0.207.2", path = "src/utils/multiformats", default-features = false } +observability = { version = "0.207.2", path = "src/utils/observability", default-features = false } +random-names = { version = "0.207.2", path = "src/utils/random-names", default-features = false } +time-source = { version = "0.207.2", path = "src/utils/time-source", default-features = false } +tracing-perfetto = { version = "0.207.2", path = "src/utils/tracing-perfetto", default-features = false } # Domain -kamu-accounts = { version = "0.207.1", path = "src/domain/accounts/domain", default-features = false } -kamu-auth-rebac = { version = "0.207.1", path = "src/domain/auth-rebac/domain", default-features = false } -kamu-core = { version = "0.207.1", path = "src/domain/core", default-features = false } -kamu-datasets = { version = "0.207.1", path = "src/domain/datasets/domain", default-features = false } -kamu-flow-system = { version = "0.207.1", path = "src/domain/flow-system/domain", default-features = false } -kamu-task-system = { version = "0.207.1", path = "src/domain/task-system/domain", default-features = false } -opendatafabric = { version = "0.207.1", path = "src/domain/opendatafabric", default-features = false } +kamu-accounts = { version = "0.207.2", path = "src/domain/accounts/domain", default-features = false } +kamu-auth-rebac = { version = "0.207.2", path = "src/domain/auth-rebac/domain", default-features = false } +kamu-core = { version = "0.207.2", path = "src/domain/core", default-features = false } +kamu-datasets = { version = "0.207.2", path = "src/domain/datasets/domain", default-features = false } +kamu-flow-system = { version = "0.207.2", path = "src/domain/flow-system/domain", default-features = false } +kamu-task-system = { version = "0.207.2", path = "src/domain/task-system/domain", default-features = false } +opendatafabric = { version = "0.207.2", path = "src/domain/opendatafabric", default-features = false } # Domain service layer -kamu-accounts-services = { version = "0.207.1", path = "src/domain/accounts/services", default-features = false } -kamu-auth-rebac-services = { version = "0.207.1", path = "src/domain/auth-rebac/services", default-features = false } -kamu-datasets-services = { version = "0.207.1", path = "src/domain/datasets/services", default-features = false } -kamu-flow-system-services = { version = "0.207.1", path = "src/domain/flow-system/services", default-features = false } -kamu-task-system-services = { version = "0.207.1", path = "src/domain/task-system/services", default-features = false } +kamu-accounts-services = { version = "0.207.2", path = "src/domain/accounts/services", default-features = false } +kamu-auth-rebac-services = { version = "0.207.2", path = "src/domain/auth-rebac/services", default-features = false } +kamu-datasets-services = { version = "0.207.2", path = "src/domain/datasets/services", default-features = false } +kamu-flow-system-services = { version = "0.207.2", path = "src/domain/flow-system/services", default-features = false } +kamu-task-system-services = { version = "0.207.2", path = "src/domain/task-system/services", default-features = false } # Infra -kamu = { version = "0.207.1", path = "src/infra/core", default-features = false } -kamu-ingest-datafusion = { version = "0.207.1", path = "src/infra/ingest-datafusion", default-features = false } +kamu = { version = "0.207.2", path = "src/infra/core", default-features = false } +kamu-ingest-datafusion = { version = "0.207.2", path = "src/infra/ingest-datafusion", default-features = false } ## Flow System -kamu-flow-system-repo-tests = { version = "0.207.1", path = "src/infra/flow-system/repo-tests", default-features = false } -kamu-flow-system-inmem = { version = "0.207.1", path = "src/infra/flow-system/inmem", default-features = false } -kamu-flow-system-postgres = { version = "0.207.1", path = "src/infra/flow-system/postgres", default-features = false } -kamu-flow-system-sqlite = { version = "0.207.1", path = "src/infra/flow-system/sqlite", default-features = false } +kamu-flow-system-repo-tests = { version = "0.207.2", path = "src/infra/flow-system/repo-tests", default-features = false } +kamu-flow-system-inmem = { version = "0.207.2", path = "src/infra/flow-system/inmem", default-features = false } +kamu-flow-system-postgres = { version = "0.207.2", path = "src/infra/flow-system/postgres", default-features = false } +kamu-flow-system-sqlite = { version = "0.207.2", path = "src/infra/flow-system/sqlite", default-features = false } ## Accounts -kamu-accounts-inmem = { version = "0.207.1", path = "src/infra/accounts/inmem", default-features = false } -kamu-accounts-mysql = { version = "0.207.1", path = "src/infra/accounts/mysql", default-features = false } -kamu-accounts-postgres = { version = "0.207.1", path = "src/infra/accounts/postgres", default-features = false } -kamu-accounts-sqlite = { version = "0.207.1", path = "src/infra/accounts/sqlite", default-features = false } -kamu-accounts-repo-tests = { version = "0.207.1", path = "src/infra/accounts/repo-tests", default-features = false } +kamu-accounts-inmem = { version = "0.207.2", path = "src/infra/accounts/inmem", default-features = false } +kamu-accounts-mysql = { version = "0.207.2", path = "src/infra/accounts/mysql", default-features = false } +kamu-accounts-postgres = { version = "0.207.2", path = "src/infra/accounts/postgres", default-features = false } +kamu-accounts-sqlite = { version = "0.207.2", path = "src/infra/accounts/sqlite", default-features = false } +kamu-accounts-repo-tests = { version = "0.207.2", path = "src/infra/accounts/repo-tests", default-features = false } ## Datasets -kamu-datasets-inmem = { version = "0.207.1", path = "src/infra/datasets/inmem", default-features = false } -kamu-datasets-postgres = { version = "0.207.1", path = "src/infra/datasets/postgres", default-features = false } -kamu-datasets-sqlite = { version = "0.207.1", path = "src/infra/datasets/sqlite", default-features = false } -kamu-datasets-repo-tests = { version = "0.207.1", path = "src/infra/datasets/repo-tests", default-features = false } +kamu-datasets-inmem = { version = "0.207.2", path = "src/infra/datasets/inmem", default-features = false } +kamu-datasets-postgres = { version = "0.207.2", path = "src/infra/datasets/postgres", default-features = false } +kamu-datasets-sqlite = { version = "0.207.2", path = "src/infra/datasets/sqlite", default-features = false } +kamu-datasets-repo-tests = { version = "0.207.2", path = "src/infra/datasets/repo-tests", default-features = false } ## Task System -kamu-task-system-inmem = { version = "0.207.1", path = "src/infra/task-system/inmem", default-features = false } -kamu-task-system-postgres = { version = "0.207.1", path = "src/infra/task-system/postgres", default-features = false } -kamu-task-system-sqlite = { version = "0.207.1", path = "src/infra/task-system/sqlite", default-features = false } -kamu-task-system-repo-tests = { version = "0.207.1", path = "src/infra/task-system/repo-tests", default-features = false } +kamu-task-system-inmem = { version = "0.207.2", path = "src/infra/task-system/inmem", default-features = false } +kamu-task-system-postgres = { version = "0.207.2", path = "src/infra/task-system/postgres", default-features = false } +kamu-task-system-sqlite = { version = "0.207.2", path = "src/infra/task-system/sqlite", default-features = false } +kamu-task-system-repo-tests = { version = "0.207.2", path = "src/infra/task-system/repo-tests", default-features = false } ## ReBAC -kamu-auth-rebac-inmem = { version = "0.207.1", path = "src/infra/auth-rebac/inmem", default-features = false } -kamu-auth-rebac-repo-tests = { version = "0.207.1", path = "src/infra/auth-rebac/repo-tests", default-features = false } -kamu-auth-rebac-postgres = { version = "0.207.1", path = "src/infra/auth-rebac/postgres", default-features = false } -kamu-auth-rebac-sqlite = { version = "0.207.1", path = "src/infra/auth-rebac/sqlite", default-features = false } +kamu-auth-rebac-inmem = { version = "0.207.2", path = "src/infra/auth-rebac/inmem", default-features = false } +kamu-auth-rebac-repo-tests = { version = "0.207.2", path = "src/infra/auth-rebac/repo-tests", default-features = false } +kamu-auth-rebac-postgres = { version = "0.207.2", path = "src/infra/auth-rebac/postgres", default-features = false } +kamu-auth-rebac-sqlite = { version = "0.207.2", path = "src/infra/auth-rebac/sqlite", default-features = false } ## Outbox -kamu-messaging-outbox-inmem = { version = "0.207.1", path = "src/infra/messaging-outbox/inmem", default-features = false } -kamu-messaging-outbox-postgres = { version = "0.207.1", path = "src/infra/messaging-outbox/postgres", default-features = false } -kamu-messaging-outbox-sqlite = { version = "0.207.1", path = "src/infra/messaging-outbox/sqlite", default-features = false } -kamu-messaging-outbox-repo-tests = { version = "0.207.1", path = "src/infra/messaging-outbox/repo-tests", default-features = false } +kamu-messaging-outbox-inmem = { version = "0.207.2", path = "src/infra/messaging-outbox/inmem", default-features = false } +kamu-messaging-outbox-postgres = { version = "0.207.2", path = "src/infra/messaging-outbox/postgres", default-features = false } +kamu-messaging-outbox-sqlite = { version = "0.207.2", path = "src/infra/messaging-outbox/sqlite", default-features = false } +kamu-messaging-outbox-repo-tests = { version = "0.207.2", path = "src/infra/messaging-outbox/repo-tests", default-features = false } # Adapters -kamu-adapter-auth-oso = { version = "0.207.1", path = "src/adapter/auth-oso", default-features = false } -kamu-adapter-flight-sql = { version = "0.207.1", path = "src/adapter/flight-sql", default-features = false } -kamu-adapter-graphql = { version = "0.207.1", path = "src/adapter/graphql", default-features = false } -kamu-adapter-http = { version = "0.207.1", path = "src/adapter/http", default-features = false } -kamu-adapter-odata = { version = "0.207.1", path = "src/adapter/odata", default-features = false } -kamu-adapter-oauth = { version = "0.207.1", path = "src/adapter/oauth", default-features = false } +kamu-adapter-auth-oso = { version = "0.207.2", path = "src/adapter/auth-oso", default-features = false } +kamu-adapter-flight-sql = { version = "0.207.2", path = "src/adapter/flight-sql", default-features = false } +kamu-adapter-graphql = { version = "0.207.2", path = "src/adapter/graphql", default-features = false } +kamu-adapter-http = { version = "0.207.2", path = "src/adapter/http", default-features = false } +kamu-adapter-odata = { version = "0.207.2", path = "src/adapter/odata", default-features = false } +kamu-adapter-oauth = { version = "0.207.2", path = "src/adapter/oauth", default-features = false } # E2E -kamu-cli-e2e-common = { version = "0.207.1", path = "src/e2e/app/cli/common", default-features = false } -kamu-cli-e2e-common-macros = { version = "0.207.1", path = "src/e2e/app/cli/common-macros", default-features = false } -kamu-cli-e2e-repo-tests = { version = "0.207.1", path = "src/e2e/app/cli/repo-tests", default-features = false } +kamu-cli-e2e-common = { version = "0.207.2", path = "src/e2e/app/cli/common", default-features = false } +kamu-cli-e2e-common-macros = { version = "0.207.2", path = "src/e2e/app/cli/common-macros", default-features = false } +kamu-cli-e2e-repo-tests = { version = "0.207.2", path = "src/e2e/app/cli/repo-tests", default-features = false } [workspace.package] -version = "0.207.1" +version = "0.207.2" edition = "2021" homepage = "https://github.com/kamu-data/kamu-cli" repository = "https://github.com/kamu-data/kamu-cli" diff --git a/LICENSE.txt b/LICENSE.txt index 356d4aae16..dcbd796d41 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -11,7 +11,7 @@ Business Source License 1.1 Licensor: Kamu Data, Inc. -Licensed Work: Kamu CLI Version 0.207.1 +Licensed Work: Kamu CLI Version 0.207.2 The Licensed Work is © 2023 Kamu Data, Inc. Additional Use Grant: You may use the Licensed Work for any purpose, diff --git a/resources/openapi-mt.json b/resources/openapi-mt.json index 7b1361d48b..43944f2ea0 100644 --- a/resources/openapi-mt.json +++ b/resources/openapi-mt.json @@ -862,7 +862,7 @@ "name": "" }, "title": "kamu-cli", - "version": "0.207.1" + "version": "0.207.2" }, "openapi": "3.1.0", "paths": { diff --git a/resources/openapi.json b/resources/openapi.json index e291c216db..3c31882d4d 100644 --- a/resources/openapi.json +++ b/resources/openapi.json @@ -862,7 +862,7 @@ "name": "" }, "title": "kamu-cli", - "version": "0.207.1" + "version": "0.207.2" }, "openapi": "3.1.0", "paths": { diff --git a/src/app/cli/src/cli_commands.rs b/src/app/cli/src/cli_commands.rs index 3dbeaf4f10..711a89d98f 100644 --- a/src/app/cli/src/cli_commands.rs +++ b/src/app/cli/src/cli_commands.rs @@ -531,6 +531,7 @@ pub fn command_needs_transaction(args: &cli::Cli) -> bool { cli::Command::Add(_) | cli::Command::Delete(_) | cli::Command::Rename(_) + | cli::Command::Push(_) | cli::Command::Pull(_) => true, _ => false, } diff --git a/src/e2e/app/cli/common-macros/src/lib.rs b/src/e2e/app/cli/common-macros/src/lib.rs index ef9f4915aa..ed4f556d5b 100644 --- a/src/e2e/app/cli/common-macros/src/lib.rs +++ b/src/e2e/app/cli/common-macros/src/lib.rs @@ -30,6 +30,8 @@ pub fn kamu_cli_execute_command_e2e_test(input: TokenStream) -> TokenStream { kamu_cli_e2e_test_impl(&harness_method, input) } +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Implementations //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// fn kamu_cli_e2e_test_impl(harness_method: &Ident, input: TokenStream) -> TokenStream { diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_add_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_add_command.rs index 01e88cf5fc..d4c0315fc7 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_add_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_add_command.rs @@ -13,35 +13,35 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_add_dataset_from_stdin + fixture = kamu_cli_e2e_repo_tests::commands::test_add_dataset_from_stdin ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_add_dataset_with_name + fixture = kamu_cli_e2e_repo_tests::commands::test_add_dataset_with_name ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_add_dataset_with_replace + fixture = kamu_cli_e2e_repo_tests::commands::test_add_dataset_with_replace ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_add_recursive + fixture = kamu_cli_e2e_repo_tests::commands::test_add_recursive ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_add_with_circular_dependency + fixture = kamu_cli_e2e_repo_tests::commands::test_add_with_circular_dependency ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_compact_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_compact_command.rs index 6ff90ee7b5..0b7227e7e6 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_compact_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_compact_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_compact_hard + fixture = kamu_cli_e2e_repo_tests::commands::test_compact_hard extra_test_groups = "engine, ingest, datafusion" ); @@ -21,7 +21,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_compact_keep_metadata_only + fixture = kamu_cli_e2e_repo_tests::commands::test_compact_keep_metadata_only extra_test_groups = "engine, ingest, datafusion" ); @@ -29,7 +29,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_compact_verify + fixture = kamu_cli_e2e_repo_tests::commands::test_compact_verify extra_test_groups = "engine, ingest, datafusion" ); diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_complete_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_complete_command.rs index 8f1a3bac5e..0ea67db949 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_complete_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_complete_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_complete_subcommand, + fixture = kamu_cli_e2e_repo_tests::commands::test_complete_subcommand, options = Options::default().with_no_workspace() ); @@ -21,7 +21,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_complete_config, + fixture = kamu_cli_e2e_repo_tests::commands::test_complete_config, options = Options::default().with_no_workspace() ); @@ -29,7 +29,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_complete_dataset_name + fixture = kamu_cli_e2e_repo_tests::commands::test_complete_dataset_name ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_config_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_config_command.rs index 9fb7d78c36..6bf62d62c0 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_config_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_config_command.rs @@ -13,28 +13,28 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_config_set_value + fixture = kamu_cli_e2e_repo_tests::commands::test_config_set_value ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_config_reset_key + fixture = kamu_cli_e2e_repo_tests::commands::test_config_reset_key ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_config_get_with_default + fixture = kamu_cli_e2e_repo_tests::commands::test_config_get_with_default ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_config_get_from_config + fixture = kamu_cli_e2e_repo_tests::commands::test_config_get_from_config options = Options::default().with_kamu_config( indoc::indoc!( r#" diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_delete_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_delete_command.rs index 0906123b31..77769f6235 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_delete_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_delete_command.rs @@ -13,21 +13,21 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_delete_dataset + fixture = kamu_cli_e2e_repo_tests::commands::test_delete_dataset ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_delete_dataset_recursive + fixture = kamu_cli_e2e_repo_tests::commands::test_delete_dataset_recursive ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_delete_dataset_all + fixture = kamu_cli_e2e_repo_tests::commands::test_delete_dataset_all ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_ingest_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_ingest_command.rs index 3687ca7d55..0d3152e346 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_ingest_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_ingest_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_push_ingest_from_file_ledger, + fixture = kamu_cli_e2e_repo_tests::commands::test_push_ingest_from_file_ledger, options = Options::default().with_frozen_system_time(), extra_test_groups = "engine, ingest, datafusion" ); @@ -22,7 +22,8 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_push_ingest_from_file_snapshot_with_event_time, + fixture = + kamu_cli_e2e_repo_tests::commands::test_push_ingest_from_file_snapshot_with_event_time, options = Options::default().with_frozen_system_time(), extra_test_groups = "engine, ingest, datafusion" ); @@ -31,7 +32,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_ingest_from_stdin, + fixture = kamu_cli_e2e_repo_tests::commands::test_ingest_from_stdin, options = Options::default().with_frozen_system_time(), extra_test_groups = "engine, ingest, datafusion" ); @@ -40,7 +41,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_ingest_recursive, + fixture = kamu_cli_e2e_repo_tests::commands::test_ingest_recursive, options = Options::default().with_frozen_system_time(), extra_test_groups = "engine, ingest, datafusion" ); @@ -49,7 +50,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_ingest_with_source_name, + fixture = kamu_cli_e2e_repo_tests::commands::test_ingest_with_source_name, options = Options::default().with_frozen_system_time(), extra_test_groups = "engine, ingest, datafusion" ); diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_init_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_init_command.rs index 7f27502c55..58162735dc 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_init_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_init_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_init_multi_tenant_creates_sqlite_database, + fixture = kamu_cli_e2e_repo_tests::commands::test_init_multi_tenant_creates_sqlite_database, options = Options::default().with_no_workspace() ); @@ -22,7 +22,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, fixture = - kamu_cli_e2e_repo_tests::test_init_multi_tenant_with_exists_ok_flag_creates_sqlite_database, + kamu_cli_e2e_repo_tests::commands::test_init_multi_tenant_with_exists_ok_flag_creates_sqlite_database, options = Options::default().with_no_workspace() ); @@ -30,7 +30,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_init_exist_ok_st, + fixture = kamu_cli_e2e_repo_tests::commands::test_init_exist_ok_st, options = Options::default().with_no_workspace() ); @@ -38,7 +38,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_init_exist_ok_mt, + fixture = kamu_cli_e2e_repo_tests::commands::test_init_exist_ok_mt, options = Options::default().with_no_workspace() ); @@ -46,7 +46,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_init_in_an_existing_workspace, + fixture = kamu_cli_e2e_repo_tests::commands::test_init_in_an_existing_workspace, options = Options::default().with_no_workspace() ); diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_inspect_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_inspect_command.rs index 0d22b67d57..1ec3cc51e4 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_inspect_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_inspect_command.rs @@ -13,14 +13,14 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_inspect_lineage, + fixture = kamu_cli_e2e_repo_tests::commands::test_inspect_lineage, ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_inspect_query, + fixture = kamu_cli_e2e_repo_tests::commands::test_inspect_query, options = Options::default().with_frozen_system_time() ); @@ -28,7 +28,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_inspect_schema, + fixture = kamu_cli_e2e_repo_tests::commands::test_inspect_schema, extra_test_groups = "engine, ingest, datafusion" ); diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_log_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_log_command.rs index d83b3b951c..f7708dc378 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_log_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_log_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_log, + fixture = kamu_cli_e2e_repo_tests::commands::test_log, options = Options::default().with_frozen_system_time(), extra_test_groups = "engine, ingest, datafusion" ); diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_login_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_login_command.rs index 8a0e3ad2cd..175f6ecacf 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_login_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_login_command.rs @@ -9,26 +9,55 @@ use kamu_cli_e2e_common::prelude::*; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_login_logout_password +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::commands::test_login_logout_password_st, +); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_run_api_server_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_login_logout_password, + fixture = kamu_cli_e2e_repo_tests::commands::test_login_logout_password_mt, ); +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_login_logout_oauth //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_run_api_server_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_login_logout_oauth, + fixture = kamu_cli_e2e_repo_tests::commands::test_login_logout_oauth_st, options = Options::default().with_multi_tenant() ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::commands::test_login_logout_oauth_mt, + options = Options::default().with_multi_tenant() +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_login_add_repo +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_login_add_repo, + fixture = kamu_cli_e2e_repo_tests::commands::test_login_add_repo_st, +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::commands::test_login_add_repo_mt, + options = Options::default().with_multi_tenant() ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_new_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_new_command.rs index fd38ab3d68..4c2eb56b49 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_new_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_new_command.rs @@ -13,14 +13,14 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_new_root, + fixture = kamu_cli_e2e_repo_tests::commands::test_new_root, ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_new_derivative, + fixture = kamu_cli_e2e_repo_tests::commands::test_new_derivative, ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_pull_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_pull_command.rs index a8790907b8..9874ef01c9 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_pull_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_pull_command.rs @@ -9,11 +9,31 @@ use kamu_cli_e2e_common::prelude::*; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_pull_env_var_template_default_value +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_env_var_template_default_value_st, + extra_test_groups = "containerized, engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_env_var_template_default_value_mt, + extra_test_groups = "containerized, engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_pull_env_var_template_default_value_missing_values //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_pull_env_var_template_default_value, + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_env_var_template_default_value_missing_values_st, extra_test_groups = "containerized, engine, ingest, datafusion" ); @@ -21,15 +41,17 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_pull_env_var_template_default_value_missing_values, + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_env_var_template_default_value_missing_values_mt, extra_test_groups = "containerized, engine, ingest, datafusion" ); +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_pull_set_watermark //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_pull_set_watermark, + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_set_watermark_st, options = Options::default().with_frozen_system_time(), ); @@ -37,7 +59,19 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_pull_reset_derivative, + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_set_watermark_mt, + options = Options::default() + .with_multi_tenant() + .with_frozen_system_time(), +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_pull_reset_derivative +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_reset_derivative_st, options = Options::default().with_frozen_system_time(), extra_test_groups = "containerized, engine, ingest, transform, datafusion" ); @@ -46,18 +80,33 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_pull_derivative, - options = Options::default().with_frozen_system_time(), + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_reset_derivative_mt, + options = Options::default() + .with_multi_tenant() + .with_frozen_system_time(), extra_test_groups = "containerized, engine, ingest, transform, datafusion" ); +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_pull_derivative //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_push_pull_s3, + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_derivative_st, options = Options::default().with_frozen_system_time(), - extra_test_groups = "containerized, engine, ingest, datafusion" + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_derivative_st, + options = Options::default() + .with_multi_tenant() + .with_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_rename_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_rename_command.rs index a8666b29bd..df119d6a6b 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_rename_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_rename_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_rename_dataset + fixture = kamu_cli_e2e_repo_tests::commands::test_rename_dataset ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_repo_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_repo_command.rs index ecbd05c727..44f6069ada 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_repo_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_repo_command.rs @@ -13,14 +13,14 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_repository_pull_aliases_commands + fixture = kamu_cli_e2e_repo_tests::commands::test_repository_pull_aliases_commands ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_repository_push_aliases_commands + fixture = kamu_cli_e2e_repo_tests::commands::test_repository_push_aliases_commands ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_reset_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_reset_command.rs index 17c46fee16..53d878a7d6 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_reset_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_reset_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_reset, + fixture = kamu_cli_e2e_repo_tests::commands::test_reset, extra_test_groups = "engine, ingest, datafusion" ); diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_search_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_search_command.rs index 3d9b2c90df..402de45391 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_search_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_search_command.rs @@ -9,11 +9,13 @@ use kamu_cli_e2e_common::prelude::*; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_search_multi_user //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_run_api_server_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_search_multi_user + fixture = kamu_cli_e2e_repo_tests::commands::test_search_multi_user_st // We need synthetic time for the tests, but the third-party JWT code // uses the current time. Assuming that the token lifetime is 24 hours, we will // use the projected date (the current day) as a workaround. @@ -39,7 +41,65 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_search_by_name + fixture = kamu_cli_e2e_repo_tests::commands::test_search_multi_user_mt + // We need synthetic time for the tests, but the third-party JWT code + // uses the current time. Assuming that the token lifetime is 24 hours, we will + // use the projected date (the current day) as a workaround. + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time() + .with_kamu_config( + indoc::indoc!( + r#" + kind: CLIConfig + version: 1 + content: + users: + predefined: + - accountName: kamu + "# + ) + ), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_search_by_name +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::commands::test_search_by_name_st + // We need synthetic time for the tests, but the third-party JWT code + // uses the current time. Assuming that the token lifetime is 24 hours, we will + // use the projected date (the current day) as a workaround. + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::commands::test_search_by_name_mt + // We need synthetic time for the tests, but the third-party JWT code + // uses the current time. Assuming that the token lifetime is 24 hours, we will + // use the projected date (the current day) as a workaround. + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_search_by_repo +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::commands::test_search_by_repo_st // We need synthetic time for the tests, but the third-party JWT code // uses the current time. Assuming that the token lifetime is 24 hours, we will // use the projected date (the current day) as a workaround. @@ -53,7 +113,7 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_search_by_repo + fixture = kamu_cli_e2e_repo_tests::commands::test_search_by_repo_mt // We need synthetic time for the tests, but the third-party JWT code // uses the current time. Assuming that the token lifetime is 24 hours, we will // use the projected date (the current day) as a workaround. diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_sql_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_sql_command.rs index 7315367ca6..a32ea7389a 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_sql_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_sql_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_datafusion_cli, + fixture = kamu_cli_e2e_repo_tests::commands::test_datafusion_cli, extra_test_groups = "engine, datafusion" ); @@ -21,7 +21,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_datafusion_cli_not_launched_in_root_ws, + fixture = kamu_cli_e2e_repo_tests::commands::test_datafusion_cli_not_launched_in_root_ws, options = Options::default().with_no_workspace(), extra_test_groups = "engine, datafusion" ); @@ -30,7 +30,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_sql_command, + fixture = kamu_cli_e2e_repo_tests::commands::test_sql_command, options = Options::default().with_frozen_system_time(), extra_test_groups = "engine, datafusion" ); diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_system_api_server_gql_query.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_system_api_server_gql_query.rs index 9d6f6a1e1e..c5cf725d00 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_system_api_server_gql_query.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_system_api_server_gql_query.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_gql_query_api_version + fixture = kamu_cli_e2e_repo_tests::commands::test_gql_query_api_version ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_system_diagnose_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_system_diagnose_command.rs index 44d8513528..909348c1de 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_system_diagnose_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_system_diagnose_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_system_diagnose + fixture = kamu_cli_e2e_repo_tests::commands::test_system_diagnose ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_system_gc_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_system_gc_command.rs index b36419994c..d830d13062 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_system_gc_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_system_gc_command.rs @@ -11,6 +11,9 @@ use kamu_cli_e2e_common::prelude::*; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -kamu_cli_execute_command_e2e_test!(storage = inmem, fixture = kamu_cli_e2e_repo_tests::test_gc); +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::commands::test_gc +); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_system_generate_token_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_system_generate_token_command.rs index 85577a0b82..594329ba02 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_system_generate_token_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_system_generate_token_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_generate_token + fixture = kamu_cli_e2e_repo_tests::commands::test_generate_token ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_system_info_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_system_info_command.rs index 00c8249687..95afe99742 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_system_info_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_system_info_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_system_info + fixture = kamu_cli_e2e_repo_tests::commands::test_system_info ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_tail_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_tail_command.rs index ad25c37f69..ce7d8a5908 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_tail_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_tail_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_tail, + fixture = kamu_cli_e2e_repo_tests::commands::test_tail, options = Options::default().with_frozen_system_time(), extra_test_groups = "engine, ingest, datafusion" ); diff --git a/src/e2e/app/cli/inmem/tests/tests/commands/test_verify_command.rs b/src/e2e/app/cli/inmem/tests/tests/commands/test_verify_command.rs index 2854462ebf..7e1aca226e 100644 --- a/src/e2e/app/cli/inmem/tests/tests/commands/test_verify_command.rs +++ b/src/e2e/app/cli/inmem/tests/tests/commands/test_verify_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_verify_regular_dataset, + fixture = kamu_cli_e2e_repo_tests::commands::test_verify_regular_dataset, extra_test_groups = "engine, ingest, datafusion" ); @@ -21,7 +21,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_verify_recursive, + fixture = kamu_cli_e2e_repo_tests::commands::test_verify_recursive, extra_test_groups = "containerized, engine, ingest, datafusion" ); @@ -29,7 +29,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_verify_integrity, + fixture = kamu_cli_e2e_repo_tests::commands::test_verify_integrity, extra_test_groups = "engine, ingest, datafusion" ); diff --git a/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs b/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs index 0193792c9f..f0a3f4b2a5 100644 --- a/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs @@ -9,14 +9,35 @@ use kamu_cli_e2e_common::prelude::*; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_smart_pull_sequence +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_smart_pull_sequence_st_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_smart_pull_sequence_st_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_run_api_server_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_sequence, - // We need synthetic time for the tests, but the third-party JWT code - // uses the current time. Assuming that the token lifetime is 24 hours, we will - // use the projected date (the current day) as a workaround. + fixture = kamu_cli_e2e_repo_tests::test_smart_push_smart_pull_sequence_mt_st, options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), @@ -27,18 +48,20 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_smart_force_push_pull, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_smart_pull_sequence_mt_mt, options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), extra_test_groups = "engine, ingest, datafusion" ); +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_force_smart_pull_force //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_run_api_server_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_add_alias, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_force_smart_pull_force_st_st, options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), @@ -49,7 +72,7 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_smart_pull_as, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_force_smart_pull_force_st_mt, options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), @@ -60,7 +83,101 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_all, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_force_smart_pull_force_mt_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_force_smart_pull_force_mt_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_no_alias_smart_pull_no_alias +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_no_alias_smart_pull_no_alias_st_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_no_alias_smart_pull_no_alias_st_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_no_alias_smart_pull_no_alias_mt_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_no_alias_smart_pull_no_alias_mt_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_pull_as +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_pull_as_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_pull_as_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_all_smart_pull_all +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_all_smart_pull_all_st_st, options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), @@ -71,7 +188,7 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_recursive, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_all_smart_pull_all_st_mt, options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), @@ -80,18 +197,36 @@ kamu_cli_run_api_server_e2e_test!( //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -kamu_cli_execute_command_e2e_test!( +kamu_cli_run_api_server_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_smart_pull_set_watermark, - options = Options::default().with_frozen_system_time(), + fixture = kamu_cli_e2e_repo_tests::test_smart_push_all_smart_pull_all_mt_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -kamu_cli_execute_command_e2e_test!( +kamu_cli_run_api_server_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_smart_pull_reset_derivative, - options = Options::default().with_frozen_system_time(), + fixture = kamu_cli_e2e_repo_tests::test_smart_push_all_smart_pull_all_mt_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_recursive_smart_pull_recursive +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_recursive_smart_pull_recursive_st_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), extra_test_groups = "containerized, engine, ingest, transform, datafusion" ); @@ -99,7 +234,42 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_smart_push_visibility, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_recursive_smart_pull_recursive_st_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_recursive_smart_pull_recursive_mt_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_recursive_smart_pull_recursive_mt_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_visibility +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_visibility_st, options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), @@ -108,9 +278,22 @@ kamu_cli_run_api_server_e2e_test!( //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_visibility_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_simple_push_to_s3_smart_pull +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_s3, + fixture = kamu_cli_e2e_repo_tests::test_simple_push_to_s3_smart_pull_st_st, options = Options::default().with_frozen_system_time(), extra_test_groups = "containerized, engine, ingest, datafusion" ); @@ -119,16 +302,40 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_smart_pull_derivative, + fixture = kamu_cli_e2e_repo_tests::test_simple_push_to_s3_smart_pull_st_mt, options = Options::default().with_frozen_system_time(), - extra_test_groups = "containerized, engine, ingest, transform, datafusion" + extra_test_groups = "containerized, engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_simple_push_to_s3_smart_pull_mt_st, + options = Options::default() + .with_multi_tenant() + .with_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_simple_push_to_s3_smart_pull_mt_mt, + options = Options::default() + .with_multi_tenant() + .with_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, datafusion" ); +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_to_registered_repo_smart_pull //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_run_api_server_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_smart_push_from_registered_repo, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_to_registered_repo_smart_pull_st_st, options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), @@ -139,11 +346,33 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = inmem, - fixture = kamu_cli_e2e_repo_tests::test_pull_mt, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_to_registered_repo_smart_pull_st_mt, options = Options::default() .with_multi_tenant() - .with_frozen_system_time(), - extra_test_groups = "containerized, engine, ingest, datafusion" + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_to_registered_repo_smart_pull_mt_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_to_registered_repo_smart_pull_mt_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/mysql/tests/tests/commands/test_system_api_server_gql_query.rs b/src/e2e/app/cli/mysql/tests/tests/commands/test_system_api_server_gql_query.rs index ed1125ef98..aebba70ebb 100644 --- a/src/e2e/app/cli/mysql/tests/tests/commands/test_system_api_server_gql_query.rs +++ b/src/e2e/app/cli/mysql/tests/tests/commands/test_system_api_server_gql_query.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = mysql, - fixture = kamu_cli_e2e_repo_tests::test_gql_query_api_version + fixture = kamu_cli_e2e_repo_tests::commands::test_gql_query_api_version ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/mysql/tests/tests/commands/test_system_generate_token_command.rs b/src/e2e/app/cli/mysql/tests/tests/commands/test_system_generate_token_command.rs index 6243ad7dec..df0f3f360a 100644 --- a/src/e2e/app/cli/mysql/tests/tests/commands/test_system_generate_token_command.rs +++ b/src/e2e/app/cli/mysql/tests/tests/commands/test_system_generate_token_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = mysql, - fixture = kamu_cli_e2e_repo_tests::test_generate_token + fixture = kamu_cli_e2e_repo_tests::commands::test_generate_token ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/mysql/tests/tests/test_smart_transfer_protocol.rs b/src/e2e/app/cli/mysql/tests/tests/test_smart_transfer_protocol.rs index 99be2b1c8e..9910594a20 100644 --- a/src/e2e/app/cli/mysql/tests/tests/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/mysql/tests/tests/test_smart_transfer_protocol.rs @@ -9,14 +9,366 @@ use kamu_cli_e2e_common::prelude::*; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_smart_pull_sequence +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_smart_pull_sequence_st_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_smart_pull_sequence_st_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_smart_pull_sequence_mt_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_smart_pull_sequence_mt_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_force_smart_pull_force +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_force_smart_pull_force_st_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_force_smart_pull_force_st_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_force_smart_pull_force_mt_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_force_smart_pull_force_mt_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_no_alias_smart_pull_no_alias +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_no_alias_smart_pull_no_alias_st_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_no_alias_smart_pull_no_alias_st_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_no_alias_smart_pull_no_alias_mt_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_no_alias_smart_pull_no_alias_mt_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_pull_as +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_smart_pull_as_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_smart_pull_as_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_all_smart_pull_all +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_all_smart_pull_all_st_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_all_smart_pull_all_st_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_all_smart_pull_all_mt_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_all_smart_pull_all_mt_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_recursive_smart_pull_recursive +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_recursive_smart_pull_recursive_st_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_recursive_smart_pull_recursive_st_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_recursive_smart_pull_recursive_mt_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_recursive_smart_pull_recursive_mt_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_visibility +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_visibility_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_visibility_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_simple_push_to_s3_smart_pull +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_simple_push_to_s3_smart_pull_st_st, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_simple_push_to_s3_smart_pull_st_mt, + options = Options::default().with_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_simple_push_to_s3_smart_pull_mt_st, + options = Options::default() + .with_multi_tenant() + .with_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_simple_push_to_s3_smart_pull_mt_mt, + options = Options::default() + .with_multi_tenant() + .with_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_to_registered_repo_smart_pull +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_to_registered_repo_smart_pull_st_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_to_registered_repo_smart_pull_st_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = mysql, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_to_registered_repo_smart_pull_mt_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_run_api_server_e2e_test!( storage = mysql, - fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_sequence, - // We need synthetic time for the tests, but the third-party JWT code - // uses the current time. Assuming that the token lifetime is 24 hours, we will - // use the projected date (the current day) as a workaround. + fixture = kamu_cli_e2e_repo_tests::test_smart_push_to_registered_repo_smart_pull_mt_mt, options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_add_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_add_command.rs index cb8df8c190..813c9cc5bb 100644 --- a/src/e2e/app/cli/postgres/tests/tests/commands/test_add_command.rs +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_add_command.rs @@ -13,35 +13,35 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_add_dataset_from_stdin + fixture = kamu_cli_e2e_repo_tests::commands::test_add_dataset_from_stdin ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_add_dataset_with_name + fixture = kamu_cli_e2e_repo_tests::commands::test_add_dataset_with_name ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_add_dataset_with_replace + fixture = kamu_cli_e2e_repo_tests::commands::test_add_dataset_with_replace ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_add_recursive + fixture = kamu_cli_e2e_repo_tests::commands::test_add_recursive ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_add_with_circular_dependency + fixture = kamu_cli_e2e_repo_tests::commands::test_add_with_circular_dependency ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_compact_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_compact_command.rs index 90d11dab3f..f1bd84984f 100644 --- a/src/e2e/app/cli/postgres/tests/tests/commands/test_compact_command.rs +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_compact_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_compact_hard + fixture = kamu_cli_e2e_repo_tests::commands::test_compact_hard extra_test_groups = "engine, ingest, datafusion" ); @@ -21,7 +21,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_compact_keep_metadata_only + fixture = kamu_cli_e2e_repo_tests::commands::test_compact_keep_metadata_only extra_test_groups = "engine, ingest, datafusion" ); @@ -29,7 +29,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_compact_verify + fixture = kamu_cli_e2e_repo_tests::commands::test_compact_verify extra_test_groups = "engine, ingest, datafusion" ); diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_delete_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_delete_command.rs index faa7a6a996..0f55b718ae 100644 --- a/src/e2e/app/cli/postgres/tests/tests/commands/test_delete_command.rs +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_delete_command.rs @@ -13,21 +13,21 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_delete_dataset + fixture = kamu_cli_e2e_repo_tests::commands::test_delete_dataset ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_delete_dataset_recursive + fixture = kamu_cli_e2e_repo_tests::commands::test_delete_dataset_recursive ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_delete_dataset_all + fixture = kamu_cli_e2e_repo_tests::commands::test_delete_dataset_all ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_ingest_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_ingest_command.rs index a9fd603d7d..399fc61f09 100644 --- a/src/e2e/app/cli/postgres/tests/tests/commands/test_ingest_command.rs +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_ingest_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_push_ingest_from_file_ledger, + fixture = kamu_cli_e2e_repo_tests::commands::test_push_ingest_from_file_ledger, options = Options::default().with_frozen_system_time(), extra_test_groups = "engine, ingest, datafusion" ); @@ -22,7 +22,8 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_push_ingest_from_file_snapshot_with_event_time, + fixture = + kamu_cli_e2e_repo_tests::commands::test_push_ingest_from_file_snapshot_with_event_time, options = Options::default().with_frozen_system_time(), extra_test_groups = "engine, ingest, datafusion" ); @@ -31,7 +32,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_ingest_from_stdin, + fixture = kamu_cli_e2e_repo_tests::commands::test_ingest_from_stdin, options = Options::default().with_frozen_system_time(), extra_test_groups = "engine, ingest, datafusion" ); @@ -40,7 +41,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_ingest_recursive, + fixture = kamu_cli_e2e_repo_tests::commands::test_ingest_recursive, options = Options::default().with_frozen_system_time(), extra_test_groups = "engine, ingest, datafusion" ); @@ -49,7 +50,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_ingest_with_source_name, + fixture = kamu_cli_e2e_repo_tests::commands::test_ingest_with_source_name, options = Options::default().with_frozen_system_time(), extra_test_groups = "engine, ingest, datafusion" ); diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_init_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_init_command.rs index 4a3ebe83c9..6fef86e961 100644 --- a/src/e2e/app/cli/postgres/tests/tests/commands/test_init_command.rs +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_init_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_init_exist_ok_st, + fixture = kamu_cli_e2e_repo_tests::commands::test_init_exist_ok_st, options = Options::default().with_no_workspace() ); @@ -21,7 +21,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_init_exist_ok_mt, + fixture = kamu_cli_e2e_repo_tests::commands::test_init_exist_ok_mt, options = Options::default().with_no_workspace() ); @@ -29,7 +29,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_init_in_an_existing_workspace, + fixture = kamu_cli_e2e_repo_tests::commands::test_init_in_an_existing_workspace, options = Options::default().with_no_workspace() ); diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_inspect_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_inspect_command.rs index 96239fd6c2..897f675418 100644 --- a/src/e2e/app/cli/postgres/tests/tests/commands/test_inspect_command.rs +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_inspect_command.rs @@ -13,14 +13,14 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_inspect_lineage, + fixture = kamu_cli_e2e_repo_tests::commands::test_inspect_lineage, ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_inspect_query, + fixture = kamu_cli_e2e_repo_tests::commands::test_inspect_query, options = Options::default().with_frozen_system_time() ); @@ -28,7 +28,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_inspect_schema, + fixture = kamu_cli_e2e_repo_tests::commands::test_inspect_schema, extra_test_groups = "engine, ingest, datafusion" ); diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_log_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_log_command.rs index c2449941d6..03654efc94 100644 --- a/src/e2e/app/cli/postgres/tests/tests/commands/test_log_command.rs +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_log_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_log, + fixture = kamu_cli_e2e_repo_tests::commands::test_log, options = Options::default().with_frozen_system_time(), extra_test_groups = "engine, ingest, datafusion" ); diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_login_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_login_command.rs index dd1e803758..36caf93bf1 100644 --- a/src/e2e/app/cli/postgres/tests/tests/commands/test_login_command.rs +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_login_command.rs @@ -9,26 +9,55 @@ use kamu_cli_e2e_common::prelude::*; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_login_logout_password +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::commands::test_login_logout_password_st, +); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_run_api_server_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_login_logout_password, + fixture = kamu_cli_e2e_repo_tests::commands::test_login_logout_password_mt, ); +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_login_logout_oauth //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_run_api_server_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_login_logout_oauth, + fixture = kamu_cli_e2e_repo_tests::commands::test_login_logout_oauth_st, options = Options::default().with_multi_tenant() ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::commands::test_login_logout_oauth_mt, + options = Options::default().with_multi_tenant() +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_login_add_repo +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_login_add_repo, + fixture = kamu_cli_e2e_repo_tests::commands::test_login_add_repo_st, +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::commands::test_login_add_repo_mt, + options = Options::default().with_multi_tenant() ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_new_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_new_command.rs index 3c65ebef48..f15f1b55df 100644 --- a/src/e2e/app/cli/postgres/tests/tests/commands/test_new_command.rs +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_new_command.rs @@ -13,14 +13,14 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_new_root, + fixture = kamu_cli_e2e_repo_tests::commands::test_new_root, ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_new_derivative, + fixture = kamu_cli_e2e_repo_tests::commands::test_new_derivative, ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_pull_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_pull_command.rs index c5d9e7b98d..72cee77a62 100644 --- a/src/e2e/app/cli/postgres/tests/tests/commands/test_pull_command.rs +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_pull_command.rs @@ -9,11 +9,13 @@ use kamu_cli_e2e_common::prelude::*; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_pull_env_var_template_default_value //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_pull_env_var_template_default_value, + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_env_var_template_default_value_st, extra_test_groups = "containerized, engine, ingest, datafusion" ); @@ -21,32 +23,55 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_pull_env_var_template_default_value_missing_values, + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_env_var_template_default_value_mt, extra_test_groups = "containerized, engine, ingest, datafusion" ); +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_pull_env_var_template_default_value_missing_values //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_pull_set_watermark, - options = Options::default().with_frozen_system_time(), + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_env_var_template_default_value_missing_values_st, + extra_test_groups = "containerized, engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_env_var_template_default_value_missing_values_mt, + extra_test_groups = "containerized, engine, ingest, datafusion" ); +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_pull_set_watermark //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_pull_reset_derivative, + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_set_watermark_st, options = Options::default().with_frozen_system_time(), - extra_test_groups = "containerized, engine, ingest, transform, datafusion" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_pull_derivative, + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_set_watermark_mt, + options = Options::default() + .with_multi_tenant() + .with_frozen_system_time(), +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_pull_reset_derivative +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_reset_derivative_st, options = Options::default().with_frozen_system_time(), extra_test_groups = "containerized, engine, ingest, transform, datafusion" ); @@ -54,10 +79,34 @@ kamu_cli_execute_command_e2e_test!( //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( - storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_push_pull_s3, + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_reset_derivative_mt, + options = Options::default() + .with_multi_tenant() + .with_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_pull_derivative +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_derivative_st, options = Options::default().with_frozen_system_time(), - extra_test_groups = "containerized, engine, ingest, datafusion" + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_derivative_st, + options = Options::default() + .with_multi_tenant() + .with_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_rename_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_rename_command.rs index 64a5565da5..845e0c235b 100644 --- a/src/e2e/app/cli/postgres/tests/tests/commands/test_rename_command.rs +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_rename_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_rename_dataset + fixture = kamu_cli_e2e_repo_tests::commands::test_rename_dataset ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_repo_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_repo_command.rs index eeec2744ef..3e2b470721 100644 --- a/src/e2e/app/cli/postgres/tests/tests/commands/test_repo_command.rs +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_repo_command.rs @@ -13,14 +13,14 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_repository_pull_aliases_commands + fixture = kamu_cli_e2e_repo_tests::commands::test_repository_pull_aliases_commands ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_repository_push_aliases_commands + fixture = kamu_cli_e2e_repo_tests::commands::test_repository_push_aliases_commands ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_reset_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_reset_command.rs index 1161e8059e..84ff9c8ce3 100644 --- a/src/e2e/app/cli/postgres/tests/tests/commands/test_reset_command.rs +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_reset_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_reset, + fixture = kamu_cli_e2e_repo_tests::commands::test_reset, extra_test_groups = "engine, ingest, datafusion" ); diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_search_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_search_command.rs index 3c2fb41048..29fc2df3dc 100644 --- a/src/e2e/app/cli/postgres/tests/tests/commands/test_search_command.rs +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_search_command.rs @@ -9,11 +9,13 @@ use kamu_cli_e2e_common::prelude::*; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_search_multi_user //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_run_api_server_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_search_multi_user + fixture = kamu_cli_e2e_repo_tests::commands::test_search_multi_user_st // We need synthetic time for the tests, but the third-party JWT code // uses the current time. Assuming that the token lifetime is 24 hours, we will // use the projected date (the current day) as a workaround. @@ -39,7 +41,65 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_search_by_name + fixture = kamu_cli_e2e_repo_tests::commands::test_search_multi_user_mt + // We need synthetic time for the tests, but the third-party JWT code + // uses the current time. Assuming that the token lifetime is 24 hours, we will + // use the projected date (the current day) as a workaround. + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time() + .with_kamu_config( + indoc::indoc!( + r#" + kind: CLIConfig + version: 1 + content: + users: + predefined: + - accountName: kamu + "# + ) + ), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_search_by_name +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::commands::test_search_by_name_st + // We need synthetic time for the tests, but the third-party JWT code + // uses the current time. Assuming that the token lifetime is 24 hours, we will + // use the projected date (the current day) as a workaround. + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::commands::test_search_by_name_mt + // We need synthetic time for the tests, but the third-party JWT code + // uses the current time. Assuming that the token lifetime is 24 hours, we will + // use the projected date (the current day) as a workaround. + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_search_by_repo +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::commands::test_search_by_repo_st // We need synthetic time for the tests, but the third-party JWT code // uses the current time. Assuming that the token lifetime is 24 hours, we will // use the projected date (the current day) as a workaround. @@ -53,7 +113,7 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_search_by_repo + fixture = kamu_cli_e2e_repo_tests::commands::test_search_by_repo_mt // We need synthetic time for the tests, but the third-party JWT code // uses the current time. Assuming that the token lifetime is 24 hours, we will // use the projected date (the current day) as a workaround. diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_sql_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_sql_command.rs index 339bc6586b..8e7b62de43 100644 --- a/src/e2e/app/cli/postgres/tests/tests/commands/test_sql_command.rs +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_sql_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_datafusion_cli, + fixture = kamu_cli_e2e_repo_tests::commands::test_datafusion_cli, extra_test_groups = "engine, datafusion" ); @@ -21,7 +21,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_datafusion_cli_not_launched_in_root_ws, + fixture = kamu_cli_e2e_repo_tests::commands::test_datafusion_cli_not_launched_in_root_ws, options = Options::default().with_no_workspace(), extra_test_groups = "engine, datafusion" ); @@ -30,7 +30,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_sql_command, + fixture = kamu_cli_e2e_repo_tests::commands::test_sql_command, options = Options::default().with_frozen_system_time(), extra_test_groups = "engine, datafusion" ); diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_system_api_server_gql_query.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_system_api_server_gql_query.rs index 3869b1c930..5dfd22b5a0 100644 --- a/src/e2e/app/cli/postgres/tests/tests/commands/test_system_api_server_gql_query.rs +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_system_api_server_gql_query.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_gql_query_api_version + fixture = kamu_cli_e2e_repo_tests::commands::test_gql_query_api_version ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_system_diagnose_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_system_diagnose_command.rs index 9d1968db8a..9630b50d9e 100644 --- a/src/e2e/app/cli/postgres/tests/tests/commands/test_system_diagnose_command.rs +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_system_diagnose_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_system_diagnose + fixture = kamu_cli_e2e_repo_tests::commands::test_system_diagnose ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_system_gc_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_system_gc_command.rs index 0a95452b54..d57b4c2baa 100644 --- a/src/e2e/app/cli/postgres/tests/tests/commands/test_system_gc_command.rs +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_system_gc_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_gc + fixture = kamu_cli_e2e_repo_tests::commands::test_gc ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_system_generate_token_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_system_generate_token_command.rs index 713df0263e..083b79d717 100644 --- a/src/e2e/app/cli/postgres/tests/tests/commands/test_system_generate_token_command.rs +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_system_generate_token_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_generate_token + fixture = kamu_cli_e2e_repo_tests::commands::test_generate_token ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_system_info_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_system_info_command.rs index 63a713ca85..ccd48f9207 100644 --- a/src/e2e/app/cli/postgres/tests/tests/commands/test_system_info_command.rs +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_system_info_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_system_info + fixture = kamu_cli_e2e_repo_tests::commands::test_system_info ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_tail_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_tail_command.rs index 9950b16af0..06dac61b7a 100644 --- a/src/e2e/app/cli/postgres/tests/tests/commands/test_tail_command.rs +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_tail_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_tail, + fixture = kamu_cli_e2e_repo_tests::commands::test_tail, options = Options::default().with_frozen_system_time(), extra_test_groups = "engine, ingest, datafusion" ); diff --git a/src/e2e/app/cli/postgres/tests/tests/commands/test_verify_command.rs b/src/e2e/app/cli/postgres/tests/tests/commands/test_verify_command.rs index 1548e1faf5..3f23287eeb 100644 --- a/src/e2e/app/cli/postgres/tests/tests/commands/test_verify_command.rs +++ b/src/e2e/app/cli/postgres/tests/tests/commands/test_verify_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_verify_regular_dataset, + fixture = kamu_cli_e2e_repo_tests::commands::test_verify_regular_dataset, extra_test_groups = "engine, ingest, datafusion" ); @@ -21,7 +21,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_verify_recursive, + fixture = kamu_cli_e2e_repo_tests::commands::test_verify_recursive, extra_test_groups = "containerized, engine, ingest, datafusion" ); @@ -29,7 +29,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_verify_integrity, + fixture = kamu_cli_e2e_repo_tests::commands::test_verify_integrity, extra_test_groups = "engine, ingest, datafusion" ); diff --git a/src/e2e/app/cli/postgres/tests/tests/test_smart_transfer_protocol.rs b/src/e2e/app/cli/postgres/tests/tests/test_smart_transfer_protocol.rs index 965433c6ae..e8ed319357 100644 --- a/src/e2e/app/cli/postgres/tests/tests/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/postgres/tests/tests/test_smart_transfer_protocol.rs @@ -9,14 +9,35 @@ use kamu_cli_e2e_common::prelude::*; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_smart_pull_sequence +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_smart_pull_sequence_st_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_smart_pull_sequence_st_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_run_api_server_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_sequence, - // We need synthetic time for the tests, but the third-party JWT code - // uses the current time. Assuming that the token lifetime is 24 hours, we will - // use the projected date (the current day) as a workaround. + fixture = kamu_cli_e2e_repo_tests::test_smart_push_smart_pull_sequence_mt_st, options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), @@ -27,18 +48,20 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_smart_force_push_pull, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_smart_pull_sequence_mt_mt, options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), extra_test_groups = "engine, ingest, datafusion" ); +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_force_smart_pull_force //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_run_api_server_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_add_alias, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_force_smart_pull_force_st_st, options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), @@ -49,7 +72,7 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_smart_pull_as, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_force_smart_pull_force_st_mt, options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), @@ -60,7 +83,101 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_all, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_force_smart_pull_force_mt_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_force_smart_pull_force_mt_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_no_alias_smart_pull_no_alias +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_no_alias_smart_pull_no_alias_st_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_no_alias_smart_pull_no_alias_st_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_no_alias_smart_pull_no_alias_mt_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_no_alias_smart_pull_no_alias_mt_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_pull_as +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_smart_pull_as_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_smart_pull_as_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_all_smart_pull_all +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_all_smart_pull_all_st_st, options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), @@ -71,7 +188,7 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_recursive, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_all_smart_pull_all_st_mt, options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), @@ -80,18 +197,36 @@ kamu_cli_run_api_server_e2e_test!( //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -kamu_cli_execute_command_e2e_test!( +kamu_cli_run_api_server_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_smart_pull_set_watermark, - options = Options::default().with_frozen_system_time(), + fixture = kamu_cli_e2e_repo_tests::test_smart_push_all_smart_pull_all_mt_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -kamu_cli_execute_command_e2e_test!( +kamu_cli_run_api_server_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_smart_pull_reset_derivative, - options = Options::default().with_frozen_system_time(), + fixture = kamu_cli_e2e_repo_tests::test_smart_push_all_smart_pull_all_mt_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_recursive_smart_pull_recursive +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_recursive_smart_pull_recursive_st_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), extra_test_groups = "containerized, engine, ingest, transform, datafusion" ); @@ -99,7 +234,42 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_smart_push_visibility, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_recursive_smart_pull_recursive_st_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_recursive_smart_pull_recursive_mt_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_recursive_smart_pull_recursive_mt_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_visibility +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_visibility_st, options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), @@ -108,9 +278,22 @@ kamu_cli_run_api_server_e2e_test!( //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_visibility_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_simple_push_to_s3_smart_pull +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_s3, + fixture = kamu_cli_e2e_repo_tests::test_simple_push_to_s3_smart_pull_st_st, options = Options::default().with_frozen_system_time(), extra_test_groups = "containerized, engine, ingest, datafusion" ); @@ -119,16 +302,40 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_smart_pull_derivative, + fixture = kamu_cli_e2e_repo_tests::test_simple_push_to_s3_smart_pull_st_mt, options = Options::default().with_frozen_system_time(), - extra_test_groups = "containerized, engine, ingest, transform, datafusion" + extra_test_groups = "containerized, engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_simple_push_to_s3_smart_pull_mt_st, + options = Options::default() + .with_multi_tenant() + .with_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_simple_push_to_s3_smart_pull_mt_mt, + options = Options::default() + .with_multi_tenant() + .with_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, datafusion" ); +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_to_registered_repo_smart_pull //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_run_api_server_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_smart_push_from_registered_repo, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_to_registered_repo_smart_pull_st_st, options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), @@ -139,11 +346,33 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = postgres, - fixture = kamu_cli_e2e_repo_tests::test_pull_mt, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_to_registered_repo_smart_pull_st_mt, options = Options::default() .with_multi_tenant() - .with_frozen_system_time(), - extra_test_groups = "containerized, engine, ingest, datafusion" + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_to_registered_repo_smart_pull_mt_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = postgres, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_to_registered_repo_smart_pull_mt_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/Cargo.toml b/src/e2e/app/cli/repo-tests/Cargo.toml index 9e0ea37f45..9be6242cdc 100644 --- a/src/e2e/app/cli/repo-tests/Cargo.toml +++ b/src/e2e/app/cli/repo-tests/Cargo.toml @@ -42,6 +42,7 @@ opendatafabric = { workspace = true } chrono = { version = "0.4", default-features = false } indoc = "2" +paste = { version = "1", default-features = false } pretty_assertions = { version = "1" } reqwest = { version = "0.12", default-features = false, features = [] } serde_json = { version = "1", default-features = false } diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_login_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_login_command.rs index 5f20a52ba7..7159d0bed3 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_login_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_login_command.rs @@ -10,17 +10,69 @@ use kamu_cli_e2e_common::{ KamuApiServerClient, KamuApiServerClientExt, + DATASET_ROOT_PLAYER_NAME, DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR, }; use kamu_cli_puppet::extensions::{KamuCliPuppetExt, RepoRecord}; use kamu_cli_puppet::KamuCliPuppet; -use opendatafabric::RepoName; +use opendatafabric as odf; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_login_logout_password +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_login_logout_password_st(kamu_node_api_client: KamuApiServerClient) { + test_login_logout_password(kamu_node_api_client, false).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_login_logout_password_mt(kamu_node_api_client: KamuApiServerClient) { + test_login_logout_password(kamu_node_api_client, true).await; +} -pub async fn test_login_logout_password(kamu_node_api_client: KamuApiServerClient) { +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_login_logout_oauth +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_login_logout_oauth_st(kamu_node_api_client: KamuApiServerClient) { + test_login_logout_oauth(kamu_node_api_client, false).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_login_logout_oauth_mt(kamu_node_api_client: KamuApiServerClient) { + test_login_logout_oauth(kamu_node_api_client, true).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_login_add_repo +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_login_add_repo_st(kamu: KamuCliPuppet) { + test_login_add_repo(kamu).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_login_add_repo_mt(kamu: KamuCliPuppet) { + test_login_add_repo(kamu).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Implementations +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +async fn test_login_logout_password( + kamu_node_api_client: KamuApiServerClient, + is_workspace_multi_tenant: bool, +) { let kamu_node_url = kamu_node_api_client.get_base_url().as_str(); - let kamu = KamuCliPuppet::new_workspace_tmp().await; + let dataset_alias = odf::DatasetAlias::new(None, DATASET_ROOT_PLAYER_NAME.clone()); + let kamu_api_server_dataset_endpoint = + kamu_node_api_client.dataset().get_endpoint(&dataset_alias); + + let kamu = KamuCliPuppet::new_workspace_tmp(is_workspace_multi_tenant).await; kamu.assert_success_command_execution( ["logout", kamu_node_url], @@ -60,7 +112,7 @@ pub async fn test_login_logout_password(kamu_node_api_client: KamuApiServerClien "push", "player-scores", "--to", - &format!("odf+{kamu_node_url}player-scores"), + kamu_api_server_dataset_endpoint.as_str(), ], None, Some(["1 dataset(s) pushed"]), @@ -77,9 +129,12 @@ pub async fn test_login_logout_password(kamu_node_api_client: KamuApiServerClien //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_login_logout_oauth(mut kamu_node_api_client: KamuApiServerClient) { +async fn test_login_logout_oauth( + mut kamu_node_api_client: KamuApiServerClient, + is_workspace_multi_tenant: bool, +) { let kamu_node_url = kamu_node_api_client.get_base_url().clone(); - let kamu = KamuCliPuppet::new_workspace_tmp().await; + let kamu = KamuCliPuppet::new_workspace_tmp(is_workspace_multi_tenant).await; kamu.assert_success_command_execution( ["logout", kamu_node_url.as_str()], @@ -133,7 +188,7 @@ pub async fn test_login_logout_oauth(mut kamu_node_api_client: KamuApiServerClie //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_login_add_repo(kamu: KamuCliPuppet) { +async fn test_login_add_repo(kamu: KamuCliPuppet) { let dummy_access_token = "dummy-access-token"; let dummy_url = "http://example.com"; @@ -165,7 +220,9 @@ pub async fn test_login_add_repo(kamu: KamuCliPuppet) { let repo_list = kamu.get_list_of_repos().await; let expected_repo_list = vec![RepoRecord { - name: RepoName::new_unchecked(url::Url::try_from(dummy_url).unwrap().host_str().unwrap()), + name: odf::RepoName::new_unchecked( + url::Url::try_from(dummy_url).unwrap().host_str().unwrap(), + ), url: url::Url::try_from(dummy_url).unwrap(), }]; @@ -193,13 +250,13 @@ pub async fn test_login_add_repo(kamu: KamuCliPuppet) { let expected_repo_list = vec![ RepoRecord { - name: RepoName::new_unchecked( + name: odf::RepoName::new_unchecked( url::Url::try_from(dummy_url).unwrap().host_str().unwrap(), ), url: url::Url::try_from(dummy_url).unwrap(), }, RepoRecord { - name: RepoName::new_unchecked(expected_repo_name), + name: odf::RepoName::new_unchecked(expected_repo_name), url: url::Url::try_from(new_dummy_url).unwrap(), }, ]; diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_pull_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_pull_command.rs index 924aee1113..90e2070998 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_pull_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_pull_command.rs @@ -7,7 +7,6 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use kamu::testing::LocalS3Server; use kamu_cli_e2e_common::{ DATASET_DERIVATIVE_LEADERBOARD_NAME, DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, @@ -18,7 +17,7 @@ use kamu_cli_e2e_common::{ }; use kamu_cli_puppet::extensions::KamuCliPuppetExt; use kamu_cli_puppet::KamuCliPuppet; -use opendatafabric::DatasetAlias; +use opendatafabric as odf; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -54,9 +53,81 @@ const DATASET_INGEST_DATA: &str = indoc::indoc!( "# ); +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_pull_env_var_template_default_value +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_pull_env_var_template_default_value_st(kamu: KamuCliPuppet) { + test_pull_env_var_template_default_value(kamu).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_pull_env_var_template_default_value_mt(kamu: KamuCliPuppet) { + test_pull_env_var_template_default_value(kamu).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_pull_env_var_template_default_value_missing_values +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_pull_env_var_template_default_value_missing_values_st(kamu: KamuCliPuppet) { + test_pull_env_var_template_default_value_missing_values(kamu).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_pull_env_var_template_default_value_missing_values_mt(kamu: KamuCliPuppet) { + test_pull_env_var_template_default_value_missing_values(kamu).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_pull_set_watermark +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_pull_set_watermark_st(kamu: KamuCliPuppet) { + test_pull_set_watermark(kamu).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_pull_set_watermark_mt(kamu: KamuCliPuppet) { + test_pull_set_watermark(kamu).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_pull_reset_derivative +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_pull_reset_derivative_st(kamu: KamuCliPuppet) { + test_pull_reset_derivative(kamu).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_pull_reset_derivative_mt(kamu: KamuCliPuppet) { + test_pull_reset_derivative(kamu).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_pull_derivative +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_pull_derivative_st(kamu: KamuCliPuppet) { + test_pull_derivative(kamu).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_pull_derivative_mt(kamu: KamuCliPuppet) { + test_pull_derivative(kamu).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Implementations //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_pull_env_var_template_default_value(kamu: KamuCliPuppet) { +async fn test_pull_env_var_template_default_value(kamu: KamuCliPuppet) { kamu.assert_success_command_execution_with_input( ["add", "--stdin"], DATASET_SNAPSHOT_STR, @@ -88,7 +159,7 @@ pub async fn test_pull_env_var_template_default_value(kamu: KamuCliPuppet) { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_pull_env_var_template_default_value_missing_values(kamu: KamuCliPuppet) { +async fn test_pull_env_var_template_default_value_missing_values(kamu: KamuCliPuppet) { kamu.assert_success_command_execution_with_input( ["add", "--stdin"], DATASET_SNAPSHOT_STR, @@ -119,31 +190,10 @@ pub async fn test_pull_env_var_template_default_value_missing_values(kamu: KamuC //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_pull_set_watermark(kamu: KamuCliPuppet) { - let dataset_name = DATASET_ROOT_PLAYER_NAME.clone(); - - kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) - .await - .success(); - - kamu.assert_success_command_execution( - [ - "pull", - dataset_name.as_str(), - "--set-watermark", - "2051-01-02T03:04:05Z", - ], - None, - Some(["Committed new block"]), - ) - .await; -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -pub async fn test_pull_reset_derivative(kamu: KamuCliPuppet) { - let dataset_name = DATASET_ROOT_PLAYER_NAME.clone(); - let dataset_derivative_name = DATASET_DERIVATIVE_LEADERBOARD_NAME.clone(); +async fn test_pull_reset_derivative(kamu: KamuCliPuppet) { + let dataset_alias = odf::DatasetAlias::new(None, DATASET_ROOT_PLAYER_NAME.clone()); + let dataset_derivative_alias = + odf::DatasetAlias::new(None, DATASET_DERIVATIVE_LEADERBOARD_NAME.clone()); kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) .await @@ -157,13 +207,13 @@ pub async fn test_pull_reset_derivative(kamu: KamuCliPuppet) { .success(); kamu.ingest_data( - &dataset_name, + &dataset_alias.dataset_name, DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, ) .await; kamu.assert_success_command_execution( - ["pull", dataset_derivative_name.as_str()], + ["pull", dataset_derivative_alias.dataset_name.as_str()], None, Some(["1 dataset(s) updated"]), ) @@ -194,7 +244,7 @@ pub async fn test_pull_reset_derivative(kamu: KamuCliPuppet) { "# ); kamu.assert_last_data_slice( - &DatasetAlias::new(None, dataset_derivative_name.clone()), + &dataset_derivative_alias, expected_derivative_schema, expected_derivative_data, ) @@ -205,7 +255,7 @@ pub async fn test_pull_reset_derivative(kamu: KamuCliPuppet) { "--yes", "system", "compact", - dataset_name.as_str(), + dataset_alias.dataset_name.as_str(), "--hard", "--keep-metadata-only", ]) @@ -214,7 +264,7 @@ pub async fn test_pull_reset_derivative(kamu: KamuCliPuppet) { // Pull derivative should fail kamu.assert_failure_command_execution( - ["pull", dataset_derivative_name.as_str()], + ["pull", dataset_derivative_alias.dataset_name.as_str()], None, Some(["Failed to update 1 dataset(s)"]), ) @@ -222,7 +272,7 @@ pub async fn test_pull_reset_derivative(kamu: KamuCliPuppet) { // Add new data to root dataset kamu.ingest_data( - &dataset_name, + &dataset_alias.dataset_name, DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, ) .await; @@ -230,7 +280,7 @@ pub async fn test_pull_reset_derivative(kamu: KamuCliPuppet) { kamu.assert_success_command_execution( [ "pull", - dataset_derivative_name.as_str(), + dataset_derivative_alias.dataset_name.as_str(), "--reset-derivatives-on-diverged-input", ], None, @@ -249,7 +299,7 @@ pub async fn test_pull_reset_derivative(kamu: KamuCliPuppet) { "# ); kamu.assert_last_data_slice( - &DatasetAlias::new(None, dataset_derivative_name), + &dataset_derivative_alias, expected_derivative_schema, expected_derivative_data, ) @@ -258,7 +308,7 @@ pub async fn test_pull_reset_derivative(kamu: KamuCliPuppet) { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_pull_derivative(kamu: KamuCliPuppet) { +async fn test_pull_derivative(kamu: KamuCliPuppet) { let dataset_name = DATASET_ROOT_PLAYER_NAME.clone(); let dataset_derivative_name = DATASET_DERIVATIVE_LEADERBOARD_NAME.clone(); @@ -313,71 +363,24 @@ pub async fn test_pull_derivative(kamu: KamuCliPuppet) { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_push_pull_s3(kamu: KamuCliPuppet) { +async fn test_pull_set_watermark(kamu: KamuCliPuppet) { let dataset_name = DATASET_ROOT_PLAYER_NAME.clone(); kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) .await .success(); - kamu.ingest_data( - &dataset_name, - DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, - ) - .await; - - let s3_server = LocalS3Server::new().await; - let dataset_url = format!("{}/e2e-user/{dataset_name}", s3_server.url); - - // Push dataset kamu.assert_success_command_execution( - ["push", dataset_name.as_str(), "--to", dataset_url.as_str()], + [ + "pull", + dataset_name.as_str(), + "--set-watermark", + "2051-01-02T03:04:05Z", + ], None, - Some(["1 dataset(s) pushed"]), + Some(["Committed new block"]), ) .await; - - { - let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; - - kamu_in_pull_workspace - .assert_success_command_execution( - ["pull", dataset_url.as_str()], - None, - Some(["1 dataset(s) updated"]), - ) - .await; - - let expected_schema = indoc::indoc!( - r#" - message arrow_schema { - REQUIRED INT64 offset; - REQUIRED INT32 op; - REQUIRED INT64 system_time (TIMESTAMP(MILLIS,true)); - OPTIONAL INT64 match_time (TIMESTAMP(MILLIS,true)); - OPTIONAL INT64 match_id; - OPTIONAL BYTE_ARRAY player_id (STRING); - OPTIONAL INT64 score; - } - "# - ); - let expected_data = indoc::indoc!( - r#" - +--------+----+----------------------+----------------------+----------+-----------+-------+ - | offset | op | system_time | match_time | match_id | player_id | score | - +--------+----+----------------------+----------------------+----------+-----------+-------+ - | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | - | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | Bob | 80 | - +--------+----+----------------------+----------------------+----------+-----------+-------+ - "# - ); - kamu.assert_last_data_slice( - &DatasetAlias::new(None, dataset_name), - expected_schema, - expected_data, - ) - .await; - } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/repo-tests/src/commands/test_search_command.rs b/src/e2e/app/cli/repo-tests/src/commands/test_search_command.rs index 50c7cfdd8b..aa86bd11b3 100644 --- a/src/e2e/app/cli/repo-tests/src/commands/test_search_command.rs +++ b/src/e2e/app/cli/repo-tests/src/commands/test_search_command.rs @@ -21,9 +21,56 @@ use kamu_cli_puppet::KamuCliPuppet; use opendatafabric::*; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_search_multi_user +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_search_multi_user_st(kamu_node_api_client: KamuApiServerClient) { + test_search_multi_user(kamu_node_api_client, false).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_search_multi_user_mt(kamu_node_api_client: KamuApiServerClient) { + test_search_multi_user(kamu_node_api_client, true).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_search_by_name +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_search_by_name_st(kamu_node_api_client: KamuApiServerClient) { + test_search_by_name(kamu_node_api_client, false).await; +} -pub async fn test_search_multi_user(mut kamu_node_api_client: KamuApiServerClient) { - let kamu = KamuCliPuppet::new_workspace_tmp().await; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_search_by_name_mt(kamu_node_api_client: KamuApiServerClient) { + test_search_by_name(kamu_node_api_client, true).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_search_by_repo +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_search_by_repo_st(kamu_node_api_client: KamuApiServerClient) { + test_search_by_repo(kamu_node_api_client, false).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_search_by_repo_mt(kamu_node_api_client: KamuApiServerClient) { + test_search_by_repo(kamu_node_api_client, true).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Implementations +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +async fn test_search_multi_user( + mut kamu_node_api_client: KamuApiServerClient, + is_workspace_multi_tenant: bool, +) { + let kamu = KamuCliPuppet::new_workspace_tmp(is_workspace_multi_tenant).await; add_repo_to_workspace(&kamu_node_api_client, &kamu, "kamu-node").await; @@ -148,8 +195,11 @@ pub async fn test_search_multi_user(mut kamu_node_api_client: KamuApiServerClien //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_search_by_name(mut kamu_node_api_client: KamuApiServerClient) { - let kamu = KamuCliPuppet::new_workspace_tmp().await; +async fn test_search_by_name( + mut kamu_node_api_client: KamuApiServerClient, + is_workspace_multi_tenant: bool, +) { + let kamu = KamuCliPuppet::new_workspace_tmp(is_workspace_multi_tenant).await; add_repo_to_workspace(&kamu_node_api_client, &kamu, "kamu-node").await; @@ -225,8 +275,11 @@ pub async fn test_search_by_name(mut kamu_node_api_client: KamuApiServerClient) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_search_by_repo(mut kamu_node_api_client: KamuApiServerClient) { - let kamu = KamuCliPuppet::new_workspace_tmp().await; +async fn test_search_by_repo( + mut kamu_node_api_client: KamuApiServerClient, + is_workspace_multi_tenant: bool, +) { + let kamu = KamuCliPuppet::new_workspace_tmp(is_workspace_multi_tenant).await; // As a test, add two repos pointing to the same node add_repo_to_workspace(&kamu_node_api_client, &kamu, "kamu-node").await; diff --git a/src/e2e/app/cli/repo-tests/src/lib.rs b/src/e2e/app/cli/repo-tests/src/lib.rs index 63510be226..805743651f 100644 --- a/src/e2e/app/cli/repo-tests/src/lib.rs +++ b/src/e2e/app/cli/repo-tests/src/lib.rs @@ -9,13 +9,12 @@ #![feature(assert_matches)] -mod commands; +pub mod commands; pub mod rest_api; mod test_flow; mod test_selftest; mod test_smart_transfer_protocol; -pub use commands::*; pub use test_flow::*; pub use test_selftest::*; pub use test_smart_transfer_protocol::*; diff --git a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs index 59591c7d02..11a2ae7eb4 100644 --- a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs @@ -24,13 +24,105 @@ use kamu_cli_e2e_common::{ E2E_USER_ACCOUNT_NAME_STR, }; use kamu_cli_puppet::extensions::{KamuCliPuppetExt, RepoAlias}; -use kamu_cli_puppet::{KamuCliPuppet, NewWorkspaceOptions}; -use opendatafabric::{AccountName, DatasetAlias, DatasetName}; +use kamu_cli_puppet::KamuCliPuppet; +use opendatafabric as odf; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_smart_push_pull_sequence(mut kamu_api_server_client: KamuApiServerClient) { - let dataset_alias = DatasetAlias::new( +macro_rules! test_smart_transfer_protocol_permutations { + ($test_name: expr) => { + paste::paste! { + pub async fn [<$test_name "_st_st">](kamu_api_server_client: KamuApiServerClient) { + $test_name(kamu_api_server_client, false, false).await; + } + + pub async fn [<$test_name "_st_mt">](kamu_api_server_client: KamuApiServerClient) { + $test_name(kamu_api_server_client, false, true).await; + } + + pub async fn [<$test_name "_mt_st">](kamu_api_server_client: KamuApiServerClient) { + $test_name(kamu_api_server_client, true, false).await; + } + + pub async fn [<$test_name "_mt_mt">](kamu_api_server_client: KamuApiServerClient) { + $test_name(kamu_api_server_client, true, true).await; + } + } + }; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +test_smart_transfer_protocol_permutations!(test_smart_push_smart_pull_sequence); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +test_smart_transfer_protocol_permutations!(test_smart_push_force_smart_pull_force); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +test_smart_transfer_protocol_permutations!(test_smart_push_no_alias_smart_pull_no_alias); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_smart_pull_as_st(kamu_api_server_client: KamuApiServerClient) { + test_smart_pull_as(kamu_api_server_client, false).await; +} + +pub async fn test_smart_pull_as_mt(kamu_api_server_client: KamuApiServerClient) { + test_smart_pull_as(kamu_api_server_client, true).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +test_smart_transfer_protocol_permutations!(test_smart_push_all_smart_pull_all); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +test_smart_transfer_protocol_permutations!(test_smart_push_recursive_smart_pull_recursive); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_smart_push_visibility_st(kamu_api_server_client: KamuApiServerClient) { + test_smart_push_visibility(kamu_api_server_client, false).await; +} + +pub async fn test_smart_push_visibility_mt(kamu_api_server_client: KamuApiServerClient) { + test_smart_push_visibility(kamu_api_server_client, true).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_simple_push_to_s3_smart_pull_st_st(kamu: KamuCliPuppet) { + test_simple_push_to_s3_smart_pull(kamu, false).await; +} + +pub async fn test_simple_push_to_s3_smart_pull_st_mt(kamu: KamuCliPuppet) { + test_simple_push_to_s3_smart_pull(kamu, true).await; +} + +pub async fn test_simple_push_to_s3_smart_pull_mt_st(kamu: KamuCliPuppet) { + test_simple_push_to_s3_smart_pull(kamu, false).await; +} + +pub async fn test_simple_push_to_s3_smart_pull_mt_mt(kamu: KamuCliPuppet) { + test_simple_push_to_s3_smart_pull(kamu, true).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +test_smart_transfer_protocol_permutations!(test_smart_push_to_registered_repo_smart_pull); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Implementations +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +async fn test_smart_push_smart_pull_sequence( + mut kamu_api_server_client: KamuApiServerClient, + is_push_workspace_multi_tenant: bool, + is_pull_workspace_multi_tenant: bool, +) { + let dataset_alias = odf::DatasetAlias::new( Some(E2E_USER_ACCOUNT_NAME.clone()), DATASET_ROOT_PLAYER_NAME.clone(), ); @@ -43,25 +135,22 @@ pub async fn test_smart_push_pull_sequence(mut kamu_api_server_client: KamuApiSe // 2. Pushing the dataset to the API server { - let kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; + let kamu_in_push_workspace = + KamuCliPuppet::new_workspace_tmp(is_push_workspace_multi_tenant).await; // 2.1. Add the dataset - { - kamu_in_push_workspace - .execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) - .await - .success(); - } + kamu_in_push_workspace + .execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) + .await + .success(); // 2.1. Ingest data to the dataset - { - kamu_in_push_workspace - .ingest_data( - &dataset_alias.dataset_name, - DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, - ) - .await; - } + kamu_in_push_workspace + .ingest_data( + &dataset_alias.dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, + ) + .await; // 2.2. Login to the API server kamu_in_push_workspace @@ -74,7 +163,48 @@ pub async fn test_smart_push_pull_sequence(mut kamu_api_server_client: KamuApiSe .await .success(); - // 2.3. Push the dataset to the API server + // Initial dataset push + kamu_in_push_workspace + .assert_success_command_execution( + [ + "push", + dataset_alias.dataset_name.as_str(), + "--to", + kamu_api_server_dataset_endpoint.as_str(), + ], + None, + Some(["1 dataset(s) pushed"]), + ) + .await; + + // Hard compact dataset + kamu_in_push_workspace + .execute([ + "--yes", + "system", + "compact", + dataset_alias.dataset_name.as_str(), + "--hard", + "--keep-metadata-only", + ]) + .await + .success(); + + // Should fail without force flag + kamu_in_push_workspace + .assert_failure_command_execution( + [ + "push", + dataset_alias.dataset_name.as_str(), + "--to", + kamu_api_server_dataset_endpoint.as_str(), + ], + None, + Some(["Failed to push 1 dataset(s)"]), + ) + .await; + + // Should successfully push with force flag kamu_in_push_workspace .assert_success_command_execution( [ @@ -82,6 +212,7 @@ pub async fn test_smart_push_pull_sequence(mut kamu_api_server_client: KamuApiSe dataset_alias.dataset_name.as_str(), "--to", kamu_api_server_dataset_endpoint.as_str(), + "--force", ], None, Some(["1 dataset(s) pushed"]), @@ -91,12 +222,45 @@ pub async fn test_smart_push_pull_sequence(mut kamu_api_server_client: KamuApiSe // 3. Pulling the dataset from the API server { - let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; + let kamu_in_pull_workspace = + KamuCliPuppet::new_workspace_tmp(is_pull_workspace_multi_tenant).await; + // Call with no-alias flag to avoid remote ingest checking in next step kamu_in_pull_workspace .assert_success_command_execution( + [ + "pull", + kamu_api_server_dataset_endpoint.as_str(), + "--no-alias", + ], + None, + Some(["1 dataset(s) updated"]), + ) + .await; + + // Ingest data in pulled dataset + + kamu_in_pull_workspace + .ingest_data( + &dataset_alias.dataset_name, + DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, + ) + .await; + + // Should fail without force flag + kamu_in_pull_workspace + .assert_failure_command_execution( ["pull", kamu_api_server_dataset_endpoint.as_str()], None, + Some(["Failed to update 1 dataset(s)"]), + ) + .await; + + // Should successfully pull with force flag + kamu_in_pull_workspace + .assert_success_command_execution( + ["pull", kamu_api_server_dataset_endpoint.as_str(), "--force"], + None, Some(["1 dataset(s) updated"]), ) .await; @@ -105,8 +269,12 @@ pub async fn test_smart_push_pull_sequence(mut kamu_api_server_client: KamuApiSe //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_smart_force_push_pull(mut kamu_api_server_client: KamuApiServerClient) { - let dataset_alias = DatasetAlias::new( +async fn test_smart_push_force_smart_pull_force( + mut kamu_api_server_client: KamuApiServerClient, + is_push_workspace_multi_tenant: bool, + is_pull_workspace_multi_tenant: bool, +) { + let dataset_alias = odf::DatasetAlias::new( Some(E2E_USER_ACCOUNT_NAME.clone()), DATASET_ROOT_PLAYER_NAME.clone(), ); @@ -119,7 +287,8 @@ pub async fn test_smart_force_push_pull(mut kamu_api_server_client: KamuApiServe // 2. Pushing the dataset to the API server { - let kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; + let kamu_in_push_workspace = + KamuCliPuppet::new_workspace_tmp(is_push_workspace_multi_tenant).await; // 2.1. Add the dataset kamu_in_push_workspace @@ -205,7 +374,8 @@ pub async fn test_smart_force_push_pull(mut kamu_api_server_client: KamuApiServe // 3. Pulling the dataset from the API server { - let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; + let kamu_in_pull_workspace = + KamuCliPuppet::new_workspace_tmp(is_pull_workspace_multi_tenant).await; // Call with no-alias flag to avoid remote ingest checking in next step kamu_in_pull_workspace @@ -251,8 +421,12 @@ pub async fn test_smart_force_push_pull(mut kamu_api_server_client: KamuApiServe //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_smart_push_pull_add_alias(mut kamu_api_server_client: KamuApiServerClient) { - let dataset_alias = DatasetAlias::new( +async fn test_smart_push_no_alias_smart_pull_no_alias( + mut kamu_api_server_client: KamuApiServerClient, + is_push_workspace_multi_tenant: bool, + is_pull_workspace_multi_tenant: bool, +) { + let dataset_alias = odf::DatasetAlias::new( Some(E2E_USER_ACCOUNT_NAME.clone()), DATASET_ROOT_PLAYER_NAME.clone(), ); @@ -265,7 +439,8 @@ pub async fn test_smart_push_pull_add_alias(mut kamu_api_server_client: KamuApiS // 2. Push command { - let kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; + let kamu_in_push_workspace = + KamuCliPuppet::new_workspace_tmp(is_push_workspace_multi_tenant).await; // Add the dataset kamu_in_push_workspace @@ -327,20 +502,33 @@ pub async fn test_smart_push_pull_add_alias(mut kamu_api_server_client: KamuApiS ) .await; - let aliases = kamu_in_push_workspace + let actual_aliases = kamu_in_push_workspace .get_list_of_repo_aliases(&dataset_alias.dataset_name.clone().into()) - .await; - let expected_aliases = vec![RepoAlias { - dataset: dataset_alias.dataset_name.clone(), - kind: "Push".to_string(), - alias: kamu_api_server_dataset_endpoint.to_string(), - }]; - pretty_assertions::assert_eq!(aliases, expected_aliases); + .await + .into_iter() + .map(|repo_alias: RepoAlias| { + ( + repo_alias.dataset.dataset_name, + repo_alias.kind, + repo_alias.alias, + ) + }) + .collect::>(); + + pretty_assertions::assert_eq!( + vec![( + dataset_alias.dataset_name.clone(), + "Push".to_string(), + kamu_api_server_dataset_endpoint.to_string() + )], + actual_aliases + ); } // 3. Pull command { - let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; + let kamu_in_pull_workspace = + KamuCliPuppet::new_workspace_tmp(is_pull_workspace_multi_tenant).await; // Dataset pull without storing alias kamu_in_pull_workspace @@ -376,22 +564,37 @@ pub async fn test_smart_push_pull_add_alias(mut kamu_api_server_client: KamuApiS ) .await; - let aliases = kamu_in_pull_workspace + let actual_aliases = kamu_in_pull_workspace .get_list_of_repo_aliases(&dataset_alias.dataset_name.clone().into()) - .await; - let expected_aliases = vec![RepoAlias { - dataset: dataset_alias.dataset_name.clone(), - kind: "Pull".to_string(), - alias: kamu_api_server_dataset_endpoint.to_string(), - }]; - pretty_assertions::assert_eq!(aliases, expected_aliases); + .await + .into_iter() + .map(|repo_alias: RepoAlias| { + ( + repo_alias.dataset.dataset_name, + repo_alias.kind, + repo_alias.alias, + ) + }) + .collect::>(); + + pretty_assertions::assert_eq!( + vec![( + dataset_alias.dataset_name.clone(), + "Pull".to_string(), + kamu_api_server_dataset_endpoint.to_string() + )], + actual_aliases + ); } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_smart_pull_as(mut kamu_api_server_client: KamuApiServerClient) { - let dataset_alias = DatasetAlias::new( +async fn test_smart_pull_as( + mut kamu_api_server_client: KamuApiServerClient, + is_pull_workspace_multi_tenant: bool, +) { + let dataset_alias = odf::DatasetAlias::new( Some(E2E_USER_ACCOUNT_NAME.clone()), DATASET_ROOT_PLAYER_NAME.clone(), ); @@ -404,14 +607,15 @@ pub async fn test_smart_pull_as(mut kamu_api_server_client: KamuApiServerClient) kamu_api_server_client .dataset() - .create_player_scores_dataset_with_data(Some(AccountName::new_unchecked( + .create_player_scores_dataset_with_data(Some(odf::AccountName::new_unchecked( E2E_USER_ACCOUNT_NAME_STR, ))) .await; { - let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; - let new_dataset_name = DatasetName::new_unchecked("foo"); + let kamu_in_pull_workspace = + KamuCliPuppet::new_workspace_tmp(is_pull_workspace_multi_tenant).await; + let new_dataset_name = odf::DatasetName::new_unchecked("foo"); kamu_in_pull_workspace .assert_success_command_execution( @@ -439,8 +643,12 @@ pub async fn test_smart_pull_as(mut kamu_api_server_client: KamuApiServerClient) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_smart_push_pull_all(mut kamu_api_server_client: KamuApiServerClient) { - let root_dataset_alias = DatasetAlias::new( +async fn test_smart_push_all_smart_pull_all( + mut kamu_api_server_client: KamuApiServerClient, + is_push_workspace_multi_tenant: bool, + is_pull_workspace_multi_tenant: bool, +) { + let root_dataset_alias = odf::DatasetAlias::new( Some(E2E_USER_ACCOUNT_NAME.clone()), DATASET_ROOT_PLAYER_NAME.clone(), ); @@ -448,7 +656,7 @@ pub async fn test_smart_push_pull_all(mut kamu_api_server_client: KamuApiServerC .dataset() .get_endpoint(&root_dataset_alias); - let derivative_dataset_alias = DatasetAlias::new( + let derivative_dataset_alias = odf::DatasetAlias::new( Some(E2E_USER_ACCOUNT_NAME.clone()), DATASET_DERIVATIVE_LEADERBOARD_NAME.clone(), ); @@ -459,7 +667,8 @@ pub async fn test_smart_push_pull_all(mut kamu_api_server_client: KamuApiServerC // 1. Grub a token let token = kamu_api_server_client.auth().login_as_e2e_user().await; - let mut kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; + let mut kamu_in_push_workspace = + KamuCliPuppet::new_workspace_tmp(is_push_workspace_multi_tenant).await; // 2. Pushing datasets to the API server { @@ -547,7 +756,8 @@ pub async fn test_smart_push_pull_all(mut kamu_api_server_client: KamuApiServerC // 3. Pulling datasets from the API server { - let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; + let kamu_in_pull_workspace = + KamuCliPuppet::new_workspace_tmp(is_pull_workspace_multi_tenant).await; // Pull datasets one by one and check data kamu_in_pull_workspace @@ -616,14 +826,14 @@ pub async fn test_smart_push_pull_all(mut kamu_api_server_client: KamuApiServerC kamu_in_pull_workspace .assert_last_data_slice( - &DatasetAlias::new(None, root_dataset_alias.dataset_name.clone()), + &odf::DatasetAlias::new(None, root_dataset_alias.dataset_name.clone()), expected_schema, expected_data, ) .await; kamu_in_pull_workspace .assert_last_data_slice( - &DatasetAlias::new(None, derivative_dataset_alias.dataset_name.clone()), + &odf::DatasetAlias::new(None, derivative_dataset_alias.dataset_name.clone()), expected_derivative_schema, expected_derivative_data, ) @@ -705,14 +915,14 @@ pub async fn test_smart_push_pull_all(mut kamu_api_server_client: KamuApiServerC kamu_in_pull_workspace .assert_last_data_slice( - &DatasetAlias::new(None, root_dataset_alias.dataset_name), + &odf::DatasetAlias::new(None, root_dataset_alias.dataset_name), expected_schema, expected_data, ) .await; kamu_in_pull_workspace .assert_last_data_slice( - &DatasetAlias::new(None, derivative_dataset_alias.dataset_name), + &odf::DatasetAlias::new(None, derivative_dataset_alias.dataset_name), expected_derivative_schema, expected_derivative_data, ) @@ -722,8 +932,12 @@ pub async fn test_smart_push_pull_all(mut kamu_api_server_client: KamuApiServerC //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_smart_push_pull_recursive(mut kamu_api_server_client: KamuApiServerClient) { - let root_dataset_alias = DatasetAlias::new( +async fn test_smart_push_recursive_smart_pull_recursive( + mut kamu_api_server_client: KamuApiServerClient, + is_push_workspace_multi_tenant: bool, + is_pull_workspace_multi_tenant: bool, +) { + let root_dataset_alias = odf::DatasetAlias::new( Some(E2E_USER_ACCOUNT_NAME.clone()), DATASET_ROOT_PLAYER_NAME.clone(), ); @@ -731,7 +945,7 @@ pub async fn test_smart_push_pull_recursive(mut kamu_api_server_client: KamuApiS .dataset() .get_endpoint(&root_dataset_alias); - let derivative_dataset_alias = DatasetAlias::new( + let derivative_dataset_alias = odf::DatasetAlias::new( Some(E2E_USER_ACCOUNT_NAME.clone()), DATASET_DERIVATIVE_LEADERBOARD_NAME.clone(), ); @@ -739,7 +953,8 @@ pub async fn test_smart_push_pull_recursive(mut kamu_api_server_client: KamuApiS // 1. Grub a token let token = kamu_api_server_client.auth().login_as_e2e_user().await; - let mut kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; + let mut kamu_in_push_workspace = + KamuCliPuppet::new_workspace_tmp(is_push_workspace_multi_tenant).await; // 2. Pushing datasets to the API server { @@ -805,7 +1020,8 @@ pub async fn test_smart_push_pull_recursive(mut kamu_api_server_client: KamuApiS // 3. Pulling datasets from the API server { - let mut kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; + let mut kamu_in_pull_workspace = + KamuCliPuppet::new_workspace_tmp(is_pull_workspace_multi_tenant).await; kamu_in_pull_workspace .set_system_time(Some(DateTime::from_str("2050-01-02T03:04:05Z").unwrap())); @@ -885,14 +1101,14 @@ pub async fn test_smart_push_pull_recursive(mut kamu_api_server_client: KamuApiS kamu_in_pull_workspace .assert_last_data_slice( - &DatasetAlias::new(None, root_dataset_alias.dataset_name.clone()), + &odf::DatasetAlias::new(None, root_dataset_alias.dataset_name.clone()), expected_schema, expected_data, ) .await; kamu_in_pull_workspace .assert_last_data_slice( - &DatasetAlias::new(None, derivative_dataset_alias.dataset_name.clone()), + &odf::DatasetAlias::new(None, derivative_dataset_alias.dataset_name.clone()), expected_derivative_schema, expected_derivative_data, ) @@ -957,14 +1173,14 @@ pub async fn test_smart_push_pull_recursive(mut kamu_api_server_client: KamuApiS kamu_in_pull_workspace .assert_last_data_slice( - &DatasetAlias::new(None, root_dataset_alias.dataset_name), + &odf::DatasetAlias::new(None, root_dataset_alias.dataset_name), expected_schema, expected_data, ) .await; kamu_in_pull_workspace .assert_last_data_slice( - &DatasetAlias::new(None, derivative_dataset_alias.dataset_name), + &odf::DatasetAlias::new(None, derivative_dataset_alias.dataset_name), expected_derivative_schema, expected_derivative_data, ) @@ -974,148 +1190,11 @@ pub async fn test_smart_push_pull_recursive(mut kamu_api_server_client: KamuApiS //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_smart_pull_set_watermark(kamu: KamuCliPuppet) { - let dataset_name = DATASET_ROOT_PLAYER_NAME.clone(); - - kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) - .await - .success(); - - kamu.assert_success_command_execution( - [ - "pull", - dataset_name.as_str(), - "--set-watermark", - "2051-01-02T03:04:05Z", - ], - None, - Some(["Committed new block"]), - ) - .await; -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -pub async fn test_smart_pull_reset_derivative(kamu: KamuCliPuppet) { - let dataset_alias = DatasetAlias::new(None, DATASET_ROOT_PLAYER_NAME.clone()); - let dataset_derivative_alias = - DatasetAlias::new(None, DATASET_DERIVATIVE_LEADERBOARD_NAME.clone()); - - kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) - .await - .success(); - - kamu.execute_with_input( - ["add", "--stdin"], - DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, - ) - .await - .success(); - - kamu.ingest_data( - &dataset_alias.dataset_name, - DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, - ) - .await; - - kamu.assert_success_command_execution( - ["pull", dataset_derivative_alias.dataset_name.as_str()], - None, - Some(["1 dataset(s) updated"]), - ) - .await; - - let expected_derivative_schema = indoc::indoc!( - r#" - message arrow_schema { - OPTIONAL INT64 offset; - REQUIRED INT32 op; - REQUIRED INT64 system_time (TIMESTAMP(MILLIS,true)); - OPTIONAL INT64 match_time (TIMESTAMP(MILLIS,true)); - OPTIONAL INT64 place (INTEGER(64,false)); - OPTIONAL INT64 match_id; - OPTIONAL BYTE_ARRAY player_id (STRING); - OPTIONAL INT64 score; - } - "# - ); - let expected_derivative_data = indoc::indoc!( - r#" - +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ - | offset | op | system_time | match_time | place | match_id | player_id | score | - +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ - | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | - | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 2 | 1 | Bob | 80 | - +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ - "# - ); - kamu.assert_last_data_slice( - &dataset_derivative_alias, - expected_derivative_schema, - expected_derivative_data, - ) - .await; - - // Compact root dataset - kamu.execute([ - "--yes", - "system", - "compact", - dataset_alias.dataset_name.as_str(), - "--hard", - "--keep-metadata-only", - ]) - .await - .success(); - - // Pull derivative should fail - kamu.assert_failure_command_execution( - ["pull", dataset_derivative_alias.dataset_name.as_str()], - None, - Some(["Failed to update 1 dataset(s)"]), - ) - .await; - - // Add new data to root dataset - kamu.ingest_data( - &dataset_alias.dataset_name, - DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_2, - ) - .await; - - kamu.assert_success_command_execution( - [ - "pull", - dataset_derivative_alias.dataset_name.as_str(), - "--reset-derivatives-on-diverged-input", - ], - None, - Some(["1 dataset(s) updated"]), - ) - .await; - - let expected_derivative_data = indoc::indoc!( - r#" - +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ - | offset | op | system_time | match_time | place | match_id | player_id | score | - +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ - | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00Z | 1 | 2 | Charlie | 90 | - | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00Z | 2 | 2 | Alice | 70 | - +--------+----+----------------------+----------------------+-------+----------+-----------+-------+ - "# - ); - kamu.assert_last_data_slice( - &dataset_derivative_alias, - expected_derivative_schema, - expected_derivative_data, - ) - .await; -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -pub async fn test_smart_push_visibility(mut kamu_api_server_client: KamuApiServerClient) { - let dataset_alias = DatasetAlias::new( +async fn test_smart_push_visibility( + mut kamu_api_server_client: KamuApiServerClient, + is_push_workspace_multi_tenant: bool, +) { + let dataset_alias = odf::DatasetAlias::new( Some(E2E_USER_ACCOUNT_NAME.clone()), DATASET_ROOT_PLAYER_NAME.clone(), ); @@ -1128,7 +1207,8 @@ pub async fn test_smart_push_visibility(mut kamu_api_server_client: KamuApiServe // 2. Pushing the dataset to the API server { - let kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; + let kamu_in_push_workspace = + KamuCliPuppet::new_workspace_tmp(is_push_workspace_multi_tenant).await; // 2.1. Add the dataset kamu_in_push_workspace @@ -1170,14 +1250,17 @@ pub async fn test_smart_push_visibility(mut kamu_api_server_client: KamuApiServe ) .await; - // ToDo add visibility check + // TODO: Private Datasets: add visibility check } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_smart_push_pull_s3(kamu: KamuCliPuppet) { - let dataset_alias = DatasetAlias::new(None, DATASET_ROOT_PLAYER_NAME.clone()); +async fn test_simple_push_to_s3_smart_pull( + kamu: KamuCliPuppet, + is_pull_workspace_multi_tenant: bool, +) { + let dataset_alias = odf::DatasetAlias::new(None, DATASET_ROOT_PLAYER_NAME.clone()); kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) .await @@ -1206,7 +1289,8 @@ pub async fn test_smart_push_pull_s3(kamu: KamuCliPuppet) { .await; { - let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; + let kamu_in_pull_workspace = + KamuCliPuppet::new_workspace_tmp(is_pull_workspace_multi_tenant).await; kamu_in_pull_workspace .assert_success_command_execution( @@ -1246,63 +1330,12 @@ pub async fn test_smart_push_pull_s3(kamu: KamuCliPuppet) { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_smart_pull_derivative(kamu: KamuCliPuppet) { - let dataset_name = DATASET_ROOT_PLAYER_NAME.clone(); - let dataset_derivative_name = DATASET_DERIVATIVE_LEADERBOARD_NAME.clone(); - - kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) - .await - .success(); - - kamu.execute_with_input( - ["add", "--stdin"], - DATASET_DERIVATIVE_LEADERBOARD_SNAPSHOT_STR, - ) - .await - .success(); - - kamu.ingest_data( - &dataset_name, - DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, - ) - .await; - - kamu.assert_failure_command_execution( - [ - "tail", - dataset_derivative_name.as_str(), - "--output-format", - "table", - ], - None, - Some(["Error: Dataset schema is not yet available: leaderboard"]), - ) - .await; - - kamu.assert_success_command_execution( - ["pull", dataset_derivative_name.as_str()], - None, - Some(["1 dataset(s) updated"]), - ) - .await; - - kamu.assert_player_scores_dataset_data(indoc::indoc!( - r#" - ┌────┬──────────────────────┬──────────────────────┬──────────┬───────────┬───────┐ - │ op │ system_time │ match_time │ match_id │ player_id │ score │ - ├────┼──────────────────────┼──────────────────────┼──────────┼───────────┼───────┤ - │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Bob │ 80 │ - │ 0 │ 2050-01-02T03:04:05Z │ 2000-01-01T00:00:00Z │ 1 │ Alice │ 100 │ - └────┴──────────────────────┴──────────────────────┴──────────┴───────────┴───────┘ - "# - )) - .await; -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -pub async fn test_smart_push_from_registered_repo(mut kamu_api_server_client: KamuApiServerClient) { - let dataset_alias = DatasetAlias::new( +async fn test_smart_push_to_registered_repo_smart_pull( + mut kamu_api_server_client: KamuApiServerClient, + is_push_workspace_multi_tenant: bool, + is_pull_workspace_multi_tenant: bool, +) { + let dataset_alias = odf::DatasetAlias::new( Some(E2E_USER_ACCOUNT_NAME.clone()), DATASET_ROOT_PLAYER_NAME.clone(), ); @@ -1315,7 +1348,8 @@ pub async fn test_smart_push_from_registered_repo(mut kamu_api_server_client: Ka // 2. Pushing the dataset to the API server { - let kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; + let kamu_in_push_workspace = + KamuCliPuppet::new_workspace_tmp(is_push_workspace_multi_tenant).await; // 2.1. Add the dataset { @@ -1359,7 +1393,8 @@ pub async fn test_smart_push_from_registered_repo(mut kamu_api_server_client: Ka // 3. Pulling the dataset from the API server { - let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; + let kamu_in_pull_workspace = + KamuCliPuppet::new_workspace_tmp(is_pull_workspace_multi_tenant).await; kamu_in_pull_workspace .assert_success_command_execution( @@ -1372,125 +1407,3 @@ pub async fn test_smart_push_from_registered_repo(mut kamu_api_server_client: Ka } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -pub async fn test_pull_mt(mut kamu_api_server_client: KamuApiServerClient) { - let dataset_alias = DatasetAlias::new( - Some(E2E_USER_ACCOUNT_NAME.clone()), - DATASET_ROOT_PLAYER_NAME.clone(), - ); - let kamu_api_server_dataset_endpoint = kamu_api_server_client - .dataset() - .get_endpoint(&dataset_alias); - - // 1. Grub a token - let token = kamu_api_server_client.auth().login_as_e2e_user().await; - - // 2. Pushing the dataset to the API server - { - let mut kamu_in_push_workspace = KamuCliPuppet::new_workspace_tmp().await; - kamu_in_push_workspace - .set_system_time(Some(DateTime::from_str("2050-01-02T03:04:05Z").unwrap())); - - // 2.1. Add the dataset - { - kamu_in_push_workspace - .execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) - .await - .success(); - } - - // 2.1. Ingest data to the dataset - { - kamu_in_push_workspace - .ingest_data( - &dataset_alias.dataset_name, - DATASET_ROOT_PLAYER_SCORES_INGEST_DATA_NDJSON_CHUNK_1, - ) - .await; - } - - // 2.2. Login to the API server - kamu_in_push_workspace - .execute([ - "login", - kamu_api_server_client.get_base_url().as_str(), - "--access-token", - token.as_str(), - ]) - .await - .success(); - - // 2.3. Push the dataset to the API server - kamu_in_push_workspace - .assert_success_command_execution( - [ - "push", - dataset_alias.dataset_name.as_str(), - "--to", - kamu_api_server_dataset_endpoint.as_str(), - ], - None, - Some(["1 dataset(s) pushed"]), - ) - .await; - } - - // 3. Pulling the dataset from the API server - { - let kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp_with(NewWorkspaceOptions { - is_multi_tenant: true, - kamu_config: None, - env_vars: vec![], - }) - .await; - - kamu_in_pull_workspace - .assert_success_command_execution( - [ - "--account", - "foo", - "pull", - kamu_api_server_dataset_endpoint.as_str(), - ], - None, - Some(["1 dataset(s) updated"]), - ) - .await; - - let expected_schema = indoc::indoc!( - r#" - message arrow_schema { - REQUIRED INT64 offset; - REQUIRED INT32 op; - REQUIRED INT64 system_time (TIMESTAMP(MILLIS,true)); - OPTIONAL INT64 match_time (TIMESTAMP(MILLIS,true)); - OPTIONAL INT64 match_id; - OPTIONAL BYTE_ARRAY player_id (STRING); - OPTIONAL INT64 score; - } - "# - ); - let expected_data = indoc::indoc!( - r#" - +--------+----+----------------------+----------------------+----------+-----------+-------+ - | offset | op | system_time | match_time | match_id | player_id | score | - +--------+----+----------------------+----------------------+----------+-----------+-------+ - | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | - | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | Bob | 80 | - +--------+----+----------------------+----------------------+----------+-----------+-------+ - "# - ); - kamu_in_pull_workspace - .assert_last_data_slice( - &DatasetAlias::new( - Some(AccountName::new_unchecked("foo")), - dataset_alias.dataset_name, - ), - expected_schema, - expected_data, - ) - .await; - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_add_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_add_command.rs index fefb11f361..07ae0cdac4 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/commands/test_add_command.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_add_command.rs @@ -13,35 +13,35 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_add_dataset_from_stdin + fixture = kamu_cli_e2e_repo_tests::commands::test_add_dataset_from_stdin ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_add_dataset_with_name + fixture = kamu_cli_e2e_repo_tests::commands::test_add_dataset_with_name ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_add_dataset_with_replace + fixture = kamu_cli_e2e_repo_tests::commands::test_add_dataset_with_replace ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_add_recursive + fixture = kamu_cli_e2e_repo_tests::commands::test_add_recursive ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_add_with_circular_dependency + fixture = kamu_cli_e2e_repo_tests::commands::test_add_with_circular_dependency ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_compact_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_compact_command.rs index fb2c2851ff..484d463784 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/commands/test_compact_command.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_compact_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_compact_hard + fixture = kamu_cli_e2e_repo_tests::commands::test_compact_hard extra_test_groups = "engine, ingest, datafusion" ); @@ -21,7 +21,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_compact_keep_metadata_only + fixture = kamu_cli_e2e_repo_tests::commands::test_compact_keep_metadata_only extra_test_groups = "engine, ingest, datafusion" ); @@ -29,7 +29,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_compact_verify + fixture = kamu_cli_e2e_repo_tests::commands::test_compact_verify extra_test_groups = "engine, ingest, datafusion" ); diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_delete_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_delete_command.rs index 4166202a21..43f4265a17 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/commands/test_delete_command.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_delete_command.rs @@ -13,21 +13,21 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_delete_dataset + fixture = kamu_cli_e2e_repo_tests::commands::test_delete_dataset ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_delete_dataset_recursive + fixture = kamu_cli_e2e_repo_tests::commands::test_delete_dataset_recursive ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_delete_dataset_all + fixture = kamu_cli_e2e_repo_tests::commands::test_delete_dataset_all ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_ingest_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_ingest_command.rs index 2c74c7d440..518f3d177e 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/commands/test_ingest_command.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_ingest_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_push_ingest_from_file_ledger, + fixture = kamu_cli_e2e_repo_tests::commands::test_push_ingest_from_file_ledger, options = Options::default().with_frozen_system_time(), extra_test_groups = "engine, ingest, datafusion" ); @@ -22,7 +22,8 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_push_ingest_from_file_snapshot_with_event_time, + fixture = + kamu_cli_e2e_repo_tests::commands::test_push_ingest_from_file_snapshot_with_event_time, options = Options::default().with_frozen_system_time(), extra_test_groups = "engine, ingest, datafusion" ); @@ -31,7 +32,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_ingest_from_stdin, + fixture = kamu_cli_e2e_repo_tests::commands::test_ingest_from_stdin, options = Options::default().with_frozen_system_time(), extra_test_groups = "engine, ingest, datafusion" ); @@ -40,7 +41,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_ingest_recursive, + fixture = kamu_cli_e2e_repo_tests::commands::test_ingest_recursive, options = Options::default().with_frozen_system_time(), extra_test_groups = "engine, ingest, datafusion" ); @@ -49,7 +50,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_ingest_with_source_name, + fixture = kamu_cli_e2e_repo_tests::commands::test_ingest_with_source_name, options = Options::default().with_frozen_system_time(), extra_test_groups = "engine, ingest, datafusion" ); diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_init_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_init_command.rs index a07699b0b3..08f1f05373 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/commands/test_init_command.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_init_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_init_multi_tenant_creates_sqlite_database, + fixture = kamu_cli_e2e_repo_tests::commands::test_init_exist_ok_st, options = Options::default().with_no_workspace() ); @@ -21,8 +21,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = - kamu_cli_e2e_repo_tests::test_init_multi_tenant_with_exists_ok_flag_creates_sqlite_database, + fixture = kamu_cli_e2e_repo_tests::commands::test_init_exist_ok_mt, options = Options::default().with_no_workspace() ); @@ -30,23 +29,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_init_exist_ok_st, - options = Options::default().with_no_workspace() -); - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -kamu_cli_execute_command_e2e_test!( - storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_init_exist_ok_mt, - options = Options::default().with_no_workspace() -); - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -kamu_cli_execute_command_e2e_test!( - storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_init_in_an_existing_workspace, + fixture = kamu_cli_e2e_repo_tests::commands::test_init_in_an_existing_workspace, options = Options::default().with_no_workspace() ); diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_inspect_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_inspect_command.rs index 9ddfcd3d54..3b5e098516 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/commands/test_inspect_command.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_inspect_command.rs @@ -13,14 +13,14 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_inspect_lineage, + fixture = kamu_cli_e2e_repo_tests::commands::test_inspect_lineage, ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_inspect_query, + fixture = kamu_cli_e2e_repo_tests::commands::test_inspect_query, options = Options::default().with_frozen_system_time() ); @@ -28,7 +28,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_inspect_schema, + fixture = kamu_cli_e2e_repo_tests::commands::test_inspect_schema, extra_test_groups = "engine, ingest, datafusion" ); diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_log_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_log_command.rs index 54bc30e5af..5dcd31cbc9 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/commands/test_log_command.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_log_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_log, + fixture = kamu_cli_e2e_repo_tests::commands::test_log, options = Options::default().with_frozen_system_time(), extra_test_groups = "engine, ingest, datafusion" ); diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_login_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_login_command.rs index fe748dfb0c..8e177edd7f 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/commands/test_login_command.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_login_command.rs @@ -9,26 +9,55 @@ use kamu_cli_e2e_common::prelude::*; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_login_logout_password +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::commands::test_login_logout_password_st, +); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_run_api_server_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_login_logout_password, + fixture = kamu_cli_e2e_repo_tests::commands::test_login_logout_password_mt, ); +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_login_logout_oauth //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_run_api_server_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_login_logout_oauth, + fixture = kamu_cli_e2e_repo_tests::commands::test_login_logout_oauth_st, options = Options::default().with_multi_tenant() ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::commands::test_login_logout_oauth_mt, + options = Options::default().with_multi_tenant() +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_login_add_repo +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_login_add_repo, + fixture = kamu_cli_e2e_repo_tests::commands::test_login_add_repo_st, +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::commands::test_login_add_repo_mt, + options = Options::default().with_multi_tenant() ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_new_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_new_command.rs index d299df3d98..2b9f3b6d52 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/commands/test_new_command.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_new_command.rs @@ -13,14 +13,14 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_new_root, + fixture = kamu_cli_e2e_repo_tests::commands::test_new_root, ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_new_derivative, + fixture = kamu_cli_e2e_repo_tests::commands::test_new_derivative, ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_pull_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_pull_command.rs index 7a48e9b8bd..f4f820c3b9 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/commands/test_pull_command.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_pull_command.rs @@ -9,11 +9,13 @@ use kamu_cli_e2e_common::prelude::*; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_pull_env_var_template_default_value //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_pull_env_var_template_default_value, + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_env_var_template_default_value_st, extra_test_groups = "containerized, engine, ingest, datafusion" ); @@ -21,32 +23,55 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_pull_env_var_template_default_value_missing_values, + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_env_var_template_default_value_mt, extra_test_groups = "containerized, engine, ingest, datafusion" ); +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_pull_env_var_template_default_value_missing_values //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_pull_set_watermark, - options = Options::default().with_frozen_system_time(), + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_env_var_template_default_value_missing_values_st, + extra_test_groups = "containerized, engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_env_var_template_default_value_missing_values_mt, + extra_test_groups = "containerized, engine, ingest, datafusion" ); +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_pull_set_watermark //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_pull_reset_derivative, + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_set_watermark_st, options = Options::default().with_frozen_system_time(), - extra_test_groups = "containerized, engine, ingest, transform, datafusion" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_pull_derivative, + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_set_watermark_mt, + options = Options::default() + .with_multi_tenant() + .with_frozen_system_time(), +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_pull_reset_derivative +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_reset_derivative_st, options = Options::default().with_frozen_system_time(), extra_test_groups = "containerized, engine, ingest, transform, datafusion" ); @@ -54,10 +79,34 @@ kamu_cli_execute_command_e2e_test!( //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( - storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_push_pull_s3, + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_reset_derivative_mt, + options = Options::default() + .with_multi_tenant() + .with_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_pull_derivative +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_derivative_st, options = Options::default().with_frozen_system_time(), - extra_test_groups = "containerized, engine, ingest, datafusion" + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = inmem, + fixture = kamu_cli_e2e_repo_tests::commands::test_pull_derivative_st, + options = Options::default() + .with_multi_tenant() + .with_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_rename_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_rename_command.rs index 889c54c2ae..68d3d07d97 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/commands/test_rename_command.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_rename_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_rename_dataset + fixture = kamu_cli_e2e_repo_tests::commands::test_rename_dataset ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_repo_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_repo_command.rs index 29b16d730a..87ae4dfe3b 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/commands/test_repo_command.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_repo_command.rs @@ -13,14 +13,14 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_repository_pull_aliases_commands + fixture = kamu_cli_e2e_repo_tests::commands::test_repository_pull_aliases_commands ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_repository_push_aliases_commands + fixture = kamu_cli_e2e_repo_tests::commands::test_repository_push_aliases_commands ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_reset_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_reset_command.rs index 08fcb2bc3f..bb4159d747 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/commands/test_reset_command.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_reset_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_reset, + fixture = kamu_cli_e2e_repo_tests::commands::test_reset, extra_test_groups = "engine, ingest, datafusion" ); diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_search_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_search_command.rs index 806c702ad1..fbc132a05e 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/commands/test_search_command.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_search_command.rs @@ -9,11 +9,13 @@ use kamu_cli_e2e_common::prelude::*; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_search_multi_user //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_run_api_server_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_search_multi_user + fixture = kamu_cli_e2e_repo_tests::commands::test_search_multi_user_st // We need synthetic time for the tests, but the third-party JWT code // uses the current time. Assuming that the token lifetime is 24 hours, we will // use the projected date (the current day) as a workaround. @@ -39,7 +41,65 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_search_by_name + fixture = kamu_cli_e2e_repo_tests::commands::test_search_multi_user_mt + // We need synthetic time for the tests, but the third-party JWT code + // uses the current time. Assuming that the token lifetime is 24 hours, we will + // use the projected date (the current day) as a workaround. + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time() + .with_kamu_config( + indoc::indoc!( + r#" + kind: CLIConfig + version: 1 + content: + users: + predefined: + - accountName: kamu + "# + ) + ), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_search_by_name +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::commands::test_search_by_name_st + // We need synthetic time for the tests, but the third-party JWT code + // uses the current time. Assuming that the token lifetime is 24 hours, we will + // use the projected date (the current day) as a workaround. + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::commands::test_search_by_name_mt + // We need synthetic time for the tests, but the third-party JWT code + // uses the current time. Assuming that the token lifetime is 24 hours, we will + // use the projected date (the current day) as a workaround. + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_search_by_repo +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::commands::test_search_by_repo_st // We need synthetic time for the tests, but the third-party JWT code // uses the current time. Assuming that the token lifetime is 24 hours, we will // use the projected date (the current day) as a workaround. @@ -53,7 +113,7 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_search_by_repo + fixture = kamu_cli_e2e_repo_tests::commands::test_search_by_repo_mt // We need synthetic time for the tests, but the third-party JWT code // uses the current time. Assuming that the token lifetime is 24 hours, we will // use the projected date (the current day) as a workaround. diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_sql_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_sql_command.rs index 8d21f99b9d..0fc9a93394 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/commands/test_sql_command.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_sql_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_datafusion_cli, + fixture = kamu_cli_e2e_repo_tests::commands::test_datafusion_cli, extra_test_groups = "engine, datafusion" ); @@ -21,7 +21,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_datafusion_cli_not_launched_in_root_ws, + fixture = kamu_cli_e2e_repo_tests::commands::test_datafusion_cli_not_launched_in_root_ws, options = Options::default().with_no_workspace(), extra_test_groups = "engine, datafusion" ); @@ -30,7 +30,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_sql_command, + fixture = kamu_cli_e2e_repo_tests::commands::test_sql_command, options = Options::default().with_frozen_system_time(), extra_test_groups = "engine, datafusion" ); diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_api_server_gql_query.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_api_server_gql_query.rs index 0a7e162b88..ad8ac2b37a 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_api_server_gql_query.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_api_server_gql_query.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_gql_query_api_version + fixture = kamu_cli_e2e_repo_tests::commands::test_gql_query_api_version ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_diagnose_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_diagnose_command.rs index df35fd6e5e..e35c2cb2f9 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_diagnose_command.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_diagnose_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_system_diagnose + fixture = kamu_cli_e2e_repo_tests::commands::test_system_diagnose ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_gc_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_gc_command.rs index 7a3c36657f..b89a290b40 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_gc_command.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_gc_command.rs @@ -11,6 +11,9 @@ use kamu_cli_e2e_common::prelude::*; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -kamu_cli_execute_command_e2e_test!(storage = sqlite, fixture = kamu_cli_e2e_repo_tests::test_gc); +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::commands::test_gc +); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_generate_token_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_generate_token_command.rs index 0b01fb3e0d..1effecee9a 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_generate_token_command.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_generate_token_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_generate_token + fixture = kamu_cli_e2e_repo_tests::commands::test_generate_token ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_info_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_info_command.rs index 8f5781110b..3b46ebe6ef 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_info_command.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_system_info_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_system_info + fixture = kamu_cli_e2e_repo_tests::commands::test_system_info ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_tail_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_tail_command.rs index ac92be7d68..c157ecd39b 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/commands/test_tail_command.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_tail_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_tail, + fixture = kamu_cli_e2e_repo_tests::commands::test_tail, options = Options::default().with_frozen_system_time(), extra_test_groups = "engine, ingest, datafusion" ); diff --git a/src/e2e/app/cli/sqlite/tests/tests/commands/test_verify_command.rs b/src/e2e/app/cli/sqlite/tests/tests/commands/test_verify_command.rs index 4df4938ec3..eab2d3abe6 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/commands/test_verify_command.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/commands/test_verify_command.rs @@ -13,7 +13,7 @@ use kamu_cli_e2e_common::prelude::*; kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_verify_regular_dataset, + fixture = kamu_cli_e2e_repo_tests::commands::test_verify_regular_dataset, extra_test_groups = "engine, ingest, datafusion" ); @@ -21,7 +21,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_verify_recursive, + fixture = kamu_cli_e2e_repo_tests::commands::test_verify_recursive, extra_test_groups = "containerized, engine, ingest, datafusion" ); @@ -29,7 +29,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_verify_integrity, + fixture = kamu_cli_e2e_repo_tests::commands::test_verify_integrity, extra_test_groups = "engine, ingest, datafusion" ); diff --git a/src/e2e/app/cli/sqlite/tests/tests/test_smart_transfer_protocol.rs b/src/e2e/app/cli/sqlite/tests/tests/test_smart_transfer_protocol.rs index 8dc69940c8..6fabbd3891 100644 --- a/src/e2e/app/cli/sqlite/tests/tests/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/sqlite/tests/tests/test_smart_transfer_protocol.rs @@ -9,14 +9,35 @@ use kamu_cli_e2e_common::prelude::*; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_smart_pull_sequence +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_smart_pull_sequence_st_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_smart_pull_sequence_st_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_run_api_server_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_sequence, - // We need synthetic time for the tests, but the third-party JWT code - // uses the current time. Assuming that the token lifetime is 24 hours, we will - // use the projected date (the current day) as a workaround. + fixture = kamu_cli_e2e_repo_tests::test_smart_push_smart_pull_sequence_mt_st, options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), @@ -27,18 +48,20 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_smart_force_push_pull, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_smart_pull_sequence_mt_mt, options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), extra_test_groups = "engine, ingest, datafusion" ); +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_force_smart_pull_force //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_run_api_server_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_add_alias, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_force_smart_pull_force_st_st, options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), @@ -49,7 +72,7 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_smart_pull_as, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_force_smart_pull_force_st_mt, options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), @@ -60,7 +83,101 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_all, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_force_smart_pull_force_mt_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_force_smart_pull_force_mt_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_no_alias_smart_pull_no_alias +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_no_alias_smart_pull_no_alias_st_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_no_alias_smart_pull_no_alias_st_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_no_alias_smart_pull_no_alias_mt_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_no_alias_smart_pull_no_alias_mt_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_pull_as +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_smart_pull_as_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_smart_pull_as_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_all_smart_pull_all +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_all_smart_pull_all_st_st, options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), @@ -71,7 +188,7 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_recursive, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_all_smart_pull_all_st_mt, options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), @@ -80,18 +197,36 @@ kamu_cli_run_api_server_e2e_test!( //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -kamu_cli_execute_command_e2e_test!( +kamu_cli_run_api_server_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_smart_pull_set_watermark, - options = Options::default().with_frozen_system_time(), + fixture = kamu_cli_e2e_repo_tests::test_smart_push_all_smart_pull_all_mt_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -kamu_cli_execute_command_e2e_test!( +kamu_cli_run_api_server_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_smart_pull_reset_derivative, - options = Options::default().with_frozen_system_time(), + fixture = kamu_cli_e2e_repo_tests::test_smart_push_all_smart_pull_all_mt_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_recursive_smart_pull_recursive +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_recursive_smart_pull_recursive_st_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), extra_test_groups = "containerized, engine, ingest, transform, datafusion" ); @@ -99,7 +234,42 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_smart_push_visibility, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_recursive_smart_pull_recursive_st_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_recursive_smart_pull_recursive_mt_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_recursive_smart_pull_recursive_mt_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, transform, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_visibility +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_visibility_st, options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), @@ -108,9 +278,22 @@ kamu_cli_run_api_server_e2e_test!( //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_visibility_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_simple_push_to_s3_smart_pull +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_s3, + fixture = kamu_cli_e2e_repo_tests::test_simple_push_to_s3_smart_pull_st_st, options = Options::default().with_frozen_system_time(), extra_test_groups = "containerized, engine, ingest, datafusion" ); @@ -119,16 +302,40 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_smart_pull_derivative, + fixture = kamu_cli_e2e_repo_tests::test_simple_push_to_s3_smart_pull_st_mt, options = Options::default().with_frozen_system_time(), - extra_test_groups = "containerized, engine, ingest, transform, datafusion" + extra_test_groups = "containerized, engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_simple_push_to_s3_smart_pull_mt_st, + options = Options::default() + .with_multi_tenant() + .with_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_execute_command_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_simple_push_to_s3_smart_pull_mt_mt, + options = Options::default() + .with_multi_tenant() + .with_frozen_system_time(), + extra_test_groups = "containerized, engine, ingest, datafusion" ); +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// test_smart_push_to_registered_repo_smart_pull //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// kamu_cli_run_api_server_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_smart_push_from_registered_repo, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_to_registered_repo_smart_pull_st_st, options = Options::default() .with_multi_tenant() .with_today_as_frozen_system_time(), @@ -139,11 +346,33 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_run_api_server_e2e_test!( storage = sqlite, - fixture = kamu_cli_e2e_repo_tests::test_pull_mt, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_to_registered_repo_smart_pull_st_mt, options = Options::default() .with_multi_tenant() - .with_frozen_system_time(), - extra_test_groups = "containerized, engine, ingest, datafusion" + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_to_registered_repo_smart_pull_mt_st, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" +); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +kamu_cli_run_api_server_e2e_test!( + storage = sqlite, + fixture = kamu_cli_e2e_repo_tests::test_smart_push_to_registered_repo_smart_pull_mt_mt, + options = Options::default() + .with_multi_tenant() + .with_today_as_frozen_system_time(), + extra_test_groups = "engine, ingest, datafusion" ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/kamu-cli-puppet/src/kamu_cli_puppet.rs b/src/utils/kamu-cli-puppet/src/kamu_cli_puppet.rs index 2186c2402e..7b98462798 100644 --- a/src/utils/kamu-cli-puppet/src/kamu_cli_puppet.rs +++ b/src/utils/kamu-cli-puppet/src/kamu_cli_puppet.rs @@ -35,8 +35,12 @@ impl KamuCliPuppet { } } - pub async fn new_workspace_tmp() -> Self { - Self::new_workspace_tmp_inner(NewWorkspaceOptions::default()).await + pub async fn new_workspace_tmp(is_multi_tenant: bool) -> Self { + Self::new_workspace_tmp_inner(NewWorkspaceOptions { + is_multi_tenant, + ..Default::default() + }) + .await } pub async fn new_workspace_tmp_with(options: NewWorkspaceOptions) -> Self { diff --git a/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs b/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs index 5138b02afc..6b3b618de6 100644 --- a/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs +++ b/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs @@ -14,18 +14,9 @@ use std::path::PathBuf; use async_trait::async_trait; use chrono::{DateTime, Utc}; use datafusion::prelude::{ParquetReadOptions, SessionContext}; +use opendatafabric as odf; use opendatafabric::serde::yaml::{YamlDatasetSnapshotSerializer, YamlMetadataBlockDeserializer}; use opendatafabric::serde::{DatasetSnapshotSerializer, MetadataBlockDeserializer}; -use opendatafabric::{ - DatasetAlias, - DatasetID, - DatasetName, - DatasetRef, - DatasetSnapshot, - MetadataBlock, - Multihash, - RepoName, -}; use serde::Deserialize; use crate::{ExecuteCommandResult, KamuCliPuppet}; @@ -86,13 +77,13 @@ pub trait KamuCliPuppetExt { async fn list_datasets(&self) -> Vec; - async fn add_dataset(&self, dataset_snapshot: DatasetSnapshot); + async fn add_dataset(&self, dataset_snapshot: odf::DatasetSnapshot); - async fn list_blocks(&self, dataset_name: &DatasetName) -> Vec; + async fn list_blocks(&self, dataset_name: &odf::DatasetName) -> Vec; - async fn ingest_data(&self, dataset_name: &DatasetName, data: &str); + async fn ingest_data(&self, dataset_name: &odf::DatasetName, data: &str); - async fn get_list_of_repo_aliases(&self, dataset_ref: &DatasetRef) -> Vec; + async fn get_list_of_repo_aliases(&self, dataset_ref: &odf::DatasetRef) -> Vec; async fn get_list_of_repos(&self) -> Vec; @@ -106,7 +97,7 @@ pub trait KamuCliPuppetExt { async fn assert_last_data_slice( &self, - dataset_alias: &DatasetAlias, + dataset_alias: &odf::DatasetAlias, expected_schema: &str, expected_data: &str, ); @@ -127,7 +118,7 @@ impl KamuCliPuppetExt for KamuCliPuppet { serde_json::from_str(stdout).unwrap() } - async fn add_dataset(&self, dataset_snapshot: DatasetSnapshot) { + async fn add_dataset(&self, dataset_snapshot: odf::DatasetSnapshot) { let content = YamlDatasetSnapshotSerializer .write_manifest(&dataset_snapshot) .unwrap(); @@ -142,7 +133,7 @@ impl KamuCliPuppetExt for KamuCliPuppet { .success(); } - async fn get_list_of_repo_aliases(&self, dataset_ref: &DatasetRef) -> Vec { + async fn get_list_of_repo_aliases(&self, dataset_ref: &odf::DatasetRef) -> Vec { let assert = self .execute([ "repo", @@ -189,7 +180,7 @@ impl KamuCliPuppetExt for KamuCliPuppet { stdout.lines().map(ToString::to_string).collect() } - async fn list_blocks(&self, dataset_name: &DatasetName) -> Vec { + async fn list_blocks(&self, dataset_name: &odf::DatasetName) -> Vec { let assert = self .execute(["log", dataset_name.as_str(), "--output-format", "yaml"]) .await @@ -219,7 +210,7 @@ impl KamuCliPuppetExt for KamuCliPuppet { .unwrap(); BlockRecord { - block_hash: Multihash::from_multibase(block_hash).unwrap(), + block_hash: odf::Multihash::from_multibase(block_hash).unwrap(), block, } }) @@ -282,7 +273,7 @@ impl KamuCliPuppetExt for KamuCliPuppet { async fn assert_last_data_slice( &self, - dataset_alias: &DatasetAlias, + dataset_alias: &odf::DatasetAlias, expected_schema: &str, expected_data: &str, ) { @@ -321,7 +312,7 @@ impl KamuCliPuppetExt for KamuCliPuppet { kamu_data_utils::testing::assert_schema_eq(df.schema(), expected_schema); } - async fn ingest_data(&self, dataset_name: &DatasetName, data: &str) { + async fn ingest_data(&self, dataset_name: &odf::DatasetName, data: &str) { self.execute_with_input(["ingest", dataset_name, "--stdin"], data) .await .success(); @@ -426,12 +417,12 @@ pub struct ServerOutput { #[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct DatasetRecord { #[serde(rename = "ID")] - pub id: DatasetID, - pub name: DatasetName, + pub id: odf::DatasetID, + pub name: odf::DatasetName, // CLI returns regular ENUM DatasetKind(Root/Derivative) for local datasets // but for remote it is Remote(DatasetKind) type pub kind: String, - pub head: Multihash, + pub head: odf::Multihash, pub pulled: Option>, pub records: usize, pub blocks: usize, @@ -442,21 +433,21 @@ pub struct DatasetRecord { #[derive(Debug, PartialEq, Eq, Deserialize)] #[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct RepoAlias { - pub dataset: DatasetName, + pub dataset: odf::DatasetAlias, pub kind: String, pub alias: String, } #[derive(Debug, PartialEq, Eq)] pub struct BlockRecord { - pub block_hash: Multihash, - pub block: MetadataBlock, + pub block_hash: odf::Multihash, + pub block: odf::MetadataBlock, } #[derive(Debug, PartialEq, Eq, Deserialize)] #[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct RepoRecord { - pub name: RepoName, + pub name: odf::RepoName, pub url: url::Url, }