From da0db9b61adab0ab2f9297bcbd888da69efdbd6c Mon Sep 17 00:00:00 2001 From: Sergei Zaychenko Date: Fri, 16 Aug 2024 11:35:13 +0300 Subject: [PATCH] v0.195.0 - Implemented outbox pattern (#741) Reliable transaction-based internal cross-domain message passing component (`MessageOutbox`), replacing `EventBus`: - Metadata-driven producer/consumer annotations - Immediate and transaction-backed message delivery - Background transactional message processor, respecting client idempotence Persistent storage for flow configuration events. Supplementary refactoring: - Introduced use case layer, encapsulating authorization checks and action validations, for first 6 basic dataset scenarios (creating, creating from snapshot, deleting, renaming, committing an event, syncing a batch of events), - Separated `DatasetRepository` on read-only and read-write parts - Isolated `time-source` library - Decoupled repositories from dependency graph. - Clarified dataset find/get interfaces in dataset repository - Unified naming of repository structs, files, harnesses --- CHANGELOG.md | 12 +- Cargo.lock | 419 +++++++++++----- Cargo.toml | 132 ++--- LICENSE.txt | 4 +- Makefile | 7 +- ...710191232_outbox_messages_consumptions.sql | 17 + ...710205713_outbox_messages_consumptions.sql | 16 + src/adapter/auth-oso/Cargo.toml | 6 +- .../auth-oso/src/oso_dataset_authorizer.rs | 3 +- .../tests/test_oso_dataset_authorizer.rs | 25 +- src/adapter/graphql/Cargo.toml | 5 +- .../src/mutations/dataset_metadata_mut.rs | 35 +- .../graphql/src/mutations/dataset_mut.rs | 12 +- .../graphql/src/mutations/datasets_mut.rs | 5 +- .../mutations/flows_mut/flows_mut_utils.rs | 7 +- .../src/mutations/metadata_chain_mut.rs | 25 +- .../graphql/src/queries/datasets/dataset.rs | 14 +- .../src/queries/datasets/dataset_data.rs | 10 +- .../src/queries/datasets/dataset_metadata.rs | 18 +- .../src/queries/datasets/metadata_chain.rs | 16 +- .../graphql/src/scalars/flow_configuration.rs | 7 +- src/adapter/graphql/src/utils.rs | 25 +- src/adapter/graphql/tests/tests/test_auth.rs | 6 +- .../tests/tests/test_error_handling.rs | 9 +- .../tests/test_gql_account_flow_configs.rs | 111 +++-- .../graphql/tests/tests/test_gql_data.rs | 21 +- .../tests/tests/test_gql_dataset_env_vars.rs | 36 +- .../tests/test_gql_dataset_flow_configs.rs | 46 +- .../tests/tests/test_gql_dataset_flow_runs.rs | 160 +++--- .../graphql/tests/tests/test_gql_datasets.rs | 36 +- .../graphql/tests/tests/test_gql_metadata.rs | 16 +- .../tests/tests/test_gql_metadata_chain.rs | 46 +- .../graphql/tests/tests/test_gql_search.rs | 15 +- src/adapter/graphql/tests/utils/auth_utils.rs | 4 +- src/adapter/http/Cargo.toml | 7 +- src/adapter/http/src/data/ingest_handler.rs | 3 +- src/adapter/http/src/data/query_handler.rs | 11 +- src/adapter/http/src/data/tail_handler.rs | 3 +- .../http/src/http_server_dataset_router.rs | 6 +- .../middleware/dataset_authorization_layer.rs | 10 +- .../src/middleware/dataset_resolver_layer.rs | 4 +- .../run_in_database_transaction_layer.rs | 2 +- .../http/src/simple_protocol/handlers.rs | 11 +- .../axum_server_pull_protocol.rs | 3 +- .../axum_server_push_protocol.rs | 76 ++- src/adapter/http/src/smart_protocol/errors.rs | 3 +- .../smart_protocol/protocol_dataset_helper.rs | 74 +-- .../smart_protocol/ws_tungstenite_client.rs | 52 +- src/adapter/http/src/upload/upload_handler.rs | 2 +- src/adapter/http/src/upload/upload_service.rs | 3 +- .../http/src/upload/upload_service_local.rs | 9 +- .../http/src/upload/upload_service_s3.rs | 2 +- src/adapter/http/src/ws_common.rs | 2 +- .../http/tests/harness/client_side_harness.rs | 27 +- .../http/tests/harness/common_harness.rs | 2 +- .../http/tests/harness/server_side_harness.rs | 19 +- .../harness/server_side_local_fs_harness.rs | 46 +- .../tests/harness/server_side_s3_harness.rs | 46 +- .../tests/tests/test_authentication_layer.rs | 2 +- .../http/tests/tests/test_data_ingest.rs | 4 +- .../http/tests/tests/test_data_query.rs | 4 +- .../tests/test_dataset_authorization_layer.rs | 25 +- .../tests/test_platform_login_validate.rs | 9 +- .../tests/test_protocol_dataset_helpers.rs | 8 +- src/adapter/http/tests/tests/test_routing.rs | 28 +- .../http/tests/tests/test_upload_local.rs | 16 +- .../http/tests/tests/test_upload_s3.rs | 10 +- ...xisting_evolved_dataset_reread_succeeds.rs | 23 +- ...rio_aborted_read_of_new_reread_succeeds.rs | 10 +- ...cenario_existing_advanced_dataset_fails.rs | 10 +- .../scenario_existing_diverged_dataset.rs | 10 +- .../scenario_existing_evolved_dataset.rs | 22 +- .../scenario_existing_up_to_date_dataset.rs | 10 +- .../scenarios/scenario_new_dataset.rs | 10 +- .../scenarios/scenario_new_empty_dataset.rs | 8 +- ...o_aborted_write_of_new_rewrite_succeeds.rs | 9 +- ...orted_write_of_updated_rewrite_succeeds.rs | 20 +- ...isting_dataset_fails_as_server_advanced.rs | 8 +- .../scenario_existing_diverged_dataset.rs | 9 +- .../scenario_existing_evolved_dataset.rs | 10 +- .../scenario_existing_ref_collision.rs | 9 +- .../scenario_existing_up_to_date_dataset.rs | 9 +- .../scenarios/scenario_new_dataset.rs | 9 +- .../scenarios/scenario_new_empty_dataset.rs | 7 +- src/adapter/oauth/Cargo.toml | 5 +- src/adapter/oauth/src/oauth_github.rs | 2 +- src/adapter/odata/Cargo.toml | 7 +- src/adapter/odata/src/context.rs | 6 +- src/adapter/odata/src/handler.rs | 5 +- .../odata/tests/tests/test_handlers.rs | 23 +- src/app/cli/Cargo.toml | 15 +- src/app/cli/src/app.rs | 44 +- src/app/cli/src/cli_commands.rs | 3 + src/app/cli/src/commands/add_command.rs | 76 ++- src/app/cli/src/commands/complete_command.rs | 1 + src/app/cli/src/commands/delete_command.rs | 6 +- src/app/cli/src/commands/ingest_command.rs | 6 +- .../cli/src/commands/inspect_query_command.rs | 5 +- src/app/cli/src/commands/list_command.rs | 3 +- src/app/cli/src/commands/log_command.rs | 5 +- src/app/cli/src/commands/rename_command.rs | 10 +- .../cli/src/commands/system_e2e_command.rs | 2 +- src/app/cli/src/commands/verify_command.rs | 7 +- src/app/cli/src/database.rs | 40 +- src/app/cli/src/error.rs | 1 + src/app/cli/src/explore/api_server.rs | 9 +- src/app/cli/src/services/config/models.rs | 33 ++ src/app/cli/src/services/gc_service.rs | 2 +- .../services/workspace/workspace_service.rs | 2 +- src/app/cli/tests/tests/test_workspace_svc.rs | 7 +- src/domain/accounts/services/Cargo.toml | 5 +- .../services/src/access_token_service_impl.rs | 2 +- .../src/authentication_service_impl.rs | 2 +- .../src/login_password_auth_provider.rs | 2 +- .../tests/test_authentication_service.rs | 8 +- src/domain/core/Cargo.toml | 3 +- src/domain/core/src/entities/dataset.rs | 18 + src/domain/core/src/entities/mod.rs | 1 - src/domain/core/src/lib.rs | 8 +- .../src/messages/core_message_consumers.rs | 18 + .../src/messages/core_message_producers.rs | 15 + .../core/src/messages/core_message_types.rs | 68 +++ src/domain/core/src/messages/mod.rs | 16 + .../core/src/repos/dataset_repository.rs | 124 +---- .../core/src/services/compaction_service.rs | 1 + .../services/ingest/polling_ingest_service.rs | 1 + .../services/ingest/push_ingest_service.rs | 1 + .../core/src/services/provenance_service.rs | 3 +- src/domain/core/src/services/pull_service.rs | 1 + src/domain/core/src/services/query_service.rs | 1 + .../core/src/services/remote_aliases.rs | 3 +- .../src/services/remote_aliases_registry.rs | 1 + .../services/remote_repository_registry.rs | 1 + src/domain/core/src/services/reset_service.rs | 1 + .../core/src/services/search_service.rs | 1 + src/domain/core/src/services/sync_service.rs | 1 + .../core/src/services/transform_service.rs | 1 + .../core/src/services/verification_service.rs | 1 + .../append_dataset_metadata_batch_use_case.rs | 26 + .../commit_dataset_event_use_case.rs | 26 + .../create_dataset_from_snapshot_use_case.rs | 24 + .../src/use_cases/create_dataset_use_case.rs | 25 + .../src/use_cases/delete_dataset_use_case.rs | 26 + src/domain/core/src/use_cases/mod.rs | 22 + .../src/use_cases/rename_dataset_use_case.rs | 25 + src/domain/core/src/utils/mod.rs | 1 - src/domain/datasets/services/Cargo.toml | 4 +- .../src/dataset_env_var_service_impl.rs | 4 +- .../src/dataset_env_var_service_null.rs | 2 +- .../src/dataset_key_value_service_impl.rs | 2 +- .../src/dataset_key_value_service_sys_env.rs | 2 +- src/domain/flow-system/domain/Cargo.toml | 1 + .../domain/src/entities/shared/schedule.rs | 2 +- .../domain/src/flow_messages_types.rs | 48 ++ src/domain/flow-system/domain/src/lib.rs | 3 + .../src/services/flow/flow_service_event.rs | 61 --- .../domain/src/services/flow/mod.rs | 2 - src/domain/flow-system/services/Cargo.toml | 7 +- .../services/src/flow/flow_service_impl.rs | 409 ++++++++------- .../flow_configuration_service_impl.rs | 91 +++- src/domain/flow-system/services/src/lib.rs | 2 + .../src/messages/flow_message_consumers.rs | 17 + .../src/messages/flow_message_producers.rs | 17 + .../flow-system/services/src/messages/mod.rs | 14 + .../test_flow_configuration_service_impl.rs | 107 ++-- .../tests/tests/test_flow_service_impl.rs | 129 +++-- .../tests/utils/flow_config_test_listener.rs | 67 +++ .../tests/tests/utils/flow_harness_shared.rs | 138 ++++-- .../tests/utils/flow_system_test_listener.rs | 45 +- .../tests/utils/manual_flow_trigger_driver.rs | 2 +- .../services/tests/tests/utils/mod.rs | 2 + .../services/tests/tests/utils/task_driver.rs | 42 +- src/domain/task-system/domain/Cargo.toml | 3 +- src/domain/task-system/domain/src/lib.rs | 2 + .../task-system/domain/src/messages/mod.rs | 14 + .../src/messages/task_message_producers.rs | 14 + .../src/messages/task_messages_types.rs | 60 +++ src/domain/task-system/services/Cargo.toml | 7 +- .../services/src/task_executor_impl.rs | 168 ++++--- .../services/src/task_scheduler_impl.rs | 4 +- .../tests/tests/test_task_aggregate.rs | 6 +- .../tests/tests/test_task_scheduler_impl.rs | 6 +- src/infra/accounts/inmem/Cargo.toml | 2 +- ...em.rs => inmem_access_token_repository.rs} | 6 +- ...y_inmem.rs => inmem_account_repository.rs} | 8 +- src/infra/accounts/inmem/src/repos/mod.rs | 8 +- src/infra/accounts/inmem/tests/repos/mod.rs | 6 +- ... => test_inmem_access_token_repository.rs} | 8 +- ...em.rs => test_inmem_account_repository.rs} | 4 +- ...=> test_inmem_password_hash_repository.rs} | 12 +- src/infra/accounts/mysql/Cargo.toml | 2 +- .../repos/mysql_access_token_repository.rs | 6 +- .../test_mysql_access_token_repository.rs | 20 +- .../test_mysql_password_hash_repository.rs | 8 +- src/infra/accounts/postgres/Cargo.toml | 2 +- .../test_postgres_access_token_repository.rs | 2 + .../test_postgres_password_hash_repository.rs | 8 +- src/infra/accounts/repo-tests/Cargo.toml | 2 +- src/infra/accounts/sqlite/Cargo.toml | 2 +- .../test_sqlite_password_hash_repository.rs | 8 +- ...ory_inmem.rs => inmem_rebac_repository.rs} | 6 +- src/infra/auth-rebac/inmem/src/repos/mod.rs | 4 +- src/infra/auth-rebac/inmem/tests/repos/mod.rs | 2 +- ...nmem.rs => test_inmem_rebac_repository.rs} | 4 +- src/infra/core/Cargo.toml | 5 +- src/infra/core/src/compaction_service_impl.rs | 7 +- .../core/src/dataset_changes_service_impl.rs | 5 +- .../src/dataset_ownership_service_inmem.rs | 72 ++- .../src/dependency_graph_repository_inmem.rs | 5 +- .../src/dependency_graph_service_inmem.rs | 105 ++-- .../core/src/engine/engine_io_strategy.rs | 9 +- src/infra/core/src/engine/engine_odf.rs | 28 +- .../src/engine/engine_provisioner_local.rs | 1 + src/infra/core/src/ingest/fetch_service.rs | 1 + src/infra/core/src/ingest/ingest_common.rs | 1 + .../src/ingest/polling_ingest_service_impl.rs | 9 +- .../core/src/ingest/polling_source_state.rs | 2 +- src/infra/core/src/ingest/prep_service.rs | 1 + .../src/ingest/push_ingest_service_impl.rs | 9 +- src/infra/core/src/lib.rs | 2 + src/infra/core/src/provenance_service_impl.rs | 1 + src/infra/core/src/pull_service_impl.rs | 8 +- src/infra/core/src/push_service_impl.rs | 1 + src/infra/core/src/query/mod.rs | 15 +- src/infra/core/src/query_service_impl.rs | 24 +- .../core/src/remote_aliases_registry_impl.rs | 3 +- .../src/remote_repository_registry_impl.rs | 1 + .../core/src/repos/dataset_factory_impl.rs | 51 +- src/infra/core/src/repos/dataset_impl.rs | 24 +- .../src/repos/dataset_repository_helpers.rs | 37 +- .../src/repos/dataset_repository_local_fs.rs | 156 ++---- .../core/src/repos/dataset_repository_s3.rs | 122 ++--- .../src/repos/dataset_repository_writer.rs | 40 ++ .../core/src/repos/metadata_chain_impl.rs | 1 + src/infra/core/src/repos/mod.rs | 2 + .../src/repos/named_object_repository_http.rs | 1 + .../named_object_repository_ipfs_http.rs | 1 + .../repos/named_object_repository_local_fs.rs | 1 + .../src/repos/named_object_repository_s3.rs | 1 + .../object_repository_caching_local_fs.rs | 1 + .../core/src/repos/object_repository_http.rs | 1 + .../src/repos/object_repository_local_fs.rs | 1 + .../core/src/repos/object_repository_s3.rs | 1 + .../repos/object_store_builder_local_fs.rs | 1 + .../core/src/repos/object_store_builder_s3.rs | 1 + .../src/repos/reference_repository_impl.rs | 1 + src/infra/core/src/reset_service_impl.rs | 10 +- src/infra/core/src/resource_loader_impl.rs | 1 + src/infra/core/src/search_service_impl.rs | 1 + src/infra/core/src/sync_service_impl.rs | 52 +- .../core/src/testing/dataset_test_helper.rs | 5 +- .../core/src/testing/metadata_factory.rs | 49 ++ .../testing/mock_dataset_action_authorizer.rs | 35 +- src/infra/core/src/transform_service_impl.rs | 49 +- ...nd_dataset_metadata_batch_use_case_impl.rs | 127 +++++ .../commit_dataset_event_use_case_impl.rs | 82 +++ ...ate_dataset_from_snapshot_use_case_impl.rs | 96 ++++ .../use_cases/create_dataset_use_case_impl.rs | 81 +++ .../use_cases/delete_dataset_use_case_impl.rs | 135 +++++ src/infra/core/src/use_cases/mod.rs | 22 + .../use_cases/rename_dataset_use_case_impl.rs | 68 +++ .../core/src/utils/datasets_filtering.rs | 10 +- src/infra/core/src/utils/ipfs_wrapper.rs | 2 +- src/infra/core/src/utils/s3_context.rs | 2 +- .../src/utils/simple_transfer_protocol.rs | 1 + .../core/src/verification_service_impl.rs | 20 +- .../parallel_simple_transfer_protocol.rs | 4 +- .../core/tests/tests/engine/test_engine_io.rs | 17 +- .../tests/engine/test_engine_transform.rs | 7 +- .../tests/tests/ingest/test_polling_ingest.rs | 13 +- .../tests/tests/ingest/test_push_ingest.rs | 17 +- .../core/tests/tests/ingest/test_writer.rs | 9 +- src/infra/core/tests/tests/mod.rs | 1 + .../tests/tests/repos/test_dataset_impl.rs | 4 +- .../repos/test_dataset_repository_local_fs.rs | 174 ++----- .../tests/repos/test_dataset_repository_s3.rs | 231 ++------- .../repos/test_dataset_repository_shared.rs | 368 +++++++------- .../tests/repos/test_metadata_chain_impl.rs | 1 + .../tests/tests/test_compact_service_impl.rs | 39 +- .../test_dataset_changes_service_impl.rs | 34 +- .../test_dataset_ownership_service_inmem.rs | 35 +- .../tests/tests/test_datasets_filtering.rs | 16 +- .../tests/test_dependency_graph_inmem.rs | 95 ++-- .../tests/tests/test_pull_service_impl.rs | 32 +- .../tests/tests/test_query_service_impl.rs | 49 +- .../tests/tests/test_reset_service_impl.rs | 23 +- .../tests/tests/test_search_service_impl.rs | 9 +- .../tests/tests/test_sync_service_impl.rs | 14 +- .../tests/test_transform_service_impl.rs | 42 +- .../tests/test_verification_service_impl.rs | 18 +- src/infra/core/tests/tests/use_cases/mod.rs | 15 + ..._append_dataset_metadata_batch_use_case.rs | 230 +++++++++ .../test_commit_dataset_event_use_case.rs | 205 ++++++++ ...t_create_dataset_from_snapshot_use_case.rs | 182 +++++++ .../use_cases/test_create_dataset_use_case.rs | 117 +++++ .../use_cases/test_delete_dataset_use_case.rs | 345 +++++++++++++ .../use_cases/test_rename_dataset_use_case.rs | 173 +++++++ src/infra/datasets/inmem/Cargo.toml | 2 +- ...rs => inmem_dataset_env_var_repository.rs} | 6 +- ...m.rs => inmem_dateset_entry_repository.rs} | 6 +- src/infra/datasets/inmem/src/repos/mod.rs | 8 +- src/infra/datasets/inmem/tests/repos/mod.rs | 4 +- ...=> test_inmem_dataset_entry_repository.rs} | 22 +- ... test_inmem_dataset_env_var_repository.rs} | 18 +- src/infra/datasets/postgres/Cargo.toml | 2 +- src/infra/datasets/repo-tests/Cargo.toml | 2 +- src/infra/datasets/sqlite/Cargo.toml | 2 +- src/infra/flow-system/inmem/Cargo.toml | 2 +- ...ore_inmem.rs => inmem_flow_event_store.rs} | 12 +- src/infra/flow-system/inmem/src/flow/mod.rs | 4 +- ...> inmem_flow_configuration_event_store.rs} | 12 +- .../inmem/src/flow_configuration/mod.rs | 4 +- .../flow-system/inmem/tests/tests/mod.rs | 4 +- ...t_inmem_flow_configuration_event_store.rs} | 6 +- ...nmem.rs => test_inmem_flow_event_store.rs} | 8 +- src/infra/flow-system/postgres/Cargo.toml | 2 +- src/infra/flow-system/postgres/src/lib.rs | 4 +- ...ostgres_flow_configuration_event_store.rs} | 8 +- .../flow-system/postgres/tests/tests/mod.rs | 2 +- ...ostgres_flow_configuration_event_store.rs} | 14 +- src/infra/flow-system/repo-tests/Cargo.toml | 2 +- src/infra/flow-system/sqlite/Cargo.toml | 2 +- src/infra/flow-system/sqlite/src/lib.rs | 4 +- ...e.rs => sqlite_flow_system_event_store.rs} | 8 +- .../flow-system/sqlite/tests/tests/mod.rs | 2 +- ..._sqlite_flow_configuration_event_store.rs} | 14 +- .../messaging-outbox/inmem}/Cargo.toml | 21 +- src/infra/messaging-outbox/inmem/src/lib.rs | 17 + ...m_outbox_message_consumption_repository.rs | 119 +++++ .../repos/inmem_outbox_message_repository.rs | 123 +++++ .../messaging-outbox/inmem/src/repos/mod.rs | 14 + .../messaging-outbox/inmem}/tests/mod.rs | 2 +- .../messaging-outbox/inmem/tests/repos/mod.rs | 11 + ...m_outbox_message_consumption_repository.rs | 73 +++ .../test_inmem_outbox_message_repository.rs | 64 +++ ...f2f87019a4ba595fb156f968c235723a6cdaf.json | 26 + ...0cc65d1e880159e85aa9008e6f78ef28a006a.json | 35 ++ ...2fde425e780689dc0117308f9e5047b0b0ab2.json | 42 ++ ...316cbd7ef346b7bbcd9c0e8aaa26fe0a2d95a.json | 32 ++ ...5328885659a631a580bf57072abba62aed5d9.json | 16 + ...7a53849db3487224c3da9cf90aa9395faa716.json | 16 + ...315ffe00a2d822f3def0c2bee5edb0168f666.json | 16 + .../messaging-outbox/postgres/Cargo.toml | 51 ++ .../messaging-outbox/postgres}/src/lib.rs | 11 +- .../postgres/src/repos/mod.rs | 14 + ...postgres_message_consumption_repository.rs | 164 ++++++ .../outbox_postgres_message_repository.rs | 128 +++++ .../messaging-outbox/postgres}/tests/mod.rs | 2 +- .../postgres/tests/repos/mod.rs | 11 + ...s_outbox_message_consumption_repository.rs | 117 +++++ ...test_postgres_outbox_message_repository.rs | 101 ++++ .../messaging-outbox/repo-tests/Cargo.toml | 33 ++ .../messaging-outbox/repo-tests/src/lib.rs | 16 + ...ssage_consumption_repository_test_suite.rs | 240 +++++++++ .../outbox_message_repository_test_suite.rs | 211 ++++++++ ...42930c873cfedd1b8d864745ed7c26fede9ac.json | 38 ++ ...0cc65d1e880159e85aa9008e6f78ef28a006a.json | 32 ++ ...316cbd7ef346b7bbcd9c0e8aaa26fe0a2d95a.json | 32 ++ ...5328885659a631a580bf57072abba62aed5d9.json | 12 + ...aee30cde75a4a9afcb4c2485825e0fec0eb7b.json | 26 + ...7a53849db3487224c3da9cf90aa9395faa716.json | 12 + ...315ffe00a2d822f3def0c2bee5edb0168f666.json | 12 + src/infra/messaging-outbox/sqlite/Cargo.toml | 50 ++ src/infra/messaging-outbox/sqlite/src/lib.rs | 17 + .../messaging-outbox/sqlite/src/repos/mod.rs | 14 + ...e_outbox_message_consumption_repository.rs | 168 +++++++ .../repos/sqlite_outbox_message_repository.rs | 133 +++++ .../messaging-outbox/sqlite/tests/mod.rs | 10 + .../sqlite/tests/repos/mod.rs | 11 + ...e_outbox_message_consumption_repository.rs | 118 +++++ .../test_sqlite_outbox_message_repository.rs | 101 ++++ src/infra/task-system/inmem/Cargo.toml | 2 +- ...em.rs => inmem_task_system_event_store.rs} | 12 +- src/infra/task-system/inmem/src/lib.rs | 4 +- .../task-system/inmem/tests/tests/mod.rs | 2 +- ... => test_inmem_task_system_event_store.rs} | 10 +- src/infra/task-system/postgres/Cargo.toml | 2 +- src/infra/task-system/postgres/src/lib.rs | 4 +- ...rs => postgres_task_system_event_store.rs} | 8 +- .../task-system/postgres/tests/tests/mod.rs | 2 +- ... test_postgres_task_system_event_store.rs} | 18 +- src/infra/task-system/repo-tests/Cargo.toml | 2 +- src/infra/task-system/sqlite/Cargo.toml | 2 +- src/infra/task-system/sqlite/src/lib.rs | 4 +- ...e.rs => sqlite_task_system_event_store.rs} | 8 +- .../task-system/sqlite/tests/tests/mod.rs | 2 +- ...=> test_sqlite_task_system_event_store.rs} | 18 +- src/utils/container-runtime/Cargo.toml | 2 +- src/utils/database-common/Cargo.toml | 2 +- .../src/plugins/sqlite_plugin.rs | 6 +- .../transactions/db_transaction_manager.rs | 20 + src/utils/event-bus/src/event_bus.rs | 82 --- src/utils/event-bus/tests/test_event_bus.rs | 108 ---- src/utils/event-sourcing/Cargo.toml | 1 + src/utils/event-sourcing/src/event_id.rs | 5 + ...nt_store_inmem.rs => inmem_event_store.rs} | 6 +- src/utils/event-sourcing/src/lib.rs | 4 +- src/utils/http-common/Cargo.toml | 1 + src/utils/http-common/src/api_error.rs | 1 + src/utils/messaging-outbox/Cargo.toml | 47 ++ .../src/consumers/message_consumer.rs} | 40 +- .../src/consumers/message_consumers_utils.rs | 183 +++++++ .../src/consumers/message_dispatcher.rs | 84 ++++ .../src/consumers/message_subscription.rs | 32 ++ .../messaging-outbox/src/consumers/mod.rs | 18 + .../messaging-outbox/src/entities/mod.rs | 14 + .../src/entities/outbox_message.rs} | 20 +- .../src/entities/outbox_message_id.rs | 52 ++ src/utils/messaging-outbox/src/lib.rs | 22 + src/utils/messaging-outbox/src/message.rs | 16 + src/utils/messaging-outbox/src/repos/mod.rs | 14 + .../outbox_message_consumption_repository.rs | 100 ++++ .../src/repos/outbox_message_repository.rs | 38 ++ .../src/services/implementation/mod.rs | 16 + .../implementation/outbox_dispatching_impl.rs | 102 ++++ .../implementation/outbox_immediate_impl.rs | 65 +++ .../outbox_transactional_impl.rs | 58 +++ .../messaging-outbox/src/services/mod.rs | 21 + .../messaging-outbox/src/services/outbox.rs | 50 ++ .../src/services/outbox_config.rs | 41 ++ .../outbox_transactional_processor.rs | 466 ++++++++++++++++++ .../src/services/testing/dummy_outbox_impl.rs | 33 ++ .../src/services/testing/mock_outbox_impl.rs | 29 ++ .../src/services/testing/mod.rs | 14 + src/utils/messaging-outbox/tests/mod.rs | 91 ++++ src/utils/messaging-outbox/tests/tests/mod.rs | 13 + .../tests/test_dispatching_outbox_impl.rs | 276 +++++++++++ .../tests/tests/test_immediate_outbox_impl.rs | 210 ++++++++ .../test_outbox_transactional_processor.rs | 366 ++++++++++++++ .../tests/test_transactional_outbox_impl.rs | 154 ++++++ src/utils/time-source/Cargo.toml | 32 ++ src/utils/time-source/src/lib.rs | 12 + .../time-source/src}/time_source.rs | 0 .../core => utils/time-source}/tests/mod.rs | 0 .../time-source/tests/tests}/mod.rs | 0 .../tests/tests}/test_time_source.rs | 2 +- 436 files changed, 11051 insertions(+), 3332 deletions(-) create mode 100644 migrations/postgres/20240710191232_outbox_messages_consumptions.sql create mode 100644 migrations/sqlite/20240710205713_outbox_messages_consumptions.sql create mode 100644 src/domain/core/src/messages/core_message_consumers.rs create mode 100644 src/domain/core/src/messages/core_message_producers.rs create mode 100644 src/domain/core/src/messages/core_message_types.rs create mode 100644 src/domain/core/src/messages/mod.rs create mode 100644 src/domain/core/src/use_cases/append_dataset_metadata_batch_use_case.rs create mode 100644 src/domain/core/src/use_cases/commit_dataset_event_use_case.rs create mode 100644 src/domain/core/src/use_cases/create_dataset_from_snapshot_use_case.rs create mode 100644 src/domain/core/src/use_cases/create_dataset_use_case.rs create mode 100644 src/domain/core/src/use_cases/delete_dataset_use_case.rs create mode 100644 src/domain/core/src/use_cases/mod.rs create mode 100644 src/domain/core/src/use_cases/rename_dataset_use_case.rs create mode 100644 src/domain/flow-system/domain/src/flow_messages_types.rs delete mode 100644 src/domain/flow-system/domain/src/services/flow/flow_service_event.rs create mode 100644 src/domain/flow-system/services/src/messages/flow_message_consumers.rs create mode 100644 src/domain/flow-system/services/src/messages/flow_message_producers.rs create mode 100644 src/domain/flow-system/services/src/messages/mod.rs create mode 100644 src/domain/flow-system/services/tests/tests/utils/flow_config_test_listener.rs create mode 100644 src/domain/task-system/domain/src/messages/mod.rs create mode 100644 src/domain/task-system/domain/src/messages/task_message_producers.rs create mode 100644 src/domain/task-system/domain/src/messages/task_messages_types.rs rename src/infra/accounts/inmem/src/repos/{access_token_repository_inmem.rs => inmem_access_token_repository.rs} (97%) rename src/infra/accounts/inmem/src/repos/{account_repository_inmem.rs => inmem_account_repository.rs} (97%) rename src/infra/accounts/inmem/tests/repos/{test_access_token_repository_inmem.rs => test_inmem_access_token_repository.rs} (89%) rename src/infra/accounts/inmem/tests/repos/{test_account_repository_inmem.rs => test_inmem_account_repository.rs} (97%) rename src/infra/accounts/inmem/tests/repos/{test_password_hash_repository_inmem.rs => test_inmem_password_hash_repository.rs} (81%) rename src/infra/auth-rebac/inmem/src/repos/{rebac_repository_inmem.rs => inmem_rebac_repository.rs} (98%) rename src/infra/auth-rebac/inmem/tests/repos/{test_rebac_repository_inmem.rs => test_inmem_rebac_repository.rs} (97%) create mode 100644 src/infra/core/src/repos/dataset_repository_writer.rs create mode 100644 src/infra/core/src/use_cases/append_dataset_metadata_batch_use_case_impl.rs create mode 100644 src/infra/core/src/use_cases/commit_dataset_event_use_case_impl.rs create mode 100644 src/infra/core/src/use_cases/create_dataset_from_snapshot_use_case_impl.rs create mode 100644 src/infra/core/src/use_cases/create_dataset_use_case_impl.rs create mode 100644 src/infra/core/src/use_cases/delete_dataset_use_case_impl.rs create mode 100644 src/infra/core/src/use_cases/mod.rs create mode 100644 src/infra/core/src/use_cases/rename_dataset_use_case_impl.rs create mode 100644 src/infra/core/tests/tests/use_cases/mod.rs create mode 100644 src/infra/core/tests/tests/use_cases/test_append_dataset_metadata_batch_use_case.rs create mode 100644 src/infra/core/tests/tests/use_cases/test_commit_dataset_event_use_case.rs create mode 100644 src/infra/core/tests/tests/use_cases/test_create_dataset_from_snapshot_use_case.rs create mode 100644 src/infra/core/tests/tests/use_cases/test_create_dataset_use_case.rs create mode 100644 src/infra/core/tests/tests/use_cases/test_delete_dataset_use_case.rs create mode 100644 src/infra/core/tests/tests/use_cases/test_rename_dataset_use_case.rs rename src/infra/datasets/inmem/src/repos/{dataset_env_var_repository_inmem.rs => inmem_dataset_env_var_repository.rs} (98%) rename src/infra/datasets/inmem/src/repos/{dateset_entry_repository_inmem.rs => inmem_dateset_entry_repository.rs} (97%) rename src/infra/datasets/inmem/tests/repos/{test_dataset_entry_repository_inmem.rs => test_inmem_dataset_entry_repository.rs} (81%) rename src/infra/datasets/inmem/tests/repos/{test_dataset_env_var_repository_inmem.rs => test_inmem_dataset_env_var_repository.rs} (80%) rename src/infra/flow-system/inmem/src/flow/{flow_event_store_inmem.rs => inmem_flow_event_store.rs} (98%) rename src/infra/flow-system/inmem/src/flow_configuration/{flow_configuration_event_store_inmem.rs => inmem_flow_configuration_event_store.rs} (89%) rename src/infra/flow-system/inmem/tests/tests/{test_flow_configuration_event_store_inmem.rs => test_inmem_flow_configuration_event_store.rs} (90%) rename src/infra/flow-system/inmem/tests/tests/{test_flow_event_store_inmem.rs => test_inmem_flow_event_store.rs} (99%) rename src/infra/flow-system/postgres/src/{flow_configuration_event_store_postgres.rs => postgres_flow_configuration_event_store.rs} (97%) rename src/infra/flow-system/postgres/tests/tests/{test_flow_configuration_event_store_postgres.rs => test_postgres_flow_configuration_event_store.rs} (86%) rename src/infra/flow-system/sqlite/src/{flow_system_event_store_sqlite.rs => sqlite_flow_system_event_store.rs} (97%) rename src/infra/flow-system/sqlite/tests/tests/{test_flow_configuration_event_store_sqlite.rs => test_sqlite_flow_configuration_event_store.rs} (87%) rename src/{utils/event-bus => infra/messaging-outbox/inmem}/Cargo.toml (57%) create mode 100644 src/infra/messaging-outbox/inmem/src/lib.rs create mode 100644 src/infra/messaging-outbox/inmem/src/repos/inmem_outbox_message_consumption_repository.rs create mode 100644 src/infra/messaging-outbox/inmem/src/repos/inmem_outbox_message_repository.rs create mode 100644 src/infra/messaging-outbox/inmem/src/repos/mod.rs rename src/{domain/core/tests => infra/messaging-outbox/inmem}/tests/mod.rs (96%) create mode 100644 src/infra/messaging-outbox/inmem/tests/repos/mod.rs create mode 100644 src/infra/messaging-outbox/inmem/tests/repos/test_inmem_outbox_message_consumption_repository.rs create mode 100644 src/infra/messaging-outbox/inmem/tests/repos/test_inmem_outbox_message_repository.rs create mode 100644 src/infra/messaging-outbox/postgres/.sqlx/query-6a7bc6be8d4f035137972579bfef2f87019a4ba595fb156f968c235723a6cdaf.json create mode 100644 src/infra/messaging-outbox/postgres/.sqlx/query-8a52d43e1c3a16c91850be1c73d0cc65d1e880159e85aa9008e6f78ef28a006a.json create mode 100644 src/infra/messaging-outbox/postgres/.sqlx/query-8b363fe5b240687390b2e6f32382fde425e780689dc0117308f9e5047b0b0ab2.json create mode 100644 src/infra/messaging-outbox/postgres/.sqlx/query-a00e0b9831fc78664e606f8f6b2316cbd7ef346b7bbcd9c0e8aaa26fe0a2d95a.json create mode 100644 src/infra/messaging-outbox/postgres/.sqlx/query-ab6e6a2d4fb596ad3eb2f4188735328885659a631a580bf57072abba62aed5d9.json create mode 100644 src/infra/messaging-outbox/postgres/.sqlx/query-e03881a854596e143a3999a3b057a53849db3487224c3da9cf90aa9395faa716.json create mode 100644 src/infra/messaging-outbox/postgres/.sqlx/query-ee89ee9f37512d9bb74dac0a142315ffe00a2d822f3def0c2bee5edb0168f666.json create mode 100644 src/infra/messaging-outbox/postgres/Cargo.toml rename src/{utils/event-bus => infra/messaging-outbox/postgres}/src/lib.rs (77%) create mode 100644 src/infra/messaging-outbox/postgres/src/repos/mod.rs create mode 100644 src/infra/messaging-outbox/postgres/src/repos/outbox_postgres_message_consumption_repository.rs create mode 100644 src/infra/messaging-outbox/postgres/src/repos/outbox_postgres_message_repository.rs rename src/{utils/event-bus => infra/messaging-outbox/postgres}/tests/mod.rs (94%) create mode 100644 src/infra/messaging-outbox/postgres/tests/repos/mod.rs create mode 100644 src/infra/messaging-outbox/postgres/tests/repos/test_postgres_outbox_message_consumption_repository.rs create mode 100644 src/infra/messaging-outbox/postgres/tests/repos/test_postgres_outbox_message_repository.rs create mode 100644 src/infra/messaging-outbox/repo-tests/Cargo.toml create mode 100644 src/infra/messaging-outbox/repo-tests/src/lib.rs create mode 100644 src/infra/messaging-outbox/repo-tests/src/outbox_message_consumption_repository_test_suite.rs create mode 100644 src/infra/messaging-outbox/repo-tests/src/outbox_message_repository_test_suite.rs create mode 100644 src/infra/messaging-outbox/sqlite/.sqlx/query-738d1c4230062bd4a670e924ef842930c873cfedd1b8d864745ed7c26fede9ac.json create mode 100644 src/infra/messaging-outbox/sqlite/.sqlx/query-8a52d43e1c3a16c91850be1c73d0cc65d1e880159e85aa9008e6f78ef28a006a.json create mode 100644 src/infra/messaging-outbox/sqlite/.sqlx/query-a00e0b9831fc78664e606f8f6b2316cbd7ef346b7bbcd9c0e8aaa26fe0a2d95a.json create mode 100644 src/infra/messaging-outbox/sqlite/.sqlx/query-ab6e6a2d4fb596ad3eb2f4188735328885659a631a580bf57072abba62aed5d9.json create mode 100644 src/infra/messaging-outbox/sqlite/.sqlx/query-d1d9fdcec93cbf10079386a1f8aaee30cde75a4a9afcb4c2485825e0fec0eb7b.json create mode 100644 src/infra/messaging-outbox/sqlite/.sqlx/query-e03881a854596e143a3999a3b057a53849db3487224c3da9cf90aa9395faa716.json create mode 100644 src/infra/messaging-outbox/sqlite/.sqlx/query-ee89ee9f37512d9bb74dac0a142315ffe00a2d822f3def0c2bee5edb0168f666.json create mode 100644 src/infra/messaging-outbox/sqlite/Cargo.toml create mode 100644 src/infra/messaging-outbox/sqlite/src/lib.rs create mode 100644 src/infra/messaging-outbox/sqlite/src/repos/mod.rs create mode 100644 src/infra/messaging-outbox/sqlite/src/repos/sqlite_outbox_message_consumption_repository.rs create mode 100644 src/infra/messaging-outbox/sqlite/src/repos/sqlite_outbox_message_repository.rs create mode 100644 src/infra/messaging-outbox/sqlite/tests/mod.rs create mode 100644 src/infra/messaging-outbox/sqlite/tests/repos/mod.rs create mode 100644 src/infra/messaging-outbox/sqlite/tests/repos/test_sqlite_outbox_message_consumption_repository.rs create mode 100644 src/infra/messaging-outbox/sqlite/tests/repos/test_sqlite_outbox_message_repository.rs rename src/infra/task-system/inmem/src/{task_system_event_store_inmem.rs => inmem_task_system_event_store.rs} (94%) rename src/infra/task-system/inmem/tests/tests/{test_task_system_event_store_inmem.rs => test_inmem_task_system_event_store.rs} (90%) rename src/infra/task-system/postgres/src/{task_system_event_store_postgres.rs => postgres_task_system_event_store.rs} (97%) rename src/infra/task-system/postgres/tests/tests/{test_task_system_event_store_postgres.rs => test_postgres_task_system_event_store.rs} (87%) rename src/infra/task-system/sqlite/src/{task_system_event_store_sqlite.rs => sqlite_task_system_event_store.rs} (97%) rename src/infra/task-system/sqlite/tests/tests/{test_task_system_event_store_sqlite.rs => test_sqlite_task_system_event_store.rs} (87%) delete mode 100644 src/utils/event-bus/src/event_bus.rs delete mode 100644 src/utils/event-bus/tests/test_event_bus.rs rename src/utils/event-sourcing/src/{event_store_inmem.rs => inmem_event_store.rs} (95%) create mode 100644 src/utils/messaging-outbox/Cargo.toml rename src/{domain/core/src/entities/events.rs => utils/messaging-outbox/src/consumers/message_consumer.rs} (58%) create mode 100644 src/utils/messaging-outbox/src/consumers/message_consumers_utils.rs create mode 100644 src/utils/messaging-outbox/src/consumers/message_dispatcher.rs create mode 100644 src/utils/messaging-outbox/src/consumers/message_subscription.rs create mode 100644 src/utils/messaging-outbox/src/consumers/mod.rs create mode 100644 src/utils/messaging-outbox/src/entities/mod.rs rename src/utils/{event-bus/src/event_handler.rs => messaging-outbox/src/entities/outbox_message.rs} (63%) create mode 100644 src/utils/messaging-outbox/src/entities/outbox_message_id.rs create mode 100644 src/utils/messaging-outbox/src/lib.rs create mode 100644 src/utils/messaging-outbox/src/message.rs create mode 100644 src/utils/messaging-outbox/src/repos/mod.rs create mode 100644 src/utils/messaging-outbox/src/repos/outbox_message_consumption_repository.rs create mode 100644 src/utils/messaging-outbox/src/repos/outbox_message_repository.rs create mode 100644 src/utils/messaging-outbox/src/services/implementation/mod.rs create mode 100644 src/utils/messaging-outbox/src/services/implementation/outbox_dispatching_impl.rs create mode 100644 src/utils/messaging-outbox/src/services/implementation/outbox_immediate_impl.rs create mode 100644 src/utils/messaging-outbox/src/services/implementation/outbox_transactional_impl.rs create mode 100644 src/utils/messaging-outbox/src/services/mod.rs create mode 100644 src/utils/messaging-outbox/src/services/outbox.rs create mode 100644 src/utils/messaging-outbox/src/services/outbox_config.rs create mode 100644 src/utils/messaging-outbox/src/services/outbox_transactional_processor.rs create mode 100644 src/utils/messaging-outbox/src/services/testing/dummy_outbox_impl.rs create mode 100644 src/utils/messaging-outbox/src/services/testing/mock_outbox_impl.rs create mode 100644 src/utils/messaging-outbox/src/services/testing/mod.rs create mode 100644 src/utils/messaging-outbox/tests/mod.rs create mode 100644 src/utils/messaging-outbox/tests/tests/mod.rs create mode 100644 src/utils/messaging-outbox/tests/tests/test_dispatching_outbox_impl.rs create mode 100644 src/utils/messaging-outbox/tests/tests/test_immediate_outbox_impl.rs create mode 100644 src/utils/messaging-outbox/tests/tests/test_outbox_transactional_processor.rs create mode 100644 src/utils/messaging-outbox/tests/tests/test_transactional_outbox_impl.rs create mode 100644 src/utils/time-source/Cargo.toml create mode 100644 src/utils/time-source/src/lib.rs rename src/{domain/core/src/utils => utils/time-source/src}/time_source.rs (100%) rename src/{domain/core => utils/time-source}/tests/mod.rs (100%) rename src/{domain/core/tests/tests/utils => utils/time-source/tests/tests}/mod.rs (100%) rename src/{domain/core/tests/tests/utils => utils/time-source/tests/tests}/test_time_source.rs (99%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5467dc41b8..e86135c97f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased +## [0.195.0] - 2024-08-16 +### Added +- Reliable transaction-based internal cross-domain message passing component (`MessageOutbox`), replacing `EventBus` + - Metadata-driven producer/consumer annotations + - Immediate and transaction-backed message delivery + - Background transactional message processor, respecting client idempotence +- Persistent storage for flow configuration events ### Changed - Upgraded to `datafusion v41` (#713) +- Introduced use case layer, encapsulating authorization checks and action validations, for first 6 basic dataset scenarios + (creating, creating from snapshot, deleting, renaming, committing an event, syncing a batch of events), +- Separated `DatasetRepository` on read-only and read-write parts +- Isolated `time-source` library ### Fixed - E2E: added additional force off colors to exclude sometimes occurring ANSI color sequences - E2E: modify a workaround for MySQL tests diff --git a/Cargo.lock b/Cargo.lock index 0e79a2cb7f..6fcd668964 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2051,9 +2051,9 @@ dependencies = [ [[package]] name = "camino" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0ec6b951b160caa93cc0c7b209e5a3bff7aae9062213451ac99493cd844c239" +checksum = "3054fea8a20d8ff3968d5b22cc27501d2b08dc4decdb31b184323f00c5ef23bb" dependencies = [ "serde", ] @@ -2201,9 +2201,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.15" +version = "4.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11d8838454fda655dafd3accb2b6e2bea645b9e4078abe84a22ceb947235c5cc" +checksum = "ed6719fffa43d0d87e5fd8caeab59be1554fb028cd30edc88fc4369b17971019" dependencies = [ "clap_builder", "clap_derive", @@ -2352,11 +2352,11 @@ checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" [[package]] name = "container-runtime" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-trait", "cfg-if", - "dill", + "dill 0.9.1", "libc", "once_cell", "random-names", @@ -2791,14 +2791,14 @@ checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" [[package]] name = "database-common" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-trait", "aws-config", "aws-credential-types", "aws-sdk-secretsmanager", "chrono", - "dill", + "dill 0.9.1", "hex", "hmac", "internal-error", @@ -2815,7 +2815,7 @@ dependencies = [ [[package]] name = "database-common-macros" -version = "0.194.1" +version = "0.195.0" dependencies = [ "quote", "syn 2.0.74", @@ -3321,8 +3321,19 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3a18eaae0c1d3dc669fa6388383dfce0420742a916950cbef6d090d13cb52a0" dependencies = [ - "dill-impl", - "multimap", + "dill-impl 0.8.1", + "multimap 0.9.1", + "thiserror", +] + +[[package]] +name = "dill" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc86c45437bf03daddf914ea76571cc341dde47ae8cfae336cc8d24803f80210" +dependencies = [ + "dill-impl 0.9.1", + "multimap 0.10.0", "thiserror", ] @@ -3337,6 +3348,17 @@ dependencies = [ "syn 2.0.74", ] +[[package]] +name = "dill-impl" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d0e68e1e07d64dbf3bb2991657979ec4e3fe13b7b3c18067b802052af1330a3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.74", +] + [[package]] name = "dirs" version = "5.0.1" @@ -3570,7 +3592,7 @@ dependencies = [ [[package]] name = "enum-variants" -version = "0.194.1" +version = "0.195.0" [[package]] name = "env_filter" @@ -3630,21 +3652,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "event-bus" -version = "0.194.1" -dependencies = [ - "async-trait", - "dill", - "futures", - "internal-error", - "test-log", - "thiserror", - "tokio", - "tracing", - "tracing-subscriber", -] - [[package]] name = "event-listener" version = "2.5.3" @@ -3653,13 +3660,14 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "event-sourcing" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-stream", "async-trait", "event-sourcing-macros", "futures", "internal-error", + "serde", "thiserror", "tokio", "tokio-stream", @@ -3668,7 +3676,7 @@ dependencies = [ [[package]] name = "event-sourcing-macros" -version = "0.194.1" +version = "0.195.0" dependencies = [ "quote", "syn 2.0.74", @@ -4335,10 +4343,11 @@ dependencies = [ [[package]] name = "http-common" -version = "0.194.1" +version = "0.195.0" dependencies = [ "axum", "http 0.2.12", + "internal-error", "kamu-core", "thiserror", "tracing", @@ -4655,7 +4664,7 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "internal-error" -version = "0.194.1" +version = "0.195.0" dependencies = [ "thiserror", ] @@ -4812,7 +4821,7 @@ dependencies = [ [[package]] name = "kamu" -version = "0.194.1" +version = "0.195.0" dependencies = [ "alloy", "async-recursion", @@ -4837,8 +4846,7 @@ dependencies = [ "datafusion-ethers", "datafusion-functions-json", "digest 0.10.7", - "dill", - "event-bus", + "dill 0.9.1", "filetime", "flatbuffers", "flate2", @@ -4860,6 +4868,7 @@ dependencies = [ "kamu-ingest-datafusion", "libc", "like", + "messaging-outbox", "mockall", "nanoid", "object_store", @@ -4883,6 +4892,7 @@ dependencies = [ "test-group", "test-log", "thiserror", + "time-source", "tokio", "tokio-stream", "tokio-util", @@ -4898,7 +4908,7 @@ dependencies = [ [[package]] name = "kamu-accounts" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-trait", "base32", @@ -4924,12 +4934,12 @@ dependencies = [ [[package]] name = "kamu-accounts-inmem" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-trait", "chrono", "database-common", - "dill", + "dill 0.9.1", "internal-error", "kamu-accounts", "kamu-accounts-repo-tests", @@ -4943,12 +4953,12 @@ dependencies = [ [[package]] name = "kamu-accounts-mysql" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-trait", "chrono", "database-common", - "dill", + "dill 0.9.1", "internal-error", "kamu-accounts", "kamu-accounts-repo-tests", @@ -4963,12 +4973,12 @@ dependencies = [ [[package]] name = "kamu-accounts-postgres" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-trait", "chrono", "database-common", - "dill", + "dill 0.9.1", "internal-error", "kamu-accounts", "kamu-accounts-repo-tests", @@ -4983,12 +4993,12 @@ dependencies = [ [[package]] name = "kamu-accounts-repo-tests" -version = "0.194.1" +version = "0.195.0" dependencies = [ "argon2", "chrono", "database-common", - "dill", + "dill 0.9.1", "kamu-accounts", "kamu-adapter-oauth", "opendatafabric", @@ -4999,18 +5009,17 @@ dependencies = [ [[package]] name = "kamu-accounts-services" -version = "0.194.1" +version = "0.195.0" dependencies = [ "argon2", "async-trait", "chrono", "database-common", - "dill", + "dill 0.9.1", "internal-error", "jsonwebtoken", "kamu-accounts", "kamu-accounts-inmem", - "kamu-core", "opendatafabric", "password-hash 0.5.0", "random-names", @@ -5018,6 +5027,7 @@ dependencies = [ "serde_json", "test-log", "thiserror", + "time-source", "tokio", "tracing", "uuid", @@ -5025,12 +5035,12 @@ dependencies = [ [[package]] name = "kamu-accounts-sqlite" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-trait", "chrono", "database-common", - "dill", + "dill 0.9.1", "internal-error", "kamu-accounts", "kamu-accounts-repo-tests", @@ -5045,19 +5055,21 @@ dependencies = [ [[package]] name = "kamu-adapter-auth-oso" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-trait", - "dill", - "event-bus", + "dill 0.9.1", + "internal-error", "kamu", "kamu-accounts", "kamu-core", + "messaging-outbox", "opendatafabric", "oso", "oso-derive", "tempfile", "test-log", + "time-source", "tokio", "tracing", "tracing-subscriber", @@ -5065,7 +5077,7 @@ dependencies = [ [[package]] name = "kamu-adapter-flight-sql" -version = "0.194.1" +version = "0.195.0" dependencies = [ "arrow-flight", "async-trait", @@ -5088,7 +5100,7 @@ dependencies = [ [[package]] name = "kamu-adapter-graphql" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-graphql", "async-trait", @@ -5097,8 +5109,7 @@ dependencies = [ "cron", "database-common", "datafusion", - "dill", - "event-bus", + "dill 0.9.1", "event-sourcing", "futures", "indoc 2.0.5", @@ -5118,6 +5129,7 @@ dependencies = [ "kamu-task-system", "kamu-task-system-inmem", "kamu-task-system-services", + "messaging-outbox", "mockall", "opendatafabric", "secrecy", @@ -5127,6 +5139,7 @@ dependencies = [ "test-group", "test-log", "thiserror", + "time-source", "tokio", "tokio-stream", "tracing", @@ -5137,7 +5150,7 @@ dependencies = [ [[package]] name = "kamu-adapter-http" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-trait", "aws-sdk-s3", @@ -5150,8 +5163,7 @@ dependencies = [ "database-common", "database-common-macros", "datafusion", - "dill", - "event-bus", + "dill 0.9.1", "flate2", "fs_extra", "futures", @@ -5159,13 +5171,16 @@ dependencies = [ "http-common", "hyper 0.14.30", "indoc 2.0.5", + "internal-error", "kamu", "kamu-accounts", "kamu-accounts-inmem", "kamu-accounts-services", + "kamu-core", "kamu-data-utils", "kamu-datasets-services", "kamu-ingest-datafusion", + "messaging-outbox", "mockall", "opendatafabric", "paste", @@ -5179,6 +5194,7 @@ dependencies = [ "test-group", "test-log", "thiserror", + "time-source", "tokio", "tokio-stream", "tokio-tungstenite 0.20.1", @@ -5193,19 +5209,18 @@ dependencies = [ [[package]] name = "kamu-adapter-oauth" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-trait", "chrono", - "dill", + "dill 0.9.1", "http 0.2.12", + "internal-error", "kamu-accounts", - "kamu-core", "opendatafabric", "reqwest 0.11.27", "serde", "serde_json", - "tempfile", "thiserror", "tokio", "tracing", @@ -5213,7 +5228,7 @@ dependencies = [ [[package]] name = "kamu-adapter-odata" -version = "0.194.1" +version = "0.195.0" dependencies = [ "axum", "chrono", @@ -5221,16 +5236,17 @@ dependencies = [ "database-common-macros", "datafusion", "datafusion-odata", - "dill", - "event-bus", + "dill 0.9.1", "futures", "http 0.2.12", "http-common", "hyper 0.14.30", "indoc 2.0.5", + "internal-error", "kamu", "kamu-accounts", "kamu-core", + "messaging-outbox", "opendatafabric", "quick-xml", "reqwest 0.11.27", @@ -5238,6 +5254,7 @@ dependencies = [ "tempfile", "test-group", "test-log", + "time-source", "tokio", "tower-http", "tracing", @@ -5246,7 +5263,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-trait", "internal-error", @@ -5256,10 +5273,10 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-inmem" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-trait", - "dill", + "dill 0.8.1", "kamu-auth-rebac", "kamu-auth-rebac-repo-tests", "test-log", @@ -5268,19 +5285,19 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-repo-tests" -version = "0.194.1" +version = "0.195.0" dependencies = [ - "dill", + "dill 0.8.1", "kamu-auth-rebac", "tokio", ] [[package]] name = "kamu-auth-rebac-services" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-trait", - "dill", + "dill 0.8.1", "futures", "kamu-auth-rebac", "opendatafabric", @@ -5288,7 +5305,7 @@ dependencies = [ [[package]] name = "kamu-cli" -version = "0.194.1" +version = "0.195.0" dependencies = [ "arrow-flight", "async-graphql", @@ -5307,10 +5324,9 @@ dependencies = [ "database-common", "database-common-macros", "datafusion", - "dill", + "dill 0.9.1", "dirs", "duration-string", - "event-bus", "fs_extra", "futures", "glob", @@ -5345,11 +5361,17 @@ dependencies = [ "kamu-datasets-services", "kamu-datasets-sqlite", "kamu-flow-system-inmem", + "kamu-flow-system-postgres", "kamu-flow-system-services", + "kamu-flow-system-sqlite", + "kamu-messaging-outbox-inmem", + "kamu-messaging-outbox-postgres", + "kamu-messaging-outbox-sqlite", "kamu-task-system-inmem", "kamu-task-system-services", "libc", "merge", + "messaging-outbox", "mime", "mime_guess", "minus", @@ -5372,6 +5394,7 @@ dependencies = [ "tempfile", "test-log", "thiserror", + "time-source", "tokio", "tokio-stream", "tokio-util", @@ -5393,7 +5416,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-common" -version = "0.194.1" +version = "0.195.0" dependencies = [ "chrono", "indoc 2.0.5", @@ -5413,7 +5436,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-common-macros" -version = "0.194.1" +version = "0.195.0" dependencies = [ "quote", "syn 2.0.74", @@ -5421,7 +5444,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-inmem" -version = "0.194.1" +version = "0.195.0" dependencies = [ "indoc 2.0.5", "kamu-cli-e2e-common", @@ -5434,7 +5457,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-mysql" -version = "0.194.1" +version = "0.195.0" dependencies = [ "indoc 2.0.5", "kamu-cli-e2e-common", @@ -5448,7 +5471,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-postgres" -version = "0.194.1" +version = "0.195.0" dependencies = [ "indoc 2.0.5", "kamu-cli-e2e-common", @@ -5462,7 +5485,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-repo-tests" -version = "0.194.1" +version = "0.195.0" dependencies = [ "chrono", "indoc 2.0.5", @@ -5478,7 +5501,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-sqlite" -version = "0.194.1" +version = "0.195.0" dependencies = [ "indoc 2.0.5", "kamu-cli-e2e-common", @@ -5492,7 +5515,7 @@ dependencies = [ [[package]] name = "kamu-cli-puppet" -version = "0.194.1" +version = "0.195.0" dependencies = [ "assert_cmd", "async-trait", @@ -5508,7 +5531,7 @@ dependencies = [ [[package]] name = "kamu-core" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-stream", "async-trait", @@ -5516,12 +5539,13 @@ dependencies = [ "chrono", "container-runtime", "datafusion", - "dill", + "dill 0.9.1", "futures", "http 0.2.12", "internal-error", "kamu-accounts", "kamu-datasets", + "messaging-outbox", "object_store", "opendatafabric", "pathdiff", @@ -5537,7 +5561,7 @@ dependencies = [ [[package]] name = "kamu-data-utils" -version = "0.194.1" +version = "0.195.0" dependencies = [ "arrow", "arrow-digest", @@ -5562,7 +5586,7 @@ dependencies = [ [[package]] name = "kamu-datafusion-cli" -version = "0.194.1" +version = "0.195.0" dependencies = [ "arrow", "async-trait", @@ -5584,7 +5608,7 @@ dependencies = [ [[package]] name = "kamu-datasets" -version = "0.194.1" +version = "0.195.0" dependencies = [ "aes-gcm", "async-trait", @@ -5603,12 +5627,12 @@ dependencies = [ [[package]] name = "kamu-datasets-inmem" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-trait", "chrono", "database-common", - "dill", + "dill 0.9.1", "internal-error", "kamu-datasets", "kamu-datasets-repo-tests", @@ -5623,12 +5647,12 @@ dependencies = [ [[package]] name = "kamu-datasets-postgres" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-trait", "chrono", "database-common", - "dill", + "dill 0.9.1", "internal-error", "kamu-datasets", "kamu-datasets-repo-tests", @@ -5644,11 +5668,11 @@ dependencies = [ [[package]] name = "kamu-datasets-repo-tests" -version = "0.194.1" +version = "0.195.0" dependencies = [ "chrono", "database-common", - "dill", + "dill 0.9.1", "kamu-datasets", "opendatafabric", "secrecy", @@ -5657,20 +5681,20 @@ dependencies = [ [[package]] name = "kamu-datasets-services" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-trait", "chrono", "database-common", - "dill", + "dill 0.9.1", "internal-error", - "kamu-core", "kamu-datasets", "opendatafabric", "secrecy", "serde", "serde_json", "thiserror", + "time-source", "tokio", "tracing", "uuid", @@ -5678,12 +5702,12 @@ dependencies = [ [[package]] name = "kamu-datasets-sqlite" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-trait", "chrono", "database-common", - "dill", + "dill 0.9.1", "internal-error", "kamu-datasets", "kamu-datasets-repo-tests", @@ -5699,7 +5723,7 @@ dependencies = [ [[package]] name = "kamu-flow-system" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-trait", "chrono", @@ -5712,6 +5736,7 @@ dependencies = [ "kamu-core", "kamu-task-system", "lazy_static", + "messaging-outbox", "opendatafabric", "serde", "serde_with", @@ -5724,13 +5749,13 @@ dependencies = [ [[package]] name = "kamu-flow-system-inmem" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-stream", "async-trait", "chrono", "cron", - "dill", + "dill 0.9.1", "futures", "kamu-flow-system", "kamu-flow-system-repo-tests", @@ -5751,13 +5776,13 @@ dependencies = [ [[package]] name = "kamu-flow-system-postgres" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-stream", "async-trait", "chrono", "database-common", - "dill", + "dill 0.9.1", "futures", "internal-error", "kamu-flow-system", @@ -5774,10 +5799,10 @@ dependencies = [ [[package]] name = "kamu-flow-system-repo-tests" -version = "0.194.1" +version = "0.195.0" dependencies = [ "chrono", - "dill", + "dill 0.9.1", "futures", "kamu-flow-system", "opendatafabric", @@ -5785,7 +5810,7 @@ dependencies = [ [[package]] name = "kamu-flow-system-services" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-stream", "async-trait", @@ -5793,10 +5818,10 @@ dependencies = [ "cron", "database-common", "database-common-macros", - "dill", - "event-bus", + "dill 0.9.1", "futures", "indoc 2.0.5", + "internal-error", "kamu", "kamu-accounts", "kamu-accounts-inmem", @@ -5807,14 +5832,17 @@ dependencies = [ "kamu-task-system", "kamu-task-system-inmem", "kamu-task-system-services", + "messaging-outbox", "mockall", "opendatafabric", "pretty_assertions", "serde", + "serde_json", "serde_with", "tempfile", "test-log", "thiserror", + "time-source", "tokio", "tokio-stream", "tracing", @@ -5824,13 +5852,13 @@ dependencies = [ [[package]] name = "kamu-flow-system-sqlite" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-stream", "async-trait", "chrono", "database-common", - "dill", + "dill 0.9.1", "futures", "internal-error", "kamu-flow-system", @@ -5847,7 +5875,7 @@ dependencies = [ [[package]] name = "kamu-ingest-datafusion" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-trait", "chrono", @@ -5882,9 +5910,83 @@ dependencies = [ "zip", ] +[[package]] +name = "kamu-messaging-outbox-inmem" +version = "0.195.0" +dependencies = [ + "async-trait", + "chrono", + "dill 0.9.1", + "internal-error", + "kamu-messaging-outbox-repo-tests", + "messaging-outbox", + "test-log", + "thiserror", + "tokio", + "tokio-stream", + "tracing", +] + +[[package]] +name = "kamu-messaging-outbox-postgres" +version = "0.195.0" +dependencies = [ + "async-stream", + "async-trait", + "chrono", + "database-common", + "dill 0.9.1", + "futures", + "internal-error", + "kamu-messaging-outbox-repo-tests", + "messaging-outbox", + "sqlx", + "test-group", + "test-log", + "thiserror", + "tokio", + "tracing", + "uuid", +] + +[[package]] +name = "kamu-messaging-outbox-repo-tests" +version = "0.195.0" +dependencies = [ + "chrono", + "database-common", + "dill 0.9.1", + "futures", + "messaging-outbox", + "rand", + "serde", + "serde_json", +] + +[[package]] +name = "kamu-messaging-outbox-sqlite" +version = "0.195.0" +dependencies = [ + "async-stream", + "async-trait", + "chrono", + "database-common", + "dill 0.9.1", + "futures", + "internal-error", + "kamu-messaging-outbox-repo-tests", + "messaging-outbox", + "sqlx", + "test-group", + "test-log", + "thiserror", + "tokio", + "tracing", +] + [[package]] name = "kamu-repo-tools" -version = "0.194.1" +version = "0.195.0" dependencies = [ "chrono", "clap", @@ -5899,13 +6001,14 @@ dependencies = [ [[package]] name = "kamu-task-system" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-trait", "chrono", "enum-variants", "event-sourcing", "kamu-core", + "messaging-outbox", "opendatafabric", "serde", "thiserror", @@ -5914,11 +6017,11 @@ dependencies = [ [[package]] name = "kamu-task-system-inmem" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-trait", "chrono", - "dill", + "dill 0.9.1", "futures", "kamu-task-system", "kamu-task-system-repo-tests", @@ -5929,13 +6032,13 @@ dependencies = [ [[package]] name = "kamu-task-system-postgres" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-stream", "async-trait", "chrono", "database-common", - "dill", + "dill 0.9.1", "futures", "internal-error", "kamu-task-system", @@ -5951,10 +6054,10 @@ dependencies = [ [[package]] name = "kamu-task-system-repo-tests" -version = "0.194.1" +version = "0.195.0" dependencies = [ "chrono", - "dill", + "dill 0.9.1", "futures", "kamu-task-system", "opendatafabric", @@ -5962,34 +6065,37 @@ dependencies = [ [[package]] name = "kamu-task-system-services" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-stream", "async-trait", "chrono", "database-common", - "dill", - "event-bus", + "dill 0.9.1", "futures", + "internal-error", "kamu-core", "kamu-datasets", "kamu-task-system", "kamu-task-system-inmem", + "messaging-outbox", "opendatafabric", + "serde_json", "test-log", + "time-source", "tokio", "tracing", ] [[package]] name = "kamu-task-system-sqlite" -version = "0.194.1" +version = "0.195.0" dependencies = [ "async-stream", "async-trait", "chrono", "database-common", - "dill", + "dill 0.9.1", "futures", "kamu-task-system", "kamu-task-system-repo-tests", @@ -6124,9 +6230,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.155" +version = "0.2.156" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "a5f43f184355eefb8d17fc948dbecf6c13be3c141f20d834ae842193a448c72a" [[package]] name = "libflate" @@ -6369,6 +6475,29 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "messaging-outbox" +version = "0.195.0" +dependencies = [ + "async-trait", + "chrono", + "database-common", + "dill 0.9.1", + "futures", + "internal-error", + "kamu-messaging-outbox-inmem", + "mockall", + "paste", + "serde", + "serde_json", + "test-log", + "thiserror", + "time-source", + "tokio", + "tokio-stream", + "tracing", +] + [[package]] name = "mime" version = "0.3.17" @@ -6486,7 +6615,7 @@ dependencies = [ [[package]] name = "multiformats" -version = "0.194.1" +version = "0.195.0" dependencies = [ "bs58", "digest 0.10.7", @@ -6508,6 +6637,15 @@ dependencies = [ "serde", ] +[[package]] +name = "multimap" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" +dependencies = [ + "serde", +] + [[package]] name = "nanoid" version = "0.4.0" @@ -6811,7 +6949,7 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "opendatafabric" -version = "0.194.1" +version = "0.195.0" dependencies = [ "arrow", "base64 0.22.1", @@ -7669,7 +7807,7 @@ dependencies = [ [[package]] name = "random-names" -version = "0.194.1" +version = "0.195.0" dependencies = [ "rand", ] @@ -8383,18 +8521,18 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.207" +version = "1.0.208" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5665e14a49a4ea1b91029ba7d3bca9f299e1f7cfa194388ccc20f14743e784f2" +checksum = "cff085d2cb684faa248efb494c39b68e522822ac0de72ccf08109abde717cfb2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.207" +version = "1.0.208" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6aea2634c86b0e8ef2cfdc0c340baede54ec27b1e46febd7f80dffb2aa44a00e" +checksum = "24008e81ff7613ed8e5ba0cfaf24e2c2f1e5b8a0495711e44fcd4882fca62bcf" dependencies = [ "proc-macro2", "quote", @@ -9296,6 +9434,17 @@ dependencies = [ "time-core", ] +[[package]] +name = "time-source" +version = "0.195.0" +dependencies = [ + "async-trait", + "chrono", + "dill 0.9.1", + "futures", + "tokio", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -9685,7 +9834,7 @@ dependencies = [ [[package]] name = "tracing-perfetto" -version = "0.194.1" +version = "0.195.0" dependencies = [ "conv", "serde", diff --git a/Cargo.toml b/Cargo.toml index 08cb9c5285..be372821b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,15 +7,16 @@ members = [ "src/utils/database-common-macros", "src/utils/datafusion-cli", "src/utils/enum-variants", - "src/utils/event-bus", "src/utils/event-sourcing", "src/utils/event-sourcing-macros", "src/utils/http-common", "src/utils/internal-error", "src/utils/kamu-cli-puppet", + "src/utils/messaging-outbox", "src/utils/multiformats", "src/utils/random-names", "src/utils/repo-tools", + "src/utils/time-source", "src/utils/tracing-perfetto", # Domain "src/domain/accounts/domain", @@ -57,6 +58,11 @@ members = [ ## ReBAC "src/infra/auth-rebac/inmem", "src/infra/auth-rebac/repo-tests", + ## Outbox + "src/infra/messaging-outbox/repo-tests", + "src/infra/messaging-outbox/inmem", + "src/infra/messaging-outbox/postgres", + "src/infra/messaging-outbox/sqlite", # Adapters "src/adapter/auth-oso", "src/adapter/flight-sql", @@ -81,84 +87,90 @@ resolver = "2" [workspace.dependencies] # Apps -kamu-cli = { version = "0.194.1", path = "src/app/cli", default-features = false } +kamu-cli = { version = "0.195.0", path = "src/app/cli", default-features = false } # Utils -container-runtime = { version = "0.194.1", path = "src/utils/container-runtime", default-features = false } -database-common = { version = "0.194.1", path = "src/utils/database-common", default-features = false } -database-common-macros = { version = "0.194.1", path = "src/utils/database-common-macros", default-features = false } -enum-variants = { version = "0.194.1", path = "src/utils/enum-variants", default-features = false } -event-bus = { version = "0.194.1", path = "src/utils/event-bus", default-features = false } -event-sourcing = { version = "0.194.1", path = "src/utils/event-sourcing", default-features = false } -event-sourcing-macros = { version = "0.194.1", path = "src/utils/event-sourcing-macros", default-features = false } -http-common = { version = "0.194.1", path = "src/utils/http-common", default-features = false } -internal-error = { version = "0.194.1", path = "src/utils/internal-error", default-features = false } -kamu-cli-puppet = { version = "0.194.1", path = "src/utils/kamu-cli-puppet", default-features = false } -kamu-data-utils = { version = "0.194.1", path = "src/utils/data-utils", default-features = false } -kamu-datafusion-cli = { version = "0.194.1", path = "src/utils/datafusion-cli", default-features = false } -multiformats = { version = "0.194.1", path = "src/utils/multiformats", default-features = false } -random-names = { version = "0.194.1", path = "src/utils/random-names", default-features = false } -tracing-perfetto = { version = "0.194.1", path = "src/utils/tracing-perfetto", default-features = false } +container-runtime = { version = "0.195.0", path = "src/utils/container-runtime", default-features = false } +database-common = { version = "0.195.0", path = "src/utils/database-common", default-features = false } +database-common-macros = { version = "0.195.0", path = "src/utils/database-common-macros", default-features = false } +enum-variants = { version = "0.195.0", path = "src/utils/enum-variants", default-features = false } +event-sourcing = { version = "0.195.0", path = "src/utils/event-sourcing", default-features = false } +event-sourcing-macros = { version = "0.195.0", path = "src/utils/event-sourcing-macros", default-features = false } +http-common = { version = "0.195.0", path = "src/utils/http-common", default-features = false } +internal-error = { version = "0.195.0", path = "src/utils/internal-error", default-features = false } +kamu-cli-puppet = { version = "0.195.0", path = "src/utils/kamu-cli-puppet", default-features = false } +kamu-data-utils = { version = "0.195.0", path = "src/utils/data-utils", default-features = false } +kamu-datafusion-cli = { version = "0.195.0", path = "src/utils/datafusion-cli", default-features = false } +messaging-outbox = { version = "0.195.0", path = "src/utils/messaging-outbox", default-features = false } +multiformats = { version = "0.195.0", path = "src/utils/multiformats", default-features = false } +random-names = { version = "0.195.0", path = "src/utils/random-names", default-features = false } +time-source = { version = "0.195.0", path = "src/utils/time-source", default-features = false } +tracing-perfetto = { version = "0.195.0", path = "src/utils/tracing-perfetto", default-features = false } # Domain -kamu-accounts = { version = "0.194.1", path = "src/domain/accounts/domain", default-features = false } -kamu-auth-rebac = { version = "0.194.1", path = "src/domain/auth-rebac/domain", default-features = false } -kamu-core = { version = "0.194.1", path = "src/domain/core", default-features = false } -kamu-datasets = { version = "0.194.1", path = "src/domain/datasets/domain", default-features = false } -kamu-flow-system = { version = "0.194.1", path = "src/domain/flow-system/domain", default-features = false } -kamu-task-system = { version = "0.194.1", path = "src/domain/task-system/domain", default-features = false } -opendatafabric = { version = "0.194.1", path = "src/domain/opendatafabric", default-features = false } +kamu-accounts = { version = "0.195.0", path = "src/domain/accounts/domain", default-features = false } +kamu-auth-rebac = { version = "0.195.0", path = "src/domain/auth-rebac/domain", default-features = false } +kamu-core = { version = "0.195.0", path = "src/domain/core", default-features = false } +kamu-datasets = { version = "0.195.0", path = "src/domain/datasets/domain", default-features = false } +kamu-flow-system = { version = "0.195.0", path = "src/domain/flow-system/domain", default-features = false } +kamu-task-system = { version = "0.195.0", path = "src/domain/task-system/domain", default-features = false } +opendatafabric = { version = "0.195.0", path = "src/domain/opendatafabric", default-features = false } # Domain service layer -kamu-accounts-services = { version = "0.194.1", path = "src/domain/accounts/services", default-features = false } -kamu-auth-rebac-services = { version = "0.194.1", path = "src/domain/auth-rebac/services", default-features = false } -kamu-datasets-services = { version = "0.194.1", path = "src/domain/datasets/services", default-features = false } -kamu-flow-system-services = { version = "0.194.1", path = "src/domain/flow-system/services", default-features = false } -kamu-task-system-services = { version = "0.194.1", path = "src/domain/task-system/services", default-features = false } +kamu-accounts-services = { version = "0.195.0", path = "src/domain/accounts/services", default-features = false } +kamu-auth-rebac-services = { version = "0.195.0", path = "src/domain/auth-rebac/services", default-features = false } +kamu-datasets-services = { version = "0.195.0", path = "src/domain/datasets/services", default-features = false } +kamu-flow-system-services = { version = "0.195.0", path = "src/domain/flow-system/services", default-features = false } +kamu-task-system-services = { version = "0.195.0", path = "src/domain/task-system/services", default-features = false } # Infra -kamu = { version = "0.194.1", path = "src/infra/core", default-features = false } -kamu-ingest-datafusion = { version = "0.194.1", path = "src/infra/ingest-datafusion", default-features = false } +kamu = { version = "0.195.0", path = "src/infra/core", default-features = false } +kamu-ingest-datafusion = { version = "0.195.0", path = "src/infra/ingest-datafusion", default-features = false } ## Flow System -kamu-flow-system-repo-tests = { version = "0.194.1", path = "src/infra/flow-system/repo-tests", default-features = false } -kamu-flow-system-inmem = { version = "0.194.1", path = "src/infra/flow-system/inmem", default-features = false } -kamu-flow-system-postgres = { version = "0.194.1", path = "src/infra/flow-system/postgres", default-features = false } -kamu-flow-system-sqlite = { version = "0.194.1", path = "src/infra/flow-system/sqlite", default-features = false } +kamu-flow-system-repo-tests = { version = "0.195.0", path = "src/infra/flow-system/repo-tests", default-features = false } +kamu-flow-system-inmem = { version = "0.195.0", path = "src/infra/flow-system/inmem", default-features = false } +kamu-flow-system-postgres = { version = "0.195.0", path = "src/infra/flow-system/postgres", default-features = false } +kamu-flow-system-sqlite = { version = "0.195.0", path = "src/infra/flow-system/sqlite", default-features = false } ## Accounts -kamu-accounts-inmem = { version = "0.194.1", path = "src/infra/accounts/inmem", default-features = false } -kamu-accounts-mysql = { version = "0.194.1", path = "src/infra/accounts/mysql", default-features = false } -kamu-accounts-postgres = { version = "0.194.1", path = "src/infra/accounts/postgres", default-features = false } -kamu-accounts-sqlite = { version = "0.194.1", path = "src/infra/accounts/sqlite", default-features = false } -kamu-accounts-repo-tests = { version = "0.194.1", path = "src/infra/accounts/repo-tests", default-features = false } +kamu-accounts-inmem = { version = "0.195.0", path = "src/infra/accounts/inmem", default-features = false } +kamu-accounts-mysql = { version = "0.195.0", path = "src/infra/accounts/mysql", default-features = false } +kamu-accounts-postgres = { version = "0.195.0", path = "src/infra/accounts/postgres", default-features = false } +kamu-accounts-sqlite = { version = "0.195.0", path = "src/infra/accounts/sqlite", default-features = false } +kamu-accounts-repo-tests = { version = "0.195.0", path = "src/infra/accounts/repo-tests", default-features = false } ## Datasets -kamu-datasets-inmem = { version = "0.194.1", path = "src/infra/datasets/inmem", default-features = false } -kamu-datasets-postgres = { version = "0.194.1", path = "src/infra/datasets/postgres", default-features = false } -kamu-datasets-sqlite = { version = "0.194.1", path = "src/infra/datasets/sqlite", default-features = false } -kamu-datasets-repo-tests = { version = "0.194.1", path = "src/infra/datasets/repo-tests", default-features = false } +kamu-datasets-inmem = { version = "0.195.0", path = "src/infra/datasets/inmem", default-features = false } +kamu-datasets-postgres = { version = "0.195.0", path = "src/infra/datasets/postgres", default-features = false } +kamu-datasets-sqlite = { version = "0.195.0", path = "src/infra/datasets/sqlite", default-features = false } +kamu-datasets-repo-tests = { version = "0.195.0", path = "src/infra/datasets/repo-tests", default-features = false } ## Task System -kamu-task-system-inmem = { version = "0.194.1", path = "src/infra/task-system/inmem", default-features = false } -kamu-task-system-postgres = { version = "0.194.1", path = "src/infra/task-system/postgres", default-features = false } -kamu-task-system-sqlite = { version = "0.194.1", path = "src/infra/task-system/sqlite", default-features = false } -kamu-task-system-repo-tests = { version = "0.194.1", path = "src/infra/task-system/repo-tests", default-features = false } +kamu-task-system-inmem = { version = "0.195.0", path = "src/infra/task-system/inmem", default-features = false } +kamu-task-system-postgres = { version = "0.195.0", path = "src/infra/task-system/postgres", default-features = false } +kamu-task-system-sqlite = { version = "0.195.0", path = "src/infra/task-system/sqlite", default-features = false } +kamu-task-system-repo-tests = { version = "0.195.0", path = "src/infra/task-system/repo-tests", default-features = false } ## ReBAC -kamu-auth-rebac-inmem = { version = "0.194.1", path = "src/infra/auth-rebac/inmem", default-features = false } -kamu-auth-rebac-repo-tests = { version = "0.194.1", path = "src/infra/auth-rebac/repo-tests", default-features = false } +kamu-auth-rebac-inmem = { version = "0.195.0", path = "src/infra/auth-rebac/inmem", default-features = false } +kamu-auth-rebac-repo-tests = { version = "0.195.0", path = "src/infra/auth-rebac/repo-tests", default-features = false } +## Outbox +kamu-messaging-outbox-inmem = { version = "0.195.0", path = "src/infra/messaging-outbox/inmem", default-features = false } +kamu-messaging-outbox-postgres = { version = "0.195.0", path = "src/infra/messaging-outbox/postgres", default-features = false } +kamu-messaging-outbox-sqlite = { version = "0.195.0", path = "src/infra/messaging-outbox/sqlite", default-features = false } +kamu-messaging-outbox-repo-tests = { version = "0.195.0", path = "src/infra/messaging-outbox/repo-tests", default-features = false } # Adapters -kamu-adapter-auth-oso = { version = "0.194.1", path = "src/adapter/auth-oso", default-features = false } -kamu-adapter-flight-sql = { version = "0.194.1", path = "src/adapter/flight-sql", default-features = false } -kamu-adapter-graphql = { version = "0.194.1", path = "src/adapter/graphql", default-features = false } -kamu-adapter-http = { version = "0.194.1", path = "src/adapter/http", default-features = false } -kamu-adapter-odata = { version = "0.194.1", path = "src/adapter/odata", default-features = false } -kamu-adapter-oauth = { version = "0.194.1", path = "src/adapter/oauth", default-features = false } +kamu-adapter-auth-oso = { version = "0.195.0", path = "src/adapter/auth-oso", default-features = false } +kamu-adapter-flight-sql = { version = "0.195.0", path = "src/adapter/flight-sql", default-features = false } +kamu-adapter-graphql = { version = "0.195.0", path = "src/adapter/graphql", default-features = false } +kamu-adapter-http = { version = "0.195.0", path = "src/adapter/http", default-features = false } +kamu-adapter-odata = { version = "0.195.0", path = "src/adapter/odata", default-features = false } +kamu-adapter-oauth = { version = "0.195.0", path = "src/adapter/oauth", default-features = false } # E2E -kamu-cli-e2e-common = { version = "0.194.1", path = "src/e2e/app/cli/common", default-features = false } -kamu-cli-e2e-common-macros = { version = "0.194.1", path = "src/e2e/app/cli/common-macros", default-features = false } -kamu-cli-e2e-repo-tests = { version = "0.194.1", path = "src/e2e/app/cli/repo-tests", default-features = false } +kamu-cli-e2e-common = { version = "0.195.0", path = "src/e2e/app/cli/common", default-features = false } +kamu-cli-e2e-common-macros = { version = "0.195.0", path = "src/e2e/app/cli/common-macros", default-features = false } +kamu-cli-e2e-repo-tests = { version = "0.195.0", path = "src/e2e/app/cli/repo-tests", default-features = false } [workspace.package] -version = "0.194.1" +version = "0.195.0" edition = "2021" homepage = "https://github.com/kamu-data/kamu-cli" repository = "https://github.com/kamu-data/kamu-cli" diff --git a/LICENSE.txt b/LICENSE.txt index b6c5874e67..660e28bcfc 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -11,7 +11,7 @@ Business Source License 1.1 Licensor: Kamu Data, Inc. -Licensed Work: Kamu CLI Version 0.194.1 +Licensed Work: Kamu CLI Version 0.195.0 The Licensed Work is © 2023 Kamu Data, Inc. Additional Use Grant: You may use the Licensed Work for any purpose, @@ -24,7 +24,7 @@ Additional Use Grant: You may use the Licensed Work for any purpose, Licensed Work where data or transformations are controlled by such third parties. -Change Date: 2028-08-13 +Change Date: 2028-08-16 Change License: Apache License, Version 2.0 diff --git a/Makefile b/Makefile index e3a6efe8a1..56cedecde9 100644 --- a/Makefile +++ b/Makefile @@ -3,9 +3,12 @@ ODF_CRATE_DIR=./src/domain/opendatafabric LICENSE_HEADER=docs/license_header.txt TEST_LOG_PARAMS=RUST_LOG_SPAN_EVENTS=new,close RUST_LOG=debug -POSTGRES_CRATES := ./src/infra/accounts/postgres ./src/infra/task-system/postgres ./src/infra/flow-system/postgres ./src/e2e/app/cli/postgres ./src/infra/datasets/postgres +POSTGRES_CRATES := ./src/infra/accounts/postgres ./src/infra/datasets/postgres ./src/infra/flow-system/postgres ./src/infra/messaging-outbox/postgres ./src/infra/task-system/postgres ./src/e2e/app/cli/postgres + MYSQL_CRATES := ./src/infra/accounts/mysql ./src/e2e/app/cli/mysql -SQLITE_CRATES := ./src/infra/accounts/sqlite ./src/infra/task-system/sqlite ./src/infra/flow-system/sqlite ./src/e2e/app/cli/sqlite ./src/infra/datasets/sqlite + +SQLITE_CRATES := ./src/infra/accounts/sqlite ./src/infra/datasets/sqlite ./src/infra/task-system/sqlite ./src/infra/flow-system/sqlite ./src/infra/messaging-outbox/sqlite ./src/e2e/app/cli/sqlite + ALL_DATABASE_CRATES := $(POSTGRES_CRATES) $(MYSQL_CRATES) $(SQLITE_CRATES) MIGRATION_DIRS := ./migrations/mysql ./migrations/postgres ./migrations/sqlite diff --git a/migrations/postgres/20240710191232_outbox_messages_consumptions.sql b/migrations/postgres/20240710191232_outbox_messages_consumptions.sql new file mode 100644 index 0000000000..80f218f8d7 --- /dev/null +++ b/migrations/postgres/20240710191232_outbox_messages_consumptions.sql @@ -0,0 +1,17 @@ +CREATE SEQUENCE outbox_message_id_seq AS BIGINT; + +CREATE TABLE outbox_messages( + message_id BIGINT PRIMARY KEY DEFAULT NEXTVAL('outbox_message_id_seq'), + producer_name VARCHAR(200) NOT NULL, + content_json JSONB NOT NULL, + occurred_on timestamptz NOT NULL +); + +CREATE INDEX outbox_messages_producer_name_idx ON outbox_messages(producer_name); + +CREATE TABLE outbox_message_consumptions( + consumer_name VARCHAR(200) NOT NULL, + producer_name VARCHAR(200) NOT NULL, + last_consumed_message_id BIGINT NOT NULL, + PRIMARY KEY (consumer_name, producer_name) +); diff --git a/migrations/sqlite/20240710205713_outbox_messages_consumptions.sql b/migrations/sqlite/20240710205713_outbox_messages_consumptions.sql new file mode 100644 index 0000000000..8ad974746d --- /dev/null +++ b/migrations/sqlite/20240710205713_outbox_messages_consumptions.sql @@ -0,0 +1,16 @@ +CREATE TABLE outbox_messages( + message_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + producer_name VARCHAR(200) NOT NULL, + content_json JSONB NOT NULL, + occurred_on timestamptz NOT NULL +); + +CREATE INDEX outbox_messages_producer_name_idx ON outbox_messages(producer_name); + +CREATE TABLE outbox_message_consumptions( + consumer_name VARCHAR(200) NOT NULL, + producer_name VARCHAR(200) NOT NULL, + last_consumed_message_id BIGINT NOT NULL, + PRIMARY KEY(consumer_name, producer_name) +); + diff --git a/src/adapter/auth-oso/Cargo.toml b/src/adapter/auth-oso/Cargo.toml index 608b000f77..f6fdd9d320 100644 --- a/src/adapter/auth-oso/Cargo.toml +++ b/src/adapter/auth-oso/Cargo.toml @@ -21,12 +21,14 @@ workspace = true doctest = false [dependencies] +internal-error = { workspace = true } +messaging-outbox = { workspace = true } opendatafabric = { workspace = true } kamu-accounts = { workspace = true } kamu-core = { workspace = true } async-trait = "0.1" -dill = "0.8" +dill = "0.9" # Authorization oso = "0.27" @@ -34,7 +36,7 @@ oso-derive = "0.27" [dev-dependencies] kamu = { workspace = true } -event-bus = { workspace = true } +time-source = { workspace = true } tempfile = "3" test-log = { version = "0.2", features = ["trace"] } diff --git a/src/adapter/auth-oso/src/oso_dataset_authorizer.rs b/src/adapter/auth-oso/src/oso_dataset_authorizer.rs index 7f622e3e8a..be600819fc 100644 --- a/src/adapter/auth-oso/src/oso_dataset_authorizer.rs +++ b/src/adapter/auth-oso/src/oso_dataset_authorizer.rs @@ -12,9 +12,10 @@ use std::str::FromStr; use std::sync::Arc; use dill::*; +use internal_error::ErrorIntoInternal; use kamu_accounts::{CurrentAccountSubject, DEFAULT_ACCOUNT_NAME_STR}; use kamu_core::auth::*; -use kamu_core::{AccessError, ErrorIntoInternal}; +use kamu_core::AccessError; use opendatafabric::DatasetHandle; use oso::Oso; diff --git a/src/adapter/auth-oso/tests/tests/test_oso_dataset_authorizer.rs b/src/adapter/auth-oso/tests/tests/test_oso_dataset_authorizer.rs index 9733ea6e7a..f3b7050ac1 100644 --- a/src/adapter/auth-oso/tests/tests/test_oso_dataset_authorizer.rs +++ b/src/adapter/auth-oso/tests/tests/test_oso_dataset_authorizer.rs @@ -11,16 +11,17 @@ use std::assert_matches::assert_matches; use std::collections::HashSet; use std::sync::Arc; -use dill::Component; -use event_bus::EventBus; +use dill::{Catalog, Component}; use kamu::testing::MetadataFactory; -use kamu::{DatasetRepositoryLocalFs, DependencyGraphServiceInMemory}; +use kamu::{CreateDatasetUseCaseImpl, DatasetRepositoryLocalFs, DatasetRepositoryWriter}; use kamu_accounts::CurrentAccountSubject; use kamu_adapter_auth_oso::{KamuAuthOso, OsoDatasetAuthorizer}; use kamu_core::auth::{DatasetAction, DatasetActionAuthorizer, DatasetActionUnauthorizedError}; -use kamu_core::{AccessError, DatasetRepository, SystemTimeSourceDefault}; +use kamu_core::{AccessError, CreateDatasetUseCase, DatasetRepository}; +use messaging_outbox::DummyOutboxImpl; use opendatafabric::{AccountID, AccountName, DatasetAlias, DatasetHandle, DatasetKind}; use tempfile::TempDir; +use time_source::SystemTimeSourceDefault; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -95,7 +96,7 @@ async fn test_guest_can_read_but_not_write() { #[allow(dead_code)] pub struct DatasetAuthorizerHarness { tempdir: TempDir, - dataset_repository: Arc, + catalog: Catalog, dataset_authorizer: Arc, } @@ -107,7 +108,7 @@ impl DatasetAuthorizerHarness { let catalog = dill::CatalogBuilder::new() .add::() - .add::() + .add::() .add_value(CurrentAccountSubject::logged( AccountID::new_seeded_ed25519(current_account_name.as_bytes()), AccountName::new_unchecked(current_account_name), @@ -115,28 +116,30 @@ impl DatasetAuthorizerHarness { )) .add::() .add::() - .add::() .add_builder( DatasetRepositoryLocalFs::builder() .with_root(datasets_dir) .with_multi_tenant(true), ) .bind::() + .bind::() + .add::() .build(); - let dataset_repository = catalog.get_one::().unwrap(); let dataset_authorizer = catalog.get_one::().unwrap(); Self { tempdir, - dataset_repository, + catalog, dataset_authorizer, } } pub async fn create_dataset(&self, alias: &DatasetAlias) -> DatasetHandle { - self.dataset_repository - .create_dataset( + let create_dataset = self.catalog.get_one::().unwrap(); + + create_dataset + .execute( alias, MetadataFactory::metadata_block(MetadataFactory::seed(DatasetKind::Root).build()) .build_typed(), diff --git a/src/adapter/graphql/Cargo.toml b/src/adapter/graphql/Cargo.toml index 96cae0d6da..c00f530d72 100644 --- a/src/adapter/graphql/Cargo.toml +++ b/src/adapter/graphql/Cargo.toml @@ -48,7 +48,7 @@ chrono = "0.4" datafusion = { version = "41", default-features = false, features = [ "serde", ] } # TODO: Currently needed for type conversions but ideally should be encapsulated by kamu-core -dill = "0.8" +dill = "0.9" futures = "0.3" secrecy = "0.8" serde = { version = "1", default-features = false } @@ -64,7 +64,7 @@ uuid = { version = "1", default-features = false } [dev-dependencies] # TODO: Limit to mock or in-memory implementations only container-runtime = { workspace = true } -event-bus = { workspace = true } +messaging-outbox = { workspace = true } kamu-accounts-inmem = { workspace = true } kamu-accounts-services = { workspace = true } kamu-datasets-services = { workspace = true } @@ -72,6 +72,7 @@ kamu-datasets-inmem = { workspace = true } kamu-flow-system-inmem = { workspace = true } kamu-task-system-inmem = { workspace = true } kamu-task-system-services = { workspace = true } +time-source = { workspace = true } indoc = "2" diff --git a/src/adapter/graphql/src/mutations/dataset_metadata_mut.rs b/src/adapter/graphql/src/mutations/dataset_metadata_mut.rs index 6632da1302..b671e9d797 100644 --- a/src/adapter/graphql/src/mutations/dataset_metadata_mut.rs +++ b/src/adapter/graphql/src/mutations/dataset_metadata_mut.rs @@ -7,13 +7,18 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use kamu_core::{self as domain, MetadataChainExt, SearchSetAttachmentsVisitor}; +use kamu_core::{ + self as domain, + CommitDatasetEventUseCase, + MetadataChainExt, + SearchSetAttachmentsVisitor, +}; use opendatafabric as odf; use super::{CommitResultAppendError, CommitResultSuccess, NoChanges}; use crate::mutations::MetadataChainMut; use crate::prelude::*; -use crate::utils::check_dataset_write_access; +use crate::utils::make_dataset_access_error; use crate::LoggedInGuard; pub struct DatasetMetadataMut { @@ -28,13 +33,10 @@ impl DatasetMetadataMut { } #[graphql(skip)] - async fn get_dataset(&self, ctx: &Context<'_>) -> Result> { + fn get_dataset(&self, ctx: &Context<'_>) -> std::sync::Arc { + // TODO: cut off this dependency - extract a higher level use case let dataset_repo = from_catalog::(ctx).unwrap(); - let dataset = dataset_repo - .get_dataset(&self.dataset_handle.as_local_ref()) - .await - .int_err()?; - Ok(dataset) + dataset_repo.get_dataset_by_handle(&self.dataset_handle) } /// Access to the mutable metadata chain of the dataset @@ -49,9 +51,7 @@ impl DatasetMetadataMut { ctx: &Context<'_>, content: Option, ) -> Result { - check_dataset_write_access(ctx, &self.dataset_handle).await?; - - let dataset = self.get_dataset(ctx).await?; + let dataset = self.get_dataset(ctx); let old_attachments = dataset .as_metadata_chain() @@ -112,8 +112,14 @@ impl DatasetMetadataMut { attachments: new_attachments.into(), }; - let result = match dataset - .commit_event(event.into(), domain::CommitOpts::default()) + let commit_event = from_catalog::(ctx).unwrap(); + + let result = match commit_event + .execute( + &self.dataset_handle, + event.into(), + domain::CommitOpts::default(), + ) .await { Ok(result) => UpdateReadmeResult::Success(CommitResultSuccess { @@ -130,6 +136,9 @@ impl DatasetMetadataMut { message: e.to_string(), }) } + Err(domain::CommitError::Access(_)) => { + return Err(make_dataset_access_error(&self.dataset_handle)) + } Err(e @ domain::CommitError::Internal(_)) => return Err(e.int_err().into()), }; diff --git a/src/adapter/graphql/src/mutations/dataset_mut.rs b/src/adapter/graphql/src/mutations/dataset_mut.rs index beea1694c3..114261fc06 100644 --- a/src/adapter/graphql/src/mutations/dataset_mut.rs +++ b/src/adapter/graphql/src/mutations/dataset_mut.rs @@ -62,9 +62,9 @@ impl DatasetMut { })); } - let dataset_repo = from_catalog::(ctx).unwrap(); - match dataset_repo - .rename_dataset(&self.dataset_handle.as_local_ref(), &new_name) + let rename_dataset = from_catalog::(ctx).unwrap(); + match rename_dataset + .execute(&self.dataset_handle.as_local_ref(), &new_name) .await { Ok(_) => Ok(RenameResult::Success(RenameResultSuccess { @@ -89,9 +89,9 @@ impl DatasetMut { /// Delete the dataset #[graphql(guard = "LoggedInGuard::new()")] async fn delete(&self, ctx: &Context<'_>) -> Result { - let dataset_repo = from_catalog::(ctx).unwrap(); - match dataset_repo - .delete_dataset(&self.dataset_handle.as_local_ref()) + let delete_dataset = from_catalog::(ctx).unwrap(); + match delete_dataset + .execute_via_handle(&self.dataset_handle) .await { Ok(_) => Ok(DeleteResult::Success(DeleteResultSuccess { diff --git a/src/adapter/graphql/src/mutations/datasets_mut.rs b/src/adapter/graphql/src/mutations/datasets_mut.rs index e8c7a750a3..0b67daaaeb 100644 --- a/src/adapter/graphql/src/mutations/datasets_mut.rs +++ b/src/adapter/graphql/src/mutations/datasets_mut.rs @@ -107,9 +107,10 @@ impl DatasetsMut { ctx: &Context<'_>, snapshot: odf::DatasetSnapshot, ) -> Result { - let dataset_repo = from_catalog::(ctx).unwrap(); + let create_from_snapshot = + from_catalog::(ctx).unwrap(); - let result = match dataset_repo.create_dataset_from_snapshot(snapshot).await { + let result = match create_from_snapshot.execute(snapshot).await { Ok(result) => { let dataset = Dataset::from_ref(ctx, &result.dataset_handle.as_local_ref()).await?; CreateDatasetFromSnapshotResult::Success(CreateDatasetResultSuccess { dataset }) diff --git a/src/adapter/graphql/src/mutations/flows_mut/flows_mut_utils.rs b/src/adapter/graphql/src/mutations/flows_mut/flows_mut_utils.rs index 69ec081d35..1497a37c9e 100644 --- a/src/adapter/graphql/src/mutations/flows_mut/flows_mut_utils.rs +++ b/src/adapter/graphql/src/mutations/flows_mut/flows_mut_utils.rs @@ -71,7 +71,7 @@ pub(crate) async fn ensure_expected_dataset_kind( let dataset_flow_type: kamu_flow_system::DatasetFlowType = dataset_flow_type.into(); match dataset_flow_type.dataset_kind_restriction() { Some(expected_kind) => { - let dataset = utils::get_dataset(ctx, dataset_handle).await?; + let dataset = utils::get_dataset(ctx, dataset_handle); let dataset_kind = dataset .get_summary(GetSummaryOpts::default()) @@ -138,10 +138,7 @@ pub(crate) async fn ensure_flow_preconditions( let dataset_repo = from_catalog::(ctx).unwrap(); - let dataset = dataset_repo - .get_dataset(&dataset_handle.as_local_ref()) - .await - .int_err()?; + let dataset = dataset_repo.get_dataset_by_handle(dataset_handle); let current_head_hash_maybe = dataset .as_metadata_chain() .try_get_ref(&kamu_core::BlockRef::Head) diff --git a/src/adapter/graphql/src/mutations/metadata_chain_mut.rs b/src/adapter/graphql/src/mutations/metadata_chain_mut.rs index d06e271e9f..596cd18373 100644 --- a/src/adapter/graphql/src/mutations/metadata_chain_mut.rs +++ b/src/adapter/graphql/src/mutations/metadata_chain_mut.rs @@ -7,11 +7,11 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use kamu_core::{self as domain}; +use kamu_core::{self as domain, CommitDatasetEventUseCase}; use opendatafabric as odf; use crate::prelude::*; -use crate::utils::check_dataset_write_access; +use crate::utils::make_dataset_access_error; use crate::LoggedInGuard; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -29,16 +29,6 @@ impl MetadataChainMut { Self { dataset_handle } } - #[graphql(skip)] - async fn get_dataset(&self, ctx: &Context<'_>) -> Result> { - let dataset_repo = from_catalog::(ctx).unwrap(); - let dataset = dataset_repo - .get_dataset(&self.dataset_handle.as_local_ref()) - .await - .int_err()?; - Ok(dataset) - } - /// Commits new event to the metadata chain #[tracing::instrument(level = "info", skip_all)] #[graphql(guard = "LoggedInGuard::new()")] @@ -48,8 +38,6 @@ impl MetadataChainMut { event: String, event_format: MetadataManifestFormat, ) -> Result { - check_dataset_write_access(ctx, &self.dataset_handle).await?; - let event = match event_format { MetadataManifestFormat::Yaml => { let de = odf::serde::yaml::YamlMetadataEventDeserializer; @@ -68,10 +56,10 @@ impl MetadataChainMut { } }; - let dataset = self.get_dataset(ctx).await?; + let commit_dataset_event = from_catalog::(ctx).unwrap(); - let result = match dataset - .commit_event(event, domain::CommitOpts::default()) + let result = match commit_dataset_event + .execute(&self.dataset_handle, event, domain::CommitOpts::default()) .await { Ok(result) => CommitResult::Success(CommitResultSuccess { @@ -88,6 +76,9 @@ impl MetadataChainMut { message: e.to_string(), }) } + Err(domain::CommitError::Access(_)) => { + return Err(make_dataset_access_error(&self.dataset_handle)) + } Err(e @ domain::CommitError::Internal(_)) => return Err(e.int_err().into()), }; diff --git a/src/adapter/graphql/src/queries/datasets/dataset.rs b/src/adapter/graphql/src/queries/datasets/dataset.rs index 865f3e23f0..8ecbace6dc 100644 --- a/src/adapter/graphql/src/queries/datasets/dataset.rs +++ b/src/adapter/graphql/src/queries/datasets/dataset.rs @@ -48,13 +48,9 @@ impl Dataset { } #[graphql(skip)] - async fn get_dataset(&self, ctx: &Context<'_>) -> Result> { + fn get_dataset(&self, ctx: &Context<'_>) -> std::sync::Arc { let dataset_repo = from_catalog::(ctx).unwrap(); - let dataset = dataset_repo - .get_dataset(&self.dataset_handle.as_local_ref()) - .await - .int_err()?; - Ok(dataset) + dataset_repo.get_dataset_by_handle(&self.dataset_handle) } /// Unique identifier of the dataset @@ -81,7 +77,7 @@ impl Dataset { /// Returns the kind of dataset (Root or Derivative) async fn kind(&self, ctx: &Context<'_>) -> Result { - let dataset = self.get_dataset(ctx).await?; + let dataset = self.get_dataset(ctx); let summary = dataset .get_summary(domain::GetSummaryOpts::default()) .await @@ -115,7 +111,7 @@ impl Dataset { // TODO: PERF: Avoid traversing the entire chain /// Creation time of the first metadata block in the chain async fn created_at(&self, ctx: &Context<'_>) -> Result> { - let dataset = self.get_dataset(ctx).await?; + let dataset = self.get_dataset(ctx); Ok(dataset .as_metadata_chain() @@ -129,7 +125,7 @@ impl Dataset { /// Creation time of the most recent metadata block in the chain async fn last_updated_at(&self, ctx: &Context<'_>) -> Result> { - let dataset = self.get_dataset(ctx).await?; + let dataset = self.get_dataset(ctx); Ok(dataset .as_metadata_chain() diff --git a/src/adapter/graphql/src/queries/datasets/dataset_data.rs b/src/adapter/graphql/src/queries/datasets/dataset_data.rs index 55690bc389..b528c2457b 100644 --- a/src/adapter/graphql/src/queries/datasets/dataset_data.rs +++ b/src/adapter/graphql/src/queries/datasets/dataset_data.rs @@ -28,10 +28,7 @@ impl DatasetData { /// Total number of records in this dataset async fn num_records_total(&self, ctx: &Context<'_>) -> Result { let dataset_repo = from_catalog::(ctx).unwrap(); - let dataset = dataset_repo - .get_dataset(&self.dataset_handle.as_local_ref()) - .await - .int_err()?; + let dataset = dataset_repo.get_dataset_by_handle(&self.dataset_handle); let summary = dataset .get_summary(GetSummaryOpts::default()) .await @@ -43,10 +40,7 @@ impl DatasetData { /// caching async fn estimated_size(&self, ctx: &Context<'_>) -> Result { let dataset_repo = from_catalog::(ctx).unwrap(); - let dataset = dataset_repo - .get_dataset(&self.dataset_handle.as_local_ref()) - .await - .int_err()?; + let dataset = dataset_repo.get_dataset_by_handle(&self.dataset_handle); let summary = dataset .get_summary(GetSummaryOpts::default()) .await diff --git a/src/adapter/graphql/src/queries/datasets/dataset_metadata.rs b/src/adapter/graphql/src/queries/datasets/dataset_metadata.rs index 8c98a7d7d4..9993e8f30a 100644 --- a/src/adapter/graphql/src/queries/datasets/dataset_metadata.rs +++ b/src/adapter/graphql/src/queries/datasets/dataset_metadata.rs @@ -33,13 +33,9 @@ impl DatasetMetadata { } #[graphql(skip)] - async fn get_dataset(&self, ctx: &Context<'_>) -> Result> { + fn get_dataset(&self, ctx: &Context<'_>) -> std::sync::Arc { let dataset_repo = from_catalog::(ctx).unwrap(); - let dataset = dataset_repo - .get_dataset(&self.dataset_handle.as_local_ref()) - .await - .int_err()?; - Ok(dataset) + dataset_repo.get_dataset_by_handle(&self.dataset_handle) } /// Access to the temporal metadata chain of the dataset @@ -49,7 +45,7 @@ impl DatasetMetadata { /// Last recorded watermark async fn current_watermark(&self, ctx: &Context<'_>) -> Result>> { - let dataset = self.get_dataset(ctx).await?; + let dataset = self.get_dataset(ctx); Ok(dataset .as_metadata_chain() @@ -197,7 +193,7 @@ impl DatasetMetadata { /// Current descriptive information about the dataset async fn current_info(&self, ctx: &Context<'_>) -> Result { - let dataset = self.get_dataset(ctx).await?; + let dataset = self.get_dataset(ctx); Ok(dataset .as_metadata_chain() @@ -217,7 +213,7 @@ impl DatasetMetadata { /// Current readme file as discovered from attachments associated with the /// dataset async fn current_readme(&self, ctx: &Context<'_>) -> Result> { - let dataset = self.get_dataset(ctx).await?; + let dataset = self.get_dataset(ctx); Ok(dataset .as_metadata_chain() @@ -238,7 +234,7 @@ impl DatasetMetadata { /// Current license associated with the dataset async fn current_license(&self, ctx: &Context<'_>) -> Result> { - let dataset = self.get_dataset(ctx).await?; + let dataset = self.get_dataset(ctx); Ok(dataset .as_metadata_chain() @@ -251,7 +247,7 @@ impl DatasetMetadata { /// Current vocabulary associated with the dataset async fn current_vocab(&self, ctx: &Context<'_>) -> Result> { - let dataset = self.get_dataset(ctx).await?; + let dataset = self.get_dataset(ctx); Ok(dataset .as_metadata_chain() diff --git a/src/adapter/graphql/src/queries/datasets/metadata_chain.rs b/src/adapter/graphql/src/queries/datasets/metadata_chain.rs index c2f8c40a8d..29d777fb64 100644 --- a/src/adapter/graphql/src/queries/datasets/metadata_chain.rs +++ b/src/adapter/graphql/src/queries/datasets/metadata_chain.rs @@ -42,19 +42,15 @@ impl MetadataChain { } #[graphql(skip)] - async fn get_dataset(&self, ctx: &Context<'_>) -> Result> { + fn get_dataset(&self, ctx: &Context<'_>) -> std::sync::Arc { let dataset_repo = from_catalog::(ctx).unwrap(); - let dataset = dataset_repo - .get_dataset(&self.dataset_handle.as_local_ref()) - .await - .int_err()?; - Ok(dataset) + dataset_repo.get_dataset_by_handle(&self.dataset_handle) } /// Returns all named metadata block references #[tracing::instrument(level = "info", skip_all)] async fn refs(&self, ctx: &Context<'_>) -> Result> { - let dataset = self.get_dataset(ctx).await?; + let dataset = self.get_dataset(ctx); Ok(vec![BlockRef { name: "head".to_owned(), block_hash: dataset @@ -73,7 +69,7 @@ impl MetadataChain { ctx: &Context<'_>, hash: Multihash, ) -> Result> { - let dataset = self.get_dataset(ctx).await?; + let dataset = self.get_dataset(ctx); let block = dataset.as_metadata_chain().try_get_block(&hash).await?; let account = Account::from_dataset_alias(ctx, &self.dataset_handle.alias) .await? @@ -92,7 +88,7 @@ impl MetadataChain { ) -> Result> { use odf::serde::MetadataBlockSerializer; - let dataset = self.get_dataset(ctx).await?; + let dataset = self.get_dataset(ctx); match dataset.as_metadata_chain().try_get_block(&hash).await? { None => Ok(None), Some(block) => match format { @@ -119,7 +115,7 @@ impl MetadataChain { let page = page.unwrap_or(0); let per_page = per_page.unwrap_or(Self::DEFAULT_BLOCKS_PER_PAGE); - let dataset = self.get_dataset(ctx).await?; + let dataset = self.get_dataset(ctx); let chain = dataset.as_metadata_chain(); let head = chain.resolve_ref(&domain::BlockRef::Head).await.int_err()?; diff --git a/src/adapter/graphql/src/scalars/flow_configuration.rs b/src/adapter/graphql/src/scalars/flow_configuration.rs index 703504045c..d768006529 100644 --- a/src/adapter/graphql/src/scalars/flow_configuration.rs +++ b/src/adapter/graphql/src/scalars/flow_configuration.rs @@ -455,13 +455,8 @@ impl FlowRunConfiguration { } DatasetFlowType::Reset => { let dataset_repo = from_catalog::(ctx).unwrap(); + let dataset = dataset_repo.get_dataset_by_handle(dataset_handle); - let dataset = dataset_repo - .get_dataset(&dataset_handle.as_local_ref()) - .await - .map_err(|_| FlowInvalidRunConfigurations { - error: "Cannot fetch default value".to_string(), - })?; // Assume unwrap safe such as we have checked this existance during // validation step let current_head_hash = dataset diff --git a/src/adapter/graphql/src/utils.rs b/src/adapter/graphql/src/utils.rs index f5824e5ad7..7023cdec6d 100644 --- a/src/adapter/graphql/src/utils.rs +++ b/src/adapter/graphql/src/utils.rs @@ -33,16 +33,9 @@ where //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub(crate) async fn get_dataset( - ctx: &Context<'_>, - dataset_handle: &DatasetHandle, -) -> Result, InternalError> { +pub(crate) fn get_dataset(ctx: &Context<'_>, dataset_handle: &DatasetHandle) -> Arc { let dataset_repo = from_catalog::(ctx).unwrap(); - let dataset = dataset_repo - .get_dataset(&dataset_handle.as_local_ref()) - .await - .int_err()?; - Ok(dataset) + dataset_repo.get_dataset_by_handle(dataset_handle) } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -91,10 +84,7 @@ pub(crate) async fn check_dataset_write_access( .check_action_allowed(dataset_handle, kamu_core::auth::DatasetAction::Write) .await .map_err(|e| match e { - DatasetActionUnauthorizedError::Access(_) => GqlError::Gql( - async_graphql::Error::new("Dataset access error") - .extend_with(|_, eev| eev.set("alias", dataset_handle.alias.to_string())), - ), + DatasetActionUnauthorizedError::Access(_) => make_dataset_access_error(dataset_handle), DatasetActionUnauthorizedError::Internal(e) => GqlError::Internal(e), })?; @@ -103,6 +93,15 @@ pub(crate) async fn check_dataset_write_access( //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +pub(crate) fn make_dataset_access_error(dataset_handle: &DatasetHandle) -> GqlError { + GqlError::Gql( + async_graphql::Error::new("Dataset access error") + .extend_with(|_, eev| eev.set("alias", dataset_handle.alias.to_string())), + ) +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + pub(crate) async fn get_task( ctx: &Context<'_>, task_id: ts::TaskID, diff --git a/src/adapter/graphql/tests/tests/test_auth.rs b/src/adapter/graphql/tests/tests/test_auth.rs index 484fce3f1a..032d45dbbc 100644 --- a/src/adapter/graphql/tests/tests/test_auth.rs +++ b/src/adapter/graphql/tests/tests/test_auth.rs @@ -17,9 +17,9 @@ use kamu_accounts::{ DEFAULT_ACCOUNT_NAME_STR, DUMMY_LOGIN_METHOD, }; -use kamu_accounts_inmem::AccessTokenRepositoryInMemory; +use kamu_accounts_inmem::InMemoryAccessTokenRepository; use kamu_accounts_services::AccessTokenServiceImpl; -use kamu_core::SystemTimeSourceDefault; +use time_source::SystemTimeSourceDefault; use crate::utils::authentication_catalogs; @@ -351,7 +351,7 @@ impl AuthGQLHarness { .bind::() .add::() .add::() - .add::() + .add::() .add::(); NoOpDatabasePlugin::init_database_components(&mut b); diff --git a/src/adapter/graphql/tests/tests/test_error_handling.rs b/src/adapter/graphql/tests/tests/test_error_handling.rs index acff7e48a9..8dc6019d38 100644 --- a/src/adapter/graphql/tests/tests/test_error_handling.rs +++ b/src/adapter/graphql/tests/tests/test_error_handling.rs @@ -8,11 +8,11 @@ // by the Apache License, Version 2.0. use dill::Component; -use event_bus::EventBus; use indoc::indoc; -use kamu::*; +use kamu::DatasetRepositoryLocalFs; use kamu_accounts::CurrentAccountSubject; -use kamu_core::*; +use kamu_core::DatasetRepository; +use time_source::SystemTimeSourceDefault; #[test_log::test(tokio::test)] async fn test_malformed_argument() { @@ -60,10 +60,7 @@ async fn test_internal_error() { let cat = dill::CatalogBuilder::new() .add::() - .add::() .add_value(CurrentAccountSubject::new_test()) - .add::() - .add::() .add_builder( DatasetRepositoryLocalFs::builder() .with_root(tempdir.path().join("datasets")) diff --git a/src/adapter/graphql/tests/tests/test_gql_account_flow_configs.rs b/src/adapter/graphql/tests/tests/test_gql_account_flow_configs.rs index 753daa555b..c8e672bdfb 100644 --- a/src/adapter/graphql/tests/tests/test_gql_account_flow_configs.rs +++ b/src/adapter/graphql/tests/tests/test_gql_account_flow_configs.rs @@ -9,13 +9,10 @@ //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -use std::sync::Arc; - use async_graphql::value; use chrono::Duration; use database_common::{DatabaseTransactionRunner, NoOpDatabasePlugin}; use dill::Component; -use event_bus::EventBus; use indoc::indoc; use kamu::testing::{ MetadataFactory, @@ -26,21 +23,25 @@ use kamu::testing::{ MockTransformService, }; use kamu::{ + CreateDatasetFromSnapshotUseCaseImpl, DatasetOwnershipServiceInMemory, DatasetOwnershipServiceInMemoryStateInitializer, DatasetRepositoryLocalFs, + DatasetRepositoryWriter, DependencyGraphServiceInMemory, }; use kamu_accounts::{JwtAuthenticationConfig, DEFAULT_ACCOUNT_NAME, DEFAULT_ACCOUNT_NAME_STR}; -use kamu_accounts_inmem::AccessTokenRepositoryInMemory; +use kamu_accounts_inmem::InMemoryAccessTokenRepository; use kamu_accounts_services::{AccessTokenServiceImpl, AuthenticationServiceImpl}; use kamu_core::*; use kamu_flow_system::FlowServiceRunConfig; -use kamu_flow_system_inmem::{FlowConfigurationEventStoreInMem, FlowEventStoreInMem}; +use kamu_flow_system_inmem::{InMemoryFlowConfigurationEventStore, InMemoryFlowEventStore}; use kamu_flow_system_services::{FlowConfigurationServiceImpl, FlowServiceImpl}; -use kamu_task_system_inmem::TaskSystemEventStoreInMemory; +use kamu_task_system_inmem::InMemoryTaskSystemEventStore; use kamu_task_system_services::TaskSchedulerImpl; +use messaging_outbox::{register_message_dispatcher, Outbox, OutboxImmediateImpl}; use opendatafabric::{AccountName, DatasetAlias, DatasetID, DatasetKind, DatasetName}; +use time_source::SystemTimeSourceDefault; use crate::utils::authentication_catalogs; @@ -623,7 +624,6 @@ struct FlowConfigHarness { _catalog_base: dill::Catalog, _catalog_anonymous: dill::Catalog, catalog_authorized: dill::Catalog, - dataset_repo: Arc, } #[derive(Default)] @@ -651,45 +651,56 @@ impl FlowConfigHarness { let catalog_base = { let mut b = dill::CatalogBuilder::new(); - b.add::() - .add_builder( - DatasetRepositoryLocalFs::builder() - .with_root(datasets_dir) - .with_multi_tenant(true), - ) - .bind::() - .add_value(dataset_changes_mock) - .bind::() - .add::() - .add_value(mock_dataset_action_authorizer) - .add::() - .add::() - .add::() - .add_value(JwtAuthenticationConfig::default()) - .bind::() - .add::() - .add_value(dependency_graph_mock) - .bind::() - .add::() - .add::() - .add::() - .add::() - .add_value(FlowServiceRunConfig::new( - Duration::try_seconds(1).unwrap(), - Duration::try_minutes(1).unwrap(), - )) - .add::() - .add::() - .add_value(transform_service_mock) - .bind::() - .add_value(polling_service_mock) - .bind::() - .add::() - .add::() - .add::(); + b.add_builder( + messaging_outbox::OutboxImmediateImpl::builder() + .with_consumer_filter(messaging_outbox::ConsumerFilter::AllConsumers), + ) + .bind::() + .add_builder( + DatasetRepositoryLocalFs::builder() + .with_root(datasets_dir) + .with_multi_tenant(true), + ) + .bind::() + .bind::() + .add::() + .add_value(dataset_changes_mock) + .bind::() + .add::() + .add_value(mock_dataset_action_authorizer) + .add::() + .add::() + .add::() + .add_value(JwtAuthenticationConfig::default()) + .bind::() + .add::() + .add_value(dependency_graph_mock) + .bind::() + .add::() + .add::() + .add::() + .add::() + .add_value(FlowServiceRunConfig::new( + Duration::try_seconds(1).unwrap(), + Duration::try_minutes(1).unwrap(), + )) + .add::() + .add::() + .add_value(transform_service_mock) + .bind::() + .add_value(polling_service_mock) + .bind::() + .add::() + .add::() + .add::(); NoOpDatabasePlugin::init_database_components(&mut b); + register_message_dispatcher::( + &mut b, + MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE, + ); + b.build() }; @@ -707,22 +718,22 @@ impl FlowConfigHarness { .await .unwrap(); - let dataset_repo = catalog_authorized - .get_one::() - .unwrap(); - Self { _tempdir: tempdir, _catalog_base: catalog_base, _catalog_anonymous: catalog_anonymous, catalog_authorized, - dataset_repo, } } async fn create_root_dataset(&self, dataset_alias: DatasetAlias) -> CreateDatasetResult { - self.dataset_repo - .create_dataset_from_snapshot( + let create_dataset_from_snapshot = self + .catalog_authorized + .get_one::() + .unwrap(); + + create_dataset_from_snapshot + .execute( MetadataFactory::dataset_snapshot() .kind(DatasetKind::Root) .name(dataset_alias) diff --git a/src/adapter/graphql/tests/tests/test_gql_data.rs b/src/adapter/graphql/tests/tests/test_gql_data.rs index 5e9a40e0f6..51a4091402 100644 --- a/src/adapter/graphql/tests/tests/test_gql_data.rs +++ b/src/adapter/graphql/tests/tests/test_gql_data.rs @@ -15,11 +15,10 @@ use datafusion::arrow::array::*; use datafusion::arrow::datatypes::{DataType, Field, Schema}; use datafusion::arrow::record_batch::RecordBatch; use dill::Component; -use event_bus::EventBus; use kamu::testing::{MetadataFactory, ParquetWriterHelper}; use kamu::*; use kamu_accounts::*; -use kamu_accounts_inmem::{AccessTokenRepositoryInMemory, AccountRepositoryInMemory}; +use kamu_accounts_inmem::{InMemoryAccessTokenRepository, InMemoryAccountRepository}; use kamu_accounts_services::{ AccessTokenServiceImpl, AuthenticationServiceImpl, @@ -27,7 +26,9 @@ use kamu_accounts_services::{ PredefinedAccountsRegistrator, }; use kamu_core::*; +use messaging_outbox::DummyOutboxImpl; use opendatafabric::*; +use time_source::SystemTimeSourceDefault; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -54,8 +55,7 @@ async fn create_catalog_with_local_workspace( let catalog = { let mut b = dill::CatalogBuilder::new(); - b.add::() - .add::() + b.add::() .add_value(current_account_subject) .add_value(predefined_accounts_config) .add_builder( @@ -64,6 +64,9 @@ async fn create_catalog_with_local_workspace( .with_multi_tenant(is_multitenant), ) .bind::() + .bind::() + .add::() + .add::() .add::() .add::() .add::() @@ -71,8 +74,8 @@ async fn create_catalog_with_local_workspace( .add::() .add::() .add::() - .add::() - .add::() + .add::() + .add::() .add_value(JwtAuthenticationConfig::default()) .add::() .add::() @@ -106,10 +109,10 @@ async fn create_test_dataset( tempdir: &Path, account_name: Option, ) { - let dataset_repo = catalog.get_one::().unwrap(); + let create_dataset = catalog.get_one::().unwrap(); - let dataset = dataset_repo - .create_dataset( + let dataset = create_dataset + .execute( &DatasetAlias::new(account_name, DatasetName::new_unchecked("foo")), MetadataFactory::metadata_block(MetadataFactory::seed(DatasetKind::Root).build()) .build_typed(), diff --git a/src/adapter/graphql/tests/tests/test_gql_dataset_env_vars.rs b/src/adapter/graphql/tests/tests/test_gql_dataset_env_vars.rs index 450c6c2892..de09cacd77 100644 --- a/src/adapter/graphql/tests/tests/test_gql_dataset_env_vars.rs +++ b/src/adapter/graphql/tests/tests/test_gql_dataset_env_vars.rs @@ -7,20 +7,24 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use std::sync::Arc; - use async_graphql::value; use database_common::{DatabaseTransactionRunner, NoOpDatabasePlugin}; use dill::Component; -use event_bus::EventBus; use indoc::indoc; use kamu::testing::MetadataFactory; -use kamu::{DatasetRepositoryLocalFs, DependencyGraphServiceInMemory}; -use kamu_core::{auth, CreateDatasetResult, DatasetRepository, SystemTimeSourceDefault}; +use kamu::{ + CreateDatasetFromSnapshotUseCaseImpl, + DatasetRepositoryLocalFs, + DatasetRepositoryWriter, + DependencyGraphServiceInMemory, +}; +use kamu_core::{auth, CreateDatasetFromSnapshotUseCase, CreateDatasetResult, DatasetRepository}; use kamu_datasets::DatasetEnvVarsConfig; -use kamu_datasets_inmem::DatasetEnvVarRepositoryInMemory; +use kamu_datasets_inmem::InMemoryDatasetEnvVarRepository; use kamu_datasets_services::DatasetEnvVarServiceImpl; +use messaging_outbox::DummyOutboxImpl; use opendatafabric::DatasetKind; +use time_source::SystemTimeSourceDefault; use crate::utils::authentication_catalogs; @@ -328,7 +332,6 @@ struct DatasetEnvVarsHarness { _tempdir: tempfile::TempDir, _catalog_anonymous: dill::Catalog, catalog_authorized: dill::Catalog, - dataset_repo: Arc, } impl DatasetEnvVarsHarness { @@ -340,7 +343,7 @@ impl DatasetEnvVarsHarness { let catalog_base = { let mut b = dill::CatalogBuilder::new(); - b.add::() + b.add::() .add_value(DatasetEnvVarsConfig::sample()) .add_builder( DatasetRepositoryLocalFs::builder() @@ -348,12 +351,14 @@ impl DatasetEnvVarsHarness { .with_multi_tenant(false), ) .bind::() + .bind::() + .add::() .add::() .add::() .add::() .add::() .add::() - .add::(); + .add::(); NoOpDatabasePlugin::init_database_components(&mut b); @@ -361,21 +366,22 @@ impl DatasetEnvVarsHarness { }; let (catalog_anonymous, catalog_authorized) = authentication_catalogs(&catalog_base).await; - let dataset_repo = catalog_authorized - .get_one::() - .unwrap(); Self { _tempdir: tempdir, _catalog_anonymous: catalog_anonymous, catalog_authorized, - dataset_repo, } } async fn create_dataset(&self) -> CreateDatasetResult { - self.dataset_repo - .create_dataset_from_snapshot( + let create_dataset_from_snapshot = self + .catalog_authorized + .get_one::() + .unwrap(); + + create_dataset_from_snapshot + .execute( MetadataFactory::dataset_snapshot() .kind(DatasetKind::Root) .name("foo") diff --git a/src/adapter/graphql/tests/tests/test_gql_dataset_flow_configs.rs b/src/adapter/graphql/tests/tests/test_gql_dataset_flow_configs.rs index e8e970506c..cb70fbc30c 100644 --- a/src/adapter/graphql/tests/tests/test_gql_dataset_flow_configs.rs +++ b/src/adapter/graphql/tests/tests/test_gql_dataset_flow_configs.rs @@ -7,26 +7,30 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use std::sync::Arc; - use async_graphql::value; use database_common::{DatabaseTransactionRunner, NoOpDatabasePlugin}; use dill::Component; -use event_bus::EventBus; use indoc::indoc; use kamu::testing::{MetadataFactory, MockPollingIngestService, MockTransformService}; -use kamu::{DatasetRepositoryLocalFs, DependencyGraphServiceInMemory}; +use kamu::{ + CreateDatasetFromSnapshotUseCaseImpl, + DatasetRepositoryLocalFs, + DatasetRepositoryWriter, + DependencyGraphServiceInMemory, +}; use kamu_core::{ auth, + CreateDatasetFromSnapshotUseCase, CreateDatasetResult, DatasetRepository, PollingIngestService, - SystemTimeSourceDefault, TransformService, }; -use kamu_flow_system_inmem::FlowConfigurationEventStoreInMem; +use kamu_flow_system_inmem::InMemoryFlowConfigurationEventStore; use kamu_flow_system_services::FlowConfigurationServiceImpl; +use messaging_outbox::DummyOutboxImpl; use opendatafabric::*; +use time_source::SystemTimeSourceDefault; use crate::utils::{authentication_catalogs, expect_anonymous_access_error}; @@ -1517,7 +1521,6 @@ struct FlowConfigHarness { _catalog_base: dill::Catalog, catalog_anonymous: dill::Catalog, catalog_authorized: dill::Catalog, - dataset_repo: Arc, } impl FlowConfigHarness { @@ -1532,13 +1535,15 @@ impl FlowConfigHarness { let catalog_base = { let mut b = dill::CatalogBuilder::new(); - b.add::() + b.add::() .add_builder( DatasetRepositoryLocalFs::builder() .with_root(datasets_dir) .with_multi_tenant(false), ) .bind::() + .bind::() + .add::() .add::() .add_value(polling_service_mock) .bind::() @@ -1547,7 +1552,7 @@ impl FlowConfigHarness { .add::() .add::() .add::() - .add::() + .add::() .add::(); NoOpDatabasePlugin::init_database_components(&mut b); @@ -1558,22 +1563,22 @@ impl FlowConfigHarness { // Init dataset with no sources let (catalog_anonymous, catalog_authorized) = authentication_catalogs(&catalog_base).await; - let dataset_repo = catalog_authorized - .get_one::() - .unwrap(); - Self { _tempdir: tempdir, _catalog_base: catalog_base, catalog_anonymous, catalog_authorized, - dataset_repo, } } async fn create_root_dataset(&self) -> CreateDatasetResult { - self.dataset_repo - .create_dataset_from_snapshot( + let create_dataset_from_snapshot = self + .catalog_authorized + .get_one::() + .unwrap(); + + create_dataset_from_snapshot + .execute( MetadataFactory::dataset_snapshot() .kind(DatasetKind::Root) .name("foo") @@ -1585,8 +1590,13 @@ impl FlowConfigHarness { } async fn create_derived_dataset(&self) -> CreateDatasetResult { - self.dataset_repo - .create_dataset_from_snapshot( + let create_dataset_from_snapshot = self + .catalog_authorized + .get_one::() + .unwrap(); + + create_dataset_from_snapshot + .execute( MetadataFactory::dataset_snapshot() .name("bar") .kind(DatasetKind::Derivative) diff --git a/src/adapter/graphql/tests/tests/test_gql_dataset_flow_runs.rs b/src/adapter/graphql/tests/tests/test_gql_dataset_flow_runs.rs index c51908ce97..09cf6fabfe 100644 --- a/src/adapter/graphql/tests/tests/test_gql_dataset_flow_runs.rs +++ b/src/adapter/graphql/tests/tests/test_gql_dataset_flow_runs.rs @@ -9,13 +9,10 @@ //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -use std::sync::Arc; - use async_graphql::value; use chrono::{DateTime, Duration, DurationRound, Utc}; use database_common::{DatabaseTransactionRunner, NoOpDatabasePlugin}; use dill::Component; -use event_bus::EventBus; use futures::TryStreamExt; use indoc::indoc; use kamu::testing::{ @@ -26,8 +23,10 @@ use kamu::testing::{ MockTransformService, }; use kamu::{ + CreateDatasetFromSnapshotUseCaseImpl, DatasetOwnershipServiceInMemory, DatasetRepositoryLocalFs, + DatasetRepositoryWriter, DependencyGraphServiceInMemory, }; use kamu_accounts::{ @@ -37,23 +36,26 @@ use kamu_accounts::{ DEFAULT_ACCOUNT_ID, DEFAULT_ACCOUNT_NAME_STR, }; -use kamu_accounts_inmem::AccessTokenRepositoryInMemory; +use kamu_accounts_inmem::InMemoryAccessTokenRepository; use kamu_accounts_services::{AccessTokenServiceImpl, AuthenticationServiceImpl}; use kamu_core::{ auth, CompactionResult, + CreateDatasetFromSnapshotUseCase, CreateDatasetResult, DatasetChangesService, DatasetIntervalIncrement, + DatasetLifecycleMessage, DatasetRepository, DependencyGraphRepository, PollingIngestService, PullResult, - SystemTimeSourceDefault, TransformService, + MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE, }; use kamu_flow_system::{ Flow, + FlowConfigurationUpdatedMessage, FlowEventStore, FlowID, FlowServiceRunConfig, @@ -61,12 +63,18 @@ use kamu_flow_system::{ FlowTrigger, FlowTriggerAutoPolling, }; -use kamu_flow_system_inmem::{FlowConfigurationEventStoreInMem, FlowEventStoreInMem}; -use kamu_flow_system_services::{FlowConfigurationServiceImpl, FlowServiceImpl}; +use kamu_flow_system_inmem::{InMemoryFlowConfigurationEventStore, InMemoryFlowEventStore}; +use kamu_flow_system_services::{ + FlowConfigurationServiceImpl, + FlowServiceImpl, + MESSAGE_PRODUCER_KAMU_FLOW_CONFIGURATION_SERVICE, +}; use kamu_task_system::{self as ts}; -use kamu_task_system_inmem::TaskSystemEventStoreInMemory; +use kamu_task_system_inmem::InMemoryTaskSystemEventStore; use kamu_task_system_services::TaskSchedulerImpl; +use messaging_outbox::{register_message_dispatcher, Outbox, OutboxExt, OutboxImmediateImpl}; use opendatafabric::{AccountID, DatasetID, DatasetKind, Multihash}; +use time_source::SystemTimeSourceDefault; use crate::utils::{authentication_catalogs, expect_anonymous_access_error}; @@ -3127,7 +3135,6 @@ struct FlowRunsHarness { _catalog_base: dill::Catalog, catalog_anonymous: dill::Catalog, catalog_authorized: dill::Catalog, - dataset_repo: Arc, } #[derive(Default)] @@ -3152,59 +3159,73 @@ impl FlowRunsHarness { let catalog_base = { let mut b = dill::CatalogBuilder::new(); - b.add::() - .add_builder( - DatasetRepositoryLocalFs::builder() - .with_root(datasets_dir) - .with_multi_tenant(false), - ) - .bind::() - .add_value(dataset_changes_mock) - .bind::() - .add::() - .add::() - .add::() - .add_value(dependency_graph_mock) - .bind::() - .add::() - .add::() - .add::() - .add::() - .add_value(FlowServiceRunConfig::new( - Duration::try_seconds(1).unwrap(), - Duration::try_minutes(1).unwrap(), - )) - .add::() - .add::() - .add_value(transform_service_mock) - .bind::() - .add_value(polling_service_mock) - .bind::() - .add::() - .add::() - .add::() - .add_value(JwtAuthenticationConfig::default()) - .add::() - .add::(); + b.add_builder( + messaging_outbox::OutboxImmediateImpl::builder() + .with_consumer_filter(messaging_outbox::ConsumerFilter::AllConsumers), + ) + .bind::() + .add_builder( + DatasetRepositoryLocalFs::builder() + .with_root(datasets_dir) + .with_multi_tenant(false), + ) + .bind::() + .bind::() + .add::() + .add_value(dataset_changes_mock) + .bind::() + .add::() + .add::() + .add::() + .add_value(dependency_graph_mock) + .bind::() + .add::() + .add::() + .add::() + .add::() + .add_value(FlowServiceRunConfig::new( + Duration::try_seconds(1).unwrap(), + Duration::try_minutes(1).unwrap(), + )) + .add::() + .add::() + .add_value(transform_service_mock) + .bind::() + .add_value(polling_service_mock) + .bind::() + .add::() + .add::() + .add::() + .add_value(JwtAuthenticationConfig::default()) + .add::() + .add::(); NoOpDatabasePlugin::init_database_components(&mut b); + register_message_dispatcher::( + &mut b, + MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE, + ); + register_message_dispatcher::( + &mut b, + ts::MESSAGE_PRODUCER_KAMU_TASK_EXECUTOR, + ); + register_message_dispatcher::( + &mut b, + MESSAGE_PRODUCER_KAMU_FLOW_CONFIGURATION_SERVICE, + ); + b.build() }; // Init dataset with no sources let (catalog_anonymous, catalog_authorized) = authentication_catalogs(&catalog_base).await; - let dataset_repo = catalog_authorized - .get_one::() - .unwrap(); - Self { _tempdir: tempdir, _catalog_base: catalog_base, catalog_anonymous, catalog_authorized, - dataset_repo, } } @@ -3222,8 +3243,13 @@ impl FlowRunsHarness { } async fn create_root_dataset(&self) -> CreateDatasetResult { - self.dataset_repo - .create_dataset_from_snapshot( + let create_dataset_from_snapshot = self + .catalog_authorized + .get_one::() + .unwrap(); + + create_dataset_from_snapshot + .execute( MetadataFactory::dataset_snapshot() .kind(DatasetKind::Root) .name("foo") @@ -3235,8 +3261,13 @@ impl FlowRunsHarness { } async fn create_derived_dataset(&self) -> CreateDatasetResult { - self.dataset_repo - .create_dataset_from_snapshot( + let create_dataset_from_snapshot = self + .catalog_authorized + .get_one::() + .unwrap(); + + create_dataset_from_snapshot + .execute( MetadataFactory::dataset_snapshot() .name("bar") .kind(DatasetKind::Derivative) @@ -3304,12 +3335,12 @@ impl FlowRunsHarness { task.run(event_time).unwrap(); task.save(task_event_store.as_ref()).await.unwrap(); - let event_bus = self.catalog_authorized.get_one::().unwrap(); - event_bus - .dispatch_event(ts::TaskEventRunning { - event_time, - task_id, - }) + let outbox = self.catalog_authorized.get_one::().unwrap(); + outbox + .post_message( + ts::MESSAGE_PRODUCER_KAMU_TASK_EXECUTOR, + ts::TaskProgressMessage::running(event_time, task_id), + ) .await .unwrap(); } @@ -3337,13 +3368,12 @@ impl FlowRunsHarness { task.finish(event_time, task_outcome.clone()).unwrap(); task.save(task_event_store.as_ref()).await.unwrap(); - let event_bus = self.catalog_authorized.get_one::().unwrap(); - event_bus - .dispatch_event(ts::TaskEventFinished { - event_time, - task_id, - outcome: task_outcome, - }) + let outbox = self.catalog_authorized.get_one::().unwrap(); + outbox + .post_message( + ts::MESSAGE_PRODUCER_KAMU_TASK_EXECUTOR, + ts::TaskProgressMessage::finished(event_time, task_id, task_outcome), + ) .await .unwrap(); } diff --git a/src/adapter/graphql/tests/tests/test_gql_datasets.rs b/src/adapter/graphql/tests/tests/test_gql_datasets.rs index 7b936eb200..3251b93cd9 100644 --- a/src/adapter/graphql/tests/tests/test_gql_datasets.rs +++ b/src/adapter/graphql/tests/tests/test_gql_datasets.rs @@ -10,16 +10,17 @@ use async_graphql::*; use database_common::NoOpDatabasePlugin; use dill::Component; -use event_bus::EventBus; use indoc::indoc; use kamu::testing::MetadataFactory; use kamu::*; use kamu_accounts::*; use kamu_core::*; +use messaging_outbox::{register_message_dispatcher, Outbox, OutboxImmediateImpl}; use mockall::predicate::eq; use opendatafabric::serde::yaml::YamlDatasetSnapshotSerializer; use opendatafabric::serde::DatasetSnapshotSerializer; use opendatafabric::*; +use time_source::SystemTimeSourceDefault; use crate::utils::{authentication_catalogs, expect_anonymous_access_error}; @@ -675,7 +676,14 @@ impl GraphQLDatasetsHarness { let mut b = dill::CatalogBuilder::new(); b.add::() - .add::() + .add_builder( + messaging_outbox::OutboxImmediateImpl::builder() + .with_consumer_filter(messaging_outbox::ConsumerFilter::AllConsumers), + ) + .bind::() + .add::() + .add::() + .add::() .add::() .add_builder( DatasetRepositoryLocalFs::builder() @@ -683,12 +691,18 @@ impl GraphQLDatasetsHarness { .with_multi_tenant(is_multi_tenant), ) .bind::() + .bind::() .add_value(mock_authentication_service) .bind::() .add::(); NoOpDatabasePlugin::init_database_components(&mut b); + register_message_dispatcher::( + &mut b, + MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE, + ); + b.build() }; @@ -722,12 +736,13 @@ impl GraphQLDatasetsHarness { account_name: Option, name: DatasetName, ) -> CreateDatasetResult { - let dataset_repo = self + let create_dataset = self .catalog_authorized - .get_one::() + .get_one::() .unwrap(); - dataset_repo - .create_dataset_from_snapshot( + + create_dataset + .execute( MetadataFactory::dataset_snapshot() .name(DatasetAlias::new(account_name, name)) .kind(DatasetKind::Root) @@ -743,12 +758,13 @@ impl GraphQLDatasetsHarness { name: DatasetName, input_dataset: &DatasetHandle, ) -> CreateDatasetResult { - let dataset_repo = self + let create_dataset = self .catalog_authorized - .get_one::() + .get_one::() .unwrap(); - dataset_repo - .create_dataset_from_snapshot( + + create_dataset + .execute( MetadataFactory::dataset_snapshot() .name(DatasetAlias::new(None, name)) .kind(DatasetKind::Derivative) diff --git a/src/adapter/graphql/tests/tests/test_gql_metadata.rs b/src/adapter/graphql/tests/tests/test_gql_metadata.rs index 10acd2408e..a2f3143589 100644 --- a/src/adapter/graphql/tests/tests/test_gql_metadata.rs +++ b/src/adapter/graphql/tests/tests/test_gql_metadata.rs @@ -10,12 +10,13 @@ use async_graphql::*; use database_common::NoOpDatabasePlugin; use dill::*; -use event_bus::EventBus; use indoc::indoc; use kamu::testing::MetadataFactory; use kamu::*; use kamu_core::*; +use messaging_outbox::DummyOutboxImpl; use opendatafabric::*; +use time_source::SystemTimeSourceDefault; use crate::utils::authentication_catalogs; @@ -31,13 +32,15 @@ async fn test_current_push_sources() { let mut b = CatalogBuilder::new(); b.add_value(RunInfoDir::new(tempdir.path().join("run"))) - .add::() + .add::() .add_builder( DatasetRepositoryLocalFs::builder() .with_root(datasets_dir) .with_multi_tenant(false), ) .bind::() + .bind::() + .add::() .add::() .add::() .add::() @@ -53,11 +56,12 @@ async fn test_current_push_sources() { // Init dataset with no sources let (_, catalog_authorized) = authentication_catalogs(&base_catalog).await; - let dataset_repo = catalog_authorized - .get_one::() + + let create_dataset_from_snapshot = catalog_authorized + .get_one::() .unwrap(); - let create_result = dataset_repo - .create_dataset_from_snapshot( + let create_result = create_dataset_from_snapshot + .execute( MetadataFactory::dataset_snapshot() .kind(DatasetKind::Root) .name("foo") diff --git a/src/adapter/graphql/tests/tests/test_gql_metadata_chain.rs b/src/adapter/graphql/tests/tests/test_gql_metadata_chain.rs index ae35d6df30..c7f3a5b15c 100644 --- a/src/adapter/graphql/tests/tests/test_gql_metadata_chain.rs +++ b/src/adapter/graphql/tests/tests/test_gql_metadata_chain.rs @@ -10,14 +10,16 @@ use std::sync::Arc; use async_graphql::*; +use chrono::Utc; use dill::Component; -use event_bus::EventBus; use indoc::indoc; use kamu::testing::MetadataFactory; use kamu::*; use kamu_core::*; +use messaging_outbox::DummyOutboxImpl; use opendatafabric::serde::yaml::YamlMetadataEventSerializer; use opendatafabric::*; +use time_source::SystemTimeSourceDefault; use crate::utils::{authentication_catalogs, expect_anonymous_access_error}; @@ -27,18 +29,24 @@ use crate::utils::{authentication_catalogs, expect_anonymous_access_error}; async fn test_metadata_chain_events() { let harness = GraphQLMetadataChainHarness::new(false).await; - // Init dataset - let dataset_repo = harness + let create_dataset = harness .catalog_authorized - .get_one::() + .get_one::() .unwrap(); - let create_result = dataset_repo - .create_dataset_from_seed( + + let create_result = create_dataset + .execute( &"foo".try_into().unwrap(), - MetadataFactory::seed(DatasetKind::Root).build(), + MetadataBlockTyped { + system_time: Utc::now(), + prev_block_hash: None, + event: MetadataFactory::seed(DatasetKind::Root).build(), + sequence_number: 0, + }, ) .await .unwrap(); + create_result .dataset .commit_event( @@ -167,12 +175,13 @@ async fn test_metadata_chain_events() { async fn metadata_chain_append_event() { let harness = GraphQLMetadataChainHarness::new(false).await; - let dataset_repo = harness + let create_dataset = harness .catalog_authorized - .get_one::() + .get_one::() .unwrap(); - let create_result = dataset_repo - .create_dataset_from_snapshot( + + let create_result = create_dataset + .execute( MetadataFactory::dataset_snapshot() .name("foo") .kind(DatasetKind::Root) @@ -250,12 +259,13 @@ async fn metadata_chain_append_event() { async fn metadata_update_readme_new() { let harness = GraphQLMetadataChainHarness::new(false).await; - let dataset_repo = harness + let create_dataset = harness .catalog_authorized - .get_one::() + .get_one::() .unwrap(); - let create_result = dataset_repo - .create_dataset_from_snapshot( + + let create_result = create_dataset + .execute( MetadataFactory::dataset_snapshot() .name("foo") .kind(DatasetKind::Root) @@ -514,7 +524,10 @@ impl GraphQLMetadataChainHarness { let mut b = dill::CatalogBuilder::new(); b.add::() - .add::() + .add::() + .add::() + .add::() + .add::() .add::() .add_builder( DatasetRepositoryLocalFs::builder() @@ -522,6 +535,7 @@ impl GraphQLMetadataChainHarness { .with_multi_tenant(is_multi_tenant), ) .bind::() + .bind::() .add::(); database_common::NoOpDatabasePlugin::init_database_components(&mut b); diff --git a/src/adapter/graphql/tests/tests/test_gql_search.rs b/src/adapter/graphql/tests/tests/test_gql_search.rs index 70fd93fcd4..515ef8dcf9 100644 --- a/src/adapter/graphql/tests/tests/test_gql_search.rs +++ b/src/adapter/graphql/tests/tests/test_gql_search.rs @@ -9,12 +9,13 @@ use async_graphql::*; use dill::Component; -use event_bus::EventBus; use kamu::testing::MetadataFactory; use kamu::*; use kamu_accounts::CurrentAccountSubject; use kamu_core::*; +use messaging_outbox::DummyOutboxImpl; use opendatafabric::*; +use time_source::SystemTimeSourceDefault; #[tokio::test] async fn test_search_query() { @@ -24,7 +25,7 @@ async fn test_search_query() { let cat = dill::CatalogBuilder::new() .add::() - .add::() + .add::() .add::() .add_value(CurrentAccountSubject::new_test()) .add::() @@ -34,11 +35,15 @@ async fn test_search_query() { .with_multi_tenant(false), ) .bind::() + .bind::() + .add::() .build(); - let dataset_repo = cat.get_one::().unwrap(); - dataset_repo - .create_dataset_from_snapshot( + let create_dataset_from_snapshot = cat + .get_one::() + .unwrap(); + create_dataset_from_snapshot + .execute( MetadataFactory::dataset_snapshot() .name("foo") .kind(DatasetKind::Root) diff --git a/src/adapter/graphql/tests/utils/auth_utils.rs b/src/adapter/graphql/tests/utils/auth_utils.rs index b8ad6233b4..658c6b9c5b 100644 --- a/src/adapter/graphql/tests/utils/auth_utils.rs +++ b/src/adapter/graphql/tests/utils/auth_utils.rs @@ -9,7 +9,7 @@ use database_common::DatabaseTransactionRunner; use kamu_accounts::*; -use kamu_accounts_inmem::AccountRepositoryInMemory; +use kamu_accounts_inmem::InMemoryAccountRepository; use kamu_accounts_services::{LoginPasswordAuthProvider, PredefinedAccountsRegistrator}; use kamu_adapter_graphql::ANONYMOUS_ACCESS_FORBIDDEN_MESSAGE; @@ -40,7 +40,7 @@ pub async fn authentication_catalogs( let catalog_authorized = dill::CatalogBuilder::new_chained(base_catalog) .add::() .add::() - .add::() + .add::() .add_value(current_account_subject) .add_value(predefined_accounts_config) .build(); diff --git a/src/adapter/http/Cargo.toml b/src/adapter/http/Cargo.toml index d6e48464ff..2df4bc46b6 100644 --- a/src/adapter/http/Cargo.toml +++ b/src/adapter/http/Cargo.toml @@ -24,13 +24,15 @@ doctest = false [dependencies] database-common = { workspace = true } database-common-macros = { workspace = true } -event-bus = { workspace = true } +internal-error = { workspace = true } http-common = { workspace = true } # TODO: Adapters should depend only on kamu-domain crate and be implementation-agnostic kamu = { workspace = true } kamu-accounts = { workspace = true } +kamu-core = { workspace = true } kamu-data-utils = { workspace = true } opendatafabric = { workspace = true } +time-source = { workspace = true } aws-sdk-s3 = { version = "0.35" } @@ -41,7 +43,7 @@ base64 = { version = "0.22", default-features = false } bytes = "1" chrono = { version = "0.4", features = ["serde"] } datafusion = { version = "41", default-features = false } # TODO: Currently needed for type conversions but ideally should be encapsulated by kamu-core -dill = "0.8" +dill = "0.9" flate2 = "1" # GZip decoder futures = "0.3" http = "0.2" @@ -79,6 +81,7 @@ kamu-accounts-services = { workspace = true } kamu-datasets-services = { workspace = true } kamu-accounts-inmem = { workspace = true } kamu-ingest-datafusion = { workspace = true } +messaging-outbox = { workspace = true } fs_extra = "1.3" # Recursive folder copy indoc = "2" diff --git a/src/adapter/http/src/data/ingest_handler.rs b/src/adapter/http/src/data/ingest_handler.rs index 9d7ffcf509..8d95fa6be3 100644 --- a/src/adapter/http/src/data/ingest_handler.rs +++ b/src/adapter/http/src/data/ingest_handler.rs @@ -22,8 +22,9 @@ use database_common_macros::transactional_handler; use dill::Catalog; use http::HeaderMap; use http_common::*; -use kamu::domain::*; +use kamu_core::*; use opendatafabric::DatasetRef; +use time_source::SystemTimeSource; use tokio::io::AsyncRead; use crate::axum_utils::ensure_authenticated_account; diff --git a/src/adapter/http/src/data/query_handler.rs b/src/adapter/http/src/data/query_handler.rs index 7bf2fea741..1b9f0ce95e 100644 --- a/src/adapter/http/src/data/query_handler.rs +++ b/src/adapter/http/src/data/query_handler.rs @@ -26,7 +26,8 @@ use datafusion::common::DFSchema; use datafusion::error::DataFusionError; use dill::Catalog; use http_common::*; -use kamu::domain::*; +use internal_error::{ErrorIntoInternal, InternalError, ResultIntoInternal}; +use kamu_core::*; use kamu_data_utils::data::format::*; use opendatafabric::{DatasetID, Multihash}; @@ -267,8 +268,8 @@ pub struct QueryDatasetState { } impl QueryState { - fn to_state(&self) -> kamu::domain::QueryState { - kamu::domain::QueryState { + fn to_state(&self) -> kamu_core::QueryState { + kamu_core::QueryState { inputs: self .inputs .iter() @@ -278,8 +279,8 @@ impl QueryState { } } -impl From for QueryState { - fn from(value: kamu::domain::QueryState) -> Self { +impl From for QueryState { + fn from(value: kamu_core::QueryState) -> Self { Self { inputs: value .inputs diff --git a/src/adapter/http/src/data/tail_handler.rs b/src/adapter/http/src/data/tail_handler.rs index d3efae9679..7ab8b00e38 100644 --- a/src/adapter/http/src/data/tail_handler.rs +++ b/src/adapter/http/src/data/tail_handler.rs @@ -21,7 +21,8 @@ use axum::response::Json; use database_common_macros::transactional_handler; use dill::Catalog; use http_common::*; -use kamu::domain::*; +use internal_error::{ErrorIntoInternal, ResultIntoInternal}; +use kamu_core::*; use opendatafabric::DatasetRef; use super::query_handler::{DataFormat, SchemaFormat}; diff --git a/src/adapter/http/src/http_server_dataset_router.rs b/src/adapter/http/src/http_server_dataset_router.rs index 5fbfb5f24c..1a8a33a994 100644 --- a/src/adapter/http/src/http_server_dataset_router.rs +++ b/src/adapter/http/src/http_server_dataset_router.rs @@ -149,11 +149,11 @@ fn is_dataset_optional_for_request(request: &http::Request) -> bool fn get_dataset_action_for_request( request: &http::Request, -) -> kamu::domain::auth::DatasetAction { +) -> kamu_core::auth::DatasetAction { if !request.method().is_safe() || request.uri().path() == "/push" { - kamu::domain::auth::DatasetAction::Write + kamu_core::auth::DatasetAction::Write } else { - kamu::domain::auth::DatasetAction::Read + kamu_core::auth::DatasetAction::Read } } diff --git a/src/adapter/http/src/middleware/dataset_authorization_layer.rs b/src/adapter/http/src/middleware/dataset_authorization_layer.rs index 4e9133e313..4dc488ddb7 100644 --- a/src/adapter/http/src/middleware/dataset_authorization_layer.rs +++ b/src/adapter/http/src/middleware/dataset_authorization_layer.rs @@ -13,8 +13,8 @@ use std::task::{Context, Poll}; use axum::body::Body; use axum::response::Response; use futures::Future; -use kamu::domain::GetDatasetError; use kamu_accounts::CurrentAccountSubject; +use kamu_core::GetDatasetError; use opendatafabric::DatasetRef; use tower::{Layer, Service}; @@ -29,7 +29,7 @@ pub struct DatasetAuthorizationLayer { impl DatasetAuthorizationLayer where - DatasetActionQuery: Fn(&http::Request) -> kamu::domain::auth::DatasetAction, + DatasetActionQuery: Fn(&http::Request) -> kamu_core::auth::DatasetAction, { pub fn new(dataset_action_query: DatasetActionQuery) -> Self { Self { @@ -77,7 +77,7 @@ where Svc: Service, Response = Response> + Send + 'static + Clone, Svc::Future: Send + 'static, DatasetActionQuery: Send + Clone + 'static, - DatasetActionQuery: Fn(&http::Request) -> kamu::domain::auth::DatasetAction, + DatasetActionQuery: Fn(&http::Request) -> kamu_core::auth::DatasetAction, { type Response = Svc::Response; type Error = Svc::Error; @@ -102,11 +102,11 @@ where .expect("Catalog not found in http server extensions"); let dataset_action_authorizer = catalog - .get_one::() + .get_one::() .unwrap(); let dataset_repo = catalog - .get_one::() + .get_one::() .unwrap(); let dataset_ref = request diff --git a/src/adapter/http/src/middleware/dataset_resolver_layer.rs b/src/adapter/http/src/middleware/dataset_resolver_layer.rs index 8c45dcbe7a..33b80c83ea 100644 --- a/src/adapter/http/src/middleware/dataset_resolver_layer.rs +++ b/src/adapter/http/src/middleware/dataset_resolver_layer.rs @@ -16,7 +16,7 @@ use axum::body::Body; use axum::extract::FromRequestParts; use axum::response::Response; use axum::RequestExt; -use kamu::domain::{DatasetRepository, GetDatasetError}; +use kamu_core::{DatasetRepository, GetDatasetError}; use opendatafabric::DatasetRef; use tower::{Layer, Service}; @@ -143,7 +143,7 @@ where let dataset_repo = catalog.get_one::().unwrap(); - let dataset = match dataset_repo.get_dataset(&dataset_ref).await { + let dataset = match dataset_repo.find_dataset_by_ref(&dataset_ref).await { Ok(ds) => ds, Err(GetDatasetError::NotFound(err)) => { tracing::warn!("Dataset not found: {:?}", err); diff --git a/src/adapter/http/src/middleware/run_in_database_transaction_layer.rs b/src/adapter/http/src/middleware/run_in_database_transaction_layer.rs index 95c8e86882..f5913033f9 100644 --- a/src/adapter/http/src/middleware/run_in_database_transaction_layer.rs +++ b/src/adapter/http/src/middleware/run_in_database_transaction_layer.rs @@ -15,7 +15,7 @@ use axum::response::{IntoResponse, Response}; use database_common::DatabaseTransactionRunner; use futures::Future; use http_common::IntoApiError; -use kamu::domain::InternalError; +use internal_error::InternalError; use tower::{Layer, Service}; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/adapter/http/src/simple_protocol/handlers.rs b/src/adapter/http/src/simple_protocol/handlers.rs index c9c04c81bc..c408b8a933 100644 --- a/src/adapter/http/src/simple_protocol/handlers.rs +++ b/src/adapter/http/src/simple_protocol/handlers.rs @@ -19,10 +19,10 @@ use std::str::FromStr; use std::sync::Arc; -use event_bus::EventBus; use http_common::*; -use kamu::domain::*; +use internal_error::ResultIntoInternal; use kamu_accounts::CurrentAccountSubject; +use kamu_core::*; use opendatafabric::serde::flatbuffers::FlatbuffersMetadataBlockSerializer; use opendatafabric::serde::MetadataBlockSerializer; use opendatafabric::{DatasetRef, Multihash}; @@ -210,7 +210,7 @@ pub async fn dataset_push_ws_upgrade_handler( let dataset_repo = catalog.get_one::().unwrap(); - let dataset = match dataset_repo.get_dataset(&dataset_ref).await { + let dataset = match dataset_repo.find_dataset_by_ref(&dataset_ref).await { Ok(ds) => Ok(Some(ds)), Err(GetDatasetError::NotFound(_)) => { // Make sure account in dataset ref being created and token account match @@ -227,13 +227,10 @@ pub async fn dataset_push_ws_upgrade_handler( Err(err) => Err(err.api_err()), }?; - let event_bus = catalog.get_one::().unwrap(); - Ok(ws.on_upgrade(|socket| { AxumServerPushProtocolInstance::new( socket, - event_bus, - dataset_repo, + catalog, dataset_ref, dataset, dataset_url, diff --git a/src/adapter/http/src/smart_protocol/axum_server_pull_protocol.rs b/src/adapter/http/src/smart_protocol/axum_server_pull_protocol.rs index 8d9edf7fa8..758f6520a3 100644 --- a/src/adapter/http/src/smart_protocol/axum_server_pull_protocol.rs +++ b/src/adapter/http/src/smart_protocol/axum_server_pull_protocol.rs @@ -9,7 +9,8 @@ use std::sync::Arc; -use kamu::domain::{BlockRef, Dataset, ErrorIntoInternal, ResultIntoInternal}; +use internal_error::{ErrorIntoInternal, ResultIntoInternal}; +use kamu_core::{BlockRef, Dataset}; use url::Url; use super::errors::*; diff --git a/src/adapter/http/src/smart_protocol/axum_server_push_protocol.rs b/src/adapter/http/src/smart_protocol/axum_server_push_protocol.rs index 5361120a35..404efd4898 100644 --- a/src/adapter/http/src/smart_protocol/axum_server_push_protocol.rs +++ b/src/adapter/http/src/smart_protocol/axum_server_push_protocol.rs @@ -11,18 +11,18 @@ use std::collections::VecDeque; use std::sync::Arc; use std::time::Duration; -use event_bus::EventBus; -use kamu::domain::events::DatasetEventDependenciesUpdated; -use kamu::domain::{ +use database_common::DatabaseTransactionRunner; +use dill::Catalog; +use internal_error::{ErrorIntoInternal, ResultIntoInternal}; +use kamu_core::{ + AppendDatasetMetadataBatchUseCase, BlockRef, CorruptedSourceError, CreateDatasetError, + CreateDatasetUseCase, Dataset, - DatasetRepository, - ErrorIntoInternal, - GetSummaryOpts, + GetRefError, HashedMetadataBlock, - ResultIntoInternal, }; use opendatafabric::{AsTypedBlock, DatasetRef}; use url::Url; @@ -42,8 +42,7 @@ const MIN_UPLOAD_PROGRESS_PING_DELAY_SEC: u64 = 10; pub struct AxumServerPushProtocolInstance { socket: axum::extract::ws::WebSocket, - event_bus: Arc, - dataset_repo: Arc, + catalog: Catalog, dataset_ref: DatasetRef, dataset: Option>, dataset_url: Url, @@ -53,8 +52,7 @@ pub struct AxumServerPushProtocolInstance { impl AxumServerPushProtocolInstance { pub fn new( socket: axum::extract::ws::WebSocket, - event_bus: Arc, - dataset_repo: Arc, + catalog: Catalog, dataset_ref: DatasetRef, dataset: Option>, dataset_url: Url, @@ -62,8 +60,7 @@ impl AxumServerPushProtocolInstance { ) -> Self { Self { socket, - event_bus, - dataset_repo, + catalog, dataset_ref, dataset, dataset_url, @@ -120,11 +117,16 @@ impl AxumServerPushProtocolInstance { }) .int_err()?; - match self - .dataset_repo - .create_dataset(dataset_alias, seed_block) - .await - { + let create_result = DatabaseTransactionRunner::new(self.catalog.clone()) + .transactional_with( + |create_dataset_use_case: Arc| async move { + create_dataset_use_case + .execute(dataset_alias, seed_block) + .await + }, + ) + .await; + match create_result { Ok(create_result) => self.dataset = Some(create_result.dataset), Err(err) => { if let CreateDatasetError::RefCollision(err) = &err { @@ -196,7 +198,7 @@ impl AxumServerPushProtocolInstance { .await { Ok(head) => Some(head), - Err(kamu::domain::GetRefError::NotFound(_)) => None, + Err(GetRefError::NotFound(_)) => None, Err(e) => return Err(PushServerError::Internal(e.int_err())), } } else { @@ -360,31 +362,17 @@ impl AxumServerPushProtocolInstance { tracing::debug!("Push client sent a complete request. Committing the dataset"); - if !new_blocks.is_empty() { - let dataset = self.dataset.as_ref().unwrap().as_ref(); - let response = dataset_append_metadata(dataset, new_blocks, force_update_if_diverged) - .await - .map_err(|e| { - tracing::debug!("Appending dataset metadata failed with error: {}", e); - PushServerError::Internal(e.int_err()) - })?; - - // TODO: encapsulate this inside dataset/chain - if !response.new_upstream_ids.is_empty() { - let summary = dataset - .get_summary(GetSummaryOpts::default()) - .await - .int_err()?; - - self.event_bus - .dispatch_event(DatasetEventDependenciesUpdated { - dataset_id: summary.id.clone(), - new_upstream_ids: response.new_upstream_ids, - }) - .await - .int_err()?; - } - } + let dataset = self.dataset.clone().unwrap(); + DatabaseTransactionRunner::new(self.catalog.clone()) + .transactional_with( + |append_dataset_metadata_batch: Arc| async move { + append_dataset_metadata_batch + .execute(dataset.as_ref(), new_blocks, force_update_if_diverged) + .await + }, + ) + .await + .int_err()?; tracing::debug!("Sending completion confirmation"); diff --git a/src/adapter/http/src/smart_protocol/errors.rs b/src/adapter/http/src/smart_protocol/errors.rs index 874fff39b7..c113d21763 100644 --- a/src/adapter/http/src/smart_protocol/errors.rs +++ b/src/adapter/http/src/smart_protocol/errors.rs @@ -9,7 +9,8 @@ use std::fmt::{self, Display}; -use kamu::domain::{InternalError, InvalidIntervalError, RefCASError, RefCollisionError}; +use internal_error::InternalError; +use kamu_core::{InvalidIntervalError, RefCASError, RefCollisionError}; use thiserror::Error; use super::phases::*; diff --git a/src/adapter/http/src/smart_protocol/protocol_dataset_helper.rs b/src/adapter/http/src/smart_protocol/protocol_dataset_helper.rs index 28b9d0615d..04793cf315 100644 --- a/src/adapter/http/src/smart_protocol/protocol_dataset_helper.rs +++ b/src/adapter/http/src/smart_protocol/protocol_dataset_helper.rs @@ -14,8 +14,9 @@ use std::str::FromStr; use bytes::Bytes; use flate2::Compression; use futures::TryStreamExt; +use internal_error::{ErrorIntoInternal, InternalError, ResultIntoInternal}; use kamu::deserialize_metadata_block; -use kamu::domain::*; +use kamu_core::*; use opendatafabric::{MetadataBlock, MetadataEvent, Multihash}; use tar::Header; use thiserror::Error; @@ -204,77 +205,6 @@ pub fn decode_metadata_batch( //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub struct AppendMetadataResponse { - pub new_upstream_ids: Vec, -} - -pub async fn dataset_append_metadata( - dataset: &dyn Dataset, - metadata: VecDeque, - force_update_if_diverged: bool, -) -> Result { - if metadata.is_empty() { - return Ok(AppendMetadataResponse { - new_upstream_ids: vec![], - }); - } - - let old_head = metadata.front().unwrap().1.prev_block_hash.clone(); - let new_head = metadata.back().unwrap().0.clone(); - - let metadata_chain = dataset.as_metadata_chain(); - - let mut new_upstream_ids: Vec = vec![]; - - for (hash, block) in metadata { - tracing::debug!(sequence_numer = %block.sequence_number, hash = %hash, "Appending block"); - - if let opendatafabric::MetadataEvent::SetTransform(transform) = &block.event { - // Collect only the latest upstream dataset IDs - new_upstream_ids.clear(); - for new_input in &transform.inputs { - if let Some(id) = new_input.dataset_ref.id() { - new_upstream_ids.push(id.clone()); - } else { - // Input references must be resolved to IDs here, but we - // ignore the errors and let the metadata chain reject this - // event - } - } - } - - metadata_chain - .append( - block, - AppendOpts { - update_ref: None, - expected_hash: Some(&hash), - ..AppendOpts::default() - }, - ) - .await?; - } - - metadata_chain - .set_ref( - &BlockRef::Head, - &new_head, - SetRefOpts { - validate_block_present: false, - check_ref_is: if force_update_if_diverged { - None - } else { - Some(old_head.as_ref()) - }, - }, - ) - .await?; - - Ok(AppendMetadataResponse { new_upstream_ids }) -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - fn unpack_dataset_metadata_batch(blocks_batch: &MetadataBlocksBatch) -> Vec<(Multihash, Vec)> { assert!( blocks_batch.media_type.eq(MEDIA_TAR_GZ), diff --git a/src/adapter/http/src/smart_protocol/ws_tungstenite_client.rs b/src/adapter/http/src/smart_protocol/ws_tungstenite_client.rs index 0515ee0da9..b6c021ab1b 100644 --- a/src/adapter/http/src/smart_protocol/ws_tungstenite_client.rs +++ b/src/adapter/http/src/smart_protocol/ws_tungstenite_client.rs @@ -10,16 +10,16 @@ use std::sync::atomic::{AtomicI32, Ordering}; use std::sync::Arc; +use database_common::DatabaseTransactionRunner; use dill::*; -use event_bus::EventBus; use futures::SinkExt; -use kamu::domain::events::DatasetEventDependenciesUpdated; -use kamu::domain::*; +use internal_error::{ErrorIntoInternal, InternalError, ResultIntoInternal}; use kamu::utils::smart_transfer_protocol::{ DatasetFactoryFn, SmartTransferProtocolClient, TransferOptions, }; +use kamu_core::*; use opendatafabric::{AsTypedBlock, Multihash}; use serde::de::DeserializeOwned; use serde::Serialize; @@ -38,7 +38,7 @@ use crate::ws_common::{self, ReadMessageError, WriteMessageError}; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub struct WsSmartTransferProtocolClient { - event_bus: Arc, + catalog: Catalog, dataset_credential_resolver: Arc, } @@ -48,11 +48,11 @@ pub struct WsSmartTransferProtocolClient { #[interface(dyn SmartTransferProtocolClient)] impl WsSmartTransferProtocolClient { pub fn new( - event_bus: Arc, + catalog: Catalog, dataset_credential_resolver: Arc, ) -> Self { Self { - event_bus, + catalog, dataset_credential_resolver, } } @@ -640,29 +640,23 @@ impl SmartTransferProtocolClient for WsSmartTransferProtocolClient { .await?; } - let response = dataset_append_metadata( - dst.as_ref(), - new_blocks, - transfer_options.force_update_if_diverged, - ) - .await - .map_err(|e| { - tracing::debug!("Appending dataset metadata failed with error: {}", e); - SyncError::Internal(e.int_err()) - })?; - - // TODO: encapsulate this inside dataset/chain - if !response.new_upstream_ids.is_empty() { - let summary = dst.get_summary(GetSummaryOpts::default()).await.int_err()?; - - self.event_bus - .dispatch_event(DatasetEventDependenciesUpdated { - dataset_id: summary.id.clone(), - new_upstream_ids: response.new_upstream_ids, - }) - .await - .int_err()?; - } + let dst_dataset = dst.clone(); + DatabaseTransactionRunner::new(self.catalog.clone()) + .transactional_with( + |append_dataset_metadata_batch: Arc< + dyn AppendDatasetMetadataBatchUseCase, + >| async move { + append_dataset_metadata_batch + .execute( + dst_dataset.as_ref(), + new_blocks, + transfer_options.force_update_if_diverged, + ) + .await + }, + ) + .await + .int_err()?; let new_dst_head = dst .as_metadata_chain() diff --git a/src/adapter/http/src/upload/upload_handler.rs b/src/adapter/http/src/upload/upload_handler.rs index e062f32a13..d2b80e29a4 100644 --- a/src/adapter/http/src/upload/upload_handler.rs +++ b/src/adapter/http/src/upload/upload_handler.rs @@ -9,7 +9,7 @@ use bytes::Bytes; use http_common::{ApiError, IntoApiError, ResultIntoApiError}; -use kamu::domain::{ErrorIntoInternal, ResultIntoInternal}; +use internal_error::{ErrorIntoInternal, ResultIntoInternal}; use serde_json::{json, Value}; use thiserror::Error; diff --git a/src/adapter/http/src/upload/upload_service.rs b/src/adapter/http/src/upload/upload_service.rs index 23483240e2..7c0d275667 100644 --- a/src/adapter/http/src/upload/upload_service.rs +++ b/src/adapter/http/src/upload/upload_service.rs @@ -9,7 +9,8 @@ use base64::Engine; use bytes::Bytes; -use kamu::domain::{InternalError, MediaType, ResultIntoInternal}; +use internal_error::{InternalError, ResultIntoInternal}; +use kamu_core::MediaType; use opendatafabric::AccountID; use serde::{Deserialize, Serialize}; use serde_json::json; diff --git a/src/adapter/http/src/upload/upload_service_local.rs b/src/adapter/http/src/upload/upload_service_local.rs index 681478ca6f..18cd5f9c6c 100644 --- a/src/adapter/http/src/upload/upload_service_local.rs +++ b/src/adapter/http/src/upload/upload_service_local.rs @@ -12,13 +12,8 @@ use std::sync::Arc; use bytes::Bytes; use dill::*; -use kamu::domain::{ - CacheDir, - ErrorIntoInternal, - InternalError, - ResultIntoInternal, - ServerUrlConfig, -}; +use internal_error::{ErrorIntoInternal, InternalError, ResultIntoInternal}; +use kamu_core::{CacheDir, ServerUrlConfig}; use opendatafabric::AccountID; use thiserror::Error; use tokio::io::AsyncRead; diff --git a/src/adapter/http/src/upload/upload_service_s3.rs b/src/adapter/http/src/upload/upload_service_s3.rs index 988dd0af9e..b3667ab6bb 100644 --- a/src/adapter/http/src/upload/upload_service_s3.rs +++ b/src/adapter/http/src/upload/upload_service_s3.rs @@ -13,7 +13,7 @@ use aws_sdk_s3::presigning::PresigningConfig; use aws_sdk_s3::types::ObjectCannedAcl; use bytes::Bytes; use dill::*; -use kamu::domain::{ErrorIntoInternal, InternalError}; +use internal_error::{ErrorIntoInternal, InternalError}; use kamu::utils::s3_context::S3Context; use opendatafabric::AccountID; use tokio::io::AsyncRead; diff --git a/src/adapter/http/src/ws_common.rs b/src/adapter/http/src/ws_common.rs index 0d365c5597..958e3dffad 100644 --- a/src/adapter/http/src/ws_common.rs +++ b/src/adapter/http/src/ws_common.rs @@ -7,7 +7,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use kamu::domain::BoxedError; +use internal_error::BoxedError; use serde::de::DeserializeOwned; use serde::Serialize; use thiserror::Error; diff --git a/src/adapter/http/tests/harness/client_side_harness.rs b/src/adapter/http/tests/harness/client_side_harness.rs index 1da62e2ad8..8254fed2d5 100644 --- a/src/adapter/http/tests/harness/client_side_harness.rs +++ b/src/adapter/http/tests/harness/client_side_harness.rs @@ -11,13 +11,14 @@ use std::path::PathBuf; use std::sync::Arc; use container_runtime::ContainerRuntime; +use database_common::NoOpDatabasePlugin; use dill::Component; -use event_bus::EventBus; use kamu::domain::*; use kamu::*; use kamu_accounts::CurrentAccountSubject; use kamu_adapter_http::SmartTransferProtocolClientWs; use kamu_datasets_services::DatasetKeyValueServiceSysEnv; +use messaging_outbox::DummyOutboxImpl; use opendatafabric::{ AccountID, AccountName, @@ -27,6 +28,7 @@ use opendatafabric::{ DatasetRefRemote, }; use tempfile::TempDir; +use time_source::SystemTimeSourceDefault; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -66,7 +68,7 @@ impl ClientSideHarness { b.add_value(CacheDir::new(cache_dir)); b.add_value(RemoteReposDir::new(repos_dir)); - b.add::(); + b.add::(); b.add::(); @@ -92,7 +94,8 @@ impl ClientSideHarness { .with_root(datasets_dir) .with_multi_tenant(options.multi_tenant), ) - .bind::(); + .bind::() + .bind::(); b.add::(); @@ -125,10 +128,16 @@ impl ClientSideHarness { b.add::(); + b.add::(); + b.add::(); + b.add::(); + b.add_value(ContainerRuntime::default()); b.add_value(kamu::utils::ipfs_wrapper::IpfsClient::default()); b.add_value(IpfsGateway::default()); + NoOpDatabasePlugin::init_database_components(&mut b); + let catalog = b.build(); let pull_service = catalog.get_one::().unwrap(); @@ -155,6 +164,18 @@ impl ClientSideHarness { self.catalog.get_one::().unwrap() } + pub fn create_dataset_from_snapshot(&self) -> Arc { + self.catalog + .get_one::() + .unwrap() + } + + pub fn commit_dataset_event(&self) -> Arc { + self.catalog + .get_one::() + .unwrap() + } + pub fn compaction_service(&self) -> Arc { self.catalog.get_one::().unwrap() } diff --git a/src/adapter/http/tests/harness/common_harness.rs b/src/adapter/http/tests/harness/common_harness.rs index d206acafd7..5568775a8c 100644 --- a/src/adapter/http/tests/harness/common_harness.rs +++ b/src/adapter/http/tests/harness/common_harness.rs @@ -159,7 +159,7 @@ pub(crate) async fn commit_add_data_event( dataset_layout: &DatasetLayout, prev_data_block_hash: Option, ) -> CommitResult { - let dataset = dataset_repo.get_dataset(dataset_ref).await.unwrap(); + let dataset = dataset_repo.find_dataset_by_ref(dataset_ref).await.unwrap(); let (prev_offset, prev_checkpoint) = if let Some(prev_data_block_hash) = prev_data_block_hash { let prev_data_block = dataset diff --git a/src/adapter/http/tests/harness/server_side_harness.rs b/src/adapter/http/tests/harness/server_side_harness.rs index d463a2f990..7c1af476e8 100644 --- a/src/adapter/http/tests/harness/server_side_harness.rs +++ b/src/adapter/http/tests/harness/server_side_harness.rs @@ -12,12 +12,19 @@ use std::sync::Arc; use chrono::Utc; +use internal_error::InternalError; use kamu::domain::auth::{ AlwaysHappyDatasetActionAuthorizer, DatasetAction, DatasetActionAuthorizer, }; -use kamu::domain::{CompactionService, DatasetRepository, InternalError, SystemTimeSourceStub}; +use kamu::domain::{ + CommitDatasetEventUseCase, + CompactionService, + CreateDatasetFromSnapshotUseCase, + CreateDatasetUseCase, + DatasetRepository, +}; use kamu::testing::MockDatasetActionAuthorizer; use kamu::DatasetLayout; use kamu_accounts::{ @@ -29,6 +36,7 @@ use kamu_accounts::{ }; use opendatafabric::{AccountID, AccountName, DatasetAlias, DatasetHandle}; use reqwest::Url; +use time_source::SystemTimeSourceStub; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -39,9 +47,16 @@ pub(crate) const SERVER_ACCOUNT_NAME: &str = "kamu-server"; #[async_trait::async_trait] pub(crate) trait ServerSideHarness { fn operating_account_name(&self) -> Option; - fn cli_dataset_repository(&self) -> Arc; + fn cli_create_dataset_use_case(&self) -> Arc; + + fn cli_create_dataset_from_snapshot_use_case( + &self, + ) -> Arc; + + fn cli_commit_dataset_event_use_case(&self) -> Arc; + fn cli_compaction_service(&self) -> Arc; fn dataset_layout(&self, dataset_handle: &DatasetHandle) -> DatasetLayout; diff --git a/src/adapter/http/tests/harness/server_side_local_fs_harness.rs b/src/adapter/http/tests/harness/server_side_local_fs_harness.rs index 7f731a05d2..bb9d1bd31f 100644 --- a/src/adapter/http/tests/harness/server_side_local_fs_harness.rs +++ b/src/adapter/http/tests/harness/server_side_local_fs_harness.rs @@ -13,29 +13,35 @@ use std::str::FromStr; use std::sync::Arc; use dill::Component; -use event_bus::EventBus; +use internal_error::{InternalError, ResultIntoInternal}; use kamu::domain::{ CacheDir, + CommitDatasetEventUseCase, CompactionService, + CreateDatasetFromSnapshotUseCase, + CreateDatasetUseCase, DatasetRepository, - InternalError, - ResultIntoInternal, RunInfoDir, ServerUrlConfig, - SystemTimeSource, - SystemTimeSourceStub, }; use kamu::{ + AppendDatasetMetadataBatchUseCaseImpl, + CommitDatasetEventUseCaseImpl, CompactionServiceImpl, + CreateDatasetFromSnapshotUseCaseImpl, + CreateDatasetUseCaseImpl, DatasetLayout, DatasetRepositoryLocalFs, + DatasetRepositoryWriter, DependencyGraphServiceInMemory, ObjectStoreBuilderLocalFs, ObjectStoreRegistryImpl, }; use kamu_accounts::{AuthenticationService, MockAuthenticationService}; +use messaging_outbox::DummyOutboxImpl; use opendatafabric::{AccountName, DatasetAlias, DatasetHandle}; use tempfile::TempDir; +use time_source::{SystemTimeSource, SystemTimeSourceStub}; use url::Url; use super::{ @@ -85,7 +91,7 @@ impl ServerSideLocalFsHarness { b.add_value(RunInfoDir::new(run_info_dir)) .add_value(CacheDir::new(cache_dir)) - .add::() + .add::() .add_value(time_source.clone()) .bind::() .add::() @@ -95,12 +101,17 @@ impl ServerSideLocalFsHarness { .with_multi_tenant(options.multi_tenant), ) .bind::() + .bind::() .add_value(server_authentication_mock()) .bind::() .add_value(ServerUrlConfig::new_test(Some(&base_url_rest))) .add::() .add::() - .add::(); + .add::() + .add::() + .add::() + .add::() + .add::(); database_common::NoOpDatabasePlugin::init_database_components(&mut b); @@ -150,6 +161,27 @@ impl ServerSideHarness for ServerSideLocalFsHarness { cli_catalog.get_one::().unwrap() } + fn cli_create_dataset_use_case(&self) -> Arc { + let cli_catalog = create_cli_user_catalog(&self.base_catalog); + cli_catalog.get_one::().unwrap() + } + + fn cli_create_dataset_from_snapshot_use_case( + &self, + ) -> Arc { + let cli_catalog = create_cli_user_catalog(&self.base_catalog); + cli_catalog + .get_one::() + .unwrap() + } + + fn cli_commit_dataset_event_use_case(&self) -> Arc { + let cli_catalog = create_cli_user_catalog(&self.base_catalog); + cli_catalog + .get_one::() + .unwrap() + } + fn cli_compaction_service(&self) -> Arc { let cli_catalog = create_cli_user_catalog(&self.base_catalog); cli_catalog.get_one::().unwrap() diff --git a/src/adapter/http/tests/harness/server_side_s3_harness.rs b/src/adapter/http/tests/harness/server_side_s3_harness.rs index a4b1e0351f..0f5a07c8d6 100644 --- a/src/adapter/http/tests/harness/server_side_s3_harness.rs +++ b/src/adapter/http/tests/harness/server_side_s3_harness.rs @@ -13,31 +13,37 @@ use std::str::FromStr; use std::sync::Arc; use dill::Component; -use event_bus::EventBus; +use internal_error::{InternalError, ResultIntoInternal}; use kamu::domain::{ + CommitDatasetEventUseCase, CompactionService, + CreateDatasetFromSnapshotUseCase, + CreateDatasetUseCase, DatasetRepository, - InternalError, ObjectStoreBuilder, - ResultIntoInternal, RunInfoDir, ServerUrlConfig, - SystemTimeSource, - SystemTimeSourceStub, }; use kamu::testing::LocalS3Server; use kamu::utils::s3_context::S3Context; use kamu::{ + AppendDatasetMetadataBatchUseCaseImpl, + CommitDatasetEventUseCaseImpl, CompactionServiceImpl, + CreateDatasetFromSnapshotUseCaseImpl, + CreateDatasetUseCaseImpl, DatasetLayout, DatasetRepositoryS3, + DatasetRepositoryWriter, DependencyGraphServiceInMemory, ObjectStoreBuilderLocalFs, ObjectStoreBuilderS3, ObjectStoreRegistryImpl, }; use kamu_accounts::{AuthenticationService, MockAuthenticationService}; +use messaging_outbox::DummyOutboxImpl; use opendatafabric::{AccountName, DatasetAlias, DatasetHandle}; +use time_source::{SystemTimeSource, SystemTimeSourceStub}; use url::Url; use super::{ @@ -83,7 +89,7 @@ impl ServerSideS3Harness { b.add_value(time_source.clone()) .add_value(RunInfoDir::new(run_info_dir)) .bind::() - .add::() + .add::() .add::() .add_builder( DatasetRepositoryS3::builder() @@ -91,6 +97,7 @@ impl ServerSideS3Harness { .with_multi_tenant(false), ) .bind::() + .bind::() .add_value(server_authentication_mock()) .bind::() .add_value(ServerUrlConfig::new_test(Some(&base_url_rest))) @@ -98,7 +105,11 @@ impl ServerSideS3Harness { .add::() .add::() .add_value(ObjectStoreBuilderS3::new(s3_context, true)) - .bind::(); + .bind::() + .add::() + .add::() + .add::() + .add::(); database_common::NoOpDatabasePlugin::init_database_components(&mut b); @@ -145,6 +156,27 @@ impl ServerSideHarness for ServerSideS3Harness { cli_catalog.get_one::().unwrap() } + fn cli_create_dataset_use_case(&self) -> Arc { + let cli_catalog = create_cli_user_catalog(&self.base_catalog); + cli_catalog.get_one::().unwrap() + } + + fn cli_create_dataset_from_snapshot_use_case( + &self, + ) -> Arc { + let cli_catalog = create_cli_user_catalog(&self.base_catalog); + cli_catalog + .get_one::() + .unwrap() + } + + fn cli_commit_dataset_event_use_case(&self) -> Arc { + let cli_catalog = create_cli_user_catalog(&self.base_catalog); + cli_catalog + .get_one::() + .unwrap() + } + fn cli_compaction_service(&self) -> Arc { let cli_catalog = create_cli_user_catalog(&self.base_catalog); cli_catalog.get_one::().unwrap() diff --git a/src/adapter/http/tests/tests/test_authentication_layer.rs b/src/adapter/http/tests/tests/test_authentication_layer.rs index f2ecd79f43..b767d04cbb 100644 --- a/src/adapter/http/tests/tests/test_authentication_layer.rs +++ b/src/adapter/http/tests/tests/test_authentication_layer.rs @@ -10,7 +10,7 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::str::FromStr; -use kamu::domain::{InternalError, ResultIntoInternal}; +use internal_error::{InternalError, ResultIntoInternal}; use kamu_accounts::{ AnonymousAccountReason, AuthenticationService, diff --git a/src/adapter/http/tests/tests/test_data_ingest.rs b/src/adapter/http/tests/tests/test_data_ingest.rs index a1e794b703..fe8749b807 100644 --- a/src/adapter/http/tests/tests/test_data_ingest.rs +++ b/src/adapter/http/tests/tests/test_data_ingest.rs @@ -534,8 +534,8 @@ impl DataIngestHarness { async fn create_population_dataset(&self, with_schema: bool) -> CreateDatasetResult { self.server_harness - .cli_dataset_repository() - .create_dataset_from_snapshot(DatasetSnapshot { + .cli_create_dataset_from_snapshot_use_case() + .execute(DatasetSnapshot { name: DatasetAlias::new( self.server_harness.operating_account_name(), DatasetName::new_unchecked("population"), diff --git a/src/adapter/http/tests/tests/test_data_query.rs b/src/adapter/http/tests/tests/test_data_query.rs index 50083cda0e..68b2c94ad6 100644 --- a/src/adapter/http/tests/tests/test_data_query.rs +++ b/src/adapter/http/tests/tests/test_data_query.rs @@ -60,8 +60,8 @@ impl Harness { DatasetName::new_unchecked("population"), ); let create_result = server_harness - .cli_dataset_repository() - .create_dataset( + .cli_create_dataset_use_case() + .execute( &alias, MetadataFactory::metadata_block(MetadataFactory::seed(DatasetKind::Root).build()) .system_time(system_time) diff --git a/src/adapter/http/tests/tests/test_dataset_authorization_layer.rs b/src/adapter/http/tests/tests/test_dataset_authorization_layer.rs index 789943c5f4..2f5d6ae73f 100644 --- a/src/adapter/http/tests/tests/test_dataset_authorization_layer.rs +++ b/src/adapter/http/tests/tests/test_dataset_authorization_layer.rs @@ -14,14 +14,21 @@ use std::str::FromStr; use database_common::{DatabaseTransactionRunner, NoOpDatabasePlugin}; use dill::{CatalogBuilder, Component}; -use event_bus::EventBus; +use internal_error::{InternalError, ResultIntoInternal}; use kamu::domain::auth::DatasetAction; -use kamu::domain::{DatasetRepository, InternalError, ResultIntoInternal, SystemTimeSourceDefault}; +use kamu::domain::{CreateDatasetUseCase, DatasetRepository}; use kamu::testing::{MetadataFactory, MockDatasetActionAuthorizer}; -use kamu::{DatasetRepositoryLocalFs, DependencyGraphServiceInMemory}; +use kamu::{ + CreateDatasetUseCaseImpl, + DatasetRepositoryLocalFs, + DatasetRepositoryWriter, + DependencyGraphServiceInMemory, +}; use kamu_accounts::*; +use messaging_outbox::DummyOutboxImpl; use mockall::predicate::{eq, function}; use opendatafabric::{DatasetAlias, DatasetHandle, DatasetKind, DatasetName, DatasetRef}; +use time_source::SystemTimeSourceDefault; use url::Url; use crate::harness::await_client_server_flow; @@ -214,7 +221,7 @@ impl ServerHarness { let mut b = dill::CatalogBuilder::new(); b.add::() - .add::() + .add::() .add::() .add_value(MockAuthenticationService::resolving_token( DUMMY_ACCESS_TOKEN, @@ -229,6 +236,8 @@ impl ServerHarness { .with_root(datasets_dir), ) .bind::() + .bind::() + .add::() .add::(); NoOpDatabasePlugin::init_database_components(&mut b); @@ -240,9 +249,11 @@ impl ServerHarness { .add_value(CurrentAccountSubject::new_test()) .build(); - let dataset_repo = catalog_system.get_one::().unwrap(); - dataset_repo - .create_dataset( + let create_dataset = catalog_system + .get_one::() + .unwrap(); + create_dataset + .execute( &Self::dataset_alias(), MetadataFactory::metadata_block(MetadataFactory::seed(DatasetKind::Root).build()) .build_typed(), diff --git a/src/adapter/http/tests/tests/test_platform_login_validate.rs b/src/adapter/http/tests/tests/test_platform_login_validate.rs index aea72affa2..4ae3bde86a 100644 --- a/src/adapter/http/tests/tests/test_platform_login_validate.rs +++ b/src/adapter/http/tests/tests/test_platform_login_validate.rs @@ -12,9 +12,9 @@ use std::sync::Arc; use chrono::{Duration, Utc}; use database_common::{DatabaseTransactionRunner, NoOpDatabasePlugin}; -use kamu::domain::{InternalError, ResultIntoInternal, SystemTimeSource, SystemTimeSourceStub}; +use internal_error::{InternalError, ResultIntoInternal}; use kamu_accounts::*; -use kamu_accounts_inmem::{AccessTokenRepositoryInMemory, AccountRepositoryInMemory}; +use kamu_accounts_inmem::{InMemoryAccessTokenRepository, InMemoryAccountRepository}; use kamu_accounts_services::{ AccessTokenServiceImpl, AuthenticationServiceImpl, @@ -24,6 +24,7 @@ use kamu_accounts_services::{ use kamu_adapter_http::{LoginRequestBody, LoginResponseBody}; use opendatafabric::AccountName; use serde_json::json; +use time_source::{SystemTimeSource, SystemTimeSourceStub}; use crate::harness::{await_client_server_flow, TestAPIServer}; @@ -63,14 +64,14 @@ impl Harness { b.add::() .add_value(predefined_accounts_config) - .add::() + .add::() .add_value(SystemTimeSourceStub::new()) .bind::() .add::() .add_value(JwtAuthenticationConfig::default()) .add::() .add::() - .add::() + .add::() .add::(); NoOpDatabasePlugin::init_database_components(&mut b); diff --git a/src/adapter/http/tests/tests/test_protocol_dataset_helpers.rs b/src/adapter/http/tests/tests/test_protocol_dataset_helpers.rs index f6c9c008f1..351b9f5e43 100644 --- a/src/adapter/http/tests/tests/test_protocol_dataset_helpers.rs +++ b/src/adapter/http/tests/tests/test_protocol_dataset_helpers.rs @@ -454,10 +454,10 @@ impl TestCase { } async fn create_test_case(server_harness: &dyn ServerSideHarness) -> TestCase { - let server_repo = server_harness.cli_dataset_repository(); + let create_dataset_from_snapshot = server_harness.cli_create_dataset_from_snapshot_use_case(); - let create_result = server_repo - .create_dataset_from_snapshot( + let create_result = create_dataset_from_snapshot + .execute( MetadataFactory::dataset_snapshot() .name("foo") .kind(DatasetKind::Root) @@ -469,7 +469,7 @@ async fn create_test_case(server_harness: &dyn ServerSideHarness) -> TestCase { .unwrap(); let commit_result = commit_add_data_event( - server_repo.as_ref(), + server_harness.cli_dataset_repository().as_ref(), &make_dataset_ref(&None, "foo"), &server_harness.dataset_layout(&create_result.dataset_handle), None, diff --git a/src/adapter/http/tests/tests/test_routing.rs b/src/adapter/http/tests/tests/test_routing.rs index af2da7ca2b..2b2f829521 100644 --- a/src/adapter/http/tests/tests/test_routing.rs +++ b/src/adapter/http/tests/tests/test_routing.rs @@ -14,13 +14,14 @@ use axum::extract::{FromRequestParts, Path}; use axum::routing::IntoMakeService; use axum::Router; use dill::Component; -use event_bus::EventBus; use hyper::server::conn::AddrIncoming; use kamu::domain::*; use kamu::testing::*; use kamu::*; use kamu_accounts::CurrentAccountSubject; +use messaging_outbox::DummyOutboxImpl; use opendatafabric::*; +use time_source::SystemTimeSourceDefault; use crate::harness::await_client_server_flow; @@ -40,7 +41,7 @@ async fn setup_repo() -> RepoFixture { let catalog = dill::CatalogBuilder::new() .add::() - .add::() + .add::() .add::() .add_builder( DatasetRepositoryLocalFs::builder() @@ -48,14 +49,18 @@ async fn setup_repo() -> RepoFixture { .with_multi_tenant(false), ) .bind::() + .bind::() .add_value(CurrentAccountSubject::new_test()) .add::() + .add::() .build(); - let dataset_repo = catalog.get_one::().unwrap(); + let create_dataset_from_snapshot = catalog + .get_one::() + .unwrap(); - let created_dataset = dataset_repo - .create_dataset_from_snapshot( + let created_dataset = create_dataset_from_snapshot + .execute( MetadataFactory::dataset_snapshot() .name("foo") .kind(DatasetKind::Root) @@ -113,18 +118,13 @@ where async fn setup_client(dataset_url: url::Url, head_expected: Multihash) { let catalog = dill::CatalogBuilder::new() - .add::() .add::() .build(); - let dataset = DatasetFactoryImpl::new( - IpfsGateway::default(), - catalog.get_one().unwrap(), - catalog.get_one().unwrap(), - ) - .get_dataset(&dataset_url, false) - .await - .unwrap(); + let dataset = DatasetFactoryImpl::new(IpfsGateway::default(), catalog.get_one().unwrap()) + .get_dataset(&dataset_url, false) + .await + .unwrap(); let head_actual = dataset .as_metadata_chain() diff --git a/src/adapter/http/tests/tests/test_upload_local.rs b/src/adapter/http/tests/tests/test_upload_local.rs index 3ad82697a7..42effb41db 100644 --- a/src/adapter/http/tests/tests/test_upload_local.rs +++ b/src/adapter/http/tests/tests/test_upload_local.rs @@ -12,15 +12,10 @@ use std::ops::Add; use std::path::{Path, PathBuf}; use database_common::{DatabaseTransactionRunner, NoOpDatabasePlugin}; -use kamu::domain::{ - CacheDir, - InternalError, - ResultIntoInternal, - ServerUrlConfig, - SystemTimeSourceDefault, -}; +use internal_error::{InternalError, ResultIntoInternal}; +use kamu::domain::{CacheDir, ServerUrlConfig}; use kamu_accounts::{JwtAuthenticationConfig, PredefinedAccountsConfig, DEFAULT_ACCOUNT_ID}; -use kamu_accounts_inmem::{AccessTokenRepositoryInMemory, AccountRepositoryInMemory}; +use kamu_accounts_inmem::{InMemoryAccessTokenRepository, InMemoryAccountRepository}; use kamu_accounts_services::{ AccessTokenServiceImpl, AuthenticationServiceImpl, @@ -33,6 +28,7 @@ use kamu_adapter_http::{ UploadContext, UploadServiceLocal, }; +use time_source::SystemTimeSourceDefault; use crate::harness::{await_client_server_flow, TestAPIServer}; @@ -63,9 +59,9 @@ impl Harness { b.add_value(CacheDir::new(cache_dir.clone())) .add_value(PredefinedAccountsConfig::single_tenant()) .add::() - .add::() + .add::() .add::() - .add::() + .add::() .add::() .add::() .add_value(JwtAuthenticationConfig::default()) diff --git a/src/adapter/http/tests/tests/test_upload_s3.rs b/src/adapter/http/tests/tests/test_upload_s3.rs index 95467561e3..daacb7fedf 100644 --- a/src/adapter/http/tests/tests/test_upload_s3.rs +++ b/src/adapter/http/tests/tests/test_upload_s3.rs @@ -12,11 +12,12 @@ use std::net::{SocketAddr, TcpListener}; use database_common::{DatabaseTransactionRunner, NoOpDatabasePlugin}; use dill::Component; use http::{HeaderMap, HeaderName, HeaderValue}; -use kamu::domain::{InternalError, ResultIntoInternal, ServerUrlConfig, SystemTimeSourceDefault}; +use internal_error::{InternalError, ResultIntoInternal}; +use kamu::domain::ServerUrlConfig; use kamu::testing::LocalS3Server; use kamu::utils::s3_context::S3Context; use kamu_accounts::{JwtAuthenticationConfig, PredefinedAccountsConfig, DEFAULT_ACCOUNT_ID}; -use kamu_accounts_inmem::{AccessTokenRepositoryInMemory, AccountRepositoryInMemory}; +use kamu_accounts_inmem::{InMemoryAccessTokenRepository, InMemoryAccountRepository}; use kamu_accounts_services::{ AccessTokenServiceImpl, AuthenticationServiceImpl, @@ -30,6 +31,7 @@ use kamu_adapter_http::{ UploadService, UploadServiceS3, }; +use time_source::SystemTimeSourceDefault; use tokio::io::AsyncReadExt; use crate::harness::{await_client_server_flow, TestAPIServer}; @@ -60,9 +62,9 @@ impl Harness { b.add_value(PredefinedAccountsConfig::single_tenant()) .add::() - .add::() + .add::() .add::() - .add::() + .add::() .add::() .add::() .add_value(JwtAuthenticationConfig::default()) diff --git a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_aborted_read_of_existing_evolved_dataset_reread_succeeds.rs b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_aborted_read_of_existing_evolved_dataset_reread_succeeds.rs index 22b5d83ed2..520b0096f0 100644 --- a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_aborted_read_of_existing_evolved_dataset_reread_succeeds.rs +++ b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_aborted_read_of_existing_evolved_dataset_reread_succeeds.rs @@ -45,9 +45,11 @@ impl ) -> Self { let server_account_name = server_harness.operating_account_name(); - let server_repo = server_harness.cli_dataset_repository(); - let server_create_result = server_repo - .create_dataset_from_snapshot( + let create_dataset_from_snapshot = + server_harness.cli_create_dataset_from_snapshot_use_case(); + + let server_create_result = create_dataset_from_snapshot + .execute( MetadataFactory::dataset_snapshot() .name(DatasetAlias::new( server_account_name.clone(), @@ -79,13 +81,18 @@ impl // Extend server-side dataset with new nodes - let server_dataset_ref = make_dataset_ref(&server_account_name, "foo"); + let server_repo = server_harness.cli_dataset_repository(); - server_repo - .get_dataset(&server_dataset_ref) + let server_dataset_ref = make_dataset_ref(&server_account_name, "foo"); + let server_dataset_handle = server_repo + .resolve_dataset_ref(&server_dataset_ref) .await - .unwrap() - .commit_event( + .unwrap(); + + let commit_dataset_event = server_harness.cli_commit_dataset_event_use_case(); + commit_dataset_event + .execute( + &server_dataset_handle, MetadataEvent::SetInfo( MetadataFactory::set_info() .description("updated description") diff --git a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_aborted_read_of_new_reread_succeeds.rs b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_aborted_read_of_new_reread_succeeds.rs index 2bc8cb514d..7f39fb5183 100644 --- a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_aborted_read_of_new_reread_succeeds.rs +++ b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_aborted_read_of_new_reread_succeeds.rs @@ -42,9 +42,11 @@ impl ) -> Self { let server_account_name = server_harness.operating_account_name(); - let server_repo = server_harness.cli_dataset_repository(); - let server_create_result = server_repo - .create_dataset_from_snapshot( + let create_dataset_from_snapshot = + server_harness.cli_create_dataset_from_snapshot_use_case(); + + let server_create_result = create_dataset_from_snapshot + .execute( MetadataFactory::dataset_snapshot() .name(DatasetAlias::new( server_account_name.clone(), @@ -62,7 +64,7 @@ impl server_harness.dataset_layout(&server_create_result.dataset_handle); let server_commit_result = commit_add_data_event( - server_repo.as_ref(), + server_harness.cli_dataset_repository().as_ref(), &make_dataset_ref(&server_account_name, "foo"), &server_dataset_layout, None, diff --git a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_advanced_dataset_fails.rs b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_advanced_dataset_fails.rs index 199023e465..f8dcec0296 100644 --- a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_advanced_dataset_fails.rs +++ b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_advanced_dataset_fails.rs @@ -37,9 +37,11 @@ impl let client_account_name = client_harness.operating_account_name(); let server_account_name = server_harness.operating_account_name(); - let server_repo = server_harness.cli_dataset_repository(); - let server_create_result = server_repo - .create_dataset_from_snapshot( + let create_dataset_from_snapshot = + server_harness.cli_create_dataset_from_snapshot_use_case(); + + let server_create_result = create_dataset_from_snapshot + .execute( MetadataFactory::dataset_snapshot() .name(DatasetAlias::new( server_account_name.clone(), @@ -72,7 +74,7 @@ impl // Extend client-side dataset with new node let client_repo = client_harness.dataset_repository(); client_repo - .get_dataset(&make_dataset_ref(&client_account_name, "foo")) + .find_dataset_by_ref(&make_dataset_ref(&client_account_name, "foo")) .await .unwrap() .commit_event( diff --git a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_diverged_dataset.rs b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_diverged_dataset.rs index 6640a85974..c8e752c09e 100644 --- a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_diverged_dataset.rs +++ b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_diverged_dataset.rs @@ -40,9 +40,11 @@ impl SmartPullExistingDivergedDatasetScenario ) -> Self { let server_account_name = server_harness.operating_account_name(); - let server_repo = server_harness.cli_dataset_repository(); - let server_create_result = server_repo - .create_dataset_from_snapshot( + let create_dataset_from_snapshot = + server_harness.cli_create_dataset_from_snapshot_use_case(); + + let server_create_result = create_dataset_from_snapshot + .execute( MetadataFactory::dataset_snapshot() .name(DatasetAlias::new( server_account_name.clone(), @@ -66,7 +68,7 @@ impl SmartPullExistingDivergedDatasetScenario for _ in 0..3 { commit_result = Some( commit_add_data_event( - server_repo.as_ref(), + server_harness.cli_dataset_repository().as_ref(), &server_dataset_ref, &server_dataset_layout, commit_result.map(|r| r.new_head), diff --git a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_evolved_dataset.rs b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_evolved_dataset.rs index 2e7e5426e4..49cb43ea30 100644 --- a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_evolved_dataset.rs +++ b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_evolved_dataset.rs @@ -40,9 +40,11 @@ impl SmartPullExistingEvolvedDatasetScenario< ) -> Self { let server_account_name = server_harness.operating_account_name(); - let server_repo = server_harness.cli_dataset_repository(); - let server_create_result = server_repo - .create_dataset_from_snapshot( + let create_dataset_from_snapshot = + server_harness.cli_create_dataset_from_snapshot_use_case(); + + let server_create_result = create_dataset_from_snapshot + .execute( MetadataFactory::dataset_snapshot() .name(DatasetAlias::new( server_account_name.clone(), @@ -73,14 +75,18 @@ impl SmartPullExistingEvolvedDatasetScenario< .await; // Extend server-side dataset with new nodes + let server_repo = server_harness.cli_dataset_repository(); let server_dataset_ref = make_dataset_ref(&server_account_name, "foo"); - - server_repo - .get_dataset(&server_dataset_ref) + let server_dataset_handle = server_repo + .resolve_dataset_ref(&server_dataset_ref) .await - .unwrap() - .commit_event( + .unwrap(); + + let commit_dataset_event = server_harness.cli_commit_dataset_event_use_case(); + commit_dataset_event + .execute( + &server_dataset_handle, MetadataEvent::SetInfo( MetadataFactory::set_info() .description("updated description") diff --git a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_up_to_date_dataset.rs b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_up_to_date_dataset.rs index ca55c852e9..7620731f53 100644 --- a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_up_to_date_dataset.rs +++ b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_existing_up_to_date_dataset.rs @@ -37,9 +37,11 @@ impl SmartPullExistingUpToDateDatasetScenario ) -> Self { let server_account_name = server_harness.operating_account_name(); - let server_repo = server_harness.cli_dataset_repository(); - let server_create_result = server_repo - .create_dataset_from_snapshot( + let create_dataset_from_snapshot = + server_harness.cli_create_dataset_from_snapshot_use_case(); + + let server_create_result = create_dataset_from_snapshot + .execute( MetadataFactory::dataset_snapshot() .name(DatasetAlias::new( server_account_name.clone(), @@ -57,7 +59,7 @@ impl SmartPullExistingUpToDateDatasetScenario server_harness.dataset_layout(&server_create_result.dataset_handle); commit_add_data_event( - server_repo.as_ref(), + server_harness.cli_dataset_repository().as_ref(), &make_dataset_ref(&server_account_name, "foo"), &server_dataset_layout, None, diff --git a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_new_dataset.rs b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_new_dataset.rs index a8850edf0e..aecf2807d6 100644 --- a/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_new_dataset.rs +++ b/src/adapter/http/tests/tests/tests_pull/scenarios/scenario_new_dataset.rs @@ -37,9 +37,11 @@ impl SmartPullNewDatasetScenario Self { let server_account_name = server_harness.operating_account_name(); - let server_repo = server_harness.cli_dataset_repository(); - let server_create_result = server_repo - .create_dataset_from_snapshot( + let create_dataset_from_snapshot = + server_harness.cli_create_dataset_from_snapshot_use_case(); + + let server_create_result = create_dataset_from_snapshot + .execute( MetadataFactory::dataset_snapshot() .name(DatasetAlias::new( server_account_name.clone(), @@ -57,7 +59,7 @@ impl SmartPullNewDatasetScenario SmartPullNewEmptyDatasetScenario Self { let server_account_name = server_harness.operating_account_name(); - let server_repo = server_harness.cli_dataset_repository(); - let server_create_result = server_repo - .create_dataset_from_snapshot( + let create_dataset_from_snapshot = + server_harness.cli_create_dataset_from_snapshot_use_case(); + + let server_create_result = create_dataset_from_snapshot + .execute( MetadataFactory::dataset_snapshot() .name(DatasetAlias::new( server_account_name.clone(), diff --git a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_aborted_write_of_new_rewrite_succeeds.rs b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_aborted_write_of_new_rewrite_succeeds.rs index 538bd18872..47b3c81c7e 100644 --- a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_aborted_write_of_new_rewrite_succeeds.rs +++ b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_aborted_write_of_new_rewrite_succeeds.rs @@ -41,10 +41,9 @@ impl SmartPushAbortedWriteOfNewWriteSucceeds< let client_account_name = client_harness.operating_account_name(); let server_account_name = server_harness.operating_account_name(); - let client_repo = client_harness.dataset_repository(); - - let client_create_result = client_repo - .create_dataset_from_snapshot( + let client_create_result = client_harness + .create_dataset_from_snapshot() + .execute( MetadataFactory::dataset_snapshot() .name(DatasetAlias::new( client_account_name.clone(), @@ -70,7 +69,7 @@ impl SmartPushAbortedWriteOfNewWriteSucceeds< let client_dataset_ref = make_dataset_ref(&client_account_name, "foo"); let client_commit_result = commit_add_data_event( - client_repo.as_ref(), + client_harness.dataset_repository().as_ref(), &client_dataset_ref, &client_dataset_layout, None, diff --git a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_aborted_write_of_updated_rewrite_succeeds.rs b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_aborted_write_of_updated_rewrite_succeeds.rs index 241be24658..904ab8cd36 100644 --- a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_aborted_write_of_updated_rewrite_succeeds.rs +++ b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_aborted_write_of_updated_rewrite_succeeds.rs @@ -45,10 +45,9 @@ impl let client_account_name = client_harness.operating_account_name(); let server_account_name = server_harness.operating_account_name(); - let client_repo = client_harness.dataset_repository(); - - let client_create_result = client_repo - .create_dataset_from_snapshot( + let client_create_result = client_harness + .create_dataset_from_snapshot() + .execute( MetadataFactory::dataset_snapshot() .name(DatasetAlias::new( client_account_name.clone(), @@ -81,13 +80,18 @@ impl ) .await; + let client_repo = client_harness.dataset_repository(); + // Extend client-side dataset with new nodes let client_dataset_ref = make_dataset_ref(&client_account_name, "foo"); - client_repo - .get_dataset(&client_dataset_ref) + let client_dataset_handle = client_repo + .resolve_dataset_ref(&client_dataset_ref) .await - .unwrap() - .commit_event( + .unwrap(); + client_harness + .commit_dataset_event() + .execute( + &client_dataset_handle, MetadataEvent::SetInfo( MetadataFactory::set_info() .description("updated description") diff --git a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_dataset_fails_as_server_advanced.rs b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_dataset_fails_as_server_advanced.rs index 35d2c90780..e74df64cb6 100644 --- a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_dataset_fails_as_server_advanced.rs +++ b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_dataset_fails_as_server_advanced.rs @@ -41,10 +41,10 @@ impl let server_account_name = server_harness.operating_account_name(); let server_repo = server_harness.cli_dataset_repository(); - let client_repo = client_harness.dataset_repository(); - let client_create_result = client_repo - .create_dataset_from_snapshot( + let client_create_result = client_harness + .create_dataset_from_snapshot() + .execute( MetadataFactory::dataset_snapshot() .name(DatasetAlias::new( client_account_name.clone(), @@ -81,7 +81,7 @@ impl // Extend server-side dataset with new node server_repo - .get_dataset(&make_dataset_ref(&server_account_name, "foo")) + .find_dataset_by_ref(&make_dataset_ref(&server_account_name, "foo")) .await .unwrap() .commit_event( diff --git a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_diverged_dataset.rs b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_diverged_dataset.rs index de640cfb81..b8b4b45634 100644 --- a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_diverged_dataset.rs +++ b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_diverged_dataset.rs @@ -42,10 +42,9 @@ impl SmartPushExistingDivergedDatasetScenario let client_account_name = client_harness.operating_account_name(); let server_account_name = server_harness.operating_account_name(); - let client_repo = client_harness.dataset_repository(); - - let client_create_result = client_repo - .create_dataset_from_snapshot( + let client_create_result = client_harness + .create_dataset_from_snapshot() + .execute( MetadataFactory::dataset_snapshot() .name(DatasetAlias::new( client_account_name.clone(), @@ -69,7 +68,7 @@ impl SmartPushExistingDivergedDatasetScenario for _ in 0..3 { commit_result = Some( commit_add_data_event( - client_repo.as_ref(), + client_harness.dataset_repository().as_ref(), &client_dataset_ref, &client_dataset_layout, commit_result.map(|r| r.new_head), diff --git a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_evolved_dataset.rs b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_evolved_dataset.rs index 1d2b8b213c..714e6f3b76 100644 --- a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_evolved_dataset.rs +++ b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_evolved_dataset.rs @@ -42,10 +42,9 @@ impl SmartPushExistingEvolvedDatasetScenario< let client_account_name = client_harness.operating_account_name(); let server_account_name = server_harness.operating_account_name(); - let client_repo = client_harness.dataset_repository(); - - let client_create_result = client_repo - .create_dataset_from_snapshot( + let client_create_result = client_harness + .create_dataset_from_snapshot() + .execute( MetadataFactory::dataset_snapshot() .name(DatasetAlias::new( client_account_name.clone(), @@ -79,9 +78,10 @@ impl SmartPushExistingEvolvedDatasetScenario< .await; // Extend client-side dataset with new nodes + let client_repo = client_harness.dataset_repository(); let client_dataset_ref = make_dataset_ref(&client_account_name, "foo"); client_repo - .get_dataset(&client_dataset_ref) + .find_dataset_by_ref(&client_dataset_ref) .await .unwrap() .commit_event( diff --git a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_ref_collision.rs b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_ref_collision.rs index 4797b18bd6..08ff5b4b5c 100644 --- a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_ref_collision.rs +++ b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_existing_ref_collision.rs @@ -36,10 +36,9 @@ impl SmartPushExistingRefCollisionScenarion SmartPushExistingRefCollisionScenarion SmartPushExistingUpToDateDatasetScenario let client_account_name = client_harness.operating_account_name(); let server_account_name = server_harness.operating_account_name(); - let client_repo = client_harness.dataset_repository(); - - let client_create_result = client_repo - .create_dataset_from_snapshot( + let client_create_result = client_harness + .create_dataset_from_snapshot() + .execute( MetadataFactory::dataset_snapshot() .name(DatasetAlias::new( client_account_name.clone(), @@ -68,7 +67,7 @@ impl SmartPushExistingUpToDateDatasetScenario let client_dataset_ref: DatasetRef = make_dataset_ref(&client_account_name, "foo"); commit_add_data_event( - client_repo.as_ref(), + client_harness.dataset_repository().as_ref(), &client_dataset_ref, &client_dataset_layout, None, diff --git a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_new_dataset.rs b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_new_dataset.rs index 00d36214ed..55bd45416f 100644 --- a/src/adapter/http/tests/tests/tests_push/scenarios/scenario_new_dataset.rs +++ b/src/adapter/http/tests/tests/tests_push/scenarios/scenario_new_dataset.rs @@ -40,10 +40,9 @@ impl SmartPushNewDatasetScenario SmartPushNewDatasetScenario SmartPushNewEmptyDatasetScenario> = Vec::new(); for dataset_handle in datasets { - let dataset = repo - .get_dataset(&dataset_handle.as_local_ref()) - .await - .unwrap(); + let dataset = repo.get_dataset_by_handle(&dataset_handle); collections.push(Arc::new(ODataCollectionContext { catalog: self.catalog.clone(), diff --git a/src/adapter/odata/src/handler.rs b/src/adapter/odata/src/handler.rs index 4243f8a52e..83bd159b1b 100644 --- a/src/adapter/odata/src/handler.rs +++ b/src/adapter/odata/src/handler.rs @@ -150,10 +150,7 @@ pub async fn odata_collection_handler_common( } .unwrap(); - let dataset = repo - .get_dataset(&dataset_handle.as_local_ref()) - .await - .unwrap(); + let dataset = repo.get_dataset_by_handle(&dataset_handle); let ctx = ODataCollectionContext::new(catalog, addr, dataset_handle, dataset); let response = datafusion_odata::handlers::odata_collection_handler( diff --git a/src/adapter/odata/tests/tests/test_handlers.rs b/src/adapter/odata/tests/tests/test_handlers.rs index 03a3a42534..a5c2227f16 100644 --- a/src/adapter/odata/tests/tests/test_handlers.rs +++ b/src/adapter/odata/tests/tests/test_handlers.rs @@ -12,13 +12,14 @@ use std::sync::Arc; use chrono::{TimeZone, Utc}; use database_common::NoOpDatabasePlugin; use dill::*; -use event_bus::EventBus; use indoc::indoc; use kamu::domain::*; use kamu::testing::*; use kamu::*; use kamu_accounts::CurrentAccountSubject; +use messaging_outbox::DummyOutboxImpl; use opendatafabric::*; +use time_source::{SystemTimeSource, SystemTimeSourceStub}; use super::test_api_server::TestAPIServer; @@ -312,7 +313,7 @@ async fn test_collection_handler_by_id_not_found() { struct TestHarness { temp_dir: tempfile::TempDir, - dataset_repo: Arc, + catalog: Catalog, push_ingest_svc: Arc, api_server: TestAPIServer, } @@ -341,8 +342,7 @@ impl TestHarness { .add::() .add::() .add::() - .add::() - .add::() + .add::() .add_value(CurrentAccountSubject::new_test()) .add_value(dataset_action_authorizer) .bind::() @@ -352,6 +352,8 @@ impl TestHarness { .with_multi_tenant(false), ) .bind::() + .bind::() + .add::() .add_value(SystemTimeSourceStub::new_set( Utc.with_ymd_and_hms(2050, 1, 1, 12, 0, 0).unwrap(), )) @@ -366,23 +368,26 @@ impl TestHarness { b.build() }; - let dataset_repo = catalog.get_one::().unwrap(); let push_ingest_svc = catalog.get_one::().unwrap(); let api_server = TestAPIServer::new(catalog.clone(), None, None, false); Self { temp_dir, - dataset_repo, + catalog, push_ingest_svc, api_server, } } async fn create_simple_dataset(&self) -> CreateDatasetResult { - let ds = self - .dataset_repo - .create_dataset_from_snapshot( + let create_dataset_from_snapshot = self + .catalog + .get_one::() + .unwrap(); + + let ds = create_dataset_from_snapshot + .execute( MetadataFactory::dataset_snapshot() .name("foo.bar") .kind(DatasetKind::Root) diff --git a/src/app/cli/Cargo.toml b/src/app/cli/Cargo.toml index 1bddc1752d..60301b323f 100644 --- a/src/app/cli/Cargo.toml +++ b/src/app/cli/Cargo.toml @@ -43,9 +43,10 @@ query-extensions-json = ["kamu/query-extensions-json"] container-runtime = { workspace = true } database-common = { workspace = true } database-common-macros = { workspace = true } -event-bus = { workspace = true } http-common = { workspace = true } internal-error = { workspace = true } +time-source = { workspace = true } + kamu = { workspace = true } kamu-data-utils = { workspace = true } @@ -61,9 +62,8 @@ random-names = { workspace = true } kamu-flow-system-services = { workspace = true } kamu-flow-system-inmem = { workspace = true } -# TODO: Activate after preparing services for transactional work -# kamu-flow-system-postgres = { workspace = true } -# kamu-flow-system-sqlite = { workspace = true } +kamu-flow-system-postgres = { workspace = true } +kamu-flow-system-sqlite = { workspace = true } kamu-task-system-services = { workspace = true } kamu-task-system-inmem = { workspace = true } @@ -84,6 +84,11 @@ kamu-datasets-inmem = { workspace = true } kamu-datasets-postgres = { workspace = true } kamu-datasets-sqlite = { workspace = true } +messaging-outbox = { workspace = true } +kamu-messaging-outbox-inmem = { workspace = true } +kamu-messaging-outbox-postgres = { workspace = true } +kamu-messaging-outbox-sqlite = { workspace = true } + # CLI chrono-humanize = "0.2" # Human readable durations clap = "4" @@ -156,7 +161,7 @@ datafusion = { version = "41", default-features = false, features = [ "unicode_expressions", "compression", ] } -dill = "0.8" +dill = "0.9" dirs = "5" fs_extra = "1.3" futures = "0.3" diff --git a/src/app/cli/src/app.rs b/src/app/cli/src/app.rs index 7540e2aa17..8dc21bd732 100644 --- a/src/app/cli/src/app.rs +++ b/src/app/cli/src/app.rs @@ -10,7 +10,7 @@ use std::path::Path; use std::sync::Arc; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, Duration, Utc}; use container_runtime::{ContainerRuntime, ContainerRuntimeConfig}; use database_common::DatabaseTransactionRunner; use dill::*; @@ -21,6 +21,11 @@ use kamu_accounts_services::PredefinedAccountsRegistrator; use kamu_adapter_http::{FileUploadLimitConfig, UploadServiceLocal}; use kamu_adapter_oauth::GithubAuthenticationConfig; use kamu_datasets::DatasetEnvVar; +use kamu_flow_system_inmem::domain::FlowConfigurationUpdatedMessage; +use kamu_flow_system_services::MESSAGE_PRODUCER_KAMU_FLOW_CONFIGURATION_SERVICE; +use kamu_task_system_inmem::domain::{TaskProgressMessage, MESSAGE_PRODUCER_KAMU_TASK_EXECUTOR}; +use messaging_outbox::{register_message_dispatcher, Outbox, OutboxDispatchingImpl}; +use time_source::{SystemTimeSource, SystemTimeSourceDefault, SystemTimeSourceStub}; use tracing::warn; use crate::accounts::AccountService; @@ -247,7 +252,6 @@ pub fn prepare_dependencies_graph_repository( // bound to CLI user. It also should be authorized to access any dataset. let special_catalog_for_graph = CatalogBuilder::new() - .add::() .add::() .add_builder( DatasetRepositoryLocalFs::builder() @@ -255,6 +259,7 @@ pub fn prepare_dependencies_graph_repository( .with_multi_tenant(multi_tenant_workspace), ) .bind::() + .bind::() .add_value(current_account_subject) .add::() .add::() @@ -289,14 +294,13 @@ pub fn configure_base_catalog( b.add::(); } - b.add::(); - b.add_builder( DatasetRepositoryLocalFs::builder() .with_root(workspace_layout.datasets_dir.clone()) .with_multi_tenant(multi_tenant_workspace), ); b.bind::(); + b.bind::(); b.add::(); @@ -353,6 +357,13 @@ pub fn configure_base_catalog( b.add::(); b.add::(); + b.add::(); + b.add::(); + b.add::(); + b.add::(); + b.add::(); + b.add::(); + b.add::(); b.add::(); b.add_value(kamu_flow_system_inmem::domain::FlowServiceRunConfig::new( @@ -386,6 +397,25 @@ pub fn configure_base_catalog( b.add::(); + b.add_builder( + messaging_outbox::OutboxImmediateImpl::builder() + .with_consumer_filter(messaging_outbox::ConsumerFilter::BestEffortConsumers), + ); + b.add::(); + b.add::(); + b.bind::(); + b.add::(); + + register_message_dispatcher::( + &mut b, + MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE, + ); + register_message_dispatcher::(&mut b, MESSAGE_PRODUCER_KAMU_TASK_EXECUTOR); + register_message_dispatcher::( + &mut b, + MESSAGE_PRODUCER_KAMU_FLOW_CONFIGURATION_SERVICE, + ); + b } @@ -608,6 +638,12 @@ pub fn register_config_in_catalog( } } } + + let outbox_config = config.outbox.as_ref().unwrap(); + catalog_builder.add_value(messaging_outbox::OutboxConfig::new( + Duration::seconds(outbox_config.awaiting_step_secs.unwrap()), + outbox_config.batch_size.unwrap(), + )); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/app/cli/src/cli_commands.rs b/src/app/cli/src/cli_commands.rs index 60e1bf2c63..56ae4f93b5 100644 --- a/src/app/cli/src/cli_commands.rs +++ b/src/app/cli/src/cli_commands.rs @@ -23,6 +23,8 @@ pub fn get_command( ) -> Result, CLIError> { let command: Box = match arg_matches.subcommand() { Some(("add", submatches)) => Box::new(AddCommand::new( + cli_catalog.get_one()?, + cli_catalog.get_one()?, cli_catalog.get_one()?, cli_catalog.get_one()?, submatches @@ -90,6 +92,7 @@ pub fn get_command( _ => return Err(CommandInterpretationFailed.into()), }, Some(("delete", submatches)) => Box::new(DeleteCommand::new( + cli_catalog.get_one()?, cli_catalog.get_one()?, validate_many_dataset_patterns( cli_catalog, diff --git a/src/app/cli/src/commands/add_command.rs b/src/app/cli/src/commands/add_command.rs index 88928ee7dc..d90f323b4f 100644 --- a/src/app/cli/src/commands/add_command.rs +++ b/src/app/cli/src/commands/add_command.rs @@ -7,6 +7,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +use std::collections::{HashSet, LinkedList}; use std::sync::Arc; use kamu::domain::*; @@ -20,6 +21,8 @@ use crate::OutputConfig; pub struct AddCommand { resource_loader: Arc, dataset_repo: Arc, + create_dataset_from_snapshot: Arc, + delete_dataset: Arc, snapshot_refs: Vec, name: Option, recursive: bool, @@ -32,6 +35,8 @@ impl AddCommand { pub fn new<'s, I>( resource_loader: Arc, dataset_repo: Arc, + create_dataset_from_snapshot: Arc, + delete_dataset: Arc, snapshot_refs_iter: I, name: Option, recursive: bool, @@ -45,6 +50,8 @@ impl AddCommand { Self { resource_loader, dataset_repo, + create_dataset_from_snapshot, + delete_dataset, snapshot_refs: snapshot_refs_iter.map(ToOwned::to_owned).collect(), name, recursive, @@ -129,6 +136,66 @@ impl AddCommand { Ok(false) } + + pub async fn create_datasets_from_snapshots( + &self, + snapshots: Vec, + ) -> Vec<( + DatasetAlias, + Result, + )> { + let snapshots_ordered = + self.sort_snapshots_in_dependency_order(snapshots.into_iter().collect()); + + let mut ret = Vec::new(); + for snapshot in snapshots_ordered { + let alias = snapshot.name.clone(); + let res = self.create_dataset_from_snapshot.execute(snapshot).await; + ret.push((alias, res)); + } + ret + } + + #[allow(clippy::linkedlist)] + fn sort_snapshots_in_dependency_order( + &self, + mut snapshots: LinkedList, + ) -> Vec { + let mut ordered = Vec::with_capacity(snapshots.len()); + let mut pending: HashSet = + snapshots.iter().map(|s| s.name.clone().into()).collect(); + let mut added: HashSet = HashSet::new(); + + // TODO: cycle detection + while !snapshots.is_empty() { + let snapshot = snapshots.pop_front().unwrap(); + + let transform = snapshot + .metadata + .iter() + .find_map(|e| e.as_variant::()); + + let has_pending_deps = if let Some(transform) = transform { + transform + .inputs + .iter() + .any(|input| pending.contains(&input.dataset_ref)) + } else { + false + }; + + if !has_pending_deps { + pending.remove(&snapshot.name.clone().into()); + added.insert(snapshot.name.clone()); + ordered.push(snapshot); + } else { + snapshots.push_back(snapshot); + } + } + ordered + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// } #[async_trait::async_trait(?Send)] @@ -211,17 +278,12 @@ impl Command for AddCommand { // TODO: delete permissions should be checked in multi-tenant scenario for hdl in already_exist { - self.dataset_repo - .delete_dataset(&hdl.as_local_ref()) - .await?; + self.delete_dataset.execute_via_handle(&hdl).await?; } } }; - let mut add_results = self - .dataset_repo - .create_datasets_from_snapshots(snapshots) - .await; + let mut add_results = self.create_datasets_from_snapshots(snapshots).await; add_results.sort_by(|(id_a, _), (id_b, _)| id_a.cmp(id_b)); diff --git a/src/app/cli/src/commands/complete_command.rs b/src/app/cli/src/commands/complete_command.rs index 8015a6ce73..3c66694b0d 100644 --- a/src/app/cli/src/commands/complete_command.rs +++ b/src/app/cli/src/commands/complete_command.rs @@ -14,6 +14,7 @@ use std::{fs, path}; use chrono::prelude::*; use futures::TryStreamExt; use glob; +use internal_error::ResultIntoInternal; use kamu::domain::*; use super::{CLIError, Command}; diff --git a/src/app/cli/src/commands/delete_command.rs b/src/app/cli/src/commands/delete_command.rs index d90b7229b3..f0d3979a0f 100644 --- a/src/app/cli/src/commands/delete_command.rs +++ b/src/app/cli/src/commands/delete_command.rs @@ -10,6 +10,7 @@ use std::sync::Arc; use futures::{future, StreamExt, TryStreamExt}; +use internal_error::ResultIntoInternal; use kamu::domain::*; use kamu::utils::datasets_filtering::filter_datasets_by_local_pattern; use opendatafabric::*; @@ -20,6 +21,7 @@ use super::{common, CLIError, Command}; pub struct DeleteCommand { dataset_repo: Arc, + delete_dataset: Arc, dataset_ref_patterns: Vec, dependency_graph_service: Arc, all: bool, @@ -30,6 +32,7 @@ pub struct DeleteCommand { impl DeleteCommand { pub fn new( dataset_repo: Arc, + delete_dataset: Arc, dataset_ref_patterns: I, dependency_graph_service: Arc, all: bool, @@ -41,6 +44,7 @@ impl DeleteCommand { { Self { dataset_repo, + delete_dataset, dataset_ref_patterns: dataset_ref_patterns.into_iter().collect(), dependency_graph_service, all, @@ -129,7 +133,7 @@ impl Command for DeleteCommand { } for dataset_ref in &dataset_refs { - match self.dataset_repo.delete_dataset(dataset_ref).await { + match self.delete_dataset.execute_via_ref(dataset_ref).await { Ok(_) => Ok(()), Err(DeleteDatasetError::DanglingReference(e)) => Err(CLIError::failure(e)), Err(DeleteDatasetError::Access(e)) => Err(CLIError::failure(e)), diff --git a/src/app/cli/src/commands/ingest_command.rs b/src/app/cli/src/commands/ingest_command.rs index 61d913fdb6..003485e5d8 100644 --- a/src/app/cli/src/commands/ingest_command.rs +++ b/src/app/cli/src/commands/ingest_command.rs @@ -12,6 +12,7 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; use chrono::{DateTime, Utc}; +use internal_error::ResultIntoInternal; use kamu::domain::*; use opendatafabric::*; @@ -103,10 +104,7 @@ impl IngestCommand { ))); } - let dataset = self - .dataset_repo - .get_dataset(&dataset_handle.as_local_ref()) - .await?; + let dataset = self.dataset_repo.get_dataset_by_handle(dataset_handle); let dataset_kind = dataset .get_summary(GetSummaryOpts::default()) .await diff --git a/src/app/cli/src/commands/inspect_query_command.rs b/src/app/cli/src/commands/inspect_query_command.rs index 7c0f6a216e..d627dc863c 100644 --- a/src/app/cli/src/commands/inspect_query_command.rs +++ b/src/app/cli/src/commands/inspect_query_command.rs @@ -46,10 +46,7 @@ impl InspectQueryCommand { output: &mut impl Write, dataset_handle: &DatasetHandle, ) -> Result<(), CLIError> { - let dataset = self - .dataset_repo - .get_dataset(&dataset_handle.as_local_ref()) - .await?; + let dataset = self.dataset_repo.get_dataset_by_handle(dataset_handle); let mut blocks = dataset.as_metadata_chain().iter_blocks(); while let Some((block_hash, block)) = blocks.try_next().await? { diff --git a/src/app/cli/src/commands/list_command.rs b/src/app/cli/src/commands/list_command.rs index d4c79b7da2..119e1e6f60 100644 --- a/src/app/cli/src/commands/list_command.rs +++ b/src/app/cli/src/commands/list_command.rs @@ -13,6 +13,7 @@ use std::sync::Arc; use chrono::{DateTime, Utc}; use chrono_humanize::HumanTime; use futures::TryStreamExt; +use internal_error::ResultIntoInternal; use kamu::domain::*; use opendatafabric::*; @@ -236,7 +237,7 @@ impl Command for ListCommand { datasets.sort_by(|a, b| a.alias.cmp(&b.alias)); for hdl in &datasets { - let dataset = self.dataset_repo.get_dataset(&hdl.as_local_ref()).await?; + let dataset = self.dataset_repo.get_dataset_by_handle(hdl); let current_head = dataset .as_metadata_chain() .resolve_ref(&BlockRef::Head) diff --git a/src/app/cli/src/commands/log_command.rs b/src/app/cli/src/commands/log_command.rs index 23a416c95f..9b3bdfa402 100644 --- a/src/app/cli/src/commands/log_command.rs +++ b/src/app/cli/src/commands/log_command.rs @@ -112,10 +112,7 @@ impl Command for LogCommand { auth::DatasetActionUnauthorizedError::Internal(e) => CLIError::critical(e), })?; - let dataset = self - .dataset_repo - .get_dataset(&dataset_handle.as_local_ref()) - .await?; + let dataset = self.dataset_repo.get_dataset_by_handle(&dataset_handle); let blocks = Box::pin( dataset diff --git a/src/app/cli/src/commands/rename_command.rs b/src/app/cli/src/commands/rename_command.rs index 7c1c9b76f2..f7b3125e1e 100644 --- a/src/app/cli/src/commands/rename_command.rs +++ b/src/app/cli/src/commands/rename_command.rs @@ -15,14 +15,14 @@ use opendatafabric::*; use super::{CLIError, Command}; pub struct RenameCommand { - dataset_repo: Arc, + rename_dataset: Arc, dataset_ref: DatasetRef, new_name: DatasetName, } impl RenameCommand { pub fn new( - dataset_repo: Arc, + rename_dataset: Arc, dataset_ref: DatasetRef, new_name: N, ) -> Self @@ -31,7 +31,7 @@ impl RenameCommand { >::Error: std::fmt::Debug, { Self { - dataset_repo, + rename_dataset, dataset_ref, new_name: new_name.try_into().unwrap(), } @@ -42,8 +42,8 @@ impl RenameCommand { impl Command for RenameCommand { async fn run(&mut self) -> Result<(), CLIError> { match self - .dataset_repo - .rename_dataset(&self.dataset_ref, &self.new_name) + .rename_dataset + .execute(&self.dataset_ref, &self.new_name) .await { Ok(_) => Ok(()), diff --git a/src/app/cli/src/commands/system_e2e_command.rs b/src/app/cli/src/commands/system_e2e_command.rs index 187996c1d4..180f6484fa 100644 --- a/src/app/cli/src/commands/system_e2e_command.rs +++ b/src/app/cli/src/commands/system_e2e_command.rs @@ -49,7 +49,7 @@ impl Command for SystemE2ECommand { return Err(CLIError::usage_error("dataset required")); }; - let dataset = self.dataset_repo.get_dataset(dataset_ref).await?; + let dataset = self.dataset_repo.find_dataset_by_ref(dataset_ref).await?; let maybe_physical_hash = dataset .as_metadata_chain() diff --git a/src/app/cli/src/commands/verify_command.rs b/src/app/cli/src/commands/verify_command.rs index 7b176ed95b..433dcc395a 100644 --- a/src/app/cli/src/commands/verify_command.rs +++ b/src/app/cli/src/commands/verify_command.rs @@ -10,6 +10,7 @@ use std::sync::Arc; use futures::{StreamExt, TryStreamExt}; +use internal_error::ResultIntoInternal; use kamu::domain::*; use kamu::utils::datasets_filtering::filter_datasets_by_local_pattern; use opendatafabric::*; @@ -184,11 +185,7 @@ impl VerifyCommand { continue; } - let dataset = self - .dataset_repo - .get_dataset(&dataset_handle.as_local_ref()) - .await - .unwrap(); + let dataset = self.dataset_repo.get_dataset_by_handle(&dataset_handle); let summary = dataset .get_summary(GetSummaryOpts::default()) .await diff --git a/src/app/cli/src/database.rs b/src/app/cli/src/database.rs index e2b7511bdb..9ee50d8bb5 100644 --- a/src/app/cli/src/database.rs +++ b/src/app/cli/src/database.rs @@ -24,12 +24,11 @@ pub fn configure_database_components( db_connection_settings: DatabaseConnectionSettings, ) { // TODO: Remove after adding implementation of FlowEventStore for databases - b.add::(); + b.add::(); // TODO: Delete after preparing services for transactional work and replace with // permanent storage options - b.add::(); - b.add::(); + b.add::(); match db_connection_settings.provider { DatabaseProvider::Postgres => { @@ -37,13 +36,26 @@ pub fn configure_database_components( b.add::(); b.add::(); + b.add::(); + + b.add::(); + + b.add::(); + b.add::(); } DatabaseProvider::MySql | DatabaseProvider::MariaDB => { MySqlPlugin::init_database_components(b); b.add::(); - b.add::(); + b.add::(); + + b.add::(); + + b.add::(); + + b.add::(); + b.add::(); // TODO: Task & Flow System MySQL versions } @@ -52,7 +64,13 @@ pub fn configure_database_components( b.add::(); b.add::(); + b.add::(); + + b.add::(); + + b.add::(); + b.add::(); } } @@ -65,12 +83,14 @@ pub fn configure_database_components( // Public only for tests pub fn configure_in_memory_components(b: &mut CatalogBuilder) { - b.add::(); - b.add::(); - b.add::(); - b.add::(); - b.add::(); - b.add::(); + b.add::(); + b.add::(); + b.add::(); + b.add::(); + b.add::(); + b.add::(); + b.add::(); + b.add::(); NoOpDatabasePlugin::init_database_components(b); } diff --git a/src/app/cli/src/error.rs b/src/app/cli/src/error.rs index 8e6a5694f2..636d388b77 100644 --- a/src/app/cli/src/error.rs +++ b/src/app/cli/src/error.rs @@ -11,6 +11,7 @@ use std::backtrace::Backtrace; use std::fmt::Display; use std::path::PathBuf; +use internal_error::{BoxedError, InternalError}; use kamu::domain::engine::normalize_logs; use kamu::domain::*; use kamu_data_utils::data::format::WriterError; diff --git a/src/app/cli/src/explore/api_server.rs b/src/app/cli/src/explore/api_server.rs index 17730c09f3..01cdfd7a6f 100644 --- a/src/app/cli/src/explore/api_server.rs +++ b/src/app/cli/src/explore/api_server.rs @@ -20,10 +20,12 @@ use dill::{Catalog, CatalogBuilder}; use http_common::ApiError; use indoc::indoc; use internal_error::*; -use kamu::domain::{Protocols, ServerUrlConfig, SystemTimeSource}; +use kamu::domain::{Protocols, ServerUrlConfig}; use kamu_adapter_http::e2e::e2e_router; use kamu_flow_system_inmem::domain::FlowService; use kamu_task_system_inmem::domain::TaskExecutor; +use messaging_outbox::OutboxTransactionalProcessor; +use time_source::SystemTimeSource; use tokio::sync::Notify; use url::Url; @@ -36,6 +38,7 @@ pub struct APIServer { >, task_executor: Arc, flow_service: Arc, + outbox_processor: Arc, time_source: Arc, maybe_shutdown_notify: Option>, } @@ -56,6 +59,8 @@ impl APIServer { let flow_service = cli_catalog.get_one().unwrap(); + let outbox_processor = cli_catalog.get_one().unwrap(); + let time_source = base_catalog.get_one().unwrap(); let gql_schema = kamu_adapter_graphql::schema(); @@ -168,6 +173,7 @@ impl APIServer { server, task_executor, flow_service, + outbox_processor, time_source, maybe_shutdown_notify, } @@ -193,6 +199,7 @@ impl APIServer { tokio::select! { res = server_run_fut => { res.int_err() }, + res = self.outbox_processor.run() => { res.int_err() }, res = self.task_executor.run() => { res.int_err() }, res = self.flow_service.run(self.time_source.now()) => { res.int_err() } } diff --git a/src/app/cli/src/services/config/models.rs b/src/app/cli/src/services/config/models.rs index 286cc8126b..9afb011433 100644 --- a/src/app/cli/src/services/config/models.rs +++ b/src/app/cli/src/services/config/models.rs @@ -54,6 +54,10 @@ pub struct CLIConfig { /// Dataset environment variables configuration #[merge(strategy = merge_recursive)] pub dataset_env_vars: Option, + + /// Messaging outbox configuration + #[merge(strategy = merge_recursive)] + pub outbox: Option, } impl CLIConfig { @@ -67,6 +71,7 @@ impl CLIConfig { database: None, uploads: None, dataset_env_vars: None, + outbox: None, } } @@ -84,6 +89,7 @@ impl CLIConfig { database: Some(DatabaseConfig::sample()), uploads: Some(UploadsConfig::sample()), dataset_env_vars: Some(DatasetEnvVarsConfig::sample()), + outbox: Some(OutboxConfig::sample()), } } } @@ -99,6 +105,7 @@ impl Default for CLIConfig { database: None, uploads: Some(UploadsConfig::default()), dataset_env_vars: Some(DatasetEnvVarsConfig::default()), + outbox: Some(OutboxConfig::default()), } } } @@ -640,6 +647,32 @@ impl Default for UploadsConfig { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +#[skip_serializing_none] +#[derive(Debug, Clone, Merge, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +#[serde(rename_all = "camelCase")] +pub struct OutboxConfig { + pub awaiting_step_secs: Option, + pub batch_size: Option, +} + +impl OutboxConfig { + pub fn sample() -> Self { + Default::default() + } +} + +impl Default for OutboxConfig { + fn default() -> Self { + Self { + awaiting_step_secs: Some(1), + batch_size: Some(20), + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ConfigScope { User, diff --git a/src/app/cli/src/services/gc_service.rs b/src/app/cli/src/services/gc_service.rs index f2c74fbeba..b56c2b2a62 100644 --- a/src/app/cli/src/services/gc_service.rs +++ b/src/app/cli/src/services/gc_service.rs @@ -10,7 +10,7 @@ use std::sync::Arc; use chrono::{DateTime, Duration, Utc}; -use kamu::domain::*; +use internal_error::{InternalError, ResultIntoInternal}; use crate::WorkspaceLayout; diff --git a/src/app/cli/src/services/workspace/workspace_service.rs b/src/app/cli/src/services/workspace/workspace_service.rs index 2bf497a1b0..019279d26d 100644 --- a/src/app/cli/src/services/workspace/workspace_service.rs +++ b/src/app/cli/src/services/workspace/workspace_service.rs @@ -10,7 +10,7 @@ use std::path::Path; use std::sync::Arc; -use kamu::domain::*; +use internal_error::{InternalError, ResultIntoInternal}; use crate::{WorkspaceConfig, WorkspaceLayout, WorkspaceVersion}; diff --git a/src/app/cli/tests/tests/test_workspace_svc.rs b/src/app/cli/tests/tests/test_workspace_svc.rs index 9544aa6caf..ae37a6ce19 100644 --- a/src/app/cli/tests/tests/test_workspace_svc.rs +++ b/src/app/cli/tests/tests/test_workspace_svc.rs @@ -10,7 +10,6 @@ use std::path::Path; use std::sync::Arc; -use event_bus::EventBus; use kamu::domain::*; use kamu::testing::{MetadataFactory, ParquetWriterHelper}; use kamu::*; @@ -91,11 +90,7 @@ async fn init_v0_workspace(workspace_path: &Path) { let dataset_name = DatasetName::new_unchecked("foo"); let dataset_dir = datasets_dir.join(&dataset_name); - let catalog = dill::CatalogBuilder::new().add::().build(); - let dataset = DatasetFactoryImpl::get_local_fs( - DatasetLayout::create(&dataset_dir).unwrap(), - catalog.get_one().unwrap(), - ); + let dataset = DatasetFactoryImpl::get_local_fs(DatasetLayout::create(&dataset_dir).unwrap()); // Metadata & refs dataset diff --git a/src/domain/accounts/services/Cargo.toml b/src/domain/accounts/services/Cargo.toml index be412a7fb2..1b13503362 100644 --- a/src/domain/accounts/services/Cargo.toml +++ b/src/domain/accounts/services/Cargo.toml @@ -25,14 +25,14 @@ doctest = false database-common = { workspace = true } internal-error = { workspace = true } kamu-accounts = { workspace = true } -kamu-core = { workspace = true } opendatafabric = { workspace = true } +time-source = { workspace = true } random-names = { workspace = true } argon2 = { version = "0.5" } async-trait = { version = "0.1", default-features = false } chrono = { version = "0.4", default-features = false } -dill = "0.8" +dill = "0.9" jsonwebtoken = "9" thiserror = { version = "1", default-features = false } password-hash = { version = "0.5", default-features = false } @@ -42,6 +42,7 @@ tokio = { version = "1", default-features = false } tracing = { version = "0.1", default-features = false } uuid = { version = "1", default-features = false } + [dev-dependencies] kamu-accounts-inmem = { workspace = true } diff --git a/src/domain/accounts/services/src/access_token_service_impl.rs b/src/domain/accounts/services/src/access_token_service_impl.rs index 9a0f888e86..84fc098323 100644 --- a/src/domain/accounts/services/src/access_token_service_impl.rs +++ b/src/domain/accounts/services/src/access_token_service_impl.rs @@ -23,8 +23,8 @@ use kamu_accounts::{ KamuAccessToken, RevokeTokenError, }; -use kamu_core::SystemTimeSource; use opendatafabric::AccountID; +use time_source::SystemTimeSource; use uuid::Uuid; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/domain/accounts/services/src/authentication_service_impl.rs b/src/domain/accounts/services/src/authentication_service_impl.rs index b98c94e3d6..599cc32ba9 100644 --- a/src/domain/accounts/services/src/authentication_service_impl.rs +++ b/src/domain/accounts/services/src/authentication_service_impl.rs @@ -17,8 +17,8 @@ use internal_error::*; use jsonwebtoken::errors::ErrorKind; use jsonwebtoken::{decode, encode, Algorithm, DecodingKey, EncodingKey, Header, Validation}; use kamu_accounts::*; -use kamu_core::SystemTimeSource; use opendatafabric::{AccountID, AccountName}; +use time_source::SystemTimeSource; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/domain/accounts/services/src/login_password_auth_provider.rs b/src/domain/accounts/services/src/login_password_auth_provider.rs index 8155d88db8..63fa9f40d3 100644 --- a/src/domain/accounts/services/src/login_password_auth_provider.rs +++ b/src/domain/accounts/services/src/login_password_auth_provider.rs @@ -12,7 +12,7 @@ use std::sync::Arc; use argon2::Argon2; use dill::*; -use kamu_core::{InternalError, ResultIntoInternal}; +use internal_error::{InternalError, ResultIntoInternal}; use opendatafabric::{AccountID, AccountName}; use password_hash::rand_core::OsRng; use password_hash::{PasswordHash, PasswordHasher, PasswordVerifier, SaltString}; diff --git a/src/domain/accounts/services/tests/tests/test_authentication_service.rs b/src/domain/accounts/services/tests/tests/test_authentication_service.rs index 9d31bcfcf5..219fac8375 100644 --- a/src/domain/accounts/services/tests/tests/test_authentication_service.rs +++ b/src/domain/accounts/services/tests/tests/test_authentication_service.rs @@ -11,10 +11,10 @@ use std::assert_matches::assert_matches; use database_common::{DatabaseTransactionRunner, NoOpDatabasePlugin}; use kamu_accounts::*; -use kamu_accounts_inmem::{AccessTokenRepositoryInMemory, AccountRepositoryInMemory}; +use kamu_accounts_inmem::{InMemoryAccessTokenRepository, InMemoryAccountRepository}; use kamu_accounts_services::{AccessTokenServiceImpl, AuthenticationServiceImpl}; -use kamu_core::{SystemTimeSource, SystemTimeSourceStub}; use opendatafabric::{AccountID, AccountName}; +use time_source::{SystemTimeSource, SystemTimeSourceStub}; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -100,9 +100,9 @@ fn make_catalog() -> dill::Catalog { b.add::() .add::() .add::() - .add::() + .add::() .add::() - .add::() + .add::() .add_value(PredefinedAccountsConfig::single_tenant()) .add_value(SystemTimeSourceStub::new()) .bind::() diff --git a/src/domain/core/Cargo.toml b/src/domain/core/Cargo.toml index a9bf0b6c45..ca04c84d67 100644 --- a/src/domain/core/Cargo.toml +++ b/src/domain/core/Cargo.toml @@ -25,6 +25,7 @@ doctest = false kamu-accounts = { workspace = true } kamu-datasets = { workspace = true } container-runtime = { workspace = true } +messaging-outbox = { workspace = true } internal-error = { workspace = true } opendatafabric = { workspace = true } @@ -32,7 +33,7 @@ async-stream = { version = "0.3", default-features = false } async-trait = { version = "0.1", default-features = false } bytes = { version = "1", default-features = false } chrono = { version = "0.4", default-features = false } -dill = { version = "0.8", default-features = false } +dill = "0.9" futures = { version = "0.3", default-features = false } http = { version = "0.2" } pathdiff = { version = "0.2", default-features = false } diff --git a/src/domain/core/src/entities/dataset.rs b/src/domain/core/src/entities/dataset.rs index 295f1ef830..714b22b308 100644 --- a/src/domain/core/src/entities/dataset.rs +++ b/src/domain/core/src/entities/dataset.rs @@ -8,7 +8,9 @@ // by the Apache License, Version 2.0. use async_trait::async_trait; +use auth::DatasetActionUnauthorizedError; use chrono::{DateTime, Utc}; +use internal_error::InternalError; use opendatafabric::*; use thiserror::Error; @@ -120,6 +122,7 @@ impl<'a> Default for CommitOpts<'a> { pub struct CommitResult { pub old_head: Option, pub new_head: Multihash, + pub new_upstream_ids: Vec, } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -162,6 +165,12 @@ pub enum CommitError { #[error(transparent)] MetadataAppendError(#[from] AppendError), #[error(transparent)] + Access( + #[from] + #[backtrace] + AccessError, + ), + #[error(transparent)] Internal( #[from] #[backtrace] @@ -169,6 +178,15 @@ pub enum CommitError { ), } +impl From for CommitError { + fn from(v: DatasetActionUnauthorizedError) -> Self { + match v { + DatasetActionUnauthorizedError::Access(e) => Self::Access(e), + DatasetActionUnauthorizedError::Internal(e) => Self::Internal(e), + } + } +} + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Commit helpers //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/domain/core/src/entities/mod.rs b/src/domain/core/src/entities/mod.rs index f3e2384e0c..a9a5137a6e 100644 --- a/src/domain/core/src/entities/mod.rs +++ b/src/domain/core/src/entities/mod.rs @@ -10,7 +10,6 @@ pub mod dataset; pub mod dataset_summary; pub mod engine; -pub mod events; pub mod metadata_chain; pub mod metadata_stream; diff --git a/src/domain/core/src/lib.rs b/src/domain/core/src/lib.rs index dfb3846cd5..46d90fcc77 100644 --- a/src/domain/core/src/lib.rs +++ b/src/domain/core/src/lib.rs @@ -10,17 +10,17 @@ #![feature(error_generic_member_access)] #![feature(let_chains)] -// Re-exports -pub use internal_error::*; - pub mod auth; pub mod entities; +pub mod messages; pub mod repos; pub mod services; +pub mod use_cases; pub mod utils; pub use entities::{SetRefError, *}; +pub use messages::*; pub use repos::{DatasetNotFoundError, *}; pub use services::*; +pub use use_cases::*; pub use utils::paths::*; -pub use utils::time_source::*; diff --git a/src/domain/core/src/messages/core_message_consumers.rs b/src/domain/core/src/messages/core_message_consumers.rs new file mode 100644 index 0000000000..758dac0b51 --- /dev/null +++ b/src/domain/core/src/messages/core_message_consumers.rs @@ -0,0 +1,18 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub const MESSAGE_CONSUMER_KAMU_CORE_DATASET_OWNERSHIP_SERVICE: &str = + "dev.kamu.domain.core.services.DatasetOwnershipService"; + +pub const MESSAGE_CONSUMER_KAMU_CORE_DEPENDENCY_GRAPH_SERVICE: &str = + "dev.kamu.domain.core.services.DependencyGraphService"; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/domain/core/src/messages/core_message_producers.rs b/src/domain/core/src/messages/core_message_producers.rs new file mode 100644 index 0000000000..84e0be9583 --- /dev/null +++ b/src/domain/core/src/messages/core_message_producers.rs @@ -0,0 +1,15 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub const MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE: &str = + "dev.kamu.domain.core.services.DatasetService"; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/domain/core/src/messages/core_message_types.rs b/src/domain/core/src/messages/core_message_types.rs new file mode 100644 index 0000000000..5f276898d3 --- /dev/null +++ b/src/domain/core/src/messages/core_message_types.rs @@ -0,0 +1,68 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use messaging_outbox::Message; +use opendatafabric::{AccountID, DatasetID}; +use serde::{Deserialize, Serialize}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum DatasetLifecycleMessage { + Created(DatasetLifecycleMessageCreated), + DependenciesUpdated(DatasetLifecycleMessageDependenciesUpdated), + Deleted(DatasetLifecycleMessageDeleted), +} + +impl DatasetLifecycleMessage { + pub fn created(dataset_id: DatasetID, owner_account_id: AccountID) -> Self { + Self::Created(DatasetLifecycleMessageCreated { + dataset_id, + owner_account_id, + }) + } + + pub fn dependencies_updated(dataset_id: DatasetID, new_upstream_ids: Vec) -> Self { + Self::DependenciesUpdated(DatasetLifecycleMessageDependenciesUpdated { + dataset_id, + new_upstream_ids, + }) + } + + pub fn deleted(dataset_id: DatasetID) -> Self { + Self::Deleted(DatasetLifecycleMessageDeleted { dataset_id }) + } +} + +impl Message for DatasetLifecycleMessage {} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DatasetLifecycleMessageCreated { + pub dataset_id: DatasetID, + pub owner_account_id: AccountID, +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DatasetLifecycleMessageDependenciesUpdated { + pub dataset_id: DatasetID, + pub new_upstream_ids: Vec, +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DatasetLifecycleMessageDeleted { + pub dataset_id: DatasetID, +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/domain/core/src/messages/mod.rs b/src/domain/core/src/messages/mod.rs new file mode 100644 index 0000000000..c1f05b9f99 --- /dev/null +++ b/src/domain/core/src/messages/mod.rs @@ -0,0 +1,16 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +mod core_message_consumers; +mod core_message_producers; +mod core_message_types; + +pub use core_message_consumers::*; +pub use core_message_producers::*; +pub use core_message_types::*; diff --git a/src/domain/core/src/repos/dataset_repository.rs b/src/domain/core/src/repos/dataset_repository.rs index 247f8b7b47..2677b3714f 100644 --- a/src/domain/core/src/repos/dataset_repository.rs +++ b/src/domain/core/src/repos/dataset_repository.rs @@ -7,13 +7,11 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use std::collections::{HashSet, LinkedList}; use std::pin::Pin; use std::sync::Arc; use async_trait::async_trait; -use chrono::Utc; -use internal_error::InternalError; +use internal_error::{ErrorIntoInternal, InternalError}; use opendatafabric::*; use thiserror::Error; use tokio_stream::Stream; @@ -41,6 +39,13 @@ impl CreateDatasetResult { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +pub struct CreateDatasetFromSnapshotResult { + pub create_dataset_result: CreateDatasetResult, + pub new_upstream_ids: Vec, +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + #[async_trait] pub trait DatasetRepository: DatasetRegistry + Sync + Send { fn is_multi_tenant(&self) -> bool; @@ -54,29 +59,12 @@ pub trait DatasetRepository: DatasetRegistry + Sync + Send { fn get_datasets_by_owner(&self, account_name: &AccountName) -> DatasetHandleStream<'_>; - async fn get_dataset( + async fn find_dataset_by_ref( &self, dataset_ref: &DatasetRef, ) -> Result, GetDatasetError>; - async fn create_dataset( - &self, - dataset_alias: &DatasetAlias, - seed_block: MetadataBlockTyped, - ) -> Result; - - async fn create_dataset_from_snapshot( - &self, - snapshot: DatasetSnapshot, - ) -> Result; - - async fn rename_dataset( - &self, - dataset_ref: &DatasetRef, - new_name: &DatasetName, - ) -> Result<(), RenameDatasetError>; - - async fn delete_dataset(&self, dataset_ref: &DatasetRef) -> Result<(), DeleteDatasetError>; + fn get_dataset_by_handle(&self, dataset_handle: &DatasetHandle) -> Arc; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -99,20 +87,6 @@ pub trait DatasetRepositoryExt: DatasetRepository { &self, dataset_ref: &DatasetRef, ) -> Result>, InternalError>; - - async fn create_dataset_from_seed( - &self, - dataset_alias: &DatasetAlias, - seed: Seed, - ) -> Result; - - async fn create_datasets_from_snapshots( - &self, - snapshots: Vec, - ) -> Vec<( - DatasetAlias, - Result, - )>; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -138,92 +112,16 @@ where &self, dataset_ref: &DatasetRef, ) -> Result>, InternalError> { - match self.get_dataset(dataset_ref).await { + match self.find_dataset_by_ref(dataset_ref).await { Ok(ds) => Ok(Some(ds)), Err(GetDatasetError::NotFound(_)) => Ok(None), Err(GetDatasetError::Internal(e)) => Err(e), } } - - async fn create_dataset_from_seed( - &self, - dataset_alias: &DatasetAlias, - seed: Seed, - ) -> Result { - // TODO: Externalize time - let system_time = Utc::now(); - - self.create_dataset( - dataset_alias, - MetadataBlockTyped { - system_time, - prev_block_hash: None, - event: seed, - sequence_number: 0, - }, - ) - .await - } - - async fn create_datasets_from_snapshots( - &self, - snapshots: Vec, - ) -> Vec<( - DatasetAlias, - Result, - )> { - let snapshots_ordered = sort_snapshots_in_dependency_order(snapshots.into_iter().collect()); - - let mut ret = Vec::new(); - for snapshot in snapshots_ordered { - let alias = snapshot.name.clone(); - let res = self.create_dataset_from_snapshot(snapshot).await; - ret.push((alias, res)); - } - ret - } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -#[allow(clippy::linkedlist)] -fn sort_snapshots_in_dependency_order( - mut snapshots: LinkedList, -) -> Vec { - let mut ordered = Vec::with_capacity(snapshots.len()); - let mut pending: HashSet = - snapshots.iter().map(|s| s.name.clone().into()).collect(); - let mut added: HashSet = HashSet::new(); - - // TODO: cycle detection - while !snapshots.is_empty() { - let snapshot = snapshots.pop_front().unwrap(); - - let transform = snapshot - .metadata - .iter() - .find_map(|e| e.as_variant::()); - - let has_pending_deps = if let Some(transform) = transform { - transform - .inputs - .iter() - .any(|input| pending.contains(&input.dataset_ref)) - } else { - false - }; - - if !has_pending_deps { - pending.remove(&snapshot.name.clone().into()); - added.insert(snapshot.name.clone()); - ordered.push(snapshot); - } else { - snapshots.push_back(snapshot); - } - } - ordered -} - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Errors //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/domain/core/src/services/compaction_service.rs b/src/domain/core/src/services/compaction_service.rs index c53de4aa23..ee93523791 100644 --- a/src/domain/core/src/services/compaction_service.rs +++ b/src/domain/core/src/services/compaction_service.rs @@ -10,6 +10,7 @@ use std::sync::Arc; use ::serde::{Deserialize, Serialize}; +use internal_error::{ErrorIntoInternal, InternalError}; use opendatafabric::*; use thiserror::Error; diff --git a/src/domain/core/src/services/ingest/polling_ingest_service.rs b/src/domain/core/src/services/ingest/polling_ingest_service.rs index 30557ad281..f38f3dd516 100644 --- a/src/domain/core/src/services/ingest/polling_ingest_service.rs +++ b/src/domain/core/src/services/ingest/polling_ingest_service.rs @@ -14,6 +14,7 @@ use std::sync::Arc; use chrono::{DateTime, Utc}; use container_runtime::ImagePullError; +use internal_error::{BoxedError, InternalError}; use kamu_datasets::DatasetEnvVar; use opendatafabric::*; use thiserror::Error; diff --git a/src/domain/core/src/services/ingest/push_ingest_service.rs b/src/domain/core/src/services/ingest/push_ingest_service.rs index 2bd794bf9d..4f10e07f01 100644 --- a/src/domain/core/src/services/ingest/push_ingest_service.rs +++ b/src/domain/core/src/services/ingest/push_ingest_service.rs @@ -10,6 +10,7 @@ use std::sync::Arc; use chrono::{DateTime, Utc}; +use internal_error::InternalError; use opendatafabric::*; use thiserror::Error; use tokio::io::AsyncRead; diff --git a/src/domain/core/src/services/provenance_service.rs b/src/domain/core/src/services/provenance_service.rs index 2a6870be03..2c4ab63be9 100644 --- a/src/domain/core/src/services/provenance_service.rs +++ b/src/domain/core/src/services/provenance_service.rs @@ -7,10 +7,11 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +use internal_error::InternalError; use opendatafabric::*; use thiserror::Error; -use crate::{auth, AccessError, DatasetNotFoundError, GetDatasetError, InternalError}; +use crate::{auth, AccessError, DatasetNotFoundError, GetDatasetError}; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/domain/core/src/services/pull_service.rs b/src/domain/core/src/services/pull_service.rs index 543fc71c54..4cb5832a60 100644 --- a/src/domain/core/src/services/pull_service.rs +++ b/src/domain/core/src/services/pull_service.rs @@ -11,6 +11,7 @@ use std::sync::Arc; use ::serde::{Deserialize, Serialize}; use chrono::{DateTime, Utc}; +use internal_error::InternalError; use opendatafabric::*; use thiserror::Error; diff --git a/src/domain/core/src/services/query_service.rs b/src/domain/core/src/services/query_service.rs index 9f31767498..ff3cce84a3 100644 --- a/src/domain/core/src/services/query_service.rs +++ b/src/domain/core/src/services/query_service.rs @@ -13,6 +13,7 @@ use datafusion::arrow; use datafusion::error::DataFusionError; use datafusion::parquet::schema::types::Type; use datafusion::prelude::{DataFrame, SessionContext}; +use internal_error::InternalError; use opendatafabric::*; use thiserror::Error; diff --git a/src/domain/core/src/services/remote_aliases.rs b/src/domain/core/src/services/remote_aliases.rs index d8612bfd1f..52d4e477a2 100644 --- a/src/domain/core/src/services/remote_aliases.rs +++ b/src/domain/core/src/services/remote_aliases.rs @@ -8,10 +8,9 @@ // by the Apache License, Version 2.0. use async_trait::async_trait; +use internal_error::InternalError; use opendatafabric::DatasetRefRemote; -use crate::InternalError; - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RemoteAliasKind { Pull, diff --git a/src/domain/core/src/services/remote_aliases_registry.rs b/src/domain/core/src/services/remote_aliases_registry.rs index 3a9cf3329b..85e5c4da08 100644 --- a/src/domain/core/src/services/remote_aliases_registry.rs +++ b/src/domain/core/src/services/remote_aliases_registry.rs @@ -8,6 +8,7 @@ // by the Apache License, Version 2.0. use async_trait::async_trait; +use internal_error::InternalError; use opendatafabric::*; use thiserror::Error; diff --git a/src/domain/core/src/services/remote_repository_registry.rs b/src/domain/core/src/services/remote_repository_registry.rs index bd50b7008a..7845490275 100644 --- a/src/domain/core/src/services/remote_repository_registry.rs +++ b/src/domain/core/src/services/remote_repository_registry.rs @@ -10,6 +10,7 @@ use ::serde::{Deserialize, Serialize}; use ::serde_with::skip_serializing_none; use async_trait::async_trait; +use internal_error::InternalError; use opendatafabric::*; use thiserror::Error; use url::Url; diff --git a/src/domain/core/src/services/reset_service.rs b/src/domain/core/src/services/reset_service.rs index 53213f8dc2..17240e7044 100644 --- a/src/domain/core/src/services/reset_service.rs +++ b/src/domain/core/src/services/reset_service.rs @@ -7,6 +7,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +use internal_error::InternalError; use opendatafabric::*; use thiserror::Error; diff --git a/src/domain/core/src/services/search_service.rs b/src/domain/core/src/services/search_service.rs index 8b8b517c5f..ca4e3592e5 100644 --- a/src/domain/core/src/services/search_service.rs +++ b/src/domain/core/src/services/search_service.rs @@ -7,6 +7,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +use internal_error::{BoxedError, InternalError}; use opendatafabric::*; use thiserror::Error; diff --git a/src/domain/core/src/services/sync_service.rs b/src/domain/core/src/services/sync_service.rs index 790af848a7..2692d8838a 100644 --- a/src/domain/core/src/services/sync_service.rs +++ b/src/domain/core/src/services/sync_service.rs @@ -9,6 +9,7 @@ use std::sync::Arc; +use internal_error::{BoxedError, InternalError}; use opendatafabric::*; use thiserror::Error; diff --git a/src/domain/core/src/services/transform_service.rs b/src/domain/core/src/services/transform_service.rs index 3cbdd245b3..fcc87498e3 100644 --- a/src/domain/core/src/services/transform_service.rs +++ b/src/domain/core/src/services/transform_service.rs @@ -9,6 +9,7 @@ use std::sync::Arc; +use internal_error::InternalError; use opendatafabric::*; use thiserror::Error; diff --git a/src/domain/core/src/services/verification_service.rs b/src/domain/core/src/services/verification_service.rs index 7e548a25ab..69fcc7e17c 100644 --- a/src/domain/core/src/services/verification_service.rs +++ b/src/domain/core/src/services/verification_service.rs @@ -10,6 +10,7 @@ use std::fmt::Display; use std::sync::Arc; +use internal_error::{ErrorIntoInternal, InternalError}; use opendatafabric::*; use thiserror::Error; diff --git a/src/domain/core/src/use_cases/append_dataset_metadata_batch_use_case.rs b/src/domain/core/src/use_cases/append_dataset_metadata_batch_use_case.rs new file mode 100644 index 0000000000..0b59f1edb0 --- /dev/null +++ b/src/domain/core/src/use_cases/append_dataset_metadata_batch_use_case.rs @@ -0,0 +1,26 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::collections::VecDeque; + +use crate::{AppendError, Dataset, HashedMetadataBlock}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[async_trait::async_trait] +pub trait AppendDatasetMetadataBatchUseCase: Send + Sync { + async fn execute( + &self, + dataset: &dyn Dataset, + new_blocks: VecDeque, + force_update_if_diverged: bool, + ) -> Result<(), AppendError>; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/domain/core/src/use_cases/commit_dataset_event_use_case.rs b/src/domain/core/src/use_cases/commit_dataset_event_use_case.rs new file mode 100644 index 0000000000..c38903c68b --- /dev/null +++ b/src/domain/core/src/use_cases/commit_dataset_event_use_case.rs @@ -0,0 +1,26 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use opendatafabric::{DatasetHandle, MetadataEvent}; + +use crate::{CommitError, CommitOpts, CommitResult}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[async_trait::async_trait] +pub trait CommitDatasetEventUseCase: Send + Sync { + async fn execute( + &self, + dataset_handle: &DatasetHandle, + event: MetadataEvent, + opts: CommitOpts<'_>, + ) -> Result; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/domain/core/src/use_cases/create_dataset_from_snapshot_use_case.rs b/src/domain/core/src/use_cases/create_dataset_from_snapshot_use_case.rs new file mode 100644 index 0000000000..e1edf185fe --- /dev/null +++ b/src/domain/core/src/use_cases/create_dataset_from_snapshot_use_case.rs @@ -0,0 +1,24 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use opendatafabric::DatasetSnapshot; + +use crate::{CreateDatasetFromSnapshotError, CreateDatasetResult}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[async_trait::async_trait] +pub trait CreateDatasetFromSnapshotUseCase: Send + Sync { + async fn execute( + &self, + snapshot: DatasetSnapshot, + ) -> Result; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/domain/core/src/use_cases/create_dataset_use_case.rs b/src/domain/core/src/use_cases/create_dataset_use_case.rs new file mode 100644 index 0000000000..8fe88db5c3 --- /dev/null +++ b/src/domain/core/src/use_cases/create_dataset_use_case.rs @@ -0,0 +1,25 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use opendatafabric::{DatasetAlias, MetadataBlockTyped, Seed}; + +use crate::{CreateDatasetError, CreateDatasetResult}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[async_trait::async_trait] +pub trait CreateDatasetUseCase: Send + Sync { + async fn execute( + &self, + dataset_alias: &DatasetAlias, + seed_block: MetadataBlockTyped, + ) -> Result; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/domain/core/src/use_cases/delete_dataset_use_case.rs b/src/domain/core/src/use_cases/delete_dataset_use_case.rs new file mode 100644 index 0000000000..9007e70b0c --- /dev/null +++ b/src/domain/core/src/use_cases/delete_dataset_use_case.rs @@ -0,0 +1,26 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use opendatafabric::{DatasetHandle, DatasetRef}; + +use crate::DeleteDatasetError; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[async_trait::async_trait] +pub trait DeleteDatasetUseCase: Send + Sync { + async fn execute_via_ref(&self, dataset_ref: &DatasetRef) -> Result<(), DeleteDatasetError>; + + async fn execute_via_handle( + &self, + dataset_handle: &DatasetHandle, + ) -> Result<(), DeleteDatasetError>; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/domain/core/src/use_cases/mod.rs b/src/domain/core/src/use_cases/mod.rs new file mode 100644 index 0000000000..be7db68249 --- /dev/null +++ b/src/domain/core/src/use_cases/mod.rs @@ -0,0 +1,22 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +mod append_dataset_metadata_batch_use_case; +mod commit_dataset_event_use_case; +mod create_dataset_from_snapshot_use_case; +mod create_dataset_use_case; +mod delete_dataset_use_case; +mod rename_dataset_use_case; + +pub use append_dataset_metadata_batch_use_case::*; +pub use commit_dataset_event_use_case::*; +pub use create_dataset_from_snapshot_use_case::*; +pub use create_dataset_use_case::*; +pub use delete_dataset_use_case::*; +pub use rename_dataset_use_case::*; diff --git a/src/domain/core/src/use_cases/rename_dataset_use_case.rs b/src/domain/core/src/use_cases/rename_dataset_use_case.rs new file mode 100644 index 0000000000..6b7d8efe17 --- /dev/null +++ b/src/domain/core/src/use_cases/rename_dataset_use_case.rs @@ -0,0 +1,25 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use opendatafabric::{DatasetName, DatasetRef}; + +use crate::RenameDatasetError; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[async_trait::async_trait] +pub trait RenameDatasetUseCase: Send + Sync { + async fn execute( + &self, + dataset_ref: &DatasetRef, + new_name: &DatasetName, + ) -> Result<(), RenameDatasetError>; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/domain/core/src/utils/mod.rs b/src/domain/core/src/utils/mod.rs index 0258813da8..c728a2d5b9 100644 --- a/src/domain/core/src/utils/mod.rs +++ b/src/domain/core/src/utils/mod.rs @@ -10,4 +10,3 @@ pub mod metadata_chain_comparator; pub mod owned_file; pub mod paths; -pub mod time_source; diff --git a/src/domain/datasets/services/Cargo.toml b/src/domain/datasets/services/Cargo.toml index 6afe64c7f3..3b4ccc7ba6 100644 --- a/src/domain/datasets/services/Cargo.toml +++ b/src/domain/datasets/services/Cargo.toml @@ -23,14 +23,14 @@ doctest = false [dependencies] database-common = { workspace = true } -kamu-core = { workspace = true } kamu-datasets = { workspace = true } internal-error = { workspace = true } opendatafabric = { workspace = true } +time-source = { workspace = true } async-trait = { version = "0.1", default-features = false } chrono = { version = "0.4", default-features = false } -dill = "0.8" +dill = "0.9" thiserror = { version = "1", default-features = false } secrecy = "0.8" serde = "1" diff --git a/src/domain/datasets/services/src/dataset_env_var_service_impl.rs b/src/domain/datasets/services/src/dataset_env_var_service_impl.rs index 32b91c40e6..69067d3dfc 100644 --- a/src/domain/datasets/services/src/dataset_env_var_service_impl.rs +++ b/src/domain/datasets/services/src/dataset_env_var_service_impl.rs @@ -11,8 +11,7 @@ use std::sync::Arc; use database_common::DatabasePaginationOpts; use dill::*; -use internal_error::ResultIntoInternal; -use kamu_core::{ErrorIntoInternal, InternalError, SystemTimeSource}; +use internal_error::{ErrorIntoInternal, InternalError, ResultIntoInternal}; use kamu_datasets::{ DatasetEnvVar, DatasetEnvVarListing, @@ -27,6 +26,7 @@ use kamu_datasets::{ }; use opendatafabric::DatasetID; use secrecy::{ExposeSecret, Secret}; +use time_source::SystemTimeSource; use uuid::Uuid; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/domain/datasets/services/src/dataset_env_var_service_null.rs b/src/domain/datasets/services/src/dataset_env_var_service_null.rs index e25493515d..7b65f5a203 100644 --- a/src/domain/datasets/services/src/dataset_env_var_service_null.rs +++ b/src/domain/datasets/services/src/dataset_env_var_service_null.rs @@ -9,7 +9,7 @@ use database_common::DatabasePaginationOpts; use dill::*; -use kamu_core::InternalError; +use internal_error::InternalError; use kamu_datasets::{ DatasetEnvVar, DatasetEnvVarListing, diff --git a/src/domain/datasets/services/src/dataset_key_value_service_impl.rs b/src/domain/datasets/services/src/dataset_key_value_service_impl.rs index 89517b37ed..7b41fcffc9 100644 --- a/src/domain/datasets/services/src/dataset_key_value_service_impl.rs +++ b/src/domain/datasets/services/src/dataset_key_value_service_impl.rs @@ -11,7 +11,7 @@ use std::collections::HashMap; use std::sync::Arc; use dill::*; -use kamu_core::ErrorIntoInternal; +use internal_error::ErrorIntoInternal; use kamu_datasets::{ DatasetEnvVar, DatasetEnvVarNotFoundError, diff --git a/src/domain/datasets/services/src/dataset_key_value_service_sys_env.rs b/src/domain/datasets/services/src/dataset_key_value_service_sys_env.rs index a30e4faa65..9b233b22f8 100644 --- a/src/domain/datasets/services/src/dataset_key_value_service_sys_env.rs +++ b/src/domain/datasets/services/src/dataset_key_value_service_sys_env.rs @@ -10,7 +10,7 @@ use std::collections::HashMap; use dill::*; -use kamu_core::ErrorIntoInternal; +use internal_error::ErrorIntoInternal; use kamu_datasets::{ DatasetEnvVar, DatasetEnvVarValue, diff --git a/src/domain/flow-system/domain/Cargo.toml b/src/domain/flow-system/domain/Cargo.toml index 2dc2ede17e..65be916b3e 100644 --- a/src/domain/flow-system/domain/Cargo.toml +++ b/src/domain/flow-system/domain/Cargo.toml @@ -25,6 +25,7 @@ doctest = false enum-variants = { workspace = true } event-sourcing = { workspace = true } internal-error = { workspace = true } +messaging-outbox = { workspace = true } opendatafabric = { workspace = true } kamu-accounts = { workspace = true } kamu-core = { workspace = true } diff --git a/src/domain/flow-system/domain/src/entities/shared/schedule.rs b/src/domain/flow-system/domain/src/entities/shared/schedule.rs index 5e2a763c4e..aa7943aae6 100644 --- a/src/domain/flow-system/domain/src/entities/shared/schedule.rs +++ b/src/domain/flow-system/domain/src/entities/shared/schedule.rs @@ -31,7 +31,7 @@ pub enum Schedule { #[serde_as] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ScheduleTimeDelta { - #[serde_as(as = "serde_with::DurationSeconds")] + #[serde_as(as = "serde_with::DurationSecondsWithFrac")] pub every: chrono::Duration, } diff --git a/src/domain/flow-system/domain/src/flow_messages_types.rs b/src/domain/flow-system/domain/src/flow_messages_types.rs new file mode 100644 index 0000000000..c81f8aae66 --- /dev/null +++ b/src/domain/flow-system/domain/src/flow_messages_types.rs @@ -0,0 +1,48 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use chrono::{DateTime, Utc}; +use messaging_outbox::Message; +use serde::{Deserialize, Serialize}; + +use crate::{FlowConfigurationRule, FlowKey}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FlowConfigurationUpdatedMessage { + pub event_time: DateTime, + pub flow_key: FlowKey, + pub paused: bool, + pub rule: FlowConfigurationRule, +} + +impl Message for FlowConfigurationUpdatedMessage {} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FlowServiceUpdatedMessage { + pub update_time: DateTime, + pub update_details: FlowServiceUpdateDetails, +} + +impl Message for FlowServiceUpdatedMessage {} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum FlowServiceUpdateDetails { + Loaded, + ExecutedTimeslot, + FlowRunning, + FlowFinished, +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/domain/flow-system/domain/src/lib.rs b/src/domain/flow-system/domain/src/lib.rs index 6db3fbf8d1..4d87f54b43 100644 --- a/src/domain/flow-system/domain/src/lib.rs +++ b/src/domain/flow-system/domain/src/lib.rs @@ -13,6 +13,8 @@ // Re-exports pub use event_sourcing::*; +mod flow_messages_types; + pub mod aggregates; pub mod dataset_flow_key; pub mod entities; @@ -22,5 +24,6 @@ pub mod services; pub use aggregates::*; pub use dataset_flow_key::*; pub use entities::*; +pub use flow_messages_types::*; pub use repos::*; pub use services::*; diff --git a/src/domain/flow-system/domain/src/services/flow/flow_service_event.rs b/src/domain/flow-system/domain/src/services/flow/flow_service_event.rs deleted file mode 100644 index 62f8f95da3..0000000000 --- a/src/domain/flow-system/domain/src/services/flow/flow_service_event.rs +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright Kamu Data, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -use chrono::{DateTime, Utc}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -#[derive(Debug, Clone)] -pub enum FlowServiceEvent { - ConfigurationLoaded(FlowServiceEventConfigurationLoaded), - ExecutedTimeSlot(FlowServiceEventExecutedTimeSlot), - FlowRunning(FlowServiceEventFlowRunning), - FlowFinished(FlowServiceEventFlowFinished), -} - -impl FlowServiceEvent { - pub fn event_time(&self) -> DateTime { - match self { - FlowServiceEvent::ConfigurationLoaded(e) => e.event_time, - FlowServiceEvent::ExecutedTimeSlot(e) => e.event_time, - FlowServiceEvent::FlowRunning(e) => e.event_time, - FlowServiceEvent::FlowFinished(e) => e.event_time, - } - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -#[derive(Debug, Clone)] -pub struct FlowServiceEventConfigurationLoaded { - pub event_time: DateTime, -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -#[derive(Debug, Clone)] -pub struct FlowServiceEventExecutedTimeSlot { - pub event_time: DateTime, -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -#[derive(Debug, Clone)] -pub struct FlowServiceEventFlowRunning { - pub event_time: DateTime, -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -#[derive(Debug, Clone)] -pub struct FlowServiceEventFlowFinished { - pub event_time: DateTime, -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/domain/flow-system/domain/src/services/flow/mod.rs b/src/domain/flow-system/domain/src/services/flow/mod.rs index 92df0069e8..71124e06bf 100644 --- a/src/domain/flow-system/domain/src/services/flow/mod.rs +++ b/src/domain/flow-system/domain/src/services/flow/mod.rs @@ -8,9 +8,7 @@ // by the Apache License, Version 2.0. mod flow_service; -mod flow_service_event; mod flow_service_test_driver; pub use flow_service::*; -pub use flow_service_event::*; pub use flow_service_test_driver::*; diff --git a/src/domain/flow-system/services/Cargo.toml b/src/domain/flow-system/services/Cargo.toml index 8dbd72b3a0..bcb26c5a62 100644 --- a/src/domain/flow-system/services/Cargo.toml +++ b/src/domain/flow-system/services/Cargo.toml @@ -24,17 +24,19 @@ doctest = false [dependencies] database-common = { workspace = true } database-common-macros = { workspace = true } -event-bus = { workspace = true } +internal-error = { workspace = true } +messaging-outbox = { workspace = true } kamu-accounts = { workspace = true } kamu-core = { workspace = true } kamu-flow-system = { workspace = true } kamu-task-system = { workspace = true } opendatafabric = { workspace = true } +time-source = { workspace = true } async-stream = "0.3" async-trait = { version = "0.1", default-features = false } chrono = { version = "0.4", default-features = false } -dill = "0.8" +dill = "0.9" futures = "0.3" thiserror = { version = "1", default-features = false } tokio = { version = "1", default-features = false, features = [] } @@ -44,6 +46,7 @@ url = { version = "2", default-features = false, features = ["serde"] } # TODO: Make serde optional serde = { version = "1", default-features = false, features = ["derive"] } +serde_json = "1" serde_with = { version = "3", default-features = false } diff --git a/src/domain/flow-system/services/src/flow/flow_service_impl.rs b/src/domain/flow-system/services/src/flow/flow_service_impl.rs index e9864bbaad..58a68e5fd5 100644 --- a/src/domain/flow-system/services/src/flow/flow_service_impl.rs +++ b/src/domain/flow-system/services/src/flow/flow_service_impl.rs @@ -13,36 +13,49 @@ use std::collections::HashSet; use std::sync::{Arc, Mutex}; use chrono::{DateTime, DurationRound, Utc}; +use database_common::DatabaseTransactionRunner; use dill::*; -use event_bus::{AsyncEventHandler, EventBus}; use futures::TryStreamExt; -use kamu_core::events::DatasetEventDeleted; +use internal_error::InternalError; use kamu_core::{ DatasetChangesService, + DatasetLifecycleMessage, DatasetOwnershipService, DependencyGraphService, - InternalError, - SystemTimeSource, + MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE, }; use kamu_flow_system::*; use kamu_task_system::*; +use messaging_outbox::{ + MessageConsumer, + MessageConsumerMeta, + MessageConsumerT, + MessageConsumptionDurability, + Outbox, + OutboxExt, +}; use opendatafabric::{AccountID, DatasetID}; +use time_source::SystemTimeSource; use tokio_stream::StreamExt; use super::active_configs_state::ActiveConfigsState; use super::flow_time_wheel::FlowTimeWheel; use super::pending_flows_state::PendingFlowsState; +use crate::{ + MESSAGE_CONSUMER_KAMU_FLOW_SERVICE, + MESSAGE_PRODUCER_KAMU_FLOW_CONFIGURATION_SERVICE, + MESSAGE_PRODUCER_KAMU_FLOW_SERVICE, +}; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub struct FlowServiceImpl { + catalog: Catalog, state: Arc>, run_config: Arc, - event_bus: Arc, flow_event_store: Arc, time_source: Arc, task_scheduler: Arc, - flow_configuration_service: Arc, dataset_changes_service: Arc, dependency_graph_service: Arc, dataset_ownership_service: Arc, @@ -63,31 +76,38 @@ struct State { #[component(pub)] #[interface(dyn FlowService)] #[interface(dyn FlowServiceTestDriver)] -#[interface(dyn AsyncEventHandler)] -#[interface(dyn AsyncEventHandler)] -#[interface(dyn AsyncEventHandler)] -#[interface(dyn AsyncEventHandler)] +#[interface(dyn MessageConsumer)] +#[interface(dyn MessageConsumerT)] +#[interface(dyn MessageConsumerT)] +#[interface(dyn MessageConsumerT)] +#[meta(MessageConsumerMeta { + consumer_name: MESSAGE_CONSUMER_KAMU_FLOW_SERVICE, + feeding_producers: &[ + MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE, + MESSAGE_PRODUCER_KAMU_TASK_EXECUTOR, + MESSAGE_PRODUCER_KAMU_FLOW_CONFIGURATION_SERVICE + ], + durability: MessageConsumptionDurability::Durable, +})] #[scope(Singleton)] impl FlowServiceImpl { pub fn new( + catalog: Catalog, run_config: Arc, - event_bus: Arc, flow_event_store: Arc, time_source: Arc, task_scheduler: Arc, - flow_configuration_service: Arc, dataset_changes_service: Arc, dependency_graph_service: Arc, dataset_ownership_service: Arc, ) -> Self { Self { + catalog, state: Arc::new(Mutex::new(State::default())), run_config, - event_bus, flow_event_store, time_source, task_scheduler, - flow_configuration_service, dataset_changes_service, dependency_graph_service, dataset_ownership_service, @@ -137,11 +157,11 @@ impl FlowServiceImpl { #[tracing::instrument(level = "debug", skip_all)] async fn initialize_auto_polling_flows_from_configurations( &self, + flow_configuration_service: &dyn FlowConfigurationService, start_time: DateTime, ) -> Result<(), InternalError> { // Query all enabled flow configurations - let enabled_configurations: Vec<_> = self - .flow_configuration_service + let enabled_configurations: Vec<_> = flow_configuration_service .list_enabled_configurations() .try_collect() .await?; @@ -920,17 +940,31 @@ impl FlowService for FlowServiceImpl { self.state.lock().unwrap().running = true; // Initial scheduling - let start_time = self.round_time(planned_start_time)?; - self.initialize_auto_polling_flows_from_configurations(start_time) - .await?; + DatabaseTransactionRunner::new(self.catalog.clone()) + .transactional_with2( + |flow_configuration_service: Arc, + outbox: Arc| async move { + let start_time = self.round_time(planned_start_time)?; + self.initialize_auto_polling_flows_from_configurations( + flow_configuration_service.as_ref(), + start_time, + ) + .await?; + + // Publish progress event + outbox + .post_message( + MESSAGE_PRODUCER_KAMU_FLOW_SERVICE, + FlowServiceUpdatedMessage { + update_time: start_time, + update_details: FlowServiceUpdateDetails::Loaded, + }, + ) + .await?; - // Publish progress event - self.event_bus - .dispatch_event(FlowServiceEvent::ConfigurationLoaded( - FlowServiceEventConfigurationLoaded { - event_time: start_time, + Ok(()) }, - )) + ) .await?; // Main scanning loop @@ -954,12 +988,18 @@ impl FlowService for FlowServiceImpl { self.run_current_timeslot(nearest_activation_time).await?; // Publish progress event - self.event_bus - .dispatch_event(FlowServiceEvent::ExecutedTimeSlot( - FlowServiceEventExecutedTimeSlot { - event_time: nearest_activation_time, - }, - )) + DatabaseTransactionRunner::new(self.catalog.clone()) + .transactional_with(|outbox: Arc| async move { + outbox + .post_message( + MESSAGE_PRODUCER_KAMU_FLOW_SERVICE, + FlowServiceUpdatedMessage { + update_time: nearest_activation_time, + update_details: FlowServiceUpdateDetails::ExecutedTimeslot, + }, + ) + .await + }) .await?; } @@ -1266,106 +1306,119 @@ impl FlowServiceTestDriver for FlowServiceImpl { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -#[async_trait::async_trait] -impl AsyncEventHandler for FlowServiceImpl { - #[tracing::instrument(level = "debug", skip_all, fields(?event))] - async fn handle(&self, event: &TaskEventRunning) -> Result<(), InternalError> { - // Is this a task associated with flows? - let maybe_flow_id = { - let state = self.state.lock().unwrap(); - if !state.running { - // Abort if running hasn't started yet - return Ok(()); - } - state.pending_flows.try_get_flow_id_by_task(event.task_id) - }; - - if let Some(flow_id) = maybe_flow_id { - let mut flow = Flow::load(flow_id, self.flow_event_store.as_ref()) - .await - .int_err()?; - flow.on_task_running(event.event_time, event.task_id) - .int_err()?; - flow.save(self.flow_event_store.as_ref()).await.int_err()?; - } - - // Publish progress event - self.event_bus - .dispatch_event(FlowServiceEvent::FlowRunning(FlowServiceEventFlowRunning { - event_time: event.event_time, - })) - .await?; - - Ok(()) - } -} +impl MessageConsumer for FlowServiceImpl {} //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[async_trait::async_trait] -impl AsyncEventHandler for FlowServiceImpl { - #[tracing::instrument(level = "debug", skip_all, fields(?event))] - async fn handle(&self, event: &TaskEventFinished) -> Result<(), InternalError> { - // Is this a task associated with flows? - let maybe_flow_id = { - let state = self.state.lock().unwrap(); - if !state.running { - // Abort if running hasn't started yet - return Ok(()); +impl MessageConsumerT for FlowServiceImpl { + #[tracing::instrument(level = "debug", skip_all, fields(?message))] + async fn consume_message( + &self, + target_catalog: &Catalog, + message: &TaskProgressMessage, + ) -> Result<(), InternalError> { + match message { + TaskProgressMessage::Running(message) => { + // Is this a task associated with flows? + let maybe_flow_id = { + let state = self.state.lock().unwrap(); + if !state.running { + // Abort if running hasn't started yet + return Ok(()); + } + state.pending_flows.try_get_flow_id_by_task(message.task_id) + }; + + if let Some(flow_id) = maybe_flow_id { + let mut flow = Flow::load(flow_id, self.flow_event_store.as_ref()) + .await + .int_err()?; + flow.on_task_running(message.event_time, message.task_id) + .int_err()?; + flow.save(self.flow_event_store.as_ref()).await.int_err()?; + + let outbox = target_catalog.get_one::().unwrap(); + outbox + .post_message( + MESSAGE_PRODUCER_KAMU_FLOW_SERVICE, + FlowServiceUpdatedMessage { + update_time: message.event_time, + update_details: FlowServiceUpdateDetails::FlowRunning, + }, + ) + .await?; + } } - state.pending_flows.try_get_flow_id_by_task(event.task_id) - }; + TaskProgressMessage::Finished(message) => { + // Is this a task associated with flows? + let maybe_flow_id = { + let state = self.state.lock().unwrap(); + if !state.running { + // Abort if running hasn't started yet + return Ok(()); + } + state.pending_flows.try_get_flow_id_by_task(message.task_id) + }; - let finish_time = self.round_time(event.event_time)?; + let finish_time = self.round_time(message.event_time)?; - if let Some(flow_id) = maybe_flow_id { - let mut flow = Flow::load(flow_id, self.flow_event_store.as_ref()) - .await - .int_err()?; - flow.on_task_finished(event.event_time, event.task_id, event.outcome.clone()) - .int_err()?; - flow.save(self.flow_event_store.as_ref()).await.int_err()?; + if let Some(flow_id) = maybe_flow_id { + let mut flow = Flow::load(flow_id, self.flow_event_store.as_ref()) + .await + .int_err()?; + flow.on_task_finished( + message.event_time, + message.task_id, + message.outcome.clone(), + ) + .int_err()?; + flow.save(self.flow_event_store.as_ref()).await.int_err()?; - { - let mut state = self.state.lock().unwrap(); - state.pending_flows.untrack_flow_by_task(event.task_id); - state.pending_flows.drop_pending_flow(&flow.flow_key); - } + { + let mut state = self.state.lock().unwrap(); + state.pending_flows.untrack_flow_by_task(message.task_id); + state.pending_flows.drop_pending_flow(&flow.flow_key); + } - // In case of success: - // - execute followup method - if let Some(flow_result) = flow.try_result_as_ref() - && !flow_result.is_empty() - { - match flow.flow_key.get_type().success_followup_method() { - FlowSuccessFollowupMethod::Ignore => {} - FlowSuccessFollowupMethod::TriggerDependent => { - self.enqueue_dependent_flows(finish_time, &flow, flow_result) - .await?; + // In case of success: + // - execute followup method + if let Some(flow_result) = flow.try_result_as_ref() + && !flow_result.is_empty() + { + match flow.flow_key.get_type().success_followup_method() { + FlowSuccessFollowupMethod::Ignore => {} + FlowSuccessFollowupMethod::TriggerDependent => { + self.enqueue_dependent_flows(finish_time, &flow, flow_result) + .await?; + } + } } - } - } - // In case of success: - // - enqueue next auto-polling flow cycle - if event.outcome.is_success() { - self.try_enqueue_scheduled_auto_polling_flow_if_enabled( - finish_time, - &flow.flow_key, - ) - .await?; - } + // In case of success: + // - enqueue next auto-polling flow cycle + if message.outcome.is_success() { + self.try_enqueue_scheduled_auto_polling_flow_if_enabled( + finish_time, + &flow.flow_key, + ) + .await?; + } - // Publish progress event - self.event_bus - .dispatch_event(FlowServiceEvent::FlowFinished( - FlowServiceEventFlowFinished { - event_time: finish_time, - }, - )) - .await?; + let outbox = target_catalog.get_one::().unwrap(); + outbox + .post_message( + MESSAGE_PRODUCER_KAMU_FLOW_SERVICE, + FlowServiceUpdatedMessage { + update_time: message.event_time, + update_details: FlowServiceUpdateDetails::FlowFinished, + }, + ) + .await?; - // TODO: retry logic in case of failed outcome + // TODO: retry logic in case of failed outcome + } + } } Ok(()) @@ -1375,10 +1428,14 @@ impl AsyncEventHandler for FlowServiceImpl { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[async_trait::async_trait] -impl AsyncEventHandler for FlowServiceImpl { - #[tracing::instrument(level = "debug", skip_all, fields(?event))] - async fn handle(&self, event: &FlowConfigurationEventModified) -> Result<(), InternalError> { - if event.paused { +impl MessageConsumerT for FlowServiceImpl { + #[tracing::instrument(level = "debug", skip_all, fields(?message))] + async fn consume_message( + &self, + _: &Catalog, + message: &FlowConfigurationUpdatedMessage, + ) -> Result<(), InternalError> { + if message.paused { let maybe_pending_flow_id = { let mut state = self.state.lock().unwrap(); if !state.running { @@ -1386,9 +1443,10 @@ impl AsyncEventHandler for FlowServiceImpl { return Ok(()); }; - state.active_configs.drop_flow_config(&event.flow_key); + state.active_configs.drop_flow_config(&message.flow_key); - let maybe_pending_flow_id = state.pending_flows.drop_pending_flow(&event.flow_key); + let maybe_pending_flow_id = + state.pending_flows.drop_pending_flow(&message.flow_key); if let Some(flow_id) = &maybe_pending_flow_id { state .time_wheel @@ -1410,11 +1468,11 @@ impl AsyncEventHandler for FlowServiceImpl { }; } - let activation_time = self.round_time(event.event_time)?; + let activation_time = self.round_time(message.event_time)?; self.activate_flow_configuration( activation_time, - event.flow_key.clone(), - event.rule.clone(), + message.flow_key.clone(), + message.rule.clone(), ) .await?; } @@ -1426,51 +1484,70 @@ impl AsyncEventHandler for FlowServiceImpl { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[async_trait::async_trait] -impl AsyncEventHandler for FlowServiceImpl { - #[tracing::instrument(level = "debug", skip_all, fields(?event))] - async fn handle(&self, event: &DatasetEventDeleted) -> Result<(), InternalError> { - let flow_ids_2_abort = { - let mut state = self.state.lock().unwrap(); - if !state.running { - // Abort if running hasn't started yet - return Ok(()); - }; +impl MessageConsumerT for FlowServiceImpl { + #[tracing::instrument(level = "debug", skip_all, fields(?message))] + async fn consume_message( + &self, + _: &Catalog, + message: &DatasetLifecycleMessage, + ) -> Result<(), InternalError> { + match message { + DatasetLifecycleMessage::Deleted(message) => { + let flow_ids_2_abort = { + let mut state = self.state.lock().unwrap(); + if !state.running { + // Abort if running hasn't started yet + return Ok(()); + }; - state.active_configs.drop_dataset_configs(&event.dataset_id); - - // For every possible dataset flow: - // - drop it from pending state - // - drop queued activations - // - collect ID of aborted flow - let mut flow_ids_2_abort: Vec<_> = Vec::with_capacity(DatasetFlowType::all().len()); - for flow_type in DatasetFlowType::all() { - if let Some(flow_id) = state - .pending_flows - .drop_dataset_pending_flow(&event.dataset_id, *flow_type) - { - flow_ids_2_abort.push(flow_id); - state.time_wheel.cancel_flow_activation(flow_id).int_err()?; + state + .active_configs + .drop_dataset_configs(&message.dataset_id); + + // For every possible dataset flow: + // - drop it from pending state + // - drop queued activations + // - collect ID of aborted flow + let mut flow_ids_2_abort: Vec<_> = + Vec::with_capacity(DatasetFlowType::all().len()); + for flow_type in DatasetFlowType::all() { + if let Some(flow_id) = state + .pending_flows + .drop_dataset_pending_flow(&message.dataset_id, *flow_type) + { + flow_ids_2_abort.push(flow_id); + state.time_wheel.cancel_flow_activation(flow_id).int_err()?; + } + } + flow_ids_2_abort + }; + + // Abort matched flows + for flow_id in flow_ids_2_abort { + let mut flow = Flow::load(flow_id, self.flow_event_store.as_ref()) + .await + .int_err()?; + flow.abort(self.time_source.now()).int_err()?; + flow.save(self.flow_event_store.as_ref()).await.int_err()?; } + + // Not deleting task->update association, it should be safe. + // Most of the time the outcome of the task will be "Cancelled". + // Even if task squeezes to succeed in between cancellations, + // it's safe: + // - we will record a successful update, no consequence + // - no further updates will be attempted (schedule + // deactivated above) + // - no dependent tasks will be launched (dependency graph + // erases neighbors) } - flow_ids_2_abort - }; - // Abort matched flows - for flow_id in flow_ids_2_abort { - let mut flow = Flow::load(flow_id, self.flow_event_store.as_ref()) - .await - .int_err()?; - flow.abort(self.time_source.now()).int_err()?; - flow.save(self.flow_event_store.as_ref()).await.int_err()?; + DatasetLifecycleMessage::Created(_) + | DatasetLifecycleMessage::DependenciesUpdated(_) => { + // No action required + } } - // Not deleting task->update association, it should be safe. - // Most of the time the outcome of the task will be "Cancelled". - // Even if task squeezes to succeed in between cancellations, it's safe: - // - we will record a successful update, no consequence - // - no further updates will be attempted (schedule deactivated above) - // - no dependent tasks will be launched (dependency graph erases neighbors) - Ok(()) } } diff --git a/src/domain/flow-system/services/src/flow_configuration/flow_configuration_service_impl.rs b/src/domain/flow-system/services/src/flow_configuration/flow_configuration_service_impl.rs index 32b5ccb96b..40f50611eb 100644 --- a/src/domain/flow-system/services/src/flow_configuration/flow_configuration_service_impl.rs +++ b/src/domain/flow-system/services/src/flow_configuration/flow_configuration_service_impl.rs @@ -11,37 +11,54 @@ use std::sync::Arc; use chrono::{DateTime, Utc}; use dill::*; -use event_bus::{AsyncEventHandler, EventBus}; use futures::TryStreamExt; -use kamu_core::events::DatasetEventDeleted; -use kamu_core::SystemTimeSource; +use kamu_core::{DatasetLifecycleMessage, MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE}; use kamu_flow_system::*; +use messaging_outbox::{ + MessageConsumer, + MessageConsumerMeta, + MessageConsumerT, + MessageConsumptionDurability, + Outbox, + OutboxExt, +}; use opendatafabric::DatasetID; +use time_source::SystemTimeSource; + +use crate::{ + MESSAGE_CONSUMER_KAMU_FLOW_CONFIGURATION_SERVICE, + MESSAGE_PRODUCER_KAMU_FLOW_CONFIGURATION_SERVICE, +}; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub struct FlowConfigurationServiceImpl { event_store: Arc, time_source: Arc, - event_bus: Arc, + outbox: Arc, } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[component(pub)] #[interface(dyn FlowConfigurationService)] -#[interface(dyn AsyncEventHandler)] -#[scope(Singleton)] +#[interface(dyn MessageConsumer)] +#[interface(dyn MessageConsumerT)] +#[meta(MessageConsumerMeta { + consumer_name: MESSAGE_CONSUMER_KAMU_FLOW_CONFIGURATION_SERVICE, + feeding_producers: &[MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE], + durability: MessageConsumptionDurability::Durable, +})] impl FlowConfigurationServiceImpl { pub fn new( event_store: Arc, time_source: Arc, - event_bus: Arc, + outbox: Arc, ) -> Self { Self { event_store, time_source, - event_bus, + outbox, } } @@ -50,13 +67,16 @@ impl FlowConfigurationServiceImpl { state: &FlowConfigurationState, request_time: DateTime, ) -> Result<(), InternalError> { - let event = FlowConfigurationEventModified { + let message = FlowConfigurationUpdatedMessage { event_time: request_time, flow_key: state.flow_key.clone(), paused: !state.is_active(), rule: state.rule.clone(), }; - self.event_bus.dispatch_event(event).await + + self.outbox + .post_message(MESSAGE_PRODUCER_KAMU_FLOW_CONFIGURATION_SERVICE, message) + .await } fn get_dataset_flow_keys( @@ -321,27 +341,44 @@ impl FlowConfigurationService for FlowConfigurationServiceImpl { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -#[async_trait::async_trait] -impl AsyncEventHandler for FlowConfigurationServiceImpl { - #[tracing::instrument(level = "debug", skip_all, fields(?event))] - async fn handle(&self, event: &DatasetEventDeleted) -> Result<(), InternalError> { - for flow_type in DatasetFlowType::all() { - let maybe_flow_configuration = FlowConfiguration::try_load( - FlowKeyDataset::new(event.dataset_id.clone(), *flow_type).into(), - self.event_store.as_ref(), - ) - .await - .int_err()?; +impl MessageConsumer for FlowConfigurationServiceImpl {} - if let Some(mut flow_configuration) = maybe_flow_configuration { - flow_configuration - .notify_dataset_removed(self.time_source.now()) - .int_err()?; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - flow_configuration - .save(self.event_store.as_ref()) +#[async_trait::async_trait] +impl MessageConsumerT for FlowConfigurationServiceImpl { + #[tracing::instrument(level = "debug", skip_all, fields(?message))] + async fn consume_message( + &self, + _: &Catalog, + message: &DatasetLifecycleMessage, + ) -> Result<(), InternalError> { + match message { + DatasetLifecycleMessage::Deleted(message) => { + for flow_type in DatasetFlowType::all() { + let maybe_flow_configuration = FlowConfiguration::try_load( + FlowKeyDataset::new(message.dataset_id.clone(), *flow_type).into(), + self.event_store.as_ref(), + ) .await .int_err()?; + + if let Some(mut flow_configuration) = maybe_flow_configuration { + flow_configuration + .notify_dataset_removed(self.time_source.now()) + .int_err()?; + + flow_configuration + .save(self.event_store.as_ref()) + .await + .int_err()?; + } + } + } + + DatasetLifecycleMessage::Created(_) + | DatasetLifecycleMessage::DependenciesUpdated(_) => { + // no action required } } diff --git a/src/domain/flow-system/services/src/lib.rs b/src/domain/flow-system/services/src/lib.rs index 4cc8efda2d..31fdf7e864 100644 --- a/src/domain/flow-system/services/src/lib.rs +++ b/src/domain/flow-system/services/src/lib.rs @@ -14,6 +14,8 @@ pub use kamu_flow_system as domain; mod flow; mod flow_configuration; +mod messages; pub use flow::*; pub use flow_configuration::*; +pub use messages::*; diff --git a/src/domain/flow-system/services/src/messages/flow_message_consumers.rs b/src/domain/flow-system/services/src/messages/flow_message_consumers.rs new file mode 100644 index 0000000000..fa31cce4be --- /dev/null +++ b/src/domain/flow-system/services/src/messages/flow_message_consumers.rs @@ -0,0 +1,17 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub const MESSAGE_CONSUMER_KAMU_FLOW_CONFIGURATION_SERVICE: &str = + "dev.kamu.domain.flow-system.FlowConfigurationService"; + +pub const MESSAGE_CONSUMER_KAMU_FLOW_SERVICE: &str = "dev.kamu.domain.flow-system.FlowService"; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/domain/flow-system/services/src/messages/flow_message_producers.rs b/src/domain/flow-system/services/src/messages/flow_message_producers.rs new file mode 100644 index 0000000000..f87d196c3a --- /dev/null +++ b/src/domain/flow-system/services/src/messages/flow_message_producers.rs @@ -0,0 +1,17 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub const MESSAGE_PRODUCER_KAMU_FLOW_CONFIGURATION_SERVICE: &str = + "dev.kamu.domain.flow-system.FlowConfigurationService"; + +pub const MESSAGE_PRODUCER_KAMU_FLOW_SERVICE: &str = "dev.kamu.domain.flow-system.FlowService"; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/domain/flow-system/services/src/messages/mod.rs b/src/domain/flow-system/services/src/messages/mod.rs new file mode 100644 index 0000000000..48f97dc9b7 --- /dev/null +++ b/src/domain/flow-system/services/src/messages/mod.rs @@ -0,0 +1,14 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +mod flow_message_consumers; +mod flow_message_producers; + +pub use flow_message_consumers::*; +pub use flow_message_producers::*; diff --git a/src/domain/flow-system/services/tests/tests/test_flow_configuration_service_impl.rs b/src/domain/flow-system/services/tests/tests/test_flow_configuration_service_impl.rs index 95aa449dc0..1e3eab5422 100644 --- a/src/domain/flow-system/services/tests/tests/test_flow_configuration_service_impl.rs +++ b/src/domain/flow-system/services/tests/tests/test_flow_configuration_service_impl.rs @@ -9,12 +9,11 @@ use std::assert_matches::assert_matches; use std::collections::HashMap; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; use chrono::{Duration, Utc}; use database_common_macros::transactional_method; use dill::*; -use event_bus::{AsyncEventHandler, EventBus}; use futures::TryStreamExt; use kamu::testing::MetadataFactory; use kamu::*; @@ -23,7 +22,11 @@ use kamu_core::*; use kamu_flow_system::*; use kamu_flow_system_inmem::*; use kamu_flow_system_services::*; +use messaging_outbox::{register_message_dispatcher, Outbox, OutboxImmediateImpl}; use opendatafabric::*; +use time_source::SystemTimeSourceDefault; + +use super::FlowConfigTestListener; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -424,7 +427,7 @@ struct FlowConfigurationHarness { dataset_repo: Arc, flow_configuration_service: Arc, flow_configuration_event_store: Arc, - config_events_listener: Arc, + config_listener: Arc, } impl FlowConfigurationHarness { @@ -436,23 +439,39 @@ impl FlowConfigurationHarness { let catalog = { let mut b = CatalogBuilder::new(); - b.add::() - .add::() - .add::() - .add::() - .add_builder( - DatasetRepositoryLocalFs::builder() - .with_root(datasets_dir) - .with_multi_tenant(false), - ) - .bind::() - .add_value(CurrentAccountSubject::new_test()) - .add::() - .add::() - .add::(); + b.add_builder( + messaging_outbox::OutboxImmediateImpl::builder() + .with_consumer_filter(messaging_outbox::ConsumerFilter::AllConsumers), + ) + .bind::() + .add::() + .add::() + .add::() + .add::() + .add_builder( + DatasetRepositoryLocalFs::builder() + .with_root(datasets_dir) + .with_multi_tenant(false), + ) + .bind::() + .bind::() + .add_value(CurrentAccountSubject::new_test()) + .add::() + .add::() + .add::() + .add::(); database_common::NoOpDatabasePlugin::init_database_components(&mut b); + register_message_dispatcher::( + &mut b, + MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE, + ); + register_message_dispatcher::( + &mut b, + MESSAGE_PRODUCER_KAMU_FLOW_CONFIGURATION_SERVICE, + ); + b.build() }; @@ -461,7 +480,7 @@ impl FlowConfigurationHarness { .get_one::() .unwrap(); let dataset_repo = catalog.get_one::().unwrap(); - let flow_config_events_listener = catalog.get_one::().unwrap(); + let flow_config_events_listener = catalog.get_one::().unwrap(); Self { _tmp_dir: tmp_dir, @@ -469,7 +488,7 @@ impl FlowConfigurationHarness { flow_configuration_service, flow_configuration_event_store, dataset_repo, - config_events_listener: flow_config_events_listener, + config_listener: flow_config_events_listener, } } @@ -616,10 +635,15 @@ impl FlowConfigurationHarness { .unwrap(); flow_configuration.into() } + async fn create_root_dataset(&self, dataset_name: &str) -> DatasetID { - let result = self - .dataset_repo - .create_dataset_from_snapshot( + let create_dataset_from_snapshot = self + .catalog + .get_one::() + .unwrap(); + + let result = create_dataset_from_snapshot + .execute( MetadataFactory::dataset_snapshot() .name(dataset_name) .kind(DatasetKind::Root) @@ -647,46 +671,15 @@ impl FlowConfigurationHarness { .unwrap(); // Do the actual deletion - self.dataset_repo - .delete_dataset(&(dataset_id.as_local_ref())) + let delete_dataset = self.catalog.get_one::().unwrap(); + delete_dataset + .execute_via_ref(&(dataset_id.as_local_ref())) .await .unwrap(); } fn configuration_events_count(&self) -> usize { - self.config_events_listener.configuration_events_count() - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -struct FlowConfigEventsListener { - configuration_modified_events: Mutex>, -} - -#[component] -#[scope(Singleton)] -#[interface(dyn AsyncEventHandler)] -impl FlowConfigEventsListener { - fn new() -> Self { - Self { - configuration_modified_events: Mutex::new(Vec::new()), - } - } - - fn configuration_events_count(&self) -> usize { - let events = self.configuration_modified_events.lock().unwrap(); - events.len() - } -} - -#[async_trait::async_trait] -impl AsyncEventHandler for FlowConfigEventsListener { - #[tracing::instrument(level = "debug", skip_all, fields(?event))] - async fn handle(&self, event: &FlowConfigurationEventModified) -> Result<(), InternalError> { - let mut events = self.configuration_modified_events.lock().unwrap(); - events.push(event.clone()); - Ok(()) + self.config_listener.configuration_events_count() } } diff --git a/src/domain/flow-system/services/tests/tests/test_flow_service_impl.rs b/src/domain/flow-system/services/tests/tests/test_flow_service_impl.rs index fe2e51f525..21b6b33c1d 100644 --- a/src/domain/flow-system/services/tests/tests/test_flow_service_impl.rs +++ b/src/domain/flow-system/services/tests/tests/test_flow_service_impl.rs @@ -33,12 +33,14 @@ async fn test_read_initial_config_and_queue_without_waiting() { let harness = FlowHarness::new().await; // Create a "foo" root dataset, and configure ingestion schedule every 60ms - let foo_id = harness + let foo_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("foo"), account_name: None, }) .await; + let foo_id = foo_create_result.dataset_handle.id; + harness .set_dataset_flow_schedule( harness.now_datetime(), @@ -155,14 +157,15 @@ async fn test_cron_config() { }) .await; - // Create a "foo" root dataset, and configure ingestion cron schedule of every - // 5s - let foo_id = harness + // Create a "foo" root dataset, configure ingestion cron schedule of every 5s + let foo_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("foo"), account_name: None, }) .await; + let foo_id = foo_create_result.dataset_handle.id; + harness .set_dataset_flow_schedule( harness.now_datetime(), @@ -248,18 +251,21 @@ async fn test_cron_config() { async fn test_manual_trigger() { let harness = FlowHarness::new().await; - let foo_id = harness + let foo_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("foo"), account_name: None, }) .await; - let bar_id = harness + let foo_id = foo_create_result.dataset_handle.id; + + let bar_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("bar"), account_name: None, }) .await; + let bar_id = bar_create_result.dataset_handle.id; // Note: only "foo" has auto-schedule, "bar" hasn't harness @@ -453,18 +459,21 @@ async fn test_manual_trigger() { async fn test_manual_trigger_compaction() { let harness = FlowHarness::new().await; - let foo_id = harness + let foo_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("foo"), account_name: None, }) .await; - let bar_id = harness + let foo_id = foo_create_result.dataset_handle.id; + + let bar_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("bar"), account_name: None, }) .await; + let bar_id = bar_create_result.dataset_handle.id; harness.eager_initialization().await; @@ -602,7 +611,7 @@ async fn test_manual_trigger_reset() { let harness = FlowHarness::new().await; let create_dataset_result = harness - .create_dataset(DatasetAlias { + .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("foo"), account_name: None, }) @@ -721,7 +730,7 @@ async fn test_reset_trigger_keep_metadata_compaction_for_derivatives() { let harness = FlowHarness::new().await; let create_foo_result = harness - .create_dataset(DatasetAlias { + .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("foo"), account_name: None, }) @@ -957,12 +966,13 @@ async fn test_manual_trigger_compaction_with_config() { let max_slice_records = 1000u64; let harness = FlowHarness::new().await; - let foo_id = harness + let foo_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("foo"), account_name: None, }) .await; + let foo_id = foo_create_result.dataset_handle.id; harness.eager_initialization().await; harness @@ -1058,12 +1068,14 @@ async fn test_full_hard_compaction_trigger_keep_metadata_compaction_for_derivati let max_slice_records = 1000u64; let harness = FlowHarness::new().await; - let foo_id = harness + let foo_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("foo"), account_name: None, }) .await; + let foo_id = foo_create_result.dataset_handle.id; + let foo_bar_id = harness .create_derived_dataset( DatasetAlias { @@ -1288,12 +1300,14 @@ async fn test_full_hard_compaction_trigger_keep_metadata_compaction_for_derivati async fn test_manual_trigger_keep_metadata_only_with_recursive_compaction() { let harness = FlowHarness::new().await; - let foo_id = harness + let foo_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("foo"), account_name: None, }) .await; + let foo_id = foo_create_result.dataset_handle.id; + let foo_bar_id = harness .create_derived_dataset( DatasetAlias { @@ -1519,12 +1533,14 @@ async fn test_manual_trigger_keep_metadata_only_with_recursive_compaction() { async fn test_manual_trigger_keep_metadata_only_without_recursive_compaction() { let harness = FlowHarness::new().await; - let foo_id = harness + let foo_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("foo"), account_name: None, }) .await; + let foo_id = foo_create_result.dataset_handle.id; + let foo_bar_id = harness .create_derived_dataset( DatasetAlias { @@ -1658,12 +1674,14 @@ async fn test_manual_trigger_keep_metadata_only_compaction_multiple_accounts() { }) .await; - let foo_id = harness + let foo_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("foo"), account_name: Some(wasya_account_name.clone()), }) .await; + let foo_id = foo_create_result.dataset_handle.id; + let foo_bar_id = harness .create_derived_dataset( DatasetAlias { @@ -1825,18 +1843,22 @@ async fn test_manual_trigger_keep_metadata_only_compaction_multiple_accounts() { async fn test_dataset_flow_configuration_paused_resumed_modified() { let harness = FlowHarness::new().await; - let foo_id = harness + let foo_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("foo"), account_name: None, }) .await; - let bar_id: DatasetID = harness + let foo_id = foo_create_result.dataset_handle.id; + + let bar_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("bar"), account_name: None, }) .await; + let bar_id = bar_create_result.dataset_handle.id; + harness .set_dataset_flow_schedule( harness.now_datetime(), @@ -2031,18 +2053,21 @@ async fn test_dataset_flow_configuration_paused_resumed_modified() { async fn test_respect_last_success_time_when_schedule_resumes() { let harness = FlowHarness::new().await; - let foo_id = harness + let foo_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("foo"), account_name: None, }) .await; - let bar_id = harness + let foo_id = foo_create_result.dataset_handle.id; + + let bar_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("bar"), account_name: None, }) .await; + let bar_id = bar_create_result.dataset_handle.id; harness .set_dataset_flow_schedule( @@ -2244,18 +2269,21 @@ async fn test_respect_last_success_time_when_schedule_resumes() { async fn test_dataset_deleted() { let harness = FlowHarness::new().await; - let foo_id = harness + let foo_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("foo"), account_name: None, }) .await; - let bar_id = harness + let foo_id = foo_create_result.dataset_handle.id; + + let bar_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("bar"), account_name: None, }) .await; + let bar_id = bar_create_result.dataset_handle.id; harness .set_dataset_flow_schedule( @@ -2429,24 +2457,29 @@ async fn test_dataset_deleted() { async fn test_task_completions_trigger_next_loop_on_success() { let harness = FlowHarness::new().await; - let foo_id = harness + let foo_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("foo"), account_name: None, }) .await; - let bar_id = harness + let foo_id = foo_create_result.dataset_handle.id; + + let bar_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("bar"), account_name: None, }) .await; - let baz_id = harness + let bar_id = bar_create_result.dataset_handle.id; + + let baz_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("baz"), account_name: None, }) .await; + let baz_id = baz_create_result.dataset_handle.id; for dataset_id in [&foo_id, &bar_id, &baz_id] { harness @@ -2646,12 +2679,14 @@ async fn test_derived_dataset_triggered_initially_and_after_input_change() { }) .await; - let foo_id = harness + let foo_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("foo"), account_name: None, }) .await; + let foo_id = foo_create_result.dataset_handle.id; + let bar_id = harness .create_derived_dataset( DatasetAlias { @@ -2894,12 +2929,13 @@ async fn test_throttling_manual_triggers() { .await; // Foo Flow - let foo_id = harness + let foo_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("foo"), account_name: None, }) .await; + let foo_id = foo_create_result.dataset_handle.id; let foo_flow_key: FlowKey = FlowKeyDataset::new(foo_id.clone(), DatasetFlowType::Ingest).into(); // Enforce dependency graph initialization @@ -3025,18 +3061,22 @@ async fn test_throttling_derived_dataset_with_2_parents() { }) .await; - let foo_id = harness + let foo_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("foo"), account_name: None, }) .await; - let bar_id = harness + let foo_id = foo_create_result.dataset_handle.id; + + let bar_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("bar"), account_name: None, }) .await; + let bar_id = bar_create_result.dataset_handle.id; + let baz_id = harness .create_derived_dataset( DatasetAlias { @@ -3497,12 +3537,14 @@ async fn test_batching_condition_records_reached() { }) .await; - let foo_id = harness + let foo_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("foo"), account_name: None, }) .await; + let foo_id = foo_create_result.dataset_handle.id; + let bar_id = harness .create_derived_dataset( DatasetAlias { @@ -3811,12 +3853,14 @@ async fn test_batching_condition_timeout() { }) .await; - let foo_id = harness + let foo_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("foo"), account_name: None, }) .await; + let foo_id = foo_create_result.dataset_handle.id; + let bar_id = harness .create_derived_dataset( DatasetAlias { @@ -4078,12 +4122,14 @@ async fn test_batching_condition_watermark() { }) .await; - let foo_id = harness + let foo_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("foo"), account_name: None, }) .await; + let foo_id = foo_create_result.dataset_handle.id; + let bar_id = harness .create_derived_dataset( DatasetAlias { @@ -4394,18 +4440,22 @@ async fn test_batching_condition_with_2_inputs() { }) .await; - let foo_id = harness + let foo_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("foo"), account_name: None, }) .await; - let bar_id = harness + let foo_id = foo_create_result.dataset_handle.id; + + let bar_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("bar"), account_name: None, }) .await; + let bar_id = bar_create_result.dataset_handle.id; + let baz_id = harness .create_derived_dataset( DatasetAlias { @@ -4868,12 +4918,14 @@ async fn test_list_all_flow_initiators() { }) .await; - let foo_id = harness + let foo_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("foo"), account_name: Some(foo_account_name.clone()), }) .await; + let foo_id = foo_create_result.dataset_handle.id; + let foo_account_id = harness .auth_svc .find_account_id_by_name(&foo_account_name) @@ -4887,12 +4939,13 @@ async fn test_list_all_flow_initiators() { .unwrap() .unwrap(); - let bar_id = harness + let bar_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("bar"), account_name: Some(bar_account_name.clone()), }) .await; + let bar_id = bar_create_result.dataset_handle.id; harness.eager_initialization().await; @@ -5017,12 +5070,13 @@ async fn test_list_all_datasets_with_flow() { }) .await; - let foo_id = harness + let foo_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("foo"), account_name: Some(foo_account_name.clone()), }) .await; + let foo_id = foo_create_result.dataset_handle.id; let _foo_bar_id = harness .create_derived_dataset( @@ -5047,12 +5101,13 @@ async fn test_list_all_datasets_with_flow() { .unwrap() .unwrap(); - let bar_id = harness + let bar_create_result = harness .create_root_dataset(DatasetAlias { dataset_name: DatasetName::new_unchecked("bar"), account_name: Some(bar_account_name.clone()), }) .await; + let bar_id = bar_create_result.dataset_handle.id; harness.eager_initialization().await; diff --git a/src/domain/flow-system/services/tests/tests/utils/flow_config_test_listener.rs b/src/domain/flow-system/services/tests/tests/utils/flow_config_test_listener.rs new file mode 100644 index 0000000000..e629568a8e --- /dev/null +++ b/src/domain/flow-system/services/tests/tests/utils/flow_config_test_listener.rs @@ -0,0 +1,67 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::sync::Mutex; + +use dill::{component, interface, meta, scope, Catalog, Singleton}; +use internal_error::InternalError; +use kamu_flow_system::FlowConfigurationUpdatedMessage; +use kamu_flow_system_services::MESSAGE_PRODUCER_KAMU_FLOW_CONFIGURATION_SERVICE; +use messaging_outbox::{ + MessageConsumer, + MessageConsumerMeta, + MessageConsumerT, + MessageConsumptionDurability, +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub(crate) struct FlowConfigTestListener { + configuration_modified_events: Mutex>, +} + +#[component(pub)] +#[scope(Singleton)] +#[interface(dyn MessageConsumer)] +#[interface(dyn MessageConsumerT)] +#[meta(MessageConsumerMeta { + consumer_name: "FlowConfigTestListener", + feeding_producers: &[MESSAGE_PRODUCER_KAMU_FLOW_CONFIGURATION_SERVICE], + durability: MessageConsumptionDurability::BestEffort, +})] +impl FlowConfigTestListener { + pub fn new() -> Self { + Self { + configuration_modified_events: Mutex::new(Vec::new()), + } + } + + pub fn configuration_events_count(&self) -> usize { + let events = self.configuration_modified_events.lock().unwrap(); + events.len() + } +} + +impl MessageConsumer for FlowConfigTestListener {} + +#[async_trait::async_trait] +impl MessageConsumerT for FlowConfigTestListener { + #[tracing::instrument(level = "debug", skip_all, fields(?message))] + async fn consume_message( + &self, + _: &Catalog, + message: &FlowConfigurationUpdatedMessage, + ) -> Result<(), InternalError> { + let mut events = self.configuration_modified_events.lock().unwrap(); + events.push(message.clone()); + Ok(()) + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/domain/flow-system/services/tests/tests/utils/flow_harness_shared.rs b/src/domain/flow-system/services/tests/tests/utils/flow_harness_shared.rs index b795a55204..0199e57a80 100644 --- a/src/domain/flow-system/services/tests/tests/utils/flow_harness_shared.rs +++ b/src/domain/flow-system/services/tests/tests/utils/flow_harness_shared.rs @@ -12,7 +12,6 @@ use std::sync::Arc; use chrono::{DateTime, Duration, TimeZone, Utc}; use database_common::{DatabaseTransactionRunner, NoOpDatabasePlugin}; use dill::*; -use event_bus::EventBus; use kamu::testing::{MetadataFactory, MockDatasetChangesService}; use kamu::*; use kamu_accounts::{ @@ -22,7 +21,7 @@ use kamu_accounts::{ JwtAuthenticationConfig, PredefinedAccountsConfig, }; -use kamu_accounts_inmem::{AccessTokenRepositoryInMemory, AccountRepositoryInMemory}; +use kamu_accounts_inmem::{InMemoryAccessTokenRepository, InMemoryAccountRepository}; use kamu_accounts_services::{ AccessTokenServiceImpl, AuthenticationServiceImpl, @@ -33,9 +32,12 @@ use kamu_core::*; use kamu_flow_system::*; use kamu_flow_system_inmem::*; use kamu_flow_system_services::*; -use kamu_task_system_inmem::TaskSystemEventStoreInMemory; +use kamu_task_system::{TaskProgressMessage, MESSAGE_PRODUCER_KAMU_TASK_EXECUTOR}; +use kamu_task_system_inmem::InMemoryTaskSystemEventStore; use kamu_task_system_services::TaskSchedulerImpl; +use messaging_outbox::{register_message_dispatcher, Outbox, OutboxImmediateImpl}; use opendatafabric::*; +use time_source::{FakeSystemTimeSource, SystemTimeSource}; use tokio::task::yield_now; use super::{ @@ -110,45 +112,69 @@ impl FlowHarness { let catalog = { let mut b = dill::CatalogBuilder::new(); - b.add::() - .add_value(FlowServiceRunConfig::new( - awaiting_step, - mandatory_throttling_period, - )) - .add::() - .add::() - .add::() - .add::() - .add_value(fake_system_time_source.clone()) - .bind::() - .add_builder( - DatasetRepositoryLocalFs::builder() - .with_root(datasets_dir) - .with_multi_tenant(overrides.is_multi_tenant), - ) - .bind::() - .add_value(mock_dataset_changes) - .bind::() - .add_value(CurrentAccountSubject::new_test()) - .add::() - .add::() - .add_value(predefined_accounts_config) - .add_value(JwtAuthenticationConfig::default()) - .add::() - .add::() - .add::() - .add::() - .add::() - .add::() - .add::() - .add::() - .add::() - .add::() - .add::() - .add::(); + b.add_builder( + messaging_outbox::OutboxImmediateImpl::builder() + .with_consumer_filter(messaging_outbox::ConsumerFilter::AllConsumers), + ) + .bind::() + .add::() + .add_value(FlowServiceRunConfig::new( + awaiting_step, + mandatory_throttling_period, + )) + .add::() + .add::() + .add::() + .add::() + .add_value(fake_system_time_source.clone()) + .bind::() + .add_builder( + DatasetRepositoryLocalFs::builder() + .with_root(datasets_dir) + .with_multi_tenant(overrides.is_multi_tenant), + ) + .bind::() + .bind::() + .add_value(mock_dataset_changes) + .bind::() + .add_value(CurrentAccountSubject::new_test()) + .add::() + .add::() + .add::() + .add::() + .add_value(predefined_accounts_config) + .add_value(JwtAuthenticationConfig::default()) + .add::() + .add::() + .add::() + .add::() + .add::() + .add::() + .add::() + .add::() + .add::() + .add::() + .add::(); NoOpDatabasePlugin::init_database_components(&mut b); + register_message_dispatcher::( + &mut b, + MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE, + ); + register_message_dispatcher::( + &mut b, + MESSAGE_PRODUCER_KAMU_TASK_EXECUTOR, + ); + register_message_dispatcher::( + &mut b, + MESSAGE_PRODUCER_KAMU_FLOW_CONFIGURATION_SERVICE, + ); + register_message_dispatcher::( + &mut b, + MESSAGE_PRODUCER_KAMU_FLOW_SERVICE, + ); + b.build() }; @@ -181,9 +207,14 @@ impl FlowHarness { } } - pub async fn create_dataset(&self, dataset_alias: DatasetAlias) -> CreateDatasetResult { - self.dataset_repo - .create_dataset_from_snapshot( + pub async fn create_root_dataset(&self, dataset_alias: DatasetAlias) -> CreateDatasetResult { + let create_dataset_from_snapshot = self + .catalog + .get_one::() + .unwrap(); + + create_dataset_from_snapshot + .execute( MetadataFactory::dataset_snapshot() .name(dataset_alias) .kind(DatasetKind::Root) @@ -194,20 +225,18 @@ impl FlowHarness { .unwrap() } - pub async fn create_root_dataset(&self, dataset_alias: DatasetAlias) -> DatasetID { - let result = self.create_dataset(dataset_alias).await; - - result.dataset_handle.id - } - pub async fn create_derived_dataset( &self, dataset_alias: DatasetAlias, input_ids: Vec, ) -> DatasetID { - let result = self - .dataset_repo - .create_dataset_from_snapshot( + let create_dataset_from_snapshot = self + .catalog + .get_one::() + .unwrap(); + + let create_result = create_dataset_from_snapshot + .execute( MetadataFactory::dataset_snapshot() .name(dataset_alias) .kind(DatasetKind::Derivative) @@ -221,7 +250,7 @@ impl FlowHarness { .await .unwrap(); - result.dataset_handle.id + create_result.dataset_handle.id } pub async fn eager_initialization(&self) { @@ -251,8 +280,9 @@ impl FlowHarness { self.eager_initialization().await; // Do the actual deletion - self.dataset_repo - .delete_dataset(&(dataset_id.as_local_ref())) + let delete_dataset = self.catalog.get_one::().unwrap(); + delete_dataset + .execute_via_ref(&(dataset_id.as_local_ref())) .await .unwrap(); } diff --git a/src/domain/flow-system/services/tests/tests/utils/flow_system_test_listener.rs b/src/domain/flow-system/services/tests/tests/utils/flow_system_test_listener.rs index 40eead43a4..1f737c8d35 100644 --- a/src/domain/flow-system/services/tests/tests/utils/flow_system_test_listener.rs +++ b/src/domain/flow-system/services/tests/tests/utils/flow_system_test_listener.rs @@ -12,20 +12,17 @@ use std::sync::{Arc, Mutex}; use chrono::{DateTime, Utc}; use dill::*; -use event_bus::AsyncEventHandler; -use kamu_core::{FakeSystemTimeSource, InternalError}; -use kamu_flow_system::{ - FlowKey, - FlowOutcome, - FlowPaginationOpts, - FlowService, - FlowServiceEvent, - FlowStartCondition, - FlowState, - FlowStatus, - FlowTrigger, +use internal_error::InternalError; +use kamu_flow_system::*; +use kamu_flow_system_services::MESSAGE_PRODUCER_KAMU_FLOW_SERVICE; +use messaging_outbox::{ + MessageConsumer, + MessageConsumerMeta, + MessageConsumerT, + MessageConsumptionDurability, }; use opendatafabric::DatasetID; +use time_source::FakeSystemTimeSource; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -45,7 +42,13 @@ struct FlowSystemTestListenerState { #[component(pub)] #[scope(Singleton)] -#[interface(dyn AsyncEventHandler)] +#[interface(dyn MessageConsumer)] +#[interface(dyn MessageConsumerT)] +#[meta(MessageConsumerMeta { + consumer_name: "FlowSystemTestListener", + feeding_producers: &[MESSAGE_PRODUCER_KAMU_FLOW_SERVICE], + durability: MessageConsumptionDurability::BestEffort, +})] impl FlowSystemTestListener { pub(crate) fn new( flow_service: Arc, @@ -58,7 +61,7 @@ impl FlowSystemTestListener { } } - pub(crate) async fn make_a_snapshot(&self, event_time: DateTime) { + pub(crate) async fn make_a_snapshot(&self, update_time: DateTime) { use futures::TryStreamExt; let flows: Vec<_> = self .flow_service @@ -82,7 +85,7 @@ impl FlowSystemTestListener { } let mut state = self.state.lock().unwrap(); - state.snapshots.push((event_time, flow_states_map)); + state.snapshots.push((update_time, flow_states_map)); } pub(crate) fn define_dataset_display_name(&self, id: DatasetID, display_name: String) { @@ -229,10 +232,16 @@ impl std::fmt::Display for FlowSystemTestListener { } } +impl MessageConsumer for FlowSystemTestListener {} + #[async_trait::async_trait] -impl AsyncEventHandler for FlowSystemTestListener { - async fn handle(&self, event: &FlowServiceEvent) -> Result<(), InternalError> { - self.make_a_snapshot(event.event_time()).await; +impl MessageConsumerT for FlowSystemTestListener { + async fn consume_message( + &self, + _: &Catalog, + message: &FlowServiceUpdatedMessage, + ) -> Result<(), InternalError> { + self.make_a_snapshot(message.update_time).await; Ok(()) } } diff --git a/src/domain/flow-system/services/tests/tests/utils/manual_flow_trigger_driver.rs b/src/domain/flow-system/services/tests/tests/utils/manual_flow_trigger_driver.rs index 8689cb98a2..219f606f02 100644 --- a/src/domain/flow-system/services/tests/tests/utils/manual_flow_trigger_driver.rs +++ b/src/domain/flow-system/services/tests/tests/utils/manual_flow_trigger_driver.rs @@ -11,9 +11,9 @@ use std::sync::Arc; use chrono::Duration; use kamu_accounts::DEFAULT_ACCOUNT_ID; -use kamu_core::SystemTimeSource; use kamu_flow_system::{FlowKey, FlowService}; use opendatafabric::AccountID; +use time_source::SystemTimeSource; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/domain/flow-system/services/tests/tests/utils/mod.rs b/src/domain/flow-system/services/tests/tests/utils/mod.rs index 7ef80a1e37..b91bd7f0ed 100644 --- a/src/domain/flow-system/services/tests/tests/utils/mod.rs +++ b/src/domain/flow-system/services/tests/tests/utils/mod.rs @@ -7,11 +7,13 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +mod flow_config_test_listener; mod flow_harness_shared; mod flow_system_test_listener; mod manual_flow_trigger_driver; mod task_driver; +pub(crate) use flow_config_test_listener::*; pub(crate) use flow_harness_shared::*; pub(crate) use flow_system_test_listener::*; pub(crate) use manual_flow_trigger_driver::*; diff --git a/src/domain/flow-system/services/tests/tests/utils/task_driver.rs b/src/domain/flow-system/services/tests/tests/utils/task_driver.rs index cc129f0d4c..a9e571b1dd 100644 --- a/src/domain/flow-system/services/tests/tests/utils/task_driver.rs +++ b/src/domain/flow-system/services/tests/tests/utils/task_driver.rs @@ -10,17 +10,17 @@ use std::sync::Arc; use chrono::Duration; -use event_bus::EventBus; -use kamu_core::SystemTimeSource; use kamu_task_system::*; +use messaging_outbox::{Outbox, OutboxExt}; use opendatafabric::DatasetID; +use time_source::SystemTimeSource; use tokio::task::yield_now; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub(crate) struct TaskDriver { time_source: Arc, - event_bus: Arc, + outbox: Arc, task_event_store: Arc, args: TaskDriverArgs, } @@ -36,13 +36,13 @@ pub(crate) struct TaskDriverArgs { impl TaskDriver { pub(crate) fn new( time_source: Arc, - event_bus: Arc, + outbox: Arc, task_event_store: Arc, args: TaskDriverArgs, ) -> Self { Self { time_source, - event_bus, + outbox, task_event_store, args, } @@ -58,23 +58,33 @@ impl TaskDriver { self.ensure_task_matches_logical_plan().await; - self.event_bus - .dispatch_event(TaskEventRunning { - event_time: start_time + self.args.run_since_start, - task_id: self.args.task_id, - }) + // Note: we can omit transaction, since this is a test-only abstraction + // with assummed immediate delivery + self.outbox + .post_message( + MESSAGE_PRODUCER_KAMU_TASK_EXECUTOR, + TaskProgressMessage::running( + start_time + self.args.run_since_start, + self.args.task_id, + ), + ) .await .unwrap(); if let Some((finish_in, with_outcome)) = self.args.finish_in_with { self.time_source.sleep(finish_in).await; - self.event_bus - .dispatch_event(TaskEventFinished { - event_time: start_time + self.args.run_since_start + finish_in, - task_id: self.args.task_id, - outcome: with_outcome, - }) + // Note: we can omit transaction, since this is a test-only abstraction + // with assummed immediate delivery + self.outbox + .post_message( + MESSAGE_PRODUCER_KAMU_TASK_EXECUTOR, + TaskProgressMessage::finished( + start_time + self.args.run_since_start + finish_in, + self.args.task_id, + with_outcome, + ), + ) .await .unwrap(); } diff --git a/src/domain/task-system/domain/Cargo.toml b/src/domain/task-system/domain/Cargo.toml index 783d8f6d4f..181061cd53 100644 --- a/src/domain/task-system/domain/Cargo.toml +++ b/src/domain/task-system/domain/Cargo.toml @@ -23,7 +23,8 @@ doctest = false [dependencies] enum-variants = { workspace = true } -event-sourcing ={ workspace = true } +event-sourcing = { workspace = true } +messaging-outbox = { workspace = true } opendatafabric = { workspace = true } kamu-core = { workspace = true } diff --git a/src/domain/task-system/domain/src/lib.rs b/src/domain/task-system/domain/src/lib.rs index 8a68b387dc..ffd068054b 100644 --- a/src/domain/task-system/domain/src/lib.rs +++ b/src/domain/task-system/domain/src/lib.rs @@ -14,10 +14,12 @@ pub use event_sourcing::*; pub mod aggregates; pub mod entities; +mod messages; pub mod repos; pub mod services; pub use aggregates::*; pub use entities::*; +pub use messages::*; pub use repos::*; pub use services::*; diff --git a/src/domain/task-system/domain/src/messages/mod.rs b/src/domain/task-system/domain/src/messages/mod.rs new file mode 100644 index 0000000000..af7b32a56e --- /dev/null +++ b/src/domain/task-system/domain/src/messages/mod.rs @@ -0,0 +1,14 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +mod task_message_producers; +mod task_messages_types; + +pub use task_message_producers::*; +pub use task_messages_types::*; diff --git a/src/domain/task-system/domain/src/messages/task_message_producers.rs b/src/domain/task-system/domain/src/messages/task_message_producers.rs new file mode 100644 index 0000000000..88c2888b91 --- /dev/null +++ b/src/domain/task-system/domain/src/messages/task_message_producers.rs @@ -0,0 +1,14 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub const MESSAGE_PRODUCER_KAMU_TASK_EXECUTOR: &str = "dev.kamu.domain.task-system.TaskExecutor"; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/domain/task-system/domain/src/messages/task_messages_types.rs b/src/domain/task-system/domain/src/messages/task_messages_types.rs new file mode 100644 index 0000000000..1675593d65 --- /dev/null +++ b/src/domain/task-system/domain/src/messages/task_messages_types.rs @@ -0,0 +1,60 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use chrono::{DateTime, Utc}; +use messaging_outbox::Message; +use serde::{Deserialize, Serialize}; + +use crate::{TaskID, TaskOutcome}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum TaskProgressMessage { + Running(TaskProgressMessageRunning), + Finished(TaskProgressMessageFinished), +} + +impl TaskProgressMessage { + pub fn running(event_time: DateTime, task_id: TaskID) -> Self { + Self::Running(TaskProgressMessageRunning { + event_time, + task_id, + }) + } + + pub fn finished(event_time: DateTime, task_id: TaskID, outcome: TaskOutcome) -> Self { + Self::Finished(TaskProgressMessageFinished { + event_time, + task_id, + outcome, + }) + } +} + +impl Message for TaskProgressMessage {} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TaskProgressMessageRunning { + pub event_time: DateTime, + pub task_id: TaskID, +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TaskProgressMessageFinished { + pub event_time: DateTime, + pub task_id: TaskID, + pub outcome: TaskOutcome, +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/domain/task-system/services/Cargo.toml b/src/domain/task-system/services/Cargo.toml index 27a94c41e4..a120cb25a9 100644 --- a/src/domain/task-system/services/Cargo.toml +++ b/src/domain/task-system/services/Cargo.toml @@ -23,16 +23,19 @@ doctest = false [dependencies] database-common = { workspace = true } -event-bus = { workspace = true } +internal-error = { workspace = true } +messaging-outbox = { workspace = true } opendatafabric = { workspace = true } kamu-core = { workspace = true } kamu-datasets = { workspace = true } +time-source = { workspace = true } kamu-task-system = { workspace = true } async-stream = "0.3" async-trait = { version = "0.1", default-features = false } -dill = "0.8" +dill = "0.9" futures = "0.3" +serde_json = "1" tokio = { version = "1", default-features = false } tracing = { version = "0.1", default-features = false } diff --git a/src/domain/task-system/services/src/task_executor_impl.rs b/src/domain/task-system/services/src/task_executor_impl.rs index b55fdabd5f..807578e312 100644 --- a/src/domain/task-system/services/src/task_executor_impl.rs +++ b/src/domain/task-system/services/src/task_executor_impl.rs @@ -12,7 +12,6 @@ use std::sync::Arc; use database_common::DatabaseTransactionRunner; use dill::*; -use event_bus::EventBus; use kamu_core::{ CompactionOptions, CompactionService, @@ -23,20 +22,19 @@ use kamu_core::{ PullService, ResetError, ResetService, - SystemTimeSource, TransformError, }; use kamu_datasets::{DatasetEnvVar, DatasetEnvVarService}; use kamu_task_system::*; +use messaging_outbox::{Outbox, OutboxExt}; +use time_source::SystemTimeSource; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub struct TaskExecutorImpl { + catalog: Catalog, task_sched: Arc, - event_store: Arc, - event_bus: Arc, time_source: Arc, - catalog: Catalog, } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -46,42 +44,105 @@ pub struct TaskExecutorImpl { #[scope(Singleton)] impl TaskExecutorImpl { pub fn new( + catalog: Catalog, task_sched: Arc, - event_store: Arc, - event_bus: Arc, time_source: Arc, - catalog: Catalog, ) -> Self { Self { + catalog, task_sched, - event_store, - event_bus, time_source, - catalog, } } - async fn publish_task_running(&self, task_id: TaskID) -> Result<(), InternalError> { - self.event_bus - .dispatch_event(TaskEventRunning { - event_time: self.time_source.now(), - task_id, - }) + async fn take_task(&self) -> Result { + let task_id = self.task_sched.take().await.int_err()?; + + DatabaseTransactionRunner::new(self.catalog.clone()) + .transactional_with2( + |event_store: Arc, outbox: Arc| async move { + let task = Task::load(task_id, event_store.as_ref()).await.int_err()?; + + tracing::info!( + %task_id, + logical_plan = ?task.logical_plan, + "Executing task", + ); + + outbox + .post_message( + MESSAGE_PRODUCER_KAMU_TASK_EXECUTOR, + TaskProgressMessage::running(self.time_source.now(), task_id), + ) + .await?; + + Ok(task) + }, + ) .await } - async fn publish_task_finished( + async fn execute_task(&self, task: &Task) -> Result { + let task_outcome = match &task.logical_plan { + LogicalPlan::UpdateDataset(upd) => self.update_dataset_logical_plan(upd).await?, + LogicalPlan::Probe(Probe { + busy_time, + end_with_outcome, + .. + }) => { + if let Some(busy_time) = busy_time { + tokio::time::sleep(*busy_time).await; + } + end_with_outcome + .clone() + .unwrap_or(TaskOutcome::Success(TaskResult::Empty)) + } + LogicalPlan::Reset(reset_args) => self.reset_dataset_logical_plan(reset_args).await?, + LogicalPlan::HardCompactionDataset(hard_compaction_args) => { + self.hard_compaction_logical_plan(hard_compaction_args) + .await? + } + }; + + tracing::info!( + task_id = %task.task_id, + logical_plan = ?task.logical_plan, + ?task_outcome, + "Task finished", + ); + + Ok(task_outcome) + } + + async fn process_task_outcome( &self, - task_id: TaskID, - outcome: TaskOutcome, + mut task: Task, + task_outcome: TaskOutcome, ) -> Result<(), InternalError> { - self.event_bus - .dispatch_event(TaskEventFinished { - event_time: self.time_source.now(), - task_id, - outcome, - }) - .await + DatabaseTransactionRunner::new(self.catalog.clone()) + .transactional_with2( + |event_store: Arc, outbox: Arc| async move { + // Refresh the task in case it was updated concurrently (e.g. late cancellation) + task.update(event_store.as_ref()).await.int_err()?; + task.finish(self.time_source.now(), task_outcome.clone()) + .int_err()?; + task.save(event_store.as_ref()).await.int_err()?; + + outbox + .post_message( + MESSAGE_PRODUCER_KAMU_TASK_EXECUTOR, + TaskProgressMessage::finished( + self.time_source.now(), + task.task_id, + task_outcome, + ), + ) + .await + }, + ) + .await?; + + Ok(()) } async fn update_dataset_logical_plan( @@ -210,56 +271,9 @@ impl TaskExecutor for TaskExecutorImpl { // TODO: Error and panic handling strategy async fn run(&self) -> Result<(), InternalError> { loop { - let task_id = self.task_sched.take().await.int_err()?; - let mut task = Task::load(task_id, self.event_store.as_ref()) - .await - .int_err()?; - - tracing::info!( - %task_id, - logical_plan = ?task.logical_plan, - "Executing task", - ); - - self.publish_task_running(task.task_id).await?; - - let outcome = match &task.logical_plan { - LogicalPlan::UpdateDataset(upd) => self.update_dataset_logical_plan(upd).await?, - LogicalPlan::Probe(Probe { - busy_time, - end_with_outcome, - .. - }) => { - if let Some(busy_time) = busy_time { - tokio::time::sleep(*busy_time).await; - } - end_with_outcome - .clone() - .unwrap_or(TaskOutcome::Success(TaskResult::Empty)) - } - LogicalPlan::HardCompactionDataset(hard_compaction_args) => { - self.hard_compaction_logical_plan(hard_compaction_args) - .await? - } - LogicalPlan::Reset(reset_args) => { - self.reset_dataset_logical_plan(reset_args).await? - } - }; - - tracing::info!( - %task_id, - logical_plan = ?task.logical_plan, - ?outcome, - "Task finished", - ); - - // Refresh the task in case it was updated concurrently (e.g. late cancellation) - task.update(self.event_store.as_ref()).await.int_err()?; - task.finish(self.time_source.now(), outcome.clone()) - .int_err()?; - task.save(self.event_store.as_ref()).await.int_err()?; - - self.publish_task_finished(task.task_id, outcome).await?; + let task = self.take_task().await?; + let task_outcome = self.execute_task(&task).await?; + self.process_task_outcome(task, task_outcome).await?; } } } diff --git a/src/domain/task-system/services/src/task_scheduler_impl.rs b/src/domain/task-system/services/src/task_scheduler_impl.rs index 3ebccdaf67..9e3ca38e70 100644 --- a/src/domain/task-system/services/src/task_scheduler_impl.rs +++ b/src/domain/task-system/services/src/task_scheduler_impl.rs @@ -11,20 +11,22 @@ use std::collections::VecDeque; use std::sync::{Arc, Mutex}; use dill::*; -use kamu_core::SystemTimeSource; use kamu_task_system::*; use opendatafabric::DatasetID; +use time_source::SystemTimeSource; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub struct TaskSchedulerImpl { state: Arc>, + // TODO: EventStore is transaction-dependent, it can't be instantiated in a singleton event_store: Arc, time_source: Arc, } #[derive(Default)] struct State { + // TODO: store in DB or something like Redis task_queue: VecDeque, } diff --git a/src/domain/task-system/services/tests/tests/test_task_aggregate.rs b/src/domain/task-system/services/tests/tests/test_task_aggregate.rs index 9543710935..3e60b1065d 100644 --- a/src/domain/task-system/services/tests/tests/test_task_aggregate.rs +++ b/src/domain/task-system/services/tests/tests/test_task_aggregate.rs @@ -17,7 +17,7 @@ use kamu_task_system_services::domain::*; #[test_log::test(tokio::test)] async fn test_task_agg_create_new() { - let event_store = TaskSystemEventStoreInMemory::new(); + let event_store = InMemoryTaskSystemEventStore::new(); let mut task = Task::new( Utc::now(), @@ -42,7 +42,7 @@ async fn test_task_agg_create_new() { #[test_log::test(tokio::test)] async fn test_task_save_load_update() { - let event_store = TaskSystemEventStoreInMemory::new(); + let event_store = InMemoryTaskSystemEventStore::new(); let task_id = event_store.new_task_id().await.unwrap(); let mut task = Task::new(Utc::now(), task_id, Probe::default().into()); @@ -86,7 +86,7 @@ async fn test_task_save_load_update() { #[test_log::test(tokio::test)] async fn test_task_agg_illegal_transition() { - let event_store = TaskSystemEventStoreInMemory::new(); + let event_store = InMemoryTaskSystemEventStore::new(); let mut task = Task::new( Utc::now(), diff --git a/src/domain/task-system/services/tests/tests/test_task_scheduler_impl.rs b/src/domain/task-system/services/tests/tests/test_task_scheduler_impl.rs index 72a79aecec..90d5f7027f 100644 --- a/src/domain/task-system/services/tests/tests/test_task_scheduler_impl.rs +++ b/src/domain/task-system/services/tests/tests/test_task_scheduler_impl.rs @@ -10,10 +10,10 @@ use std::assert_matches::assert_matches; use std::sync::Arc; -use kamu_core::SystemTimeSourceStub; use kamu_task_system::{LogicalPlan, Probe, TaskScheduler, TaskState, TaskStatus}; -use kamu_task_system_inmem::TaskSystemEventStoreInMemory; +use kamu_task_system_inmem::InMemoryTaskSystemEventStore; use kamu_task_system_services::TaskSchedulerImpl; +use time_source::SystemTimeSourceStub; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -85,7 +85,7 @@ async fn test_task_cancellation() { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// fn create_task_scheduler() -> impl TaskScheduler { - let event_store = Arc::new(TaskSystemEventStoreInMemory::new()); + let event_store = Arc::new(InMemoryTaskSystemEventStore::new()); let time_source = Arc::new(SystemTimeSourceStub::new()); TaskSchedulerImpl::new(event_store, time_source) diff --git a/src/infra/accounts/inmem/Cargo.toml b/src/infra/accounts/inmem/Cargo.toml index b0aa358e44..d61fb05a27 100644 --- a/src/infra/accounts/inmem/Cargo.toml +++ b/src/infra/accounts/inmem/Cargo.toml @@ -29,7 +29,7 @@ internal-error = { workspace = true } async-trait = { version = "0.1", default-features = false } chrono = { version = "0.4", default-features = false } -dill = "0.8" +dill = "0.9" thiserror = { version = "1", default-features = false } tracing = { version = "0.1", default-features = false } uuid = "1" diff --git a/src/infra/accounts/inmem/src/repos/access_token_repository_inmem.rs b/src/infra/accounts/inmem/src/repos/inmem_access_token_repository.rs similarity index 97% rename from src/infra/accounts/inmem/src/repos/access_token_repository_inmem.rs rename to src/infra/accounts/inmem/src/repos/inmem_access_token_repository.rs index 8415b64100..86b211fdb0 100644 --- a/src/infra/accounts/inmem/src/repos/access_token_repository_inmem.rs +++ b/src/infra/accounts/inmem/src/repos/inmem_access_token_repository.rs @@ -23,7 +23,7 @@ use crate::domain::*; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub struct AccessTokenRepositoryInMemory { +pub struct InMemoryAccessTokenRepository { state: Arc>, account_repository: Arc, } @@ -52,7 +52,7 @@ impl State { #[component(pub)] #[interface(dyn AccessTokenRepository)] #[scope(Singleton)] -impl AccessTokenRepositoryInMemory { +impl InMemoryAccessTokenRepository { pub fn new(account_repository: Arc) -> Self { Self { account_repository, @@ -64,7 +64,7 @@ impl AccessTokenRepositoryInMemory { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[async_trait::async_trait] -impl AccessTokenRepository for AccessTokenRepositoryInMemory { +impl AccessTokenRepository for InMemoryAccessTokenRepository { async fn save_access_token( &self, access_token: &AccessToken, diff --git a/src/infra/accounts/inmem/src/repos/account_repository_inmem.rs b/src/infra/accounts/inmem/src/repos/inmem_account_repository.rs similarity index 97% rename from src/infra/accounts/inmem/src/repos/account_repository_inmem.rs rename to src/infra/accounts/inmem/src/repos/inmem_account_repository.rs index 36d41e8dfb..ca0e09ca2c 100644 --- a/src/infra/accounts/inmem/src/repos/account_repository_inmem.rs +++ b/src/infra/accounts/inmem/src/repos/inmem_account_repository.rs @@ -17,7 +17,7 @@ use crate::domain::*; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub struct AccountRepositoryInMemory { +pub struct InMemoryAccountRepository { state: Arc>, } @@ -48,7 +48,7 @@ impl State { #[interface(dyn AccountRepository)] #[interface(dyn PasswordHashRepository)] #[scope(Singleton)] -impl AccountRepositoryInMemory { +impl InMemoryAccountRepository { pub fn new() -> Self { Self { state: Arc::new(Mutex::new(State::new())), @@ -59,7 +59,7 @@ impl AccountRepositoryInMemory { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[async_trait::async_trait] -impl AccountRepository for AccountRepositoryInMemory { +impl AccountRepository for InMemoryAccountRepository { async fn create_account(&self, account: &Account) -> Result<(), CreateAccountError> { let mut guard = self.state.lock().unwrap(); if guard.accounts_by_id.contains_key(&account.id) { @@ -188,7 +188,7 @@ impl AccountRepository for AccountRepositoryInMemory { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[async_trait::async_trait] -impl PasswordHashRepository for AccountRepositoryInMemory { +impl PasswordHashRepository for InMemoryAccountRepository { async fn save_password_hash( &self, account_name: &AccountName, diff --git a/src/infra/accounts/inmem/src/repos/mod.rs b/src/infra/accounts/inmem/src/repos/mod.rs index 1f77ca679b..e08cfed6a2 100644 --- a/src/infra/accounts/inmem/src/repos/mod.rs +++ b/src/infra/accounts/inmem/src/repos/mod.rs @@ -7,8 +7,8 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -mod access_token_repository_inmem; -mod account_repository_inmem; +mod inmem_access_token_repository; +mod inmem_account_repository; -pub use access_token_repository_inmem::*; -pub use account_repository_inmem::*; +pub use inmem_access_token_repository::*; +pub use inmem_account_repository::*; diff --git a/src/infra/accounts/inmem/tests/repos/mod.rs b/src/infra/accounts/inmem/tests/repos/mod.rs index 90e1166100..55c6f7d54a 100644 --- a/src/infra/accounts/inmem/tests/repos/mod.rs +++ b/src/infra/accounts/inmem/tests/repos/mod.rs @@ -7,6 +7,6 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -mod test_access_token_repository_inmem; -mod test_account_repository_inmem; -mod test_password_hash_repository_inmem; +mod test_inmem_access_token_repository; +mod test_inmem_account_repository; +mod test_inmem_password_hash_repository; diff --git a/src/infra/accounts/inmem/tests/repos/test_access_token_repository_inmem.rs b/src/infra/accounts/inmem/tests/repos/test_inmem_access_token_repository.rs similarity index 89% rename from src/infra/accounts/inmem/tests/repos/test_access_token_repository_inmem.rs rename to src/infra/accounts/inmem/tests/repos/test_inmem_access_token_repository.rs index 4a49a895d5..478c4598dd 100644 --- a/src/infra/accounts/inmem/tests/repos/test_access_token_repository_inmem.rs +++ b/src/infra/accounts/inmem/tests/repos/test_inmem_access_token_repository.rs @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0. use dill::{Catalog, CatalogBuilder}; -use kamu_accounts_inmem::{AccessTokenRepositoryInMemory, AccountRepositoryInMemory}; +use kamu_accounts_inmem::{InMemoryAccessTokenRepository, InMemoryAccountRepository}; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -67,11 +67,13 @@ struct InmemAccessTokenRepositoryHarness { impl InmemAccessTokenRepositoryHarness { pub fn new() -> Self { let mut catalog_builder = CatalogBuilder::new(); - catalog_builder.add::(); - catalog_builder.add::(); + catalog_builder.add::(); + catalog_builder.add::(); Self { catalog: catalog_builder.build(), } } } + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/accounts/inmem/tests/repos/test_account_repository_inmem.rs b/src/infra/accounts/inmem/tests/repos/test_inmem_account_repository.rs similarity index 97% rename from src/infra/accounts/inmem/tests/repos/test_account_repository_inmem.rs rename to src/infra/accounts/inmem/tests/repos/test_inmem_account_repository.rs index f86eadb171..9507601587 100644 --- a/src/infra/accounts/inmem/tests/repos/test_account_repository_inmem.rs +++ b/src/infra/accounts/inmem/tests/repos/test_inmem_account_repository.rs @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0. use dill::{Catalog, CatalogBuilder}; -use kamu_accounts_inmem::AccountRepositoryInMemory; +use kamu_accounts_inmem::InMemoryAccountRepository; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -109,7 +109,7 @@ struct InmemAccountRepositoryHarness { impl InmemAccountRepositoryHarness { pub fn new() -> Self { let mut catalog_builder = CatalogBuilder::new(); - catalog_builder.add::(); + catalog_builder.add::(); Self { catalog: catalog_builder.build(), diff --git a/src/infra/accounts/inmem/tests/repos/test_password_hash_repository_inmem.rs b/src/infra/accounts/inmem/tests/repos/test_inmem_password_hash_repository.rs similarity index 81% rename from src/infra/accounts/inmem/tests/repos/test_password_hash_repository_inmem.rs rename to src/infra/accounts/inmem/tests/repos/test_inmem_password_hash_repository.rs index d42b9fde46..722b3e1cbf 100644 --- a/src/infra/accounts/inmem/tests/repos/test_password_hash_repository_inmem.rs +++ b/src/infra/accounts/inmem/tests/repos/test_inmem_password_hash_repository.rs @@ -8,13 +8,13 @@ // by the Apache License, Version 2.0. use dill::{Catalog, CatalogBuilder}; -use kamu_accounts_inmem::AccountRepositoryInMemory; +use kamu_accounts_inmem::InMemoryAccountRepository; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[test_log::test(tokio::test)] async fn test_no_password_stored() { - let harness = InmemAccountRepositoryHarness::new(); + let harness = InmemPasswordHashRepositoryHarness::new(); kamu_accounts_repo_tests::test_no_password_stored(&harness.catalog).await; } @@ -22,20 +22,20 @@ async fn test_no_password_stored() { #[test_log::test(tokio::test)] async fn test_store_couple_account_passwords() { - let harness = InmemAccountRepositoryHarness::new(); + let harness = InmemPasswordHashRepositoryHarness::new(); kamu_accounts_repo_tests::test_store_couple_account_passwords(&harness.catalog).await; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -struct InmemAccountRepositoryHarness { +struct InmemPasswordHashRepositoryHarness { catalog: Catalog, } -impl InmemAccountRepositoryHarness { +impl InmemPasswordHashRepositoryHarness { pub fn new() -> Self { let mut catalog_builder = CatalogBuilder::new(); - catalog_builder.add::(); + catalog_builder.add::(); Self { catalog: catalog_builder.build(), diff --git a/src/infra/accounts/mysql/Cargo.toml b/src/infra/accounts/mysql/Cargo.toml index 57fbbfd407..d0d717960a 100644 --- a/src/infra/accounts/mysql/Cargo.toml +++ b/src/infra/accounts/mysql/Cargo.toml @@ -29,7 +29,7 @@ opendatafabric = { workspace = true, features = ["sqlx-mysql"] } async-trait = { version = "0.1", default-features = false } chrono = { version = "0.4", default-features = false } -dill = "0.8" +dill = "0.9" sqlx = { version = "0.7", default-features = false, features = [ "runtime-tokio-rustls", "macros", diff --git a/src/infra/accounts/mysql/src/repos/mysql_access_token_repository.rs b/src/infra/accounts/mysql/src/repos/mysql_access_token_repository.rs index 5bb3b68d13..32269ac468 100644 --- a/src/infra/accounts/mysql/src/repos/mysql_access_token_repository.rs +++ b/src/infra/accounts/mysql/src/repos/mysql_access_token_repository.rs @@ -19,13 +19,13 @@ use crate::domain::*; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub struct MysqlAccessTokenRepository { +pub struct MySqlAccessTokenRepository { transaction: TransactionRefT, } #[component(pub)] #[interface(dyn AccessTokenRepository)] -impl MysqlAccessTokenRepository { +impl MySqlAccessTokenRepository { pub fn new(transaction: TransactionRef) -> Self { Self { transaction: transaction.into(), @@ -61,7 +61,7 @@ impl MysqlAccessTokenRepository { } #[async_trait::async_trait] -impl AccessTokenRepository for MysqlAccessTokenRepository { +impl AccessTokenRepository for MySqlAccessTokenRepository { async fn save_access_token( &self, access_token: &AccessToken, diff --git a/src/infra/accounts/mysql/tests/repos/test_mysql_access_token_repository.rs b/src/infra/accounts/mysql/tests/repos/test_mysql_access_token_repository.rs index fe8b424a6f..30498188ee 100644 --- a/src/infra/accounts/mysql/tests/repos/test_mysql_access_token_repository.rs +++ b/src/infra/accounts/mysql/tests/repos/test_mysql_access_token_repository.rs @@ -10,7 +10,7 @@ use database_common::{DatabaseTransactionRunner, MySqlTransactionManager}; use dill::{Catalog, CatalogBuilder}; use internal_error::InternalError; -use kamu_accounts_mysql::{MySqlAccountRepository, MysqlAccessTokenRepository}; +use kamu_accounts_mysql::{MySqlAccessTokenRepository, MySqlAccountRepository}; use sqlx::MySqlPool; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -18,7 +18,7 @@ use sqlx::MySqlPool; #[test_group::group(database, mysql)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/mysql"))] async fn test_missing_access_token_not_found(mysql_pool: MySqlPool) { - let harness = MysqlAccessTokenRepositoryHarness::new(mysql_pool); + let harness = MySqlAccessTokenRepositoryHarness::new(mysql_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -34,7 +34,7 @@ async fn test_missing_access_token_not_found(mysql_pool: MySqlPool) { #[test_group::group(database, mysql)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/mysql"))] async fn test_insert_and_locate_access_token(mysql_pool: MySqlPool) { - let harness = MysqlAccessTokenRepositoryHarness::new(mysql_pool); + let harness = MySqlAccessTokenRepositoryHarness::new(mysql_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -50,7 +50,7 @@ async fn test_insert_and_locate_access_token(mysql_pool: MySqlPool) { #[test_group::group(database, mysql)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/mysql"))] async fn test_insert_and_locate_multiple_access_tokens(mysql_pool: MySqlPool) { - let harness = MysqlAccessTokenRepositoryHarness::new(mysql_pool); + let harness = MySqlAccessTokenRepositoryHarness::new(mysql_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -66,7 +66,7 @@ async fn test_insert_and_locate_multiple_access_tokens(mysql_pool: MySqlPool) { #[test_group::group(database, mysql)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/mysql"))] async fn test_mark_existing_access_token_revorked(mysql_pool: MySqlPool) { - let harness = MysqlAccessTokenRepositoryHarness::new(mysql_pool); + let harness = MySqlAccessTokenRepositoryHarness::new(mysql_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -82,7 +82,7 @@ async fn test_mark_existing_access_token_revorked(mysql_pool: MySqlPool) { #[test_group::group(database, mysql)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/mysql"))] async fn test_mark_non_existing_access_token_revorked(mysql_pool: MySqlPool) { - let harness = MysqlAccessTokenRepositoryHarness::new(mysql_pool); + let harness = MySqlAccessTokenRepositoryHarness::new(mysql_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -98,7 +98,7 @@ async fn test_mark_non_existing_access_token_revorked(mysql_pool: MySqlPool) { #[test_group::group(database, mysql)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/mysql"))] async fn test_find_account_by_active_token_id(mysql_pool: MySqlPool) { - let harness = MysqlAccessTokenRepositoryHarness::new(mysql_pool); + let harness = MySqlAccessTokenRepositoryHarness::new(mysql_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -111,17 +111,17 @@ async fn test_find_account_by_active_token_id(mysql_pool: MySqlPool) { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -struct MysqlAccessTokenRepositoryHarness { +struct MySqlAccessTokenRepositoryHarness { catalog: Catalog, } -impl MysqlAccessTokenRepositoryHarness { +impl MySqlAccessTokenRepositoryHarness { pub fn new(mysql_pool: MySqlPool) -> Self { // Initialize catalog with predefined Postgres pool let mut catalog_builder = CatalogBuilder::new(); catalog_builder.add_value(mysql_pool); catalog_builder.add::(); - catalog_builder.add::(); + catalog_builder.add::(); catalog_builder.add::(); Self { diff --git a/src/infra/accounts/mysql/tests/repos/test_mysql_password_hash_repository.rs b/src/infra/accounts/mysql/tests/repos/test_mysql_password_hash_repository.rs index fe1447d457..4b829c1292 100644 --- a/src/infra/accounts/mysql/tests/repos/test_mysql_password_hash_repository.rs +++ b/src/infra/accounts/mysql/tests/repos/test_mysql_password_hash_repository.rs @@ -18,7 +18,7 @@ use sqlx::MySqlPool; #[test_group::group(database, mysql)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/mysql"))] async fn test_no_password_stored(mysql_pool: MySqlPool) { - let harness = MySqlAccountRepositoryHarness::new(mysql_pool); + let harness = MySqlPasswordHashRepositoryHarness::new(mysql_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -34,7 +34,7 @@ async fn test_no_password_stored(mysql_pool: MySqlPool) { #[test_group::group(database, mysql)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/mysql"))] async fn test_store_couple_account_passwords(mysql_pool: MySqlPool) { - let harness = MySqlAccountRepositoryHarness::new(mysql_pool); + let harness = MySqlPasswordHashRepositoryHarness::new(mysql_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -47,11 +47,11 @@ async fn test_store_couple_account_passwords(mysql_pool: MySqlPool) { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -struct MySqlAccountRepositoryHarness { +struct MySqlPasswordHashRepositoryHarness { catalog: Catalog, } -impl MySqlAccountRepositoryHarness { +impl MySqlPasswordHashRepositoryHarness { pub fn new(mysql_pool: MySqlPool) -> Self { // Initialize catalog with predefined MySql pool let mut catalog_builder = CatalogBuilder::new(); diff --git a/src/infra/accounts/postgres/Cargo.toml b/src/infra/accounts/postgres/Cargo.toml index d761de5e1e..50708acc89 100644 --- a/src/infra/accounts/postgres/Cargo.toml +++ b/src/infra/accounts/postgres/Cargo.toml @@ -29,7 +29,7 @@ opendatafabric = { workspace = true, features = ["sqlx-postgres"] } async-trait = { version = "0.1", default-features = false } chrono = { version = "0.4", default-features = false } -dill = "0.8" +dill = "0.9" sqlx = { version = "0.7", default-features = false, features = [ "runtime-tokio-rustls", "macros", diff --git a/src/infra/accounts/postgres/tests/repos/test_postgres_access_token_repository.rs b/src/infra/accounts/postgres/tests/repos/test_postgres_access_token_repository.rs index 65088804fb..070b2d5e46 100644 --- a/src/infra/accounts/postgres/tests/repos/test_postgres_access_token_repository.rs +++ b/src/infra/accounts/postgres/tests/repos/test_postgres_access_token_repository.rs @@ -129,3 +129,5 @@ impl PostgresAccessTokenRepositoryHarness { } } } + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/accounts/postgres/tests/repos/test_postgres_password_hash_repository.rs b/src/infra/accounts/postgres/tests/repos/test_postgres_password_hash_repository.rs index 518289e7f5..5d620672b6 100644 --- a/src/infra/accounts/postgres/tests/repos/test_postgres_password_hash_repository.rs +++ b/src/infra/accounts/postgres/tests/repos/test_postgres_password_hash_repository.rs @@ -18,7 +18,7 @@ use sqlx::PgPool; #[test_group::group(database, postgres)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/postgres"))] async fn test_no_password_stored(pg_pool: PgPool) { - let harness = PostgresAccountRepositoryHarness::new(pg_pool); + let harness = PostgresPasswordHashRepositoryHarness::new(pg_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -34,7 +34,7 @@ async fn test_no_password_stored(pg_pool: PgPool) { #[test_group::group(database, postgres)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/postgres"))] async fn test_store_couple_account_passwords(pg_pool: PgPool) { - let harness = PostgresAccountRepositoryHarness::new(pg_pool); + let harness = PostgresPasswordHashRepositoryHarness::new(pg_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -47,11 +47,11 @@ async fn test_store_couple_account_passwords(pg_pool: PgPool) { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -struct PostgresAccountRepositoryHarness { +struct PostgresPasswordHashRepositoryHarness { catalog: Catalog, } -impl PostgresAccountRepositoryHarness { +impl PostgresPasswordHashRepositoryHarness { pub fn new(pg_pool: PgPool) -> Self { // Initialize catalog with predefined Postgres pool let mut catalog_builder = CatalogBuilder::new(); diff --git a/src/infra/accounts/repo-tests/Cargo.toml b/src/infra/accounts/repo-tests/Cargo.toml index 62b8976b55..389c24c277 100644 --- a/src/infra/accounts/repo-tests/Cargo.toml +++ b/src/infra/accounts/repo-tests/Cargo.toml @@ -29,7 +29,7 @@ kamu-adapter-oauth = { workspace = true } argon2 = { version = "0.5" } chrono = { version = "0.4", default-features = false } -dill = "0.8" +dill = "0.9" password-hash = { version = "0.5", default-features = false } uuid = "1" rand = "0.8" diff --git a/src/infra/accounts/sqlite/Cargo.toml b/src/infra/accounts/sqlite/Cargo.toml index 42d8960c36..b888e19d1b 100644 --- a/src/infra/accounts/sqlite/Cargo.toml +++ b/src/infra/accounts/sqlite/Cargo.toml @@ -29,7 +29,7 @@ opendatafabric = { workspace = true, features = ["sqlx-sqlite"] } async-trait = { version = "0.1", default-features = false } chrono = { version = "0.4", default-features = false } -dill = "0.8" +dill = "0.9" sqlx = { version = "0.7", default-features = false, features = [ "runtime-tokio-rustls", "macros", diff --git a/src/infra/accounts/sqlite/tests/repos/test_sqlite_password_hash_repository.rs b/src/infra/accounts/sqlite/tests/repos/test_sqlite_password_hash_repository.rs index e2344f12f6..238fe6b297 100644 --- a/src/infra/accounts/sqlite/tests/repos/test_sqlite_password_hash_repository.rs +++ b/src/infra/accounts/sqlite/tests/repos/test_sqlite_password_hash_repository.rs @@ -18,7 +18,7 @@ use sqlx::SqlitePool; #[test_group::group(database, sqlite)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/sqlite"))] async fn test_no_password_stored(sqlite_pool: SqlitePool) { - let harness = SqliteAccountRepositoryHarness::new(sqlite_pool); + let harness = SqlitePasswordHashRepositoryHarness::new(sqlite_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -34,7 +34,7 @@ async fn test_no_password_stored(sqlite_pool: SqlitePool) { #[test_group::group(database, sqlite)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/sqlite"))] async fn test_store_couple_account_passwords(sqlite_pool: SqlitePool) { - let harness = SqliteAccountRepositoryHarness::new(sqlite_pool); + let harness = SqlitePasswordHashRepositoryHarness::new(sqlite_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -47,11 +47,11 @@ async fn test_store_couple_account_passwords(sqlite_pool: SqlitePool) { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -struct SqliteAccountRepositoryHarness { +struct SqlitePasswordHashRepositoryHarness { catalog: Catalog, } -impl SqliteAccountRepositoryHarness { +impl SqlitePasswordHashRepositoryHarness { pub fn new(sqlite_pool: SqlitePool) -> Self { // Initialize catalog with predefined Postgres pool let mut catalog_builder = CatalogBuilder::new(); diff --git a/src/infra/auth-rebac/inmem/src/repos/rebac_repository_inmem.rs b/src/infra/auth-rebac/inmem/src/repos/inmem_rebac_repository.rs similarity index 98% rename from src/infra/auth-rebac/inmem/src/repos/rebac_repository_inmem.rs rename to src/infra/auth-rebac/inmem/src/repos/inmem_rebac_repository.rs index c74bdaf3a1..691e626916 100644 --- a/src/infra/auth-rebac/inmem/src/repos/rebac_repository_inmem.rs +++ b/src/infra/auth-rebac/inmem/src/repos/inmem_rebac_repository.rs @@ -46,14 +46,14 @@ struct State { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub struct RebacRepositoryInMem { +pub struct InMemoryRebacRepository { state: Arc>, } #[component(pub)] #[interface(dyn RebacRepository)] #[scope(Singleton)] -impl RebacRepositoryInMem { +impl InMemoryRebacRepository { pub fn new() -> Self { Self { state: Default::default(), @@ -75,7 +75,7 @@ impl RebacRepositoryInMem { } #[async_trait::async_trait] -impl RebacRepository for RebacRepositoryInMem { +impl RebacRepository for InMemoryRebacRepository { async fn set_entity_property( &self, entity: &Entity, diff --git a/src/infra/auth-rebac/inmem/src/repos/mod.rs b/src/infra/auth-rebac/inmem/src/repos/mod.rs index 086d82eb86..404b050b86 100644 --- a/src/infra/auth-rebac/inmem/src/repos/mod.rs +++ b/src/infra/auth-rebac/inmem/src/repos/mod.rs @@ -7,6 +7,6 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -mod rebac_repository_inmem; +mod inmem_rebac_repository; -pub use rebac_repository_inmem::*; +pub use inmem_rebac_repository::*; diff --git a/src/infra/auth-rebac/inmem/tests/repos/mod.rs b/src/infra/auth-rebac/inmem/tests/repos/mod.rs index e3abf8991f..95480d950a 100644 --- a/src/infra/auth-rebac/inmem/tests/repos/mod.rs +++ b/src/infra/auth-rebac/inmem/tests/repos/mod.rs @@ -7,4 +7,4 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -mod test_rebac_repository_inmem; +mod test_inmem_rebac_repository; diff --git a/src/infra/auth-rebac/inmem/tests/repos/test_rebac_repository_inmem.rs b/src/infra/auth-rebac/inmem/tests/repos/test_inmem_rebac_repository.rs similarity index 97% rename from src/infra/auth-rebac/inmem/tests/repos/test_rebac_repository_inmem.rs rename to src/infra/auth-rebac/inmem/tests/repos/test_inmem_rebac_repository.rs index d85b266bbf..851817fbb5 100644 --- a/src/infra/auth-rebac/inmem/tests/repos/test_rebac_repository_inmem.rs +++ b/src/infra/auth-rebac/inmem/tests/repos/test_inmem_rebac_repository.rs @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0. use dill::{Catalog, CatalogBuilder}; -use kamu_auth_rebac_inmem::RebacRepositoryInMem; +use kamu_auth_rebac_inmem::InMemoryRebacRepository; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -95,7 +95,7 @@ impl InmemRebacRepositoryHarness { pub fn new() -> Self { let mut catalog_builder = CatalogBuilder::new(); - catalog_builder.add::(); + catalog_builder.add::(); Self { catalog: catalog_builder.build(), diff --git a/src/infra/core/Cargo.toml b/src/infra/core/Cargo.toml index 721ed8246f..ef87a03c7a 100644 --- a/src/infra/core/Cargo.toml +++ b/src/infra/core/Cargo.toml @@ -36,12 +36,13 @@ internal-error = { workspace = true } container-runtime = { workspace = true } kamu-data-utils = { workspace = true } opendatafabric = { workspace = true } -event-bus = { workspace = true } kamu-accounts = { workspace = true } kamu-datasets = { workspace = true } kamu-core = { workspace = true } kamu-ingest-datafusion = { workspace = true } +messaging-outbox = { workspace = true } random-names = { workspace = true } +time-source = { workspace = true } # Serialization flatbuffers = "24" @@ -90,7 +91,7 @@ bytes = "1" cfg-if = "1" # Conditional compilation chrono = { version = "0.4", features = ["serde"] } dashmap = { version = "6", default-features = false } -dill = "0.8" +dill = "0.9" futures = "0.3" glob = "0.3" # Used for glob fetch hyper = "0.14" diff --git a/src/infra/core/src/compaction_service_impl.rs b/src/infra/core/src/compaction_service_impl.rs index 828d218884..e1ad61c905 100644 --- a/src/infra/core/src/compaction_service_impl.rs +++ b/src/infra/core/src/compaction_service_impl.rs @@ -29,6 +29,7 @@ use domain::{ DEFAULT_MAX_SLICE_SIZE, }; use futures::stream::TryStreamExt; +use internal_error::ResultIntoInternal; use kamu_core::*; use opendatafabric::{ Checkpoint, @@ -44,6 +45,7 @@ use opendatafabric::{ SourceState, }; use random_names::get_random_name; +use time_source::SystemTimeSource; use url::Url; use crate::utils::datasets_filtering::filter_datasets_by_local_pattern; @@ -470,10 +472,7 @@ impl CompactionService for CompactionServiceImpl { .check_action_allowed(dataset_handle, domain::auth::DatasetAction::Write) .await?; - let dataset = self - .dataset_repo - .get_dataset(&dataset_handle.as_local_ref()) - .await?; + let dataset = self.dataset_repo.get_dataset_by_handle(dataset_handle); let dataset_kind = dataset .get_summary(GetSummaryOpts::default()) diff --git a/src/infra/core/src/dataset_changes_service_impl.rs b/src/infra/core/src/dataset_changes_service_impl.rs index a3bb3a6c5a..aba602796c 100644 --- a/src/infra/core/src/dataset_changes_service_impl.rs +++ b/src/infra/core/src/dataset_changes_service_impl.rs @@ -11,7 +11,7 @@ use std::sync::Arc; use dill::*; use futures::TryStreamExt; -use internal_error::ResultIntoInternal; +use internal_error::{InternalError, ResultIntoInternal}; use kamu_core::{ BlockRef, Dataset, @@ -21,7 +21,6 @@ use kamu_core::{ GetDatasetError, GetIncrementError, GetRefError, - InternalError, MetadataChainExt, SearchSingleDataBlockVisitor, }; @@ -47,7 +46,7 @@ impl DatasetChangesServiceImpl { dataset_id: &DatasetID, ) -> Result, GetIncrementError> { self.dataset_repo - .get_dataset(&dataset_id.as_local_ref()) + .find_dataset_by_ref(&dataset_id.as_local_ref()) .await .map_err(|e| match e { GetDatasetError::NotFound(e) => GetIncrementError::DatasetNotFound(e), diff --git a/src/infra/core/src/dataset_ownership_service_inmem.rs b/src/infra/core/src/dataset_ownership_service_inmem.rs index 9c678d8fc7..9126d34386 100644 --- a/src/infra/core/src/dataset_ownership_service_inmem.rs +++ b/src/infra/core/src/dataset_ownership_service_inmem.rs @@ -11,10 +11,15 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; use dill::*; -use event_bus::AsyncEventHandler; +use internal_error::InternalError; use kamu_accounts::{AuthenticationService, CurrentAccountSubject}; -use kamu_core::events::{DatasetEventCreated, DatasetEventDeleted}; use kamu_core::*; +use messaging_outbox::{ + MessageConsumer, + MessageConsumerMeta, + MessageConsumerT, + MessageConsumptionDurability, +}; use opendatafabric::{AccountID, AccountName, DatasetID}; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -34,8 +39,13 @@ struct State { #[component(pub)] #[interface(dyn DatasetOwnershipService)] -#[interface(dyn AsyncEventHandler)] -#[interface(dyn AsyncEventHandler)] +#[interface(dyn MessageConsumer)] +#[interface(dyn MessageConsumerT)] +#[meta(MessageConsumerMeta { + consumer_name: MESSAGE_CONSUMER_KAMU_CORE_DATASET_OWNERSHIP_SERVICE, + feeding_producers: &[MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE], + durability: MessageConsumptionDurability::BestEffort, +})] #[scope(Singleton)] impl DatasetOwnershipServiceInMemory { pub fn new() -> Self { @@ -135,32 +145,46 @@ impl DatasetOwnershipService for DatasetOwnershipServiceInMemory { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -#[async_trait::async_trait] -impl AsyncEventHandler for DatasetOwnershipServiceInMemory { - #[tracing::instrument(level = "debug", skip_all, fields(?event))] - async fn handle(&self, event: &DatasetEventCreated) -> Result<(), InternalError> { - let mut guard = self.state.write().await; - self.insert_dataset_record(&mut guard, &event.dataset_id, &event.owner_account_id); - Ok(()) - } -} +impl MessageConsumer for DatasetOwnershipServiceInMemory {} //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[async_trait::async_trait] -impl AsyncEventHandler for DatasetOwnershipServiceInMemory { - #[tracing::instrument(level = "debug", skip_all, fields(?event))] - async fn handle(&self, event: &DatasetEventDeleted) -> Result<(), InternalError> { - let account_ids = self.get_dataset_owners(&event.dataset_id).await?; - if !account_ids.is_empty() { - let mut guard = self.state.write().await; - for account_id in account_ids { - if let Some(dataset_ids) = guard.dataset_ids_by_account_id.get_mut(&account_id) { - dataset_ids.remove(&event.dataset_id); +impl MessageConsumerT for DatasetOwnershipServiceInMemory { + #[tracing::instrument(level = "debug", skip_all, fields(?message))] + async fn consume_message( + &self, + _: &Catalog, + message: &DatasetLifecycleMessage, + ) -> Result<(), InternalError> { + match message { + DatasetLifecycleMessage::Created(message) => { + let mut guard = self.state.write().await; + self.insert_dataset_record( + &mut guard, + &message.dataset_id, + &message.owner_account_id, + ); + } + DatasetLifecycleMessage::Deleted(message) => { + let account_ids = self.get_dataset_owners(&message.dataset_id).await?; + if !account_ids.is_empty() { + let mut guard = self.state.write().await; + for account_id in account_ids { + if let Some(dataset_ids) = + guard.dataset_ids_by_account_id.get_mut(&account_id) + { + dataset_ids.remove(&message.dataset_id); + } + } + guard.account_ids_by_dataset_id.remove(&message.dataset_id); } } - guard.account_ids_by_dataset_id.remove(&event.dataset_id); + DatasetLifecycleMessage::DependenciesUpdated(_) => { + // No action required + } } + Ok(()) } } @@ -251,3 +275,5 @@ impl DatasetOwnershipServiceInMemoryStateInitializer { Ok(()) } } + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/core/src/dependency_graph_repository_inmem.rs b/src/infra/core/src/dependency_graph_repository_inmem.rs index 3f84182146..e8b4bc3184 100644 --- a/src/infra/core/src/dependency_graph_repository_inmem.rs +++ b/src/infra/core/src/dependency_graph_repository_inmem.rs @@ -9,6 +9,7 @@ use std::sync::Arc; +use internal_error::ResultIntoInternal; use kamu_core::*; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -39,9 +40,7 @@ impl DependencyGraphRepository for DependencyGraphRepositoryInMemory { let summary = self .dataset_repo - .get_dataset(&dataset_handle.as_local_ref()) - .await - .int_err()? + .get_dataset_by_handle(&dataset_handle) .get_summary(GetSummaryOpts::default()) .await .int_err()?; diff --git a/src/infra/core/src/dependency_graph_service_inmem.rs b/src/infra/core/src/dependency_graph_service_inmem.rs index e628e29e47..9aee37aac4 100644 --- a/src/infra/core/src/dependency_graph_service_inmem.rs +++ b/src/infra/core/src/dependency_graph_service_inmem.rs @@ -11,14 +11,14 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; use dill::*; -use event_bus::AsyncEventHandler; -use internal_error::InternalError; -use kamu_core::events::{ - DatasetEventCreated, - DatasetEventDeleted, - DatasetEventDependenciesUpdated, -}; +use internal_error::{InternalError, ResultIntoInternal}; use kamu_core::*; +use messaging_outbox::{ + MessageConsumer, + MessageConsumerMeta, + MessageConsumerT, + MessageConsumptionDurability, +}; use opendatafabric::DatasetID; use petgraph::stable_graph::{NodeIndex, StableDiGraph}; use petgraph::visit::{depth_first_search, Bfs, DfsEvent, Reversed}; @@ -70,9 +70,13 @@ impl State { #[component(pub)] #[interface(dyn DependencyGraphService)] -#[interface(dyn AsyncEventHandler)] -#[interface(dyn AsyncEventHandler)] -#[interface(dyn AsyncEventHandler)] +#[interface(dyn MessageConsumer)] +#[interface(dyn MessageConsumerT)] +#[meta(MessageConsumerMeta { + consumer_name: MESSAGE_CONSUMER_KAMU_CORE_DEPENDENCY_GRAPH_SERVICE, + feeding_producers: &[MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE], + durability: MessageConsumptionDurability::BestEffort, + })] #[scope(Singleton)] impl DependencyGraphServiceInMemory { pub fn new(repository: Option>) -> Self { @@ -373,63 +377,58 @@ impl DependencyGraphService for DependencyGraphServiceInMemory { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -#[async_trait::async_trait] -impl AsyncEventHandler for DependencyGraphServiceInMemory { - #[tracing::instrument(level = "debug", skip_all, fields(?event))] - async fn handle(&self, event: &DatasetEventCreated) -> Result<(), InternalError> { - let mut state = self.state.write().await; - state.get_or_create_dataset_node(&event.dataset_id); - Ok(()) - } -} +impl MessageConsumer for DependencyGraphServiceInMemory {} //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[async_trait::async_trait] -impl AsyncEventHandler for DependencyGraphServiceInMemory { - #[tracing::instrument(level = "debug", skip_all, fields(?event))] - async fn handle(&self, event: &DatasetEventDeleted) -> Result<(), InternalError> { +impl MessageConsumerT for DependencyGraphServiceInMemory { + #[tracing::instrument(level = "debug", skip_all, fields(?message))] + async fn consume_message( + &self, + _: &Catalog, + message: &DatasetLifecycleMessage, + ) -> Result<(), InternalError> { let mut state = self.state.write().await; - let node_index = state.get_dataset_node(&event.dataset_id).int_err()?; - - state.datasets_graph.remove_node(node_index); - state.dataset_node_indices.remove(&event.dataset_id); - - Ok(()) - } -} + match message { + DatasetLifecycleMessage::Created(message) => { + state.get_or_create_dataset_node(&message.dataset_id); + } -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + DatasetLifecycleMessage::Deleted(message) => { + let node_index = state.get_dataset_node(&message.dataset_id).int_err()?; -#[async_trait::async_trait] -impl AsyncEventHandler for DependencyGraphServiceInMemory { - #[tracing::instrument(level = "debug", skip_all, fields(?event))] - async fn handle(&self, event: &DatasetEventDependenciesUpdated) -> Result<(), InternalError> { - let mut state = self.state.write().await; + state.datasets_graph.remove_node(node_index); + state.dataset_node_indices.remove(&message.dataset_id); + } - let node_index = state.get_dataset_node(&event.dataset_id).int_err()?; + DatasetLifecycleMessage::DependenciesUpdated(message) => { + let node_index = state.get_dataset_node(&message.dataset_id).int_err()?; - let existing_upstream_ids: HashSet<_> = state - .datasets_graph - .neighbors_directed(node_index, Direction::Incoming) - .map(|node_index| { - state + let existing_upstream_ids: HashSet<_> = state .datasets_graph - .node_weight(node_index) - .unwrap() - .clone() - }) - .collect(); + .neighbors_directed(node_index, Direction::Incoming) + .map(|node_index| { + state + .datasets_graph + .node_weight(node_index) + .unwrap() + .clone() + }) + .collect(); - let new_upstream_ids: HashSet<_> = event.new_upstream_ids.iter().cloned().collect(); + let new_upstream_ids: HashSet<_> = + message.new_upstream_ids.iter().cloned().collect(); - for obsolete_upstream_id in existing_upstream_ids.difference(&new_upstream_ids) { - self.remove_dependency(&mut state, obsolete_upstream_id, &event.dataset_id); - } + for obsolete_upstream_id in existing_upstream_ids.difference(&new_upstream_ids) { + self.remove_dependency(&mut state, obsolete_upstream_id, &message.dataset_id); + } - for added_id in new_upstream_ids.difference(&existing_upstream_ids) { - self.add_dependency(&mut state, added_id, &event.dataset_id); + for added_id in new_upstream_ids.difference(&existing_upstream_ids) { + self.add_dependency(&mut state, added_id, &message.dataset_id); + } + } } Ok(()) diff --git a/src/infra/core/src/engine/engine_io_strategy.rs b/src/infra/core/src/engine/engine_io_strategy.rs index c6986e7fbf..6fdcb9864d 100644 --- a/src/infra/core/src/engine/engine_io_strategy.rs +++ b/src/infra/core/src/engine/engine_io_strategy.rs @@ -11,6 +11,7 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use container_runtime::*; +use internal_error::{InternalError, ResultIntoInternal}; use kamu_core::engine::*; use kamu_core::*; use opendatafabric::*; @@ -119,9 +120,7 @@ impl EngineIoStrategy for EngineIoStrategyLocalVolume { for input in request.inputs { let input_dataset = self .dataset_repo - .get_dataset(&input.dataset_handle.as_local_ref()) - .await - .int_err()?; + .get_dataset_by_handle(&input.dataset_handle); let mut schema_file = None; let mut data_paths = Vec::new(); @@ -300,9 +299,7 @@ impl EngineIoStrategy for EngineIoStrategyRemoteProxy { for input in request.inputs { let input_dataset = self .dataset_repo - .get_dataset(&input.dataset_handle.as_local_ref()) - .await - .int_err()?; + .get_dataset_by_handle(&input.dataset_handle); let mut schema_file = None; let mut data_paths = Vec::new(); diff --git a/src/infra/core/src/engine/engine_odf.rs b/src/infra/core/src/engine/engine_odf.rs index d3741fca39..44ca64540c 100644 --- a/src/infra/core/src/engine/engine_odf.rs +++ b/src/infra/core/src/engine/engine_odf.rs @@ -14,6 +14,7 @@ use std::sync::Arc; use container_runtime::*; use datafusion::config::{ParquetOptions, TableParquetOptions}; +use internal_error::ResultIntoInternal; use kamu_core::engine::*; use kamu_core::*; use odf::engine::{EngineGrpcClient, ExecuteRawQueryError, ExecuteTransformError}; @@ -52,25 +53,20 @@ impl ODFEngine { // TODO: Currently we are always proxying remote inputs, but in future we should // have a capabilities mechanism for engines to declare that they can work // with some remote storages directly without us needing to proxy data. - async fn get_io_strategy( - &self, - request: &TransformRequestExt, - ) -> Result, InternalError> { + fn get_io_strategy(&self, request: &TransformRequestExt) -> Arc { let dataset = self .dataset_repo - .get_dataset(&request.dataset_handle.as_local_ref()) - .await - .int_err()?; + .get_dataset_by_handle(&request.dataset_handle); match dataset.as_data_repo().protocol() { - ObjectRepositoryProtocol::LocalFs { .. } => Ok(Arc::new( - EngineIoStrategyLocalVolume::new(self.dataset_repo.clone()), - )), + ObjectRepositoryProtocol::LocalFs { .. } => { + Arc::new(EngineIoStrategyLocalVolume::new(self.dataset_repo.clone())) + } ObjectRepositoryProtocol::Memory | ObjectRepositoryProtocol::Http - | ObjectRepositoryProtocol::S3 => Ok(Arc::new(EngineIoStrategyRemoteProxy::new( - self.dataset_repo.clone(), - ))), + | ObjectRepositoryProtocol::S3 => { + Arc::new(EngineIoStrategyRemoteProxy::new(self.dataset_repo.clone())) + } } } @@ -340,9 +336,7 @@ impl Engine for ODFEngine { ) -> Result { let dataset = self .dataset_repo - .get_dataset(&request.dataset_handle.as_local_ref()) - .await - .int_err()?; + .get_dataset_by_handle(&request.dataset_handle); let operation_id = request.operation_id.clone(); let operation_dir = self @@ -352,7 +346,7 @@ impl Engine for ODFEngine { std::fs::create_dir(&operation_dir).int_err()?; std::fs::create_dir(&logs_dir).int_err()?; - let io_strategy = self.get_io_strategy(&request).await.int_err()?; + let io_strategy = self.get_io_strategy(&request); let materialized_request = io_strategy .materialize_request(dataset.as_ref(), request, &operation_dir) diff --git a/src/infra/core/src/engine/engine_provisioner_local.rs b/src/infra/core/src/engine/engine_provisioner_local.rs index 2709f8be5b..1e8aaae3b3 100644 --- a/src/infra/core/src/engine/engine_provisioner_local.rs +++ b/src/infra/core/src/engine/engine_provisioner_local.rs @@ -12,6 +12,7 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; use container_runtime::*; +use internal_error::{ErrorIntoInternal, ResultIntoInternal}; use kamu_core::engine::*; use kamu_core::*; diff --git a/src/infra/core/src/ingest/fetch_service.rs b/src/infra/core/src/ingest/fetch_service.rs index 7227c016f0..255807f372 100644 --- a/src/infra/core/src/ingest/fetch_service.rs +++ b/src/infra/core/src/ingest/fetch_service.rs @@ -16,6 +16,7 @@ use std::sync::Arc; use chrono::{DateTime, SubsecRound, TimeZone, Utc}; use container_runtime::*; use futures::TryStreamExt; +use internal_error::{ErrorIntoInternal, InternalError, ResultIntoInternal}; use kamu_core::engine::ProcessError; use kamu_core::*; use kamu_datasets::{DatasetEnvVar, DatasetKeyValueService}; diff --git a/src/infra/core/src/ingest/ingest_common.rs b/src/infra/core/src/ingest/ingest_common.rs index bdf61d4561..913d71873f 100644 --- a/src/infra/core/src/ingest/ingest_common.rs +++ b/src/infra/core/src/ingest/ingest_common.rs @@ -10,6 +10,7 @@ use std::sync::Arc; use datafusion::prelude::*; +use internal_error::ResultIntoInternal; use kamu_core::engine::*; use kamu_core::{ObjectStoreRegistry, *}; use opendatafabric::*; diff --git a/src/infra/core/src/ingest/polling_ingest_service_impl.rs b/src/infra/core/src/ingest/polling_ingest_service_impl.rs index ed1bdee298..ca2ec05aba 100644 --- a/src/infra/core/src/ingest/polling_ingest_service_impl.rs +++ b/src/infra/core/src/ingest/polling_ingest_service_impl.rs @@ -12,12 +12,14 @@ use std::sync::Arc; use chrono::{DateTime, Utc}; use datafusion::prelude::{DataFrame, SessionContext}; +use internal_error::{InternalError, ResultIntoInternal}; use kamu_core::ingest::*; use kamu_core::*; use kamu_ingest_datafusion::DataWriterDataFusion; use opendatafabric::serde::yaml::Manifest; use opendatafabric::*; use random_names::get_random_name; +use time_source::SystemTimeSource; use super::*; @@ -76,10 +78,7 @@ impl PollingIngestServiceImpl { .check_action_allowed(&dataset_handle, auth::DatasetAction::Write) .await?; - let dataset = self - .dataset_repo - .get_dataset(&dataset_handle.as_local_ref()) - .await?; + let dataset = self.dataset_repo.get_dataset_by_handle(&dataset_handle); let listener = get_listener(&dataset_handle).unwrap_or_else(|| Arc::new(NullPollingIngestListener)); @@ -586,7 +585,7 @@ impl PollingIngestService for PollingIngestServiceImpl { &self, dataset_ref: &DatasetRef, ) -> Result)>, GetDatasetError> { - let dataset = self.dataset_repo.get_dataset(dataset_ref).await?; + let dataset = self.dataset_repo.find_dataset_by_ref(dataset_ref).await?; // TODO: Support source evolution Ok(dataset diff --git a/src/infra/core/src/ingest/polling_source_state.rs b/src/infra/core/src/ingest/polling_source_state.rs index 7f7ce1df9b..ef83dfb0ad 100644 --- a/src/infra/core/src/ingest/polling_source_state.rs +++ b/src/infra/core/src/ingest/polling_source_state.rs @@ -10,7 +10,7 @@ use std::path::{Path, PathBuf}; use chrono::{DateTime, SecondsFormat, Utc}; -use kamu_core::{InternalError, ResultIntoInternal}; +use internal_error::{InternalError, ResultIntoInternal}; use opendatafabric::serde::yaml::{datetime_rfc3339, datetime_rfc3339_opt, SourceStateDef}; use opendatafabric::SourceState; use serde::{Deserialize, Deserializer, Serialize, Serializer}; diff --git a/src/infra/core/src/ingest/prep_service.rs b/src/infra/core/src/ingest/prep_service.rs index 2b004e310a..3ec92c674a 100644 --- a/src/infra/core/src/ingest/prep_service.rs +++ b/src/infra/core/src/ingest/prep_service.rs @@ -15,6 +15,7 @@ use std::process; use std::process::{Command, Stdio}; use std::sync::Arc; +use internal_error::ResultIntoInternal; use kamu_core::*; use opendatafabric::*; use thiserror::Error; diff --git a/src/infra/core/src/ingest/push_ingest_service_impl.rs b/src/infra/core/src/ingest/push_ingest_service_impl.rs index 8667ad417c..faede4e245 100644 --- a/src/infra/core/src/ingest/push_ingest_service_impl.rs +++ b/src/infra/core/src/ingest/push_ingest_service_impl.rs @@ -12,11 +12,13 @@ use std::sync::Arc; use chrono::{DateTime, Utc}; use datafusion::prelude::{DataFrame, SessionContext}; +use internal_error::{ErrorIntoInternal, InternalError, ResultIntoInternal}; use kamu_core::ingest::*; use kamu_core::*; use kamu_ingest_datafusion::*; use opendatafabric::*; use random_names::get_random_name; +use time_source::SystemTimeSource; use tokio::io::AsyncRead; use super::ingest_common; @@ -72,10 +74,7 @@ impl PushIngestServiceImpl { .check_action_allowed(&dataset_handle, auth::DatasetAction::Write) .await?; - let dataset = self - .dataset_repo - .get_dataset(&dataset_handle.as_local_ref()) - .await?; + let dataset = self.dataset_repo.get_dataset_by_handle(&dataset_handle); let operation_id = get_random_name(None, 10); let operation_dir = self.run_info_dir.join(format!("ingest-{operation_id}")); @@ -413,7 +412,7 @@ impl PushIngestService for PushIngestServiceImpl { use futures::TryStreamExt; // TODO: Support source disabling and evolution - let dataset = self.dataset_repo.get_dataset(dataset_ref).await?; + let dataset = self.dataset_repo.find_dataset_by_ref(dataset_ref).await?; let stream = dataset .as_metadata_chain() .iter_blocks() diff --git a/src/infra/core/src/lib.rs b/src/infra/core/src/lib.rs index 3896e72b72..3a4efc0cd3 100644 --- a/src/infra/core/src/lib.rs +++ b/src/infra/core/src/lib.rs @@ -23,6 +23,7 @@ pub mod ingest; mod query; mod repos; pub mod testing; // TODO: Put under feature flag +mod use_cases; pub mod utils; mod compaction_service_impl; @@ -66,4 +67,5 @@ pub use resource_loader_impl::*; pub use search_service_impl::*; pub use sync_service_impl::*; pub use transform_service_impl::*; +pub use use_cases::*; pub use verification_service_impl::*; diff --git a/src/infra/core/src/provenance_service_impl.rs b/src/infra/core/src/provenance_service_impl.rs index c8a4181dad..e4f3417fdd 100644 --- a/src/infra/core/src/provenance_service_impl.rs +++ b/src/infra/core/src/provenance_service_impl.rs @@ -13,6 +13,7 @@ use std::marker::PhantomData; use std::sync::Arc; use dill::*; +use internal_error::ResultIntoInternal; use kamu_core::*; use opendatafabric::*; diff --git a/src/infra/core/src/pull_service_impl.rs b/src/infra/core/src/pull_service_impl.rs index b38bc044e4..aa89d6d956 100644 --- a/src/infra/core/src/pull_service_impl.rs +++ b/src/infra/core/src/pull_service_impl.rs @@ -13,10 +13,12 @@ use std::sync::Arc; use chrono::prelude::*; use dill::*; +use internal_error::{ErrorIntoInternal, InternalError, ResultIntoInternal}; use kamu_accounts::CurrentAccountSubject; use kamu_core::*; use kamu_ingest_datafusion::DataWriterDataFusion; use opendatafabric::*; +use time_source::SystemTimeSource; use url::Url; pub struct PullServiceImpl { @@ -198,9 +200,7 @@ impl PullServiceImpl { let summary = self .dataset_repo - .get_dataset(&local_handle.as_local_ref()) - .await - .int_err()? + .get_dataset_by_handle(&local_handle) .get_summary(GetSummaryOpts::default()) .await .int_err()?; @@ -627,7 +627,7 @@ impl PullService for PullServiceImpl { .check_action_allowed(&dataset_handle, auth::DatasetAction::Write) .await?; - let dataset = self.dataset_repo.get_dataset(dataset_ref).await?; + let dataset = self.dataset_repo.find_dataset_by_ref(dataset_ref).await?; let summary = dataset .get_summary(GetSummaryOpts::default()) .await diff --git a/src/infra/core/src/push_service_impl.rs b/src/infra/core/src/push_service_impl.rs index 21985be631..8f2c048e96 100644 --- a/src/infra/core/src/push_service_impl.rs +++ b/src/infra/core/src/push_service_impl.rs @@ -10,6 +10,7 @@ use std::sync::Arc; use dill::*; +use internal_error::ResultIntoInternal; use kamu_core::*; use opendatafabric::*; diff --git a/src/infra/core/src/query/mod.rs b/src/infra/core/src/query/mod.rs index f484ad8577..068bd86754 100644 --- a/src/infra/core/src/query/mod.rs +++ b/src/infra/core/src/query/mod.rs @@ -24,6 +24,7 @@ use datafusion::logical_expr::LogicalPlan; use datafusion::physical_plan::ExecutionPlan; use datafusion::prelude::*; use futures::stream::{self, StreamExt, TryStreamExt}; +use internal_error::{InternalError, ResultIntoInternal}; use kamu_core::*; use opendatafabric::*; @@ -130,12 +131,7 @@ impl KamuSchema { .await .int_err()?; - let dataset = self - .inner - .dataset_repo - .get_dataset(&hdl.as_local_ref()) - .await - .unwrap(); + let dataset = self.inner.dataset_repo.get_dataset_by_handle(&hdl); let as_of = self .inner @@ -187,12 +183,7 @@ impl KamuSchema { .await .is_ok() { - let dataset = self - .inner - .dataset_repo - .get_dataset(&hdl.as_local_ref()) - .await - .unwrap(); + let dataset = self.inner.dataset_repo.get_dataset_by_handle(&hdl); let as_of = inputs_state.get(&hdl.id).cloned(); diff --git a/src/infra/core/src/query_service_impl.rs b/src/infra/core/src/query_service_impl.rs index 3057b82ad4..5d28686e47 100644 --- a/src/infra/core/src/query_service_impl.rs +++ b/src/infra/core/src/query_service_impl.rs @@ -20,6 +20,7 @@ use datafusion::parquet::schema::types::Type; use datafusion::prelude::*; use datafusion::sql::TableReference; use dill::*; +use internal_error::{InternalError, ResultIntoInternal}; use kamu_core::auth::{DatasetAction, DatasetActionAuthorizer}; use kamu_core::*; use opendatafabric::*; @@ -95,7 +96,7 @@ impl QueryServiceImpl { inputs: BTreeMap::new(), }; for id in aliases.values() { - let dataset = self.dataset_repo.get_dataset(&id.into()).await?; + let dataset = self.dataset_repo.find_dataset_by_ref(&id.into()).await?; // TODO: Do we leak any info by not checking read permissions here? let hash = dataset @@ -152,11 +153,7 @@ impl QueryServiceImpl { tracing::warn!(?dataset_ref, "Ignoring table with unresolvable alias"); continue; }; - let dataset = self - .dataset_repo - .get_dataset(&hdl.as_local_ref()) - .await - .int_err()?; + let dataset = self.dataset_repo.get_dataset_by_handle(&hdl); let hash = dataset .as_metadata_chain() .resolve_ref(&BlockRef::Head) @@ -185,10 +182,7 @@ impl QueryServiceImpl { .check_action_allowed(&dataset_handle, DatasetAction::Read) .await?; - let dataset = self - .dataset_repo - .get_dataset(&dataset_handle.as_local_ref()) - .await?; + let dataset = self.dataset_repo.get_dataset_by_handle(&dataset_handle); let summary = dataset .get_summary(GetSummaryOpts::default()) @@ -234,10 +228,7 @@ impl QueryServiceImpl { .check_action_allowed(&dataset_handle, DatasetAction::Read) .await?; - let dataset = self - .dataset_repo - .get_dataset(&dataset_handle.as_local_ref()) - .await?; + let dataset = self.dataset_repo.get_dataset_by_handle(&dataset_handle); let schema_opt = dataset .as_metadata_chain() @@ -265,10 +256,7 @@ impl QueryServiceImpl { .check_action_allowed(&dataset_handle, DatasetAction::Read) .await?; - let dataset = self - .dataset_repo - .get_dataset(&dataset_handle.as_local_ref()) - .await?; + let dataset = self.dataset_repo.get_dataset_by_handle(&dataset_handle); // TODO: Update to use SetDataSchema event let maybe_last_data_slice_hash = dataset diff --git a/src/infra/core/src/remote_aliases_registry_impl.rs b/src/infra/core/src/remote_aliases_registry_impl.rs index 2d391bea83..3b712ac681 100644 --- a/src/infra/core/src/remote_aliases_registry_impl.rs +++ b/src/infra/core/src/remote_aliases_registry_impl.rs @@ -10,6 +10,7 @@ use std::sync::Arc; use dill::*; +use internal_error::{ErrorIntoInternal, InternalError, ResultIntoInternal}; use kamu_core::*; use opendatafabric::serde::yaml::Manifest; use opendatafabric::*; @@ -73,7 +74,7 @@ impl RemoteAliasesRegistry for RemoteAliasesRegistryImpl { &self, dataset_ref: &DatasetRef, ) -> Result, GetAliasesError> { - let dataset = self.dataset_repo.get_dataset(dataset_ref).await?; + let dataset = self.dataset_repo.find_dataset_by_ref(dataset_ref).await?; let config = Self::read_config(dataset.clone()).await?; Ok(Box::new(RemoteAliasesImpl::new(dataset, config))) } diff --git a/src/infra/core/src/remote_repository_registry_impl.rs b/src/infra/core/src/remote_repository_registry_impl.rs index d3fc3f46bb..cd48a2063f 100644 --- a/src/infra/core/src/remote_repository_registry_impl.rs +++ b/src/infra/core/src/remote_repository_registry_impl.rs @@ -12,6 +12,7 @@ use std::ops::Deref; use std::path::{Path, PathBuf}; use std::sync::Arc; +use internal_error::{ErrorIntoInternal, ResultIntoInternal}; use kamu_core::*; use opendatafabric::serde::yaml::Manifest; use opendatafabric::*; diff --git a/src/infra/core/src/repos/dataset_factory_impl.rs b/src/infra/core/src/repos/dataset_factory_impl.rs index 464e015954..3c51da7cf2 100644 --- a/src/infra/core/src/repos/dataset_factory_impl.rs +++ b/src/infra/core/src/repos/dataset_factory_impl.rs @@ -10,7 +10,7 @@ use std::sync::Arc; use dill::*; -use event_bus::EventBus; +use internal_error::{ErrorIntoInternal, InternalError, ResultIntoInternal}; use kamu_core::*; use url::Url; @@ -22,7 +22,6 @@ use crate::*; pub struct DatasetFactoryImpl { ipfs_gateway: IpfsGateway, access_token_resolver: Arc, - event_bus: Arc, } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -48,18 +47,15 @@ impl DatasetFactoryImpl { pub fn new( ipfs_gateway: IpfsGateway, access_token_resolver: Arc, - event_bus: Arc, ) -> Self { Self { ipfs_gateway, access_token_resolver, - event_bus, } } - pub fn get_local_fs(layout: DatasetLayout, event_bus: Arc) -> DatasetImplLocalFS { + pub fn get_local_fs(layout: DatasetLayout) -> DatasetImplLocalFS { DatasetImpl::new( - event_bus, MetadataChainImpl::new( MetadataBlockRepositoryCachingInMem::new(MetadataBlockRepositoryImpl::new( ObjectRepositoryLocalFSSha3::new(layout.blocks_dir), @@ -72,15 +68,10 @@ impl DatasetFactoryImpl { ) } - fn get_http( - base_url: &Url, - header_map: http::HeaderMap, - event_bus: Arc, - ) -> impl Dataset { + fn get_http(base_url: &Url, header_map: http::HeaderMap) -> impl Dataset { let client = reqwest::Client::new(); DatasetImpl::new( - event_bus, MetadataChainImpl::new( MetadataBlockRepositoryCachingInMem::new(MetadataBlockRepositoryImpl::new( ObjectRepositoryHttp::new( @@ -119,27 +110,20 @@ impl DatasetFactoryImpl { /// credential resolution from scratch which can be very expensive. If you /// already have an established [S3Context] use /// [DatasetFactoryImpl::get_s3_from_context()] function instead. - pub async fn get_s3_from_url( - base_url: Url, - event_bus: Arc, - ) -> Result { + pub async fn get_s3_from_url(base_url: Url) -> Result { // TODO: We should ensure optimal credential reuse. Perhaps in future we should // create a cache of S3Contexts keyed by an endpoint. let s3_context = S3Context::from_url(&base_url).await; - Self::get_s3_from_context(s3_context, event_bus) + Self::get_s3_from_context(s3_context) } - pub fn get_s3_from_context( - s3_context: S3Context, - event_bus: Arc, - ) -> Result { + pub fn get_s3_from_context(s3_context: S3Context) -> Result { let client = s3_context.client; let endpoint = s3_context.endpoint; let bucket = s3_context.bucket; let key_prefix = s3_context.key_prefix; Ok(DatasetImpl::new( - event_bus, MetadataChainImpl::new( MetadataBlockRepositoryCachingInMem::new(MetadataBlockRepositoryImpl::new( ObjectRepositoryS3Sha3::new(S3Context::new( @@ -177,11 +161,7 @@ impl DatasetFactoryImpl { )) } - async fn get_ipfs_http( - &self, - base_url: Url, - event_bus: Arc, - ) -> Result { + async fn get_ipfs_http(&self, base_url: Url) -> Result { // Resolve IPNS DNSLink names if configured let dataset_url = match base_url.scheme() { "ipns" if self.ipfs_gateway.pre_resolve_dnslink => { @@ -236,7 +216,6 @@ impl DatasetFactoryImpl { let client = reqwest::Client::new(); Ok(DatasetImpl::new( - event_bus, MetadataChainImpl::new( MetadataBlockRepositoryCachingInMem::new(MetadataBlockRepositoryImpl::new( ObjectRepositoryHttp::new( @@ -325,31 +304,25 @@ impl DatasetFactory for DatasetFactoryImpl { } else { DatasetLayout::new(path) }; - let ds = Self::get_local_fs(layout, self.event_bus.clone()); + let ds = Self::get_local_fs(layout); Ok(Arc::new(ds) as Arc) } "http" | "https" => { - let ds = Self::get_http(url, self.build_header_map(url), self.event_bus.clone()); + let ds = Self::get_http(url, self.build_header_map(url)); Ok(Arc::new(ds)) } "odf+http" | "odf+https" => { // TODO: PERF: Consider what speedups are possible in smart protocol let http_url = Url::parse(url.as_str().strip_prefix("odf+").unwrap()).unwrap(); - let ds = Self::get_http( - &http_url, - self.build_header_map(&http_url), - self.event_bus.clone(), - ); + let ds = Self::get_http(&http_url, self.build_header_map(&http_url)); Ok(Arc::new(ds)) } "ipfs" | "ipns" | "ipfs+http" | "ipfs+https" | "ipns+http" | "ipns+https" => { - let ds = self - .get_ipfs_http(url.clone(), self.event_bus.clone()) - .await?; + let ds = self.get_ipfs_http(url.clone()).await?; Ok(Arc::new(ds)) } "s3" | "s3+http" | "s3+https" => { - let ds = Self::get_s3_from_url(url.clone(), self.event_bus.clone()).await?; + let ds = Self::get_s3_from_url(url.clone()).await?; Ok(Arc::new(ds)) } _ => Err(UnsupportedProtocolError { diff --git a/src/infra/core/src/repos/dataset_impl.rs b/src/infra/core/src/repos/dataset_impl.rs index ad1c216f6a..f75d7cc85f 100644 --- a/src/infra/core/src/repos/dataset_impl.rs +++ b/src/infra/core/src/repos/dataset_impl.rs @@ -7,12 +7,9 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use std::sync::Arc; - use async_trait::async_trait; use chrono::{DateTime, Utc}; -use event_bus::EventBus; -use kamu_core::events::DatasetEventDependenciesUpdated; +use internal_error::{ErrorIntoInternal, InternalError, ResultIntoInternal}; use kamu_core::*; use opendatafabric::serde::yaml::Manifest; use opendatafabric::*; @@ -20,7 +17,6 @@ use opendatafabric::*; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub struct DatasetImpl { - event_bus: Arc, metadata_chain: MetaChain, data_repo: DataRepo, checkpoint_repo: CheckpointRepo, @@ -38,14 +34,12 @@ where InfoRepo: NamedObjectRepository + Sync + Send, { pub fn new( - event_bus: Arc, metadata_chain: MetaChain, data_repo: DataRepo, checkpoint_repo: CheckpointRepo, info_repo: InfoRepo, ) -> Self { Self { - event_bus, metadata_chain, data_repo, checkpoint_repo, @@ -473,24 +467,10 @@ where tracing::info!(%new_head, "Committed new block"); - if !new_upstream_ids.is_empty() { - let summary = self - .get_summary(GetSummaryOpts::default()) - .await - .int_err()?; - - self.event_bus - .dispatch_event(DatasetEventDependenciesUpdated { - dataset_id: summary.id.clone(), - new_upstream_ids, - }) - .await - .int_err()?; - } - Ok(CommitResult { old_head: prev_block_hash, new_head, + new_upstream_ids, }) } diff --git a/src/infra/core/src/repos/dataset_repository_helpers.rs b/src/infra/core/src/repos/dataset_repository_helpers.rs index 8d8e50c8c3..80fd38d089 100644 --- a/src/infra/core/src/repos/dataset_repository_helpers.rs +++ b/src/infra/core/src/repos/dataset_repository_helpers.rs @@ -7,16 +7,14 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - use chrono::{DateTime, Utc}; -use event_bus::EventBus; use internal_error::*; -use kamu_core::events::DatasetEventDependenciesUpdated; use kamu_core::*; use opendatafabric::*; use random_names::get_random_name; +use crate::DatasetRepositoryWriter; + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub fn get_staging_name() -> String { @@ -26,12 +24,13 @@ pub fn get_staging_name() -> String { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn create_dataset_from_snapshot_impl( - dataset_repo: &dyn DatasetRepositoryExt, - event_bus: &EventBus, +pub(crate) async fn create_dataset_from_snapshot_impl< + TRepository: DatasetRepositoryExt + DatasetRepositoryWriter, +>( + dataset_repo: &TRepository, mut snapshot: DatasetSnapshot, system_time: DateTime, -) -> Result { +) -> Result { // Validate / resolve events for event in &mut snapshot.metadata { match event { @@ -154,7 +153,7 @@ pub async fn create_dataset_from_snapshot_impl( Err(e) => { // Attempt to clean up dataset let _ = dataset_repo - .delete_dataset(&create_result.dataset_handle.as_local_ref()) + .delete_dataset(&create_result.dataset_handle) .await; Err(e) } @@ -175,20 +174,12 @@ pub async fn create_dataset_from_snapshot_impl( .await .int_err()?; - // TODO: encapsulate this inside dataset/chain - if !new_upstream_ids.is_empty() { - event_bus - .dispatch_event(DatasetEventDependenciesUpdated { - dataset_id: create_result.dataset_handle.id.clone(), - new_upstream_ids, - }) - .await - .int_err()?; - } - - Ok(CreateDatasetResult { - head, - ..create_result + Ok(CreateDatasetFromSnapshotResult { + create_dataset_result: CreateDatasetResult { + head, + ..create_result + }, + new_upstream_ids, }) } diff --git a/src/infra/core/src/repos/dataset_repository_local_fs.rs b/src/infra/core/src/repos/dataset_repository_local_fs.rs index 9eac1e2a9a..d2f7e2f043 100644 --- a/src/infra/core/src/repos/dataset_repository_local_fs.rs +++ b/src/infra/core/src/repos/dataset_repository_local_fs.rs @@ -12,11 +12,11 @@ use std::sync::Arc; use async_trait::async_trait; use dill::*; -use domain::auth::{DatasetAction, DatasetActionAuthorizer}; -use event_bus::EventBus; +use internal_error::{ErrorIntoInternal, InternalError, ResultIntoInternal}; use kamu_accounts::{CurrentAccountSubject, DEFAULT_ACCOUNT_NAME_STR}; use kamu_core::*; use opendatafabric::*; +use time_source::SystemTimeSource; use url::Url; use crate::*; @@ -24,11 +24,7 @@ use crate::*; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub struct DatasetRepositoryLocalFs { - current_account_subject: Arc, storage_strategy: Box, - dataset_action_authorizer: Arc, - dependency_graph_service: Arc, - event_bus: Arc, thrash_lock: tokio::sync::Mutex<()>, system_time_source: Arc, } @@ -40,29 +36,18 @@ impl DatasetRepositoryLocalFs { pub fn new( root: PathBuf, current_account_subject: Arc, - dataset_action_authorizer: Arc, - dependency_graph_service: Arc, - event_bus: Arc, multi_tenant: bool, system_time_source: Arc, ) -> Self { Self { - current_account_subject: current_account_subject.clone(), storage_strategy: if multi_tenant { Box::new(DatasetMultiTenantStorageStrategy::new( root, current_account_subject, - event_bus.clone(), )) } else { - Box::new(DatasetSingleTenantStorageStrategy::new( - root, - event_bus.clone(), - )) + Box::new(DatasetSingleTenantStorageStrategy::new(root)) }, - dataset_action_authorizer, - dependency_graph_service, - event_bus, thrash_lock: tokio::sync::Mutex::new(()), system_time_source, } @@ -71,9 +56,6 @@ impl DatasetRepositoryLocalFs { pub fn create( root: impl Into, current_account_subject: Arc, - dataset_action_authorizer: Arc, - dependency_graph_service: Arc, - event_bus: Arc, multi_tenant: bool, system_time_source: Arc, ) -> Result { @@ -82,17 +64,13 @@ impl DatasetRepositoryLocalFs { Ok(Self::new( root, current_account_subject, - dataset_action_authorizer, - dependency_graph_service, - event_bus, multi_tenant, system_time_source, )) } - fn build_dataset(layout: DatasetLayout, event_bus: Arc) -> Arc { + fn build_dataset(layout: DatasetLayout) -> Arc { Arc::new(DatasetImpl::new( - event_bus, MetadataChainImpl::new( MetadataBlockRepositoryCachingInMem::new(MetadataBlockRepositoryImpl::new( ObjectRepositoryLocalFSSha3::new(layout.blocks_dir), @@ -105,12 +83,6 @@ impl DatasetRepositoryLocalFs { )) } - // TODO: Make dataset factory (and thus the hashing algo) configurable - fn get_dataset_impl(&self, dataset_handle: &DatasetHandle) -> Arc { - let layout = DatasetLayout::new(self.storage_strategy.get_dataset_path(dataset_handle)); - Self::build_dataset(layout, self.event_bus.clone()) - } - // TODO: Used only for testing, but should be removed it in future to discourage // file-based access pub async fn get_dataset_layout( @@ -201,15 +173,23 @@ impl DatasetRepository for DatasetRepositoryLocalFs { self.storage_strategy.get_datasets_by_owner(account_name) } - async fn get_dataset( + async fn find_dataset_by_ref( &self, dataset_ref: &DatasetRef, ) -> Result, GetDatasetError> { let dataset_handle = self.resolve_dataset_ref(dataset_ref).await?; - let dataset = self.get_dataset_impl(&dataset_handle); + let dataset = self.get_dataset_by_handle(&dataset_handle); Ok(dataset) } + fn get_dataset_by_handle(&self, dataset_handle: &DatasetHandle) -> Arc { + let layout = DatasetLayout::new(self.storage_strategy.get_dataset_path(dataset_handle)); + Self::build_dataset(layout) + } +} + +#[async_trait] +impl DatasetRepositoryWriter for DatasetRepositoryLocalFs { async fn create_dataset( &self, dataset_alias: &DatasetAlias, @@ -231,10 +211,7 @@ impl DatasetRepository for DatasetRepositoryLocalFs { // - Dataset existed before (has valid head) - we should error out with name // collision if let Some(existing_dataset_handle) = maybe_existing_dataset_handle { - let existing_dataset = self - .get_dataset(&existing_dataset_handle.as_local_ref()) - .await - .int_err()?; + let existing_dataset = self.get_dataset_by_handle(&existing_dataset_handle); match existing_dataset .as_metadata_chain() @@ -269,7 +246,7 @@ impl DatasetRepository for DatasetRepositoryLocalFs { let dataset_path = self.storage_strategy.get_dataset_path(&dataset_handle); let layout = DatasetLayout::create(&dataset_path).int_err()?; - let dataset = Self::build_dataset(layout, self.event_bus.clone()); + let dataset = Self::build_dataset(layout); // There are three possibilities at this point: // - Dataset did not exist before - continue normally @@ -314,18 +291,6 @@ impl DatasetRepository for DatasetRepositoryLocalFs { "Created new dataset", ); - self.event_bus - .dispatch_event(events::DatasetEventCreated { - dataset_id: dataset_handle.id.clone(), - owner_account_id: match self.current_account_subject.as_ref() { - CurrentAccountSubject::Anonymous(_) => { - panic!("Anonymous account cannot create dataset"); - } - CurrentAccountSubject::Logged(l) => l.account_id.clone(), - }, - }) - .await?; - Ok(CreateDatasetResult { dataset_handle, dataset, @@ -336,25 +301,19 @@ impl DatasetRepository for DatasetRepositoryLocalFs { async fn create_dataset_from_snapshot( &self, snapshot: DatasetSnapshot, - ) -> Result { - create_dataset_from_snapshot_impl( - self, - self.event_bus.as_ref(), - snapshot, - self.system_time_source.now(), - ) - .await + ) -> Result { + create_dataset_from_snapshot_impl(self, snapshot, self.system_time_source.now()).await } async fn rename_dataset( &self, - dataset_ref: &DatasetRef, + dataset_handle: &DatasetHandle, new_name: &DatasetName, ) -> Result<(), RenameDatasetError> { - let dataset_handle = self.resolve_dataset_ref(dataset_ref).await?; - let new_alias = DatasetAlias::new(dataset_handle.alias.account_name.clone(), new_name.clone()); + + // Note: should collision check be moved to use case level? match self .storage_strategy .resolve_dataset_alias(&new_alias) @@ -370,55 +329,17 @@ impl DatasetRepository for DatasetRepositoryLocalFs { Err(ResolveDatasetError::NotFound(_)) => Ok(()), }?; - self.dataset_action_authorizer - .check_action_allowed(&dataset_handle, DatasetAction::Write) - .await?; - self.storage_strategy - .handle_dataset_renamed(&dataset_handle, new_name) + .handle_dataset_renamed(dataset_handle, new_name) .await?; Ok(()) } - // TODO: PERF: Need fast inverse dependency lookup - async fn delete_dataset(&self, dataset_ref: &DatasetRef) -> Result<(), DeleteDatasetError> { - let dataset_handle = match self.resolve_dataset_ref(dataset_ref).await { - Ok(h) => Ok(h), - Err(GetDatasetError::NotFound(e)) => Err(DeleteDatasetError::NotFound(e)), - Err(GetDatasetError::Internal(e)) => Err(DeleteDatasetError::Internal(e)), - }?; - - use tokio_stream::StreamExt; - let downstream_dataset_ids: Vec<_> = self - .dependency_graph_service - .get_downstream_dependencies(&dataset_handle.id) - .await - .int_err()? - .collect() - .await; - - if !downstream_dataset_ids.is_empty() { - let mut children = Vec::with_capacity(downstream_dataset_ids.len()); - for downstream_dataset_id in downstream_dataset_ids { - let hdl = self - .resolve_dataset_ref(&downstream_dataset_id.as_local_ref()) - .await - .int_err()?; - children.push(hdl); - } - - return Err(DanglingReferenceError { - dataset_handle, - children, - } - .into()); - } - - self.dataset_action_authorizer - .check_action_allowed(&dataset_handle, DatasetAction::Write) - .await?; - + async fn delete_dataset( + &self, + dataset_handle: &DatasetHandle, + ) -> Result<(), DeleteDatasetError> { // // Update repo info // let mut repo_info = self.read_repo_info().await?; // let index = repo_info @@ -430,15 +351,9 @@ impl DatasetRepository for DatasetRepositoryLocalFs { // repo_info.datasets.remove(index); // self.write_repo_info(repo_info).await?; - let dataset_dir = self.storage_strategy.get_dataset_path(&dataset_handle); + let dataset_dir = self.storage_strategy.get_dataset_path(dataset_handle); tokio::fs::remove_dir_all(dataset_dir).await.int_err()?; - self.event_bus - .dispatch_event(events::DatasetEventDeleted { - dataset_id: dataset_handle.id, - }) - .await?; - Ok(()) } } @@ -503,15 +418,11 @@ enum ResolveDatasetError { struct DatasetSingleTenantStorageStrategy { root: PathBuf, - event_bus: Arc, } impl DatasetSingleTenantStorageStrategy { - pub fn new(root: impl Into, event_bus: Arc) -> Self { - Self { - root: root.into(), - event_bus, - } + pub fn new(root: impl Into) -> Self { + Self { root: root.into() } } fn dataset_name<'a>(&self, dataset_alias: &'a DatasetAlias) -> &'a DatasetName { @@ -529,7 +440,7 @@ impl DatasetSingleTenantStorageStrategy { dataset_alias: &DatasetAlias, ) -> Result<(DatasetSummary, DatasetAlias), ResolveDatasetError> { let layout = DatasetLayout::new(dataset_path); - let dataset = DatasetRepositoryLocalFs::build_dataset(layout, self.event_bus.clone()); + let dataset = DatasetRepositoryLocalFs::build_dataset(layout); let dataset_summary = dataset .get_summary(GetSummaryOpts::default()) @@ -713,19 +624,16 @@ impl DatasetStorageStrategy for DatasetSingleTenantStorageStrategy { struct DatasetMultiTenantStorageStrategy { root: PathBuf, current_account_subject: Arc, - event_bus: Arc, } impl DatasetMultiTenantStorageStrategy { pub fn new( root: impl Into, current_account_subject: Arc, - event_bus: Arc, ) -> Self { Self { root: root.into(), current_account_subject, - event_bus, } } @@ -749,7 +657,7 @@ impl DatasetMultiTenantStorageStrategy { dataset_id: &DatasetID, ) -> Result { let layout = DatasetLayout::new(dataset_path); - let dataset = DatasetRepositoryLocalFs::build_dataset(layout, self.event_bus.clone()); + let dataset = DatasetRepositoryLocalFs::build_dataset(layout); match dataset.as_info_repo().get("alias").await { Ok(bytes) => { let dataset_alias_str = std::str::from_utf8(&bytes[..]).int_err()?.trim(); @@ -1014,7 +922,7 @@ impl DatasetStorageStrategy for DatasetMultiTenantStorageStrategy { ) -> Result<(), InternalError> { let dataset_path = self.get_dataset_path(dataset_handle); let layout = DatasetLayout::new(dataset_path); - let dataset = DatasetRepositoryLocalFs::build_dataset(layout, self.event_bus.clone()); + let dataset = DatasetRepositoryLocalFs::build_dataset(layout); let new_alias = DatasetAlias::new(dataset_handle.alias.account_name.clone(), new_name.clone()); diff --git a/src/infra/core/src/repos/dataset_repository_s3.rs b/src/infra/core/src/repos/dataset_repository_s3.rs index 032092aaee..d819b571ba 100644 --- a/src/infra/core/src/repos/dataset_repository_s3.rs +++ b/src/infra/core/src/repos/dataset_repository_s3.rs @@ -13,11 +13,11 @@ use std::sync::Arc; use async_trait::async_trait; use chrono::{DateTime, Utc}; use dill::*; -use event_bus::EventBus; +use internal_error::{ErrorIntoInternal, InternalError, ResultIntoInternal}; use kamu_accounts::{CurrentAccountSubject, DEFAULT_ACCOUNT_NAME_STR}; -use kamu_core::auth::{DatasetAction, DatasetActionAuthorizer}; use kamu_core::*; use opendatafabric::*; +use time_source::SystemTimeSource; use tokio::sync::Mutex; use url::Url; @@ -29,9 +29,6 @@ use crate::*; pub struct DatasetRepositoryS3 { s3_context: S3Context, current_account_subject: Arc, - dataset_action_authorizer: Arc, - dependency_graph_service: Arc, - event_bus: Arc, multi_tenant: bool, registry_cache: Option>, metadata_cache_local_fs_path: Option>, @@ -54,9 +51,6 @@ impl DatasetRepositoryS3 { pub fn new( s3_context: S3Context, current_account_subject: Arc, - dataset_action_authorizer: Arc, - dependency_graph_service: Arc, - event_bus: Arc, multi_tenant: bool, registry_cache: Option>, metadata_cache_local_fs_path: Option>, @@ -65,9 +59,6 @@ impl DatasetRepositoryS3 { Self { s3_context, current_account_subject, - dataset_action_authorizer, - dependency_graph_service, - event_bus, multi_tenant, registry_cache, metadata_cache_local_fs_path, @@ -89,7 +80,6 @@ impl DatasetRepositoryS3 { // configurability if let Some(metadata_cache_local_fs_path) = &self.metadata_cache_local_fs_path { Arc::new(DatasetImpl::new( - self.event_bus.clone(), MetadataChainImpl::new( MetadataBlockRepositoryCachingInMem::new(MetadataBlockRepositoryImpl::new( ObjectRepositoryCachingLocalFs::new( @@ -130,7 +120,6 @@ impl DatasetRepositoryS3 { )) } else { Arc::new(DatasetImpl::new( - self.event_bus.clone(), MetadataChainImpl::new( MetadataBlockRepositoryCachingInMem::new(MetadataBlockRepositoryImpl::new( ObjectRepositoryS3Sha3::new(S3Context::new( @@ -362,7 +351,7 @@ impl DatasetRepository for DatasetRepositoryS3 { }) } - async fn get_dataset( + async fn find_dataset_by_ref( &self, dataset_ref: &DatasetRef, ) -> Result, GetDatasetError> { @@ -371,6 +360,13 @@ impl DatasetRepository for DatasetRepositoryS3 { Ok(dataset) } + fn get_dataset_by_handle(&self, dataset_handle: &DatasetHandle) -> Arc { + self.get_dataset_impl(&dataset_handle.id) + } +} + +#[async_trait] +impl DatasetRepositoryWriter for DatasetRepositoryS3 { async fn create_dataset( &self, dataset_alias: &DatasetAlias, @@ -397,10 +393,7 @@ impl DatasetRepository for DatasetRepositoryS3 { // - Dataset existed before (has valid head) - we should error out with name // collision if let Some(existing_dataset_handle) = maybe_existing_dataset_handle { - let existing_dataset = self - .get_dataset(&existing_dataset_handle.as_local_ref()) - .await - .int_err()?; + let existing_dataset = self.get_dataset_by_handle(&existing_dataset_handle); match existing_dataset .as_metadata_chain() @@ -472,18 +465,6 @@ impl DatasetRepository for DatasetRepositoryS3 { cache.datasets.push(dataset_handle.clone()); } - self.event_bus - .dispatch_event(events::DatasetEventCreated { - dataset_id: dataset_handle.id.clone(), - owner_account_id: match self.current_account_subject.as_ref() { - CurrentAccountSubject::Anonymous(_) => { - panic!("Anonymous account cannot create dataset"); - } - CurrentAccountSubject::Logged(l) => l.account_id.clone(), - }, - }) - .await?; - tracing::info!( id = %dataset_handle.id, alias = %dataset_handle.alias, @@ -501,40 +482,32 @@ impl DatasetRepository for DatasetRepositoryS3 { async fn create_dataset_from_snapshot( &self, snapshot: DatasetSnapshot, - ) -> Result { - create_dataset_from_snapshot_impl( - self, - self.event_bus.as_ref(), - snapshot, - self.system_time_source.now(), - ) - .await + ) -> Result { + create_dataset_from_snapshot_impl(self, snapshot, self.system_time_source.now()).await } async fn rename_dataset( &self, - dataset_ref: &DatasetRef, + dataset_handle: &DatasetHandle, new_name: &DatasetName, ) -> Result<(), RenameDatasetError> { - let old_handle = self.resolve_dataset_ref(dataset_ref).await?; - - let dataset = self.get_dataset_impl(&old_handle.id); + let dataset = self.get_dataset_impl(&dataset_handle.id); - let new_alias = DatasetAlias::new(old_handle.alias.account_name.clone(), new_name.clone()); + let new_alias = + DatasetAlias::new(dataset_handle.alias.account_name.clone(), new_name.clone()); - // Check against possible name collisions + // Note: should collision check be moved to use case level? match self.resolve_dataset_ref(&new_alias.as_local_ref()).await { Ok(_) => Err(RenameDatasetError::NameCollision(NameCollisionError { - alias: DatasetAlias::new(old_handle.alias.account_name.clone(), new_name.clone()), + alias: DatasetAlias::new( + dataset_handle.alias.account_name.clone(), + new_name.clone(), + ), })), Err(GetDatasetError::Internal(e)) => Err(RenameDatasetError::Internal(e)), Err(GetDatasetError::NotFound(_)) => Ok(()), }?; - self.dataset_action_authorizer - .check_action_allowed(&old_handle, DatasetAction::Write) - .await?; - // It's safe to rename dataset self.save_dataset_alias(dataset.as_ref(), &new_alias) .await?; @@ -542,52 +515,19 @@ impl DatasetRepository for DatasetRepositoryS3 { // Update cache if enabled if let Some(cache) = &self.registry_cache { let mut cache = cache.state.lock().await; - cache.datasets.retain(|h| h.id != old_handle.id); + cache.datasets.retain(|h| h.id != dataset_handle.id); cache .datasets - .push(DatasetHandle::new(old_handle.id, new_alias)); + .push(DatasetHandle::new(dataset_handle.id.clone(), new_alias)); } Ok(()) } - async fn delete_dataset(&self, dataset_ref: &DatasetRef) -> Result<(), DeleteDatasetError> { - let dataset_handle = match self.resolve_dataset_ref(dataset_ref).await { - Ok(dataset_handle) => dataset_handle, - Err(GetDatasetError::NotFound(e)) => return Err(DeleteDatasetError::NotFound(e)), - Err(GetDatasetError::Internal(e)) => return Err(DeleteDatasetError::Internal(e)), - }; - - use tokio_stream::StreamExt; - let downstream_dataset_ids: Vec<_> = self - .dependency_graph_service - .get_downstream_dependencies(&dataset_handle.id) - .await - .int_err()? - .collect() - .await; - - if !downstream_dataset_ids.is_empty() { - let mut children = Vec::with_capacity(downstream_dataset_ids.len()); - for downstream_dataset_id in downstream_dataset_ids { - let hdl = self - .resolve_dataset_ref(&downstream_dataset_id.as_local_ref()) - .await - .int_err()?; - children.push(hdl); - } - - return Err(DanglingReferenceError { - dataset_handle, - children, - } - .into()); - } - - self.dataset_action_authorizer - .check_action_allowed(&dataset_handle, DatasetAction::Write) - .await?; - + async fn delete_dataset( + &self, + dataset_handle: &DatasetHandle, + ) -> Result<(), DeleteDatasetError> { self.delete_dataset_s3_objects(&dataset_handle.id) .await .map_err(DeleteDatasetError::Internal)?; @@ -598,12 +538,6 @@ impl DatasetRepository for DatasetRepositoryS3 { cache.datasets.retain(|h| h.id != dataset_handle.id); } - self.event_bus - .dispatch_event(events::DatasetEventDeleted { - dataset_id: dataset_handle.id, - }) - .await?; - Ok(()) } } diff --git a/src/infra/core/src/repos/dataset_repository_writer.rs b/src/infra/core/src/repos/dataset_repository_writer.rs new file mode 100644 index 0000000000..a9ddb86f2e --- /dev/null +++ b/src/infra/core/src/repos/dataset_repository_writer.rs @@ -0,0 +1,40 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use kamu_core::*; +use opendatafabric::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[async_trait::async_trait] +pub trait DatasetRepositoryWriter: Sync + Send { + async fn create_dataset( + &self, + dataset_alias: &DatasetAlias, + seed_block: MetadataBlockTyped, + ) -> Result; + + async fn create_dataset_from_snapshot( + &self, + snapshot: DatasetSnapshot, + ) -> Result; + + async fn rename_dataset( + &self, + dataset_handle: &DatasetHandle, + new_name: &DatasetName, + ) -> Result<(), RenameDatasetError>; + + async fn delete_dataset( + &self, + dataset_handle: &DatasetHandle, + ) -> Result<(), DeleteDatasetError>; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/core/src/repos/metadata_chain_impl.rs b/src/infra/core/src/repos/metadata_chain_impl.rs index 799c4a6875..e5cac8cef0 100644 --- a/src/infra/core/src/repos/metadata_chain_impl.rs +++ b/src/infra/core/src/repos/metadata_chain_impl.rs @@ -8,6 +8,7 @@ // by the Apache License, Version 2.0. use async_trait::async_trait; +use internal_error::ErrorIntoInternal; use kamu_core::*; use opendatafabric::*; diff --git a/src/infra/core/src/repos/mod.rs b/src/infra/core/src/repos/mod.rs index 31b037977f..27688d99a7 100644 --- a/src/infra/core/src/repos/mod.rs +++ b/src/infra/core/src/repos/mod.rs @@ -12,6 +12,7 @@ mod dataset_impl; mod dataset_repository_helpers; mod dataset_repository_local_fs; mod dataset_repository_s3; +mod dataset_repository_writer; mod metadata_block_repository_caching_inmem; mod metadata_block_repository_helpers; mod metadata_block_repository_impl; @@ -37,6 +38,7 @@ pub use dataset_impl::*; pub use dataset_repository_helpers::*; pub use dataset_repository_local_fs::*; pub use dataset_repository_s3::*; +pub use dataset_repository_writer::*; pub use metadata_block_repository_caching_inmem::*; pub use metadata_block_repository_helpers::*; pub use metadata_block_repository_impl::*; diff --git a/src/infra/core/src/repos/named_object_repository_http.rs b/src/infra/core/src/repos/named_object_repository_http.rs index 17713c78e0..05c01ca685 100644 --- a/src/infra/core/src/repos/named_object_repository_http.rs +++ b/src/infra/core/src/repos/named_object_repository_http.rs @@ -9,6 +9,7 @@ use async_trait::async_trait; use bytes::Bytes; +use internal_error::{ErrorIntoInternal, ResultIntoInternal}; use kamu_core::*; use reqwest::Client; use url::Url; diff --git a/src/infra/core/src/repos/named_object_repository_ipfs_http.rs b/src/infra/core/src/repos/named_object_repository_ipfs_http.rs index 7014b6bcfa..8b3b9c6344 100644 --- a/src/infra/core/src/repos/named_object_repository_ipfs_http.rs +++ b/src/infra/core/src/repos/named_object_repository_ipfs_http.rs @@ -9,6 +9,7 @@ use async_trait::async_trait; use bytes::Bytes; +use internal_error::{ErrorIntoInternal, ResultIntoInternal}; use kamu_core::*; use reqwest::Client; use url::Url; diff --git a/src/infra/core/src/repos/named_object_repository_local_fs.rs b/src/infra/core/src/repos/named_object_repository_local_fs.rs index fcab938836..4ed62dc81d 100644 --- a/src/infra/core/src/repos/named_object_repository_local_fs.rs +++ b/src/infra/core/src/repos/named_object_repository_local_fs.rs @@ -11,6 +11,7 @@ use std::path::PathBuf; use async_trait::async_trait; use bytes::Bytes; +use internal_error::{ErrorIntoInternal, ResultIntoInternal}; use kamu_core::*; use super::get_staging_name; diff --git a/src/infra/core/src/repos/named_object_repository_s3.rs b/src/infra/core/src/repos/named_object_repository_s3.rs index 059ff09661..e9535f276f 100644 --- a/src/infra/core/src/repos/named_object_repository_s3.rs +++ b/src/infra/core/src/repos/named_object_repository_s3.rs @@ -10,6 +10,7 @@ use async_trait::async_trait; use aws_sdk_s3::operation::get_object::GetObjectError; use bytes::Bytes; +use internal_error::{ErrorIntoInternal, ResultIntoInternal}; use kamu_core::*; use crate::utils::s3_context::S3Context; diff --git a/src/infra/core/src/repos/object_repository_caching_local_fs.rs b/src/infra/core/src/repos/object_repository_caching_local_fs.rs index 36960cc4d0..b714ebc6e8 100644 --- a/src/infra/core/src/repos/object_repository_caching_local_fs.rs +++ b/src/infra/core/src/repos/object_repository_caching_local_fs.rs @@ -13,6 +13,7 @@ use std::sync::Arc; use async_trait::async_trait; use bytes::Bytes; +use internal_error::{ErrorIntoInternal, ResultIntoInternal}; use kamu_core::*; use opendatafabric::Multihash; use tokio::io::{AsyncSeekExt, AsyncWriteExt}; diff --git a/src/infra/core/src/repos/object_repository_http.rs b/src/infra/core/src/repos/object_repository_http.rs index 2fe473bf6c..0da73cd545 100644 --- a/src/infra/core/src/repos/object_repository_http.rs +++ b/src/infra/core/src/repos/object_repository_http.rs @@ -11,6 +11,7 @@ use std::path::Path; use async_trait::async_trait; use bytes::Bytes; +use internal_error::{ErrorIntoInternal, ResultIntoInternal}; use kamu_core::*; use opendatafabric::Multihash; use reqwest::Client; diff --git a/src/infra/core/src/repos/object_repository_local_fs.rs b/src/infra/core/src/repos/object_repository_local_fs.rs index 4d4510a67c..ecdf3a1461 100644 --- a/src/infra/core/src/repos/object_repository_local_fs.rs +++ b/src/infra/core/src/repos/object_repository_local_fs.rs @@ -12,6 +12,7 @@ use std::path::{Path, PathBuf}; use async_trait::async_trait; use bytes::Bytes; +use internal_error::ResultIntoInternal; use kamu_core::*; use opendatafabric::{Multicodec, Multihash}; use tokio::io::{AsyncRead, AsyncWriteExt}; diff --git a/src/infra/core/src/repos/object_repository_s3.rs b/src/infra/core/src/repos/object_repository_s3.rs index e905270b74..42db26ea92 100644 --- a/src/infra/core/src/repos/object_repository_s3.rs +++ b/src/infra/core/src/repos/object_repository_s3.rs @@ -16,6 +16,7 @@ use aws_sdk_s3::operation::get_object::GetObjectError; use aws_sdk_s3::operation::head_object::HeadObjectError; use aws_sdk_s3::presigning::PresigningConfig; use bytes::Bytes; +use internal_error::{ErrorIntoInternal, ResultIntoInternal}; use kamu_core::*; use opendatafabric::{Multicodec, Multihash}; use url::Url; diff --git a/src/infra/core/src/repos/object_store_builder_local_fs.rs b/src/infra/core/src/repos/object_store_builder_local_fs.rs index ee45ff1dee..7b922734fd 100644 --- a/src/infra/core/src/repos/object_store_builder_local_fs.rs +++ b/src/infra/core/src/repos/object_store_builder_local_fs.rs @@ -10,6 +10,7 @@ use std::sync::Arc; use dill::*; +use internal_error::InternalError; use kamu_core::*; use url::Url; diff --git a/src/infra/core/src/repos/object_store_builder_s3.rs b/src/infra/core/src/repos/object_store_builder_s3.rs index 81be1f6339..59bdd2f994 100644 --- a/src/infra/core/src/repos/object_store_builder_s3.rs +++ b/src/infra/core/src/repos/object_store_builder_s3.rs @@ -11,6 +11,7 @@ use std::sync::{Arc, Mutex}; use aws_credential_types::provider::SharedCredentialsProvider; use dill::*; +use internal_error::{InternalError, ResultIntoInternal}; use kamu_core::*; use object_store::aws::{AmazonS3Builder, AwsCredential}; use object_store::CredentialProvider; diff --git a/src/infra/core/src/repos/reference_repository_impl.rs b/src/infra/core/src/repos/reference_repository_impl.rs index 1bbf3bc4ee..d9bee9a04c 100644 --- a/src/infra/core/src/repos/reference_repository_impl.rs +++ b/src/infra/core/src/repos/reference_repository_impl.rs @@ -8,6 +8,7 @@ // by the Apache License, Version 2.0. use async_trait::async_trait; +use internal_error::ResultIntoInternal; use kamu_core::repos::reference_repository::SetRefError; use kamu_core::*; use opendatafabric::Multihash; diff --git a/src/infra/core/src/reset_service_impl.rs b/src/infra/core/src/reset_service_impl.rs index 82b2a01669..bf7bf6211c 100644 --- a/src/infra/core/src/reset_service_impl.rs +++ b/src/infra/core/src/reset_service_impl.rs @@ -10,9 +10,12 @@ use std::sync::Arc; use dill::*; +use internal_error::ResultIntoInternal; use kamu_core::*; use opendatafabric::*; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + pub struct ResetServiceImpl { dataset_repo: Arc, dataset_action_authorizer: Arc, @@ -44,10 +47,7 @@ impl ResetService for ResetServiceImpl { .check_action_allowed(dataset_handle, auth::DatasetAction::Write) .await?; - let dataset = self - .dataset_repo - .get_dataset(&dataset_handle.as_local_ref()) - .await?; + let dataset = self.dataset_repo.get_dataset_by_handle(dataset_handle); let new_head = if let Some(new_head) = new_head_maybe { new_head @@ -89,3 +89,5 @@ impl ResetService for ResetServiceImpl { Ok(new_head.clone()) } } + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/core/src/resource_loader_impl.rs b/src/infra/core/src/resource_loader_impl.rs index a973960c5f..ddab215215 100644 --- a/src/infra/core/src/resource_loader_impl.rs +++ b/src/infra/core/src/resource_loader_impl.rs @@ -10,6 +10,7 @@ use std::path::Path; use dill::{component, interface}; +use internal_error::ResultIntoInternal; use kamu_core::*; use opendatafabric::serde::yaml::*; use opendatafabric::*; diff --git a/src/infra/core/src/search_service_impl.rs b/src/infra/core/src/search_service_impl.rs index 633c43af1b..dd9f4ae72a 100644 --- a/src/infra/core/src/search_service_impl.rs +++ b/src/infra/core/src/search_service_impl.rs @@ -10,6 +10,7 @@ use std::sync::Arc; use dill::*; +use internal_error::{ErrorIntoInternal, ResultIntoInternal}; use kamu_core::*; use opendatafabric::*; use serde_json::json; diff --git a/src/infra/core/src/sync_service_impl.rs b/src/infra/core/src/sync_service_impl.rs index 4e556f5bba..e4abaa3d1d 100644 --- a/src/infra/core/src/sync_service_impl.rs +++ b/src/infra/core/src/sync_service_impl.rs @@ -10,6 +10,7 @@ use std::sync::Arc; use dill::*; +use internal_error::{ErrorIntoInternal, InternalError, ResultIntoInternal}; use kamu_core::services::sync_service::DatasetNotFoundError; use kamu_core::utils::metadata_chain_comparator::*; use kamu_core::*; @@ -20,12 +21,14 @@ use super::utils::smart_transfer_protocol::SmartTransferProtocolClient; use crate::utils::ipfs_wrapper::*; use crate::utils::simple_transfer_protocol::{DatasetFactoryFn, SimpleTransferProtocol}; use crate::utils::smart_transfer_protocol::TransferOptions; +use crate::DatasetRepositoryWriter; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// pub struct SyncServiceImpl { remote_repo_reg: Arc, dataset_repo: Arc, + dataset_repo_writer: Arc, dataset_action_authorizer: Arc, dataset_factory: Arc, smart_transfer_protocol: Arc, @@ -40,6 +43,7 @@ impl SyncServiceImpl { pub fn new( remote_repo_reg: Arc, dataset_repo: Arc, + dataset_repo_writer: Arc, dataset_action_authorizer: Arc, dataset_factory: Arc, smart_transfer_protocol: Arc, @@ -48,6 +52,7 @@ impl SyncServiceImpl { Self { remote_repo_reg, dataset_repo, + dataset_repo_writer, dataset_action_authorizer, dataset_factory, smart_transfer_protocol, @@ -96,7 +101,7 @@ impl SyncServiceImpl { .check_action_allowed(&dataset_handle, auth::DatasetAction::Read) .await?; - self.dataset_repo.get_dataset(local_ref).await? + self.dataset_repo.find_dataset_by_ref(local_ref).await? } SyncRef::Remote(url) => { // TODO: implement authorization checks somehow @@ -127,27 +132,32 @@ impl SyncServiceImpl { create_if_not_exists: bool, ) -> Result<(Option>, Option), SyncError> { match dataset_ref { - SyncRef::Local(local_ref) => match self.dataset_repo.get_dataset(local_ref).await { - Ok(dataset) => { - let dataset_handle = self.dataset_repo.resolve_dataset_ref(local_ref).await?; - self.dataset_action_authorizer - .check_action_allowed(&dataset_handle, auth::DatasetAction::Write) - .await?; + SyncRef::Local(local_ref) => { + match self.dataset_repo.find_dataset_by_ref(local_ref).await { + Ok(dataset) => { + let dataset_handle = + self.dataset_repo.resolve_dataset_ref(local_ref).await?; + self.dataset_action_authorizer + .check_action_allowed(&dataset_handle, auth::DatasetAction::Write) + .await?; - Ok((Some(dataset), None)) - } - Err(GetDatasetError::NotFound(_)) if create_if_not_exists => { - let alias = local_ref.alias().unwrap().clone(); - let repo = self.dataset_repo.clone(); - Ok(( - None, - Some(Box::new(move |seed_block| { - Box::pin(async move { repo.create_dataset(&alias, seed_block).await }) - })), - )) + Ok((Some(dataset), None)) + } + Err(GetDatasetError::NotFound(_)) if create_if_not_exists => { + let alias = local_ref.alias().unwrap().clone(); + let repo_writer = self.dataset_repo_writer.clone(); + Ok(( + None, + Some(Box::new(move |seed_block| { + Box::pin(async move { + repo_writer.create_dataset(&alias, seed_block).await + }) + })), + )) + } + Err(err) => Err(err.into()), } - Err(err) => Err(err.into()), - }, + } SyncRef::Remote(url) => { // TODO: implement authorization checks somehow let dataset = self @@ -316,7 +326,7 @@ impl SyncServiceImpl { .await?; // Resolve and compare heads - let src_dataset = self.dataset_repo.get_dataset(src).await?; + let src_dataset = self.dataset_repo.find_dataset_by_ref(src).await?; let src_head = src_dataset .as_metadata_chain() .resolve_ref(&BlockRef::Head) diff --git a/src/infra/core/src/testing/dataset_test_helper.rs b/src/infra/core/src/testing/dataset_test_helper.rs index 639a014f4a..99577e6eac 100644 --- a/src/infra/core/src/testing/dataset_test_helper.rs +++ b/src/infra/core/src/testing/dataset_test_helper.rs @@ -87,7 +87,10 @@ impl DatasetTestHelper { ) -> Multihash { let tmp_dir = tempfile::tempdir().unwrap(); - let ds = dataset_repo.get_dataset(&dataset_ref.into()).await.unwrap(); + let ds = dataset_repo + .find_dataset_by_ref(&dataset_ref.into()) + .await + .unwrap(); let prev_data = ds .as_metadata_chain() diff --git a/src/infra/core/src/testing/metadata_factory.rs b/src/infra/core/src/testing/metadata_factory.rs index 08109ea4a0..c21c560170 100644 --- a/src/infra/core/src/testing/metadata_factory.rs +++ b/src/infra/core/src/testing/metadata_factory.rs @@ -25,6 +25,10 @@ impl MetadataFactory { SetInfoBuilder::new() } + pub fn set_license() -> SetLicenseBuilder { + SetLicenseBuilder::new() + } + pub fn transform() -> TransformSqlBuilder { TransformSqlBuilder::new() } @@ -136,6 +140,51 @@ impl SetInfoBuilder { } } +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// SetLicense Builder +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub struct SetLicenseBuilder { + v: SetLicense, +} + +impl SetLicenseBuilder { + fn new() -> Self { + Self { + v: SetLicense { + short_name: String::from("TEST"), + name: String::from("TEST LICENSE"), + spdx_id: None, + website_url: String::from("http://example.com"), + }, + } + } + + pub fn short_name(mut self, short_name: &str) -> Self { + self.v.short_name = String::from(short_name); + self + } + + pub fn name(mut self, name: &str) -> Self { + self.v.name = String::from(name); + self + } + + pub fn spdx_id(mut self, spdx_id: &str) -> Self { + self.v.spdx_id = Some(String::from(spdx_id)); + self + } + + pub fn website_url(mut self, website_url: &str) -> Self { + self.v.website_url = String::from(website_url); + self + } + + pub fn build(self) -> SetLicense { + self.v + } +} + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Transform Builder //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/core/src/testing/mock_dataset_action_authorizer.rs b/src/infra/core/src/testing/mock_dataset_action_authorizer.rs index 04135985c2..9670b576ce 100644 --- a/src/infra/core/src/testing/mock_dataset_action_authorizer.rs +++ b/src/infra/core/src/testing/mock_dataset_action_authorizer.rs @@ -67,28 +67,42 @@ impl MockDatasetActionAuthorizer { mock_dataset_action_authorizer } - pub fn expect_check_read_dataset(self, dataset_alias: DatasetAlias, times: usize) -> Self { + pub fn expect_check_read_dataset( + self, + dataset_alias: &DatasetAlias, + times: usize, + success: bool, + ) -> Self { + let dataset_alias = dataset_alias.clone(); self.expect_check_action_allowed_internal( function(move |dh: &DatasetHandle| dh.alias == dataset_alias), DatasetAction::Read, times, + success, ) } - pub fn expect_check_write_dataset(self, dataset_alias: DatasetAlias, times: usize) -> Self { + pub fn expect_check_write_dataset( + self, + dataset_alias: &DatasetAlias, + times: usize, + success: bool, + ) -> Self { + let dataset_alias = dataset_alias.clone(); self.expect_check_action_allowed_internal( function(move |dh: &DatasetHandle| dh.alias == dataset_alias), DatasetAction::Write, times, + success, ) } - pub fn expect_check_read_a_dataset(self, times: usize) -> Self { - self.expect_check_action_allowed_internal(always(), DatasetAction::Read, times) + pub fn expect_check_read_a_dataset(self, times: usize, success: bool) -> Self { + self.expect_check_action_allowed_internal(always(), DatasetAction::Read, times, success) } - pub fn expect_check_write_a_dataset(self, times: usize) -> Self { - self.expect_check_action_allowed_internal(always(), DatasetAction::Write, times) + pub fn expect_check_write_a_dataset(self, times: usize, success: bool) -> Self { + self.expect_check_action_allowed_internal(always(), DatasetAction::Write, times, success) } fn expect_check_action_allowed_internal

( @@ -96,6 +110,7 @@ impl MockDatasetActionAuthorizer { dataset_handle_predicate: P, action: auth::DatasetAction, times: usize, + success: bool, ) -> Self where P: Predicate + Sync + Send + 'static, @@ -104,7 +119,13 @@ impl MockDatasetActionAuthorizer { self.expect_check_action_allowed() .with(dataset_handle_predicate, eq(action)) .times(times) - .returning(|_, _| Ok(())); + .returning(move |hdl, action| { + if success { + Ok(()) + } else { + Err(Self::denying_error(hdl, action)) + } + }); } else { self.expect_check_action_allowed() .with(dataset_handle_predicate, eq(action)) diff --git a/src/infra/core/src/transform_service_impl.rs b/src/infra/core/src/transform_service_impl.rs index 73b204e5af..187739b113 100644 --- a/src/infra/core/src/transform_service_impl.rs +++ b/src/infra/core/src/transform_service_impl.rs @@ -13,12 +13,14 @@ use std::sync::Arc; use chrono::{DateTime, Utc}; use dill::*; use futures::{StreamExt, TryFutureExt, TryStreamExt}; +use internal_error::{ErrorIntoInternal, InternalError, ResultIntoInternal}; use itertools::Itertools; use kamu_core::engine::*; use kamu_core::*; use kamu_ingest_datafusion::DataWriterDataFusion; use opendatafabric::*; use random_names::get_random_name; +use time_source::SystemTimeSource; pub struct TransformServiceImpl { dataset_repo: Arc, @@ -117,10 +119,7 @@ impl TransformServiceImpl { let old_head = request.head.clone(); - let dataset = dataset_repo - .get_dataset(&request.dataset_handle.as_local_ref()) - .await - .int_err()?; + let dataset = dataset_repo.get_dataset_by_handle(&request.dataset_handle); // Read new schema let new_schema = if let Some(out_data) = &response.new_data { @@ -198,11 +197,7 @@ impl TransformServiceImpl { dataset_handle: &DatasetHandle, system_time: DateTime, ) -> Result, TransformError> { - let dataset = self - .dataset_repo - .get_dataset(&dataset_handle.as_local_ref()) - .await - .int_err()?; + let dataset = self.dataset_repo.get_dataset_by_handle(dataset_handle); let output_chain = dataset.as_metadata_chain(); @@ -304,7 +299,11 @@ impl TransformServiceImpl { // TODO: Allow derivative datasets to function with inputs containing no data // This will require passing the schema explicitly instead of relying on a file async fn is_never_pulled(&self, dataset_ref: &DatasetRef) -> Result { - let dataset = self.dataset_repo.get_dataset(dataset_ref).await.int_err()?; + let dataset = self + .dataset_repo + .find_dataset_by_ref(dataset_ref) + .await + .int_err()?; Ok(dataset .as_metadata_chain() @@ -331,11 +330,7 @@ impl TransformServiceImpl { .resolve_dataset_ref(&dataset_id.as_local_ref()) .await .int_err()?; - let dataset = self - .dataset_repo - .get_dataset(&dataset_handle.as_local_ref()) - .await - .int_err()?; + let dataset = self.dataset_repo.get_dataset_by_handle(&dataset_handle); let input_chain = dataset.as_metadata_chain(); // Determine last processed input block and offset @@ -395,11 +390,7 @@ impl TransformServiceImpl { .check_action_allowed(&dataset_handle, auth::DatasetAction::Read) .await?; - let dataset = self - .dataset_repo - .get_dataset(&dataset_handle.as_local_ref()) - .await - .int_err()?; + let dataset = self.dataset_repo.get_dataset_by_handle(&dataset_handle); let input_chain = dataset.as_metadata_chain(); // Collect unprocessed input blocks @@ -481,7 +472,11 @@ impl TransformServiceImpl { &self, dataset_ref: &DatasetRef, ) -> Result { - let dataset = self.dataset_repo.get_dataset(dataset_ref).await.int_err()?; + let dataset = self + .dataset_repo + .find_dataset_by_ref(dataset_ref) + .await + .int_err()?; Ok(dataset .as_metadata_chain() @@ -501,10 +496,7 @@ impl TransformServiceImpl { dataset_handle: &DatasetHandle, block_range: (Option, Option), ) -> Result, VerificationError> { - let dataset = self - .dataset_repo - .get_dataset(&dataset_handle.as_local_ref()) - .await?; + let dataset = self.dataset_repo.get_dataset_by_handle(dataset_handle); let metadata_chain = dataset.as_metadata_chain(); let head = match block_range.1 { @@ -763,7 +755,7 @@ impl TransformService for TransformServiceImpl { &self, dataset_ref: &DatasetRef, ) -> Result)>, GetDatasetError> { - let dataset = self.dataset_repo.get_dataset(dataset_ref).await?; + let dataset = self.dataset_repo.find_dataset_by_ref(dataset_ref).await?; // TODO: Support transform evolution Ok(dataset @@ -830,10 +822,7 @@ impl TransformService for TransformServiceImpl { // VerificationService. But permissions for input datasets have to be // checked here - let dataset = self - .dataset_repo - .get_dataset(&dataset_handle.as_local_ref()) - .await?; + let dataset = self.dataset_repo.get_dataset_by_handle(&dataset_handle); let verification_plan = self .get_verification_plan(&dataset_handle, block_range) diff --git a/src/infra/core/src/use_cases/append_dataset_metadata_batch_use_case_impl.rs b/src/infra/core/src/use_cases/append_dataset_metadata_batch_use_case_impl.rs new file mode 100644 index 0000000000..d5f1bd60d0 --- /dev/null +++ b/src/infra/core/src/use_cases/append_dataset_metadata_batch_use_case_impl.rs @@ -0,0 +1,127 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::collections::VecDeque; +use std::sync::Arc; + +use dill::{component, interface}; +use internal_error::ResultIntoInternal; +use kamu_core::{ + AppendDatasetMetadataBatchUseCase, + AppendError, + AppendOpts, + BlockRef, + Dataset, + DatasetLifecycleMessage, + GetSummaryOpts, + HashedMetadataBlock, + SetRefOpts, + MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE, +}; +use messaging_outbox::{Outbox, OutboxExt}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub struct AppendDatasetMetadataBatchUseCaseImpl { + outbox: Arc, +} + +#[component(pub)] +#[interface(dyn AppendDatasetMetadataBatchUseCase)] +impl AppendDatasetMetadataBatchUseCaseImpl { + pub fn new(outbox: Arc) -> Self { + Self { outbox } + } +} + +#[async_trait::async_trait] +impl AppendDatasetMetadataBatchUseCase for AppendDatasetMetadataBatchUseCaseImpl { + async fn execute( + &self, + dataset: &dyn Dataset, + new_blocks: VecDeque, + force_update_if_diverged: bool, + ) -> Result<(), AppendError> { + if new_blocks.is_empty() { + return Ok(()); + } + + let old_head = new_blocks.front().unwrap().1.prev_block_hash.clone(); + let new_head = new_blocks.back().unwrap().0.clone(); + + let metadata_chain = dataset.as_metadata_chain(); + + let mut new_upstream_ids: Vec = vec![]; + + for (hash, block) in new_blocks { + tracing::debug!(sequence_numer = %block.sequence_number, hash = %hash, "Appending block"); + + if let opendatafabric::MetadataEvent::SetTransform(transform) = &block.event { + // Collect only the latest upstream dataset IDs + new_upstream_ids.clear(); + for new_input in &transform.inputs { + if let Some(id) = new_input.dataset_ref.id() { + new_upstream_ids.push(id.clone()); + } else { + // Input references must be resolved to IDs here, but we + // ignore the errors and let the metadata chain reject + // this event + } + } + } + + metadata_chain + .append( + block, + AppendOpts { + update_ref: None, + expected_hash: Some(&hash), + ..AppendOpts::default() + }, + ) + .await?; + } + + metadata_chain + .set_ref( + &BlockRef::Head, + &new_head, + SetRefOpts { + validate_block_present: false, + check_ref_is: if force_update_if_diverged { + None + } else { + Some(old_head.as_ref()) + }, + }, + ) + .await?; + + if !new_upstream_ids.is_empty() { + let summary = dataset + .get_summary(GetSummaryOpts::default()) + .await + .int_err()?; + + self.outbox + .post_message( + MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE, + DatasetLifecycleMessage::dependencies_updated( + summary.id.clone(), + new_upstream_ids, + ), + ) + .await?; + } + + Ok(()) + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/core/src/use_cases/commit_dataset_event_use_case_impl.rs b/src/infra/core/src/use_cases/commit_dataset_event_use_case_impl.rs new file mode 100644 index 0000000000..d23f3b9f7d --- /dev/null +++ b/src/infra/core/src/use_cases/commit_dataset_event_use_case_impl.rs @@ -0,0 +1,82 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::sync::Arc; + +use dill::{component, interface}; +use kamu_core::auth::{DatasetAction, DatasetActionAuthorizer}; +use kamu_core::{ + CommitDatasetEventUseCase, + CommitError, + CommitOpts, + CommitResult, + DatasetLifecycleMessage, + DatasetRepository, + MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE, +}; +use messaging_outbox::{Outbox, OutboxExt}; +use opendatafabric::{DatasetHandle, MetadataEvent}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[component(pub)] +#[interface(dyn CommitDatasetEventUseCase)] +pub struct CommitDatasetEventUseCaseImpl { + dataset_repo: Arc, + dataset_action_authorizer: Arc, + outbox: Arc, +} + +impl CommitDatasetEventUseCaseImpl { + pub fn new( + dataset_repo: Arc, + dataset_action_authorizer: Arc, + outbox: Arc, + ) -> Self { + Self { + dataset_repo, + dataset_action_authorizer, + outbox, + } + } +} + +#[async_trait::async_trait] +impl CommitDatasetEventUseCase for CommitDatasetEventUseCaseImpl { + async fn execute( + &self, + dataset_handle: &DatasetHandle, + event: MetadataEvent, + opts: CommitOpts<'_>, + ) -> Result { + self.dataset_action_authorizer + .check_action_allowed(dataset_handle, DatasetAction::Write) + .await?; + + let dataset = self.dataset_repo.get_dataset_by_handle(dataset_handle); + + let commit_result = dataset.commit_event(event, opts).await?; + + if !commit_result.new_upstream_ids.is_empty() { + self.outbox + .post_message( + MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE, + DatasetLifecycleMessage::dependencies_updated( + dataset_handle.id.clone(), + commit_result.new_upstream_ids.clone(), + ), + ) + .await?; + } + + Ok(commit_result) + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/core/src/use_cases/create_dataset_from_snapshot_use_case_impl.rs b/src/infra/core/src/use_cases/create_dataset_from_snapshot_use_case_impl.rs new file mode 100644 index 0000000000..4aab963ecb --- /dev/null +++ b/src/infra/core/src/use_cases/create_dataset_from_snapshot_use_case_impl.rs @@ -0,0 +1,96 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::sync::Arc; + +use dill::{component, interface}; +use kamu_accounts::CurrentAccountSubject; +use kamu_core::{ + CreateDatasetFromSnapshotError, + CreateDatasetFromSnapshotResult, + CreateDatasetFromSnapshotUseCase, + CreateDatasetResult, + DatasetLifecycleMessage, + MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE, +}; +use messaging_outbox::{Outbox, OutboxExt}; +use opendatafabric::DatasetSnapshot; + +use crate::DatasetRepositoryWriter; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[component(pub)] +#[interface(dyn CreateDatasetFromSnapshotUseCase)] +pub struct CreateDatasetFromSnapshotUseCaseImpl { + current_account_subject: Arc, + dataset_repo_writer: Arc, + outbox: Arc, +} + +impl CreateDatasetFromSnapshotUseCaseImpl { + pub fn new( + current_account_subject: Arc, + dataset_repo_writer: Arc, + outbox: Arc, + ) -> Self { + Self { + current_account_subject, + dataset_repo_writer, + outbox, + } + } +} + +#[async_trait::async_trait] +impl CreateDatasetFromSnapshotUseCase for CreateDatasetFromSnapshotUseCaseImpl { + async fn execute( + &self, + snapshot: DatasetSnapshot, + ) -> Result { + let CreateDatasetFromSnapshotResult { + create_dataset_result, + new_upstream_ids, + } = self + .dataset_repo_writer + .create_dataset_from_snapshot(snapshot) + .await?; + + self.outbox + .post_message( + MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE, + DatasetLifecycleMessage::created( + create_dataset_result.dataset_handle.id.clone(), + match self.current_account_subject.as_ref() { + CurrentAccountSubject::Anonymous(_) => { + panic!("Anonymous account cannot create dataset"); + } + CurrentAccountSubject::Logged(l) => l.account_id.clone(), + }, + ), + ) + .await?; + + if !new_upstream_ids.is_empty() { + self.outbox + .post_message( + MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE, + DatasetLifecycleMessage::dependencies_updated( + create_dataset_result.dataset_handle.id.clone(), + new_upstream_ids, + ), + ) + .await?; + } + + Ok(create_dataset_result) + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/core/src/use_cases/create_dataset_use_case_impl.rs b/src/infra/core/src/use_cases/create_dataset_use_case_impl.rs new file mode 100644 index 0000000000..d4ecc76aa6 --- /dev/null +++ b/src/infra/core/src/use_cases/create_dataset_use_case_impl.rs @@ -0,0 +1,81 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::sync::Arc; + +use dill::{component, interface}; +use kamu_accounts::CurrentAccountSubject; +use kamu_core::{ + CreateDatasetError, + CreateDatasetResult, + CreateDatasetUseCase, + DatasetLifecycleMessage, + MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE, +}; +use messaging_outbox::{Outbox, OutboxExt}; +use opendatafabric::{DatasetAlias, MetadataBlockTyped, Seed}; + +use crate::DatasetRepositoryWriter; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[component(pub)] +#[interface(dyn CreateDatasetUseCase)] +pub struct CreateDatasetUseCaseImpl { + current_account_subject: Arc, + dataset_repo_writer: Arc, + outbox: Arc, +} + +impl CreateDatasetUseCaseImpl { + pub fn new( + current_account_subject: Arc, + dataset_repo_writer: Arc, + outbox: Arc, + ) -> Self { + Self { + current_account_subject, + dataset_repo_writer, + outbox, + } + } +} + +#[async_trait::async_trait] +impl CreateDatasetUseCase for CreateDatasetUseCaseImpl { + async fn execute( + &self, + dataset_alias: &DatasetAlias, + seed_block: MetadataBlockTyped, + ) -> Result { + let create_result = self + .dataset_repo_writer + .create_dataset(dataset_alias, seed_block) + .await?; + + self.outbox + .post_message( + MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE, + DatasetLifecycleMessage::created( + create_result.dataset_handle.id.clone(), + match self.current_account_subject.as_ref() { + CurrentAccountSubject::Anonymous(_) => { + panic!("Anonymous account cannot create dataset"); + } + CurrentAccountSubject::Logged(l) => l.account_id.clone(), + }, + ), + ) + .await?; + + Ok(create_result) + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/core/src/use_cases/delete_dataset_use_case_impl.rs b/src/infra/core/src/use_cases/delete_dataset_use_case_impl.rs new file mode 100644 index 0000000000..9bdbfad506 --- /dev/null +++ b/src/infra/core/src/use_cases/delete_dataset_use_case_impl.rs @@ -0,0 +1,135 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::sync::Arc; + +use dill::{component, interface}; +use internal_error::ResultIntoInternal; +use kamu_core::auth::{DatasetAction, DatasetActionAuthorizer}; +use kamu_core::{ + DanglingReferenceError, + DatasetLifecycleMessage, + DatasetRepository, + DeleteDatasetError, + DeleteDatasetUseCase, + DependencyGraphService, + GetDatasetError, + MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE, +}; +use messaging_outbox::{Outbox, OutboxExt}; +use opendatafabric::{DatasetHandle, DatasetRef}; + +use crate::DatasetRepositoryWriter; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[component(pub)] +#[interface(dyn DeleteDatasetUseCase)] +pub struct DeleteDatasetUseCaseImpl { + dataset_repo: Arc, + dataset_repo_writer: Arc, + dataset_action_authorizer: Arc, + dependency_graph_service: Arc, + outbox: Arc, +} + +impl DeleteDatasetUseCaseImpl { + pub fn new( + dataset_repo: Arc, + dataset_repo_writer: Arc, + dataset_action_authorizer: Arc, + dependency_graph_service: Arc, + outbox: Arc, + ) -> Self { + Self { + dataset_repo, + dataset_repo_writer, + dataset_action_authorizer, + dependency_graph_service, + outbox, + } + } + + async fn ensure_no_dangling_references( + &self, + dataset_handle: &DatasetHandle, + ) -> Result<(), DeleteDatasetError> { + use tokio_stream::StreamExt; + let downstream_dataset_ids: Vec<_> = self + .dependency_graph_service + .get_downstream_dependencies(&dataset_handle.id) + .await + .int_err()? + .collect() + .await; + + if !downstream_dataset_ids.is_empty() { + let mut children = Vec::with_capacity(downstream_dataset_ids.len()); + for downstream_dataset_id in downstream_dataset_ids { + let hdl = self + .dataset_repo + .resolve_dataset_ref(&downstream_dataset_id.as_local_ref()) + .await + .int_err()?; + children.push(hdl); + } + + return Err(DanglingReferenceError { + dataset_handle: dataset_handle.clone(), + children, + } + .into()); + } + + Ok(()) + } +} + +#[async_trait::async_trait] +impl DeleteDatasetUseCase for DeleteDatasetUseCaseImpl { + async fn execute_via_ref(&self, dataset_ref: &DatasetRef) -> Result<(), DeleteDatasetError> { + let dataset_handle = match self.dataset_repo.resolve_dataset_ref(dataset_ref).await { + Ok(h) => Ok(h), + Err(GetDatasetError::NotFound(e)) => Err(DeleteDatasetError::NotFound(e)), + Err(GetDatasetError::Internal(e)) => Err(DeleteDatasetError::Internal(e)), + }?; + + self.execute_via_handle(&dataset_handle).await + } + + async fn execute_via_handle( + &self, + dataset_handle: &DatasetHandle, + ) -> Result<(), DeleteDatasetError> { + // Permission check + self.dataset_action_authorizer + .check_action_allowed(dataset_handle, DatasetAction::Write) + .await?; + + // Validate against dangling ref + self.ensure_no_dangling_references(dataset_handle).await?; + + // Do actual delete + self.dataset_repo_writer + .delete_dataset(dataset_handle) + .await?; + + // Notify interested parties + self.outbox + .post_message( + MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE, + DatasetLifecycleMessage::deleted(dataset_handle.id.clone()), + ) + .await?; + + Ok(()) + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/core/src/use_cases/mod.rs b/src/infra/core/src/use_cases/mod.rs new file mode 100644 index 0000000000..5a46a4d4b4 --- /dev/null +++ b/src/infra/core/src/use_cases/mod.rs @@ -0,0 +1,22 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +mod append_dataset_metadata_batch_use_case_impl; +mod commit_dataset_event_use_case_impl; +mod create_dataset_from_snapshot_use_case_impl; +mod create_dataset_use_case_impl; +mod delete_dataset_use_case_impl; +mod rename_dataset_use_case_impl; + +pub use append_dataset_metadata_batch_use_case_impl::*; +pub use commit_dataset_event_use_case_impl::*; +pub use create_dataset_from_snapshot_use_case_impl::*; +pub use create_dataset_use_case_impl::*; +pub use delete_dataset_use_case_impl::*; +pub use rename_dataset_use_case_impl::*; diff --git a/src/infra/core/src/use_cases/rename_dataset_use_case_impl.rs b/src/infra/core/src/use_cases/rename_dataset_use_case_impl.rs new file mode 100644 index 0000000000..1fe796f006 --- /dev/null +++ b/src/infra/core/src/use_cases/rename_dataset_use_case_impl.rs @@ -0,0 +1,68 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::sync::Arc; + +use dill::{component, interface}; +use kamu_core::auth::{DatasetAction, DatasetActionAuthorizer}; +use kamu_core::{DatasetRepository, GetDatasetError, RenameDatasetError, RenameDatasetUseCase}; +use opendatafabric::{DatasetName, DatasetRef}; + +use crate::DatasetRepositoryWriter; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub struct RenameDatasetUseCaseImpl { + dataset_repo: Arc, + dataset_repo_writer: Arc, + dataset_action_authorizer: Arc, +} + +#[component(pub)] +#[interface(dyn RenameDatasetUseCase)] +impl RenameDatasetUseCaseImpl { + pub fn new( + dataset_repo: Arc, + dataset_repo_writer: Arc, + dataset_action_authorizer: Arc, + ) -> Self { + Self { + dataset_repo, + dataset_repo_writer, + dataset_action_authorizer, + } + } +} + +#[async_trait::async_trait] +impl RenameDatasetUseCase for RenameDatasetUseCaseImpl { + async fn execute( + &self, + dataset_ref: &DatasetRef, + new_name: &DatasetName, + ) -> Result<(), RenameDatasetError> { + let dataset_handle = match self.dataset_repo.resolve_dataset_ref(dataset_ref).await { + Ok(h) => Ok(h), + Err(GetDatasetError::NotFound(e)) => Err(RenameDatasetError::NotFound(e)), + Err(GetDatasetError::Internal(e)) => Err(RenameDatasetError::Internal(e)), + }?; + + self.dataset_action_authorizer + .check_action_allowed(&dataset_handle, DatasetAction::Write) + .await?; + + self.dataset_repo_writer + .rename_dataset(&dataset_handle, new_name) + .await?; + + Ok(()) + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/core/src/utils/datasets_filtering.rs b/src/infra/core/src/utils/datasets_filtering.rs index 7a9b736aa7..befe0400d4 100644 --- a/src/infra/core/src/utils/datasets_filtering.rs +++ b/src/infra/core/src/utils/datasets_filtering.rs @@ -11,14 +11,8 @@ use std::pin::Pin; use std::sync::Arc; use futures::{future, StreamExt, TryStreamExt}; -use kamu_core::{ - DatasetRepository, - GetDatasetError, - InternalError, - SearchError, - SearchOptions, - SearchService, -}; +use internal_error::InternalError; +use kamu_core::{DatasetRepository, GetDatasetError, SearchError, SearchOptions, SearchService}; use opendatafabric::{ AccountName, DatasetAliasRemote, diff --git a/src/infra/core/src/utils/ipfs_wrapper.rs b/src/infra/core/src/utils/ipfs_wrapper.rs index 448101fbf9..d71f0f9b48 100644 --- a/src/infra/core/src/utils/ipfs_wrapper.rs +++ b/src/infra/core/src/utils/ipfs_wrapper.rs @@ -9,7 +9,7 @@ use std::path::{Path, PathBuf}; -use kamu_core::*; +use internal_error::{ErrorIntoInternal, InternalError, ResultIntoInternal}; #[derive(Default)] pub struct IpfsClient { diff --git a/src/infra/core/src/utils/s3_context.rs b/src/infra/core/src/utils/s3_context.rs index d5bddda38d..060d4dab46 100644 --- a/src/infra/core/src/utils/s3_context.rs +++ b/src/infra/core/src/utils/s3_context.rs @@ -17,7 +17,7 @@ use aws_sdk_s3::operation::head_object::{HeadObjectError, HeadObjectOutput}; use aws_sdk_s3::operation::put_object::{PutObjectError, PutObjectOutput}; use aws_sdk_s3::types::{CommonPrefix, Delete, ObjectIdentifier}; use aws_sdk_s3::Client; -use kamu_core::*; +use internal_error::{ErrorIntoInternal, InternalError, ResultIntoInternal}; use tokio::io::AsyncRead; use tokio_util::io::ReaderStream; use url::Url; diff --git a/src/infra/core/src/utils/simple_transfer_protocol.rs b/src/infra/core/src/utils/simple_transfer_protocol.rs index 212b783529..0aa70c06e1 100644 --- a/src/infra/core/src/utils/simple_transfer_protocol.rs +++ b/src/infra/core/src/utils/simple_transfer_protocol.rs @@ -11,6 +11,7 @@ use std::pin::Pin; use std::sync::{Arc, Mutex}; use futures::{stream, Future, StreamExt, TryStreamExt}; +use internal_error::ErrorIntoInternal; use kamu_core::sync_service::DatasetNotFoundError; use kamu_core::utils::metadata_chain_comparator::*; use kamu_core::*; diff --git a/src/infra/core/src/verification_service_impl.rs b/src/infra/core/src/verification_service_impl.rs index a7a43d57c6..ef4460ca60 100644 --- a/src/infra/core/src/verification_service_impl.rs +++ b/src/infra/core/src/verification_service_impl.rs @@ -11,6 +11,7 @@ use std::sync::Arc; use dill::*; use futures::TryStreamExt; +use internal_error::{ErrorIntoInternal, ResultIntoInternal}; use kamu_core::*; use opendatafabric::*; @@ -47,10 +48,7 @@ impl VerificationServiceImpl { check_logical_hashes: bool, listener: Arc, ) -> Result<(), VerificationError> { - let dataset = self - .dataset_repo - .get_dataset(&dataset_handle.as_local_ref()) - .await?; + let dataset = self.dataset_repo.get_dataset_by_handle(dataset_handle); let chain = dataset.as_metadata_chain(); @@ -201,10 +199,7 @@ impl VerificationServiceImpl { block_range: (Option, Option), listener: Arc, ) -> Result<(), VerificationError> { - let dataset = self - .dataset_repo - .get_dataset(&dataset_handle.as_local_ref()) - .await?; + let dataset = self.dataset_repo.get_dataset_by_handle(dataset_handle); let chain = dataset.as_metadata_chain(); @@ -277,14 +272,7 @@ impl VerificationService for VerificationServiceImpl { Err(e) => return VerificationResult::err(dataset_handle, e), }; - let dataset = match self - .dataset_repo - .get_dataset(&dataset_handle.as_local_ref()) - .await - { - Ok(v) => v, - Err(e) => return VerificationResult::err(dataset_handle, e), - }; + let dataset = self.dataset_repo.get_dataset_by_handle(&dataset_handle); let dataset_kind = match dataset.get_summary(GetSummaryOpts::default()).await { Ok(summary) => summary.kind, diff --git a/src/infra/core/tests/benches/parallel_simple_transfer_protocol.rs b/src/infra/core/tests/benches/parallel_simple_transfer_protocol.rs index 24f75269e7..e13f770dd5 100644 --- a/src/infra/core/tests/benches/parallel_simple_transfer_protocol.rs +++ b/src/infra/core/tests/benches/parallel_simple_transfer_protocol.rs @@ -13,7 +13,6 @@ use std::sync::Arc; use criterion::{criterion_group, criterion_main, Criterion}; use dill::*; -use event_bus::EventBus; use kamu::domain::*; use kamu::testing::{ DatasetTestHelper, @@ -26,6 +25,7 @@ use kamu::utils::simple_transfer_protocol::ENV_VAR_SIMPLE_PROTOCOL_MAX_PARALLEL_ use kamu::{ DatasetFactoryImpl, DatasetRepositoryLocalFs, + DatasetRepositoryWriter, DependencyGraphServiceInMemory, IpfsGateway, RemoteReposDir, @@ -49,7 +49,6 @@ async fn setup_dataset( let (ipfs_gateway, ipfs_client) = ipfs.unwrap_or_default(); let catalog = dill::CatalogBuilder::new() - .add::() .add::() .add_value(ipfs_gateway) .add_value(ipfs_client) @@ -83,6 +82,7 @@ async fn setup_dataset( .create_dataset_from_snapshot(snapshot) .await .unwrap() + .create_dataset_result .head; append_data_to_dataset( diff --git a/src/infra/core/tests/tests/engine/test_engine_io.rs b/src/infra/core/tests/tests/engine/test_engine_io.rs index e4e49a3eff..b7000a1288 100644 --- a/src/infra/core/tests/tests/engine/test_engine_io.rs +++ b/src/infra/core/tests/tests/engine/test_engine_io.rs @@ -13,7 +13,6 @@ use std::sync::Arc; use container_runtime::ContainerRuntime; use dill::Component; -use event_bus::EventBus; use indoc::indoc; use kamu::domain::*; use kamu::testing::*; @@ -21,10 +20,13 @@ use kamu::*; use kamu_accounts::CurrentAccountSubject; use kamu_datasets_services::DatasetKeyValueServiceSysEnv; use opendatafabric::*; +use time_source::SystemTimeSourceDefault; -async fn test_engine_io_common( +async fn test_engine_io_common< + TDatasetRepo: DatasetRepository + DatasetRepositoryWriter + 'static, +>( object_stores: Vec>, - dataset_repo: Arc, + dataset_repo: Arc, run_info_dir: &Path, cache_dir: &Path, transform: Transform, @@ -154,6 +156,7 @@ async fn test_engine_io_common( .create_dataset_from_snapshot(deriv_snapshot) .await .unwrap() + .create_dataset_result .dataset; let block_hash = match transform_svc @@ -260,9 +263,7 @@ async fn test_engine_io_local_file_mount() { let catalog = dill::CatalogBuilder::new() .add::() - .add::() .add::() - .add::() .add::() .add_value(CurrentAccountSubject::new_test()) .add_builder( @@ -273,7 +274,7 @@ async fn test_engine_io_local_file_mount() { .bind::() .build(); - let dataset_repo = catalog.get_one::().unwrap(); + let dataset_repo = catalog.get_one::().unwrap(); test_engine_io_common( vec![Arc::new(ObjectStoreBuilderLocalFs::new())], @@ -304,9 +305,7 @@ async fn test_engine_io_s3_to_local_file_mount_proxy() { let catalog = dill::CatalogBuilder::new() .add::() - .add::() .add::() - .add::() .add_value(CurrentAccountSubject::new_test()) .add_builder( DatasetRepositoryS3::builder() @@ -316,7 +315,7 @@ async fn test_engine_io_s3_to_local_file_mount_proxy() { .bind::() .build(); - let dataset_repo = catalog.get_one::().unwrap(); + let dataset_repo = catalog.get_one::().unwrap(); test_engine_io_common( vec![ diff --git a/src/infra/core/tests/tests/engine/test_engine_transform.rs b/src/infra/core/tests/tests/engine/test_engine_transform.rs index 148eb7bb34..393676bede 100644 --- a/src/infra/core/tests/tests/engine/test_engine_transform.rs +++ b/src/infra/core/tests/tests/engine/test_engine_transform.rs @@ -16,7 +16,6 @@ use container_runtime::*; use datafusion::arrow::record_batch::RecordBatch; use datafusion::common::DFSchema; use dill::Component; -use event_bus::EventBus; use futures::StreamExt; use indoc::indoc; use kamu::domain::*; @@ -25,6 +24,7 @@ use kamu::*; use kamu_accounts::CurrentAccountSubject; use kamu_datasets_services::DatasetKeyValueServiceSysEnv; use opendatafabric::*; +use time_source::{SystemTimeSource, SystemTimeSourceStub}; struct DatasetHelper { dataset: Arc, @@ -225,9 +225,7 @@ async fn test_transform_common(transform: Transform, test_retractions: bool) { .add_value(RunInfoDir::new(run_info_dir)) .add_value(CacheDir::new(cache_dir)) .add::() - .add::() .add::() - .add::() .add_value(CurrentAccountSubject::new_test()) .add_builder( DatasetRepositoryLocalFs::builder() @@ -253,7 +251,7 @@ async fn test_transform_common(transform: Transform, test_retractions: bool) { .bind::() .build(); - let dataset_repo = catalog.get_one::().unwrap(); + let dataset_repo = catalog.get_one::().unwrap(); let ingest_svc = catalog.get_one::().unwrap(); let transform_svc = catalog.get_one::().unwrap(); @@ -335,6 +333,7 @@ async fn test_transform_common(transform: Transform, test_retractions: bool) { .create_dataset_from_snapshot(deriv_snapshot) .await .unwrap() + .create_dataset_result .dataset; let deriv_helper = DatasetHelper::new(dataset.clone(), tempdir.path()); diff --git a/src/infra/core/tests/tests/ingest/test_polling_ingest.rs b/src/infra/core/tests/tests/ingest/test_polling_ingest.rs index dc4f6f59a2..44cb5b866d 100644 --- a/src/infra/core/tests/tests/ingest/test_polling_ingest.rs +++ b/src/infra/core/tests/tests/ingest/test_polling_ingest.rs @@ -14,7 +14,6 @@ use chrono::{TimeZone, Utc}; use container_runtime::*; use datafusion::prelude::*; use dill::Component; -use event_bus::EventBus; use indoc::indoc; use kamu::domain::*; use kamu::testing::*; @@ -23,6 +22,7 @@ use kamu_accounts::CurrentAccountSubject; use kamu_datasets_services::DatasetKeyValueServiceSysEnv; use opendatafabric::*; use tempfile::TempDir; +use time_source::{SystemTimeSource, SystemTimeSourceStub}; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -1029,8 +1029,9 @@ async fn test_ingest_polling_preprocess_with_flink() { async fn test_ingest_checks_auth() { let harness = IngestTestHarness::new_with_authorizer( MockDatasetActionAuthorizer::new().expect_check_write_dataset( - DatasetAlias::new(None, DatasetName::new_unchecked("foo.bar")), + &DatasetAlias::new(None, DatasetName::new_unchecked("foo.bar")), 1, + true, ), ); let src_path = harness.temp_dir.path().join("data.json"); @@ -1070,7 +1071,7 @@ async fn test_ingest_checks_auth() { struct IngestTestHarness { temp_dir: TempDir, - dataset_repo: Arc, + dataset_repo: Arc, ingest_svc: Arc, time_source: Arc, ctx: SessionContext, @@ -1099,8 +1100,6 @@ impl IngestTestHarness { .add::() .add::() .add::() - .add::() - .add::() .add_value(CurrentAccountSubject::new_test()) .add_value(dataset_action_authorizer) .bind::() @@ -1122,7 +1121,7 @@ impl IngestTestHarness { .add::() .build(); - let dataset_repo = catalog.get_one::().unwrap(); + let dataset_repo = catalog.get_one::().unwrap(); let ingest_svc = catalog.get_one::().unwrap(); let time_source = catalog.get_one::().unwrap(); @@ -1158,7 +1157,7 @@ impl IngestTestHarness { async fn dataset_data_helper(&self, dataset_alias: &DatasetAlias) -> DatasetDataHelper { let dataset = self .dataset_repo - .get_dataset(&dataset_alias.as_local_ref()) + .find_dataset_by_ref(&dataset_alias.as_local_ref()) .await .unwrap(); diff --git a/src/infra/core/tests/tests/ingest/test_push_ingest.rs b/src/infra/core/tests/tests/ingest/test_push_ingest.rs index 88bacbdfff..06bebb88e9 100644 --- a/src/infra/core/tests/tests/ingest/test_push_ingest.rs +++ b/src/infra/core/tests/tests/ingest/test_push_ingest.rs @@ -12,7 +12,6 @@ use std::sync::Arc; use chrono::{TimeZone, Utc}; use datafusion::prelude::*; use dill::Component; -use event_bus::EventBus; use indoc::indoc; use kamu::domain::*; use kamu::testing::*; @@ -20,6 +19,7 @@ use kamu::*; use kamu_accounts::CurrentAccountSubject; use opendatafabric::*; use tempfile::TempDir; +use time_source::{SystemTimeSource, SystemTimeSourceStub}; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -473,19 +473,13 @@ async fn test_ingest_push_schema_stability() { struct IngestTestHarness { temp_dir: TempDir, - dataset_repo: Arc, + dataset_repo: Arc, push_ingest_svc: Arc, ctx: SessionContext, } impl IngestTestHarness { fn new() -> Self { - Self::new_with_authorizer(kamu_core::auth::AlwaysHappyDatasetActionAuthorizer::new()) - } - - fn new_with_authorizer( - dataset_action_authorizer: TDatasetAuthorizer, - ) -> Self { let temp_dir = tempfile::tempdir().unwrap(); let run_info_dir = temp_dir.path().join("run"); let cache_dir = temp_dir.path().join("cache"); @@ -497,11 +491,8 @@ impl IngestTestHarness { let catalog = dill::CatalogBuilder::new() .add_value(RunInfoDir::new(run_info_dir)) .add_value(CacheDir::new(cache_dir)) - .add::() - .add::() .add_value(CurrentAccountSubject::new_test()) - .add_value(dataset_action_authorizer) - .bind::() + .add::() .add_builder( DatasetRepositoryLocalFs::builder() .with_root(datasets_dir) @@ -537,7 +528,7 @@ impl IngestTestHarness { async fn dataset_data_helper(&self, dataset_alias: &DatasetAlias) -> DatasetDataHelper { let dataset = self .dataset_repo - .get_dataset(&dataset_alias.as_local_ref()) + .find_dataset_by_ref(&dataset_alias.as_local_ref()) .await .unwrap(); diff --git a/src/infra/core/tests/tests/ingest/test_writer.rs b/src/infra/core/tests/tests/ingest/test_writer.rs index 18583b4edf..85e2591886 100644 --- a/src/infra/core/tests/tests/ingest/test_writer.rs +++ b/src/infra/core/tests/tests/ingest/test_writer.rs @@ -15,16 +15,16 @@ use chrono::{DateTime, TimeZone, Utc}; use datafusion::arrow::datatypes::SchemaRef; use datafusion::prelude::*; use dill::Component; -use event_bus::EventBus; use indoc::indoc; use kamu::testing::MetadataFactory; -use kamu::{DatasetRepositoryLocalFs, DependencyGraphServiceInMemory}; +use kamu::{DatasetRepositoryLocalFs, DatasetRepositoryWriter}; use kamu_accounts::CurrentAccountSubject; use kamu_core::*; use kamu_data_utils::testing::{assert_data_eq, assert_schema_eq}; use kamu_ingest_datafusion::*; use odf::{AsTypedBlock, DatasetAlias}; use opendatafabric as odf; +use time_source::SystemTimeSourceDefault; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // TODO: This test belongs in kamu-ingest-datafusion crate. @@ -941,9 +941,6 @@ impl Harness { let catalog = dill::CatalogBuilder::new() .add::() - .add::() - .add::() - .add::() .add_value(CurrentAccountSubject::new_test()) .add_builder( DatasetRepositoryLocalFs::builder() @@ -953,7 +950,7 @@ impl Harness { .bind::() .build(); - let dataset_repo = catalog.get_one::().unwrap(); + let dataset_repo = catalog.get_one::().unwrap(); let dataset = dataset_repo .create_dataset( diff --git a/src/infra/core/tests/tests/mod.rs b/src/infra/core/tests/tests/mod.rs index f652690b7e..a1ea4bd2dc 100644 --- a/src/infra/core/tests/tests/mod.rs +++ b/src/infra/core/tests/tests/mod.rs @@ -27,3 +27,4 @@ mod test_setup; mod test_sync_service_impl; mod test_transform_service_impl; mod test_verification_service_impl; +mod use_cases; diff --git a/src/infra/core/tests/tests/repos/test_dataset_impl.rs b/src/infra/core/tests/tests/repos/test_dataset_impl.rs index 940ee51801..1fb4f26008 100644 --- a/src/infra/core/tests/tests/repos/test_dataset_impl.rs +++ b/src/infra/core/tests/tests/repos/test_dataset_impl.rs @@ -9,7 +9,6 @@ use std::assert_matches::assert_matches; -use event_bus::EventBus; use kamu::domain::*; use kamu::testing::*; use kamu::*; @@ -20,8 +19,7 @@ async fn test_summary_updates() { let tmp_dir = tempfile::tempdir().unwrap(); let layout = DatasetLayout::create(tmp_dir.path()).unwrap(); - let catalog = dill::CatalogBuilder::new().add::().build(); - let ds = DatasetFactoryImpl::get_local_fs(layout, catalog.get_one().unwrap()); + let ds = DatasetFactoryImpl::get_local_fs(layout); assert_matches!( ds.get_summary(GetSummaryOpts::default()).await, diff --git a/src/infra/core/tests/tests/repos/test_dataset_repository_local_fs.rs b/src/infra/core/tests/tests/repos/test_dataset_repository_local_fs.rs index 268d6fc9a9..750ed6bfb1 100644 --- a/src/infra/core/tests/tests/repos/test_dataset_repository_local_fs.rs +++ b/src/infra/core/tests/tests/repos/test_dataset_repository_local_fs.rs @@ -10,67 +10,58 @@ use std::sync::Arc; use dill::Component; -use domain::{DatasetRepository, DependencyGraphService, SystemTimeSourceDefault}; -use event_bus::EventBus; -use kamu::testing::MockDatasetActionAuthorizer; +use domain::DatasetRepository; use kamu::*; use kamu_accounts::{CurrentAccountSubject, DEFAULT_ACCOUNT_NAME}; -use kamu_core::auth; +use kamu_core::CreateDatasetFromSnapshotUseCase; +use messaging_outbox::{Outbox, OutboxImmediateImpl}; use tempfile::TempDir; +use time_source::SystemTimeSourceDefault; use super::test_dataset_repository_shared; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// struct LocalFsRepoHarness { - catalog: dill::Catalog, - dataset_repo: Arc, + _catalog: dill::Catalog, + dataset_repo: Arc, + create_dataset_from_snapshot: Arc, } impl LocalFsRepoHarness { - pub fn create( - tempdir: &TempDir, - dataset_action_authorizer: TDatasetActionAuthorizer, - multi_tenant: bool, - ) -> Self { + pub fn create(tempdir: &TempDir, multi_tenant: bool) -> Self { let datasets_dir = tempdir.path().join("datasets"); std::fs::create_dir(&datasets_dir).unwrap(); - let catalog = dill::CatalogBuilder::new() - .add::() - .add::() - .add::() + let mut b = dill::CatalogBuilder::new(); + b.add::() + .add_builder( + messaging_outbox::OutboxImmediateImpl::builder() + .with_consumer_filter(messaging_outbox::ConsumerFilter::AllConsumers), + ) + .bind::() .add_value(CurrentAccountSubject::new_test()) - .add_value(dataset_action_authorizer) - .bind::() .add_builder( DatasetRepositoryLocalFs::builder() .with_root(datasets_dir) .with_multi_tenant(multi_tenant), ) .bind::() - .build(); + .bind::() + .add::(); + + let catalog = b.build(); let dataset_repo = catalog.get_one().unwrap(); + let create_dataset_from_snapshot = catalog.get_one().unwrap(); + Self { - catalog, + _catalog: catalog, dataset_repo, + create_dataset_from_snapshot, } } - - pub async fn dependencies_eager_initialization(&self) { - let dependency_graph_service = self - .catalog - .get_one::() - .unwrap(); - dependency_graph_service - .eager_initialization(&DependencyGraphRepositoryInMemory::new( - self.dataset_repo.clone(), - )) - .await - .unwrap(); - } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -78,11 +69,7 @@ impl LocalFsRepoHarness { #[tokio::test] async fn test_create_dataset() { let tempdir = tempfile::tempdir().unwrap(); - let harness = LocalFsRepoHarness::create( - &tempdir, - auth::AlwaysHappyDatasetActionAuthorizer::new(), - false, - ); + let harness = LocalFsRepoHarness::create(&tempdir, false); test_dataset_repository_shared::test_create_dataset(harness.dataset_repo.as_ref(), None).await; } @@ -92,11 +79,7 @@ async fn test_create_dataset() { #[tokio::test] async fn test_create_dataset_multi_tenant() { let tempdir = tempfile::tempdir().unwrap(); - let harness = LocalFsRepoHarness::create( - &tempdir, - auth::AlwaysHappyDatasetActionAuthorizer::new(), - true, - ); + let harness = LocalFsRepoHarness::create(&tempdir, true); test_dataset_repository_shared::test_create_dataset( harness.dataset_repo.as_ref(), @@ -110,11 +93,7 @@ async fn test_create_dataset_multi_tenant() { #[tokio::test] async fn test_create_dataset_same_name_multiple_tenants() { let tempdir = tempfile::tempdir().unwrap(); - let harness = LocalFsRepoHarness::create( - &tempdir, - auth::AlwaysHappyDatasetActionAuthorizer::new(), - true, - ); + let harness = LocalFsRepoHarness::create(&tempdir, true); test_dataset_repository_shared::test_create_dataset_same_name_multiple_tenants( harness.dataset_repo.as_ref(), @@ -127,11 +106,7 @@ async fn test_create_dataset_same_name_multiple_tenants() { #[tokio::test] async fn test_create_dataset_from_snapshot() { let tempdir = tempfile::tempdir().unwrap(); - let harness = LocalFsRepoHarness::create( - &tempdir, - auth::AlwaysHappyDatasetActionAuthorizer::new(), - false, - ); + let harness = LocalFsRepoHarness::create(&tempdir, false); test_dataset_repository_shared::test_create_dataset_from_snapshot( harness.dataset_repo.as_ref(), @@ -145,11 +120,7 @@ async fn test_create_dataset_from_snapshot() { #[tokio::test] async fn test_create_dataset_from_snapshot_multi_tenant() { let tempdir = tempfile::tempdir().unwrap(); - let harness = LocalFsRepoHarness::create( - &tempdir, - auth::AlwaysHappyDatasetActionAuthorizer::new(), - true, - ); + let harness = LocalFsRepoHarness::create(&tempdir, true); test_dataset_repository_shared::test_create_dataset_from_snapshot( harness.dataset_repo.as_ref(), @@ -163,11 +134,7 @@ async fn test_create_dataset_from_snapshot_multi_tenant() { #[tokio::test] async fn test_rename_dataset() { let tempdir = tempfile::tempdir().unwrap(); - let harness = LocalFsRepoHarness::create( - &tempdir, - MockDatasetActionAuthorizer::new().expect_check_write_a_dataset(1), - false, - ); + let harness = LocalFsRepoHarness::create(&tempdir, false); test_dataset_repository_shared::test_rename_dataset(harness.dataset_repo.as_ref(), None).await; } @@ -177,11 +144,7 @@ async fn test_rename_dataset() { #[tokio::test] async fn test_rename_dataset_multi_tenant() { let tempdir = tempfile::tempdir().unwrap(); - let harness = LocalFsRepoHarness::create( - &tempdir, - MockDatasetActionAuthorizer::new().expect_check_write_a_dataset(1), - true, - ); + let harness = LocalFsRepoHarness::create(&tempdir, true); test_dataset_repository_shared::test_rename_dataset( harness.dataset_repo.as_ref(), @@ -195,11 +158,7 @@ async fn test_rename_dataset_multi_tenant() { #[tokio::test] async fn test_rename_dataset_same_name_multiple_tenants() { let tempdir = tempfile::tempdir().unwrap(); - let harness = LocalFsRepoHarness::create( - &tempdir, - MockDatasetActionAuthorizer::new().expect_check_write_a_dataset(1), - true, - ); + let harness = LocalFsRepoHarness::create(&tempdir, true); test_dataset_repository_shared::test_rename_dataset_same_name_multiple_tenants( harness.dataset_repo.as_ref(), @@ -210,13 +169,13 @@ async fn test_rename_dataset_same_name_multiple_tenants() { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[tokio::test] -async fn test_rename_unauthorized() { +async fn test_delete_dataset() { let tempdir = tempfile::tempdir().unwrap(); - let harness = - LocalFsRepoHarness::create(&tempdir, MockDatasetActionAuthorizer::denying(), true); + let harness = LocalFsRepoHarness::create(&tempdir, false); - test_dataset_repository_shared::test_rename_dataset_unauthorized( + test_dataset_repository_shared::test_delete_dataset( harness.dataset_repo.as_ref(), + harness.create_dataset_from_snapshot.as_ref(), None, ) .await; @@ -224,33 +183,14 @@ async fn test_rename_unauthorized() { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -#[tokio::test] -async fn test_delete_dataset() { - let tempdir = tempfile::tempdir().unwrap(); - let harness = LocalFsRepoHarness::create( - &tempdir, - auth::AlwaysHappyDatasetActionAuthorizer::new(), - false, - ); - harness.dependencies_eager_initialization().await; - - test_dataset_repository_shared::test_delete_dataset(harness.dataset_repo.as_ref(), None).await; -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - #[tokio::test] async fn test_delete_dataset_multi_tenant() { let tempdir = tempfile::tempdir().unwrap(); - let harness = LocalFsRepoHarness::create( - &tempdir, - auth::AlwaysHappyDatasetActionAuthorizer::new(), - true, - ); - harness.dependencies_eager_initialization().await; + let harness = LocalFsRepoHarness::create(&tempdir, true); test_dataset_repository_shared::test_delete_dataset( harness.dataset_repo.as_ref(), + harness.create_dataset_from_snapshot.as_ref(), Some(DEFAULT_ACCOUNT_NAME.clone()), ) .await; @@ -258,30 +198,10 @@ async fn test_delete_dataset_multi_tenant() { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -#[tokio::test] -async fn test_delete_unauthorized() { - let tempdir = tempfile::tempdir().unwrap(); - let harness = - LocalFsRepoHarness::create(&tempdir, MockDatasetActionAuthorizer::denying(), true); - harness.dependencies_eager_initialization().await; - - test_dataset_repository_shared::test_delete_dataset_unauthorized( - harness.dataset_repo.as_ref(), - None, - ) - .await; -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - #[tokio::test] async fn test_iterate_datasets() { let tempdir = tempfile::tempdir().unwrap(); - let harness = LocalFsRepoHarness::create( - &tempdir, - auth::AlwaysHappyDatasetActionAuthorizer::new(), - false, - ); + let harness = LocalFsRepoHarness::create(&tempdir, false); test_dataset_repository_shared::test_iterate_datasets(harness.dataset_repo.as_ref()).await; } @@ -291,11 +211,7 @@ async fn test_iterate_datasets() { #[tokio::test] async fn test_iterate_datasets_multi_tenant() { let tempdir = tempfile::tempdir().unwrap(); - let harness = LocalFsRepoHarness::create( - &tempdir, - auth::AlwaysHappyDatasetActionAuthorizer::new(), - true, - ); + let harness = LocalFsRepoHarness::create(&tempdir, true); test_dataset_repository_shared::test_iterate_datasets_multi_tenant( harness.dataset_repo.as_ref(), @@ -308,11 +224,7 @@ async fn test_iterate_datasets_multi_tenant() { #[tokio::test] async fn test_create_and_get_case_insensetive_dataset() { let tempdir = tempfile::tempdir().unwrap(); - let harness = LocalFsRepoHarness::create( - &tempdir, - auth::AlwaysHappyDatasetActionAuthorizer::new(), - false, - ); + let harness = LocalFsRepoHarness::create(&tempdir, false); test_dataset_repository_shared::test_create_and_get_case_insensetive_dataset( harness.dataset_repo.as_ref(), @@ -326,11 +238,7 @@ async fn test_create_and_get_case_insensetive_dataset() { #[tokio::test] async fn test_create_and_get_case_insensetive_dataset_multi_tenant() { let tempdir = tempfile::tempdir().unwrap(); - let harness = LocalFsRepoHarness::create( - &tempdir, - auth::AlwaysHappyDatasetActionAuthorizer::new(), - true, - ); + let harness = LocalFsRepoHarness::create(&tempdir, true); test_dataset_repository_shared::test_create_and_get_case_insensetive_dataset( harness.dataset_repo.as_ref(), diff --git a/src/infra/core/tests/tests/repos/test_dataset_repository_s3.rs b/src/infra/core/tests/tests/repos/test_dataset_repository_s3.rs index 924b4fb5f3..ca06b98ffb 100644 --- a/src/infra/core/tests/tests/repos/test_dataset_repository_s3.rs +++ b/src/infra/core/tests/tests/repos/test_dataset_repository_s3.rs @@ -9,80 +9,68 @@ use std::sync::Arc; -use dill::Component; -use event_bus::EventBus; -use kamu::domain::auth; -use kamu::testing::{LocalS3Server, MockDatasetActionAuthorizer}; +use dill::*; +use kamu::testing::LocalS3Server; use kamu::utils::s3_context::S3Context; use kamu::{ + CreateDatasetFromSnapshotUseCaseImpl, DatasetRepositoryS3, - DependencyGraphRepositoryInMemory, - DependencyGraphServiceInMemory, + DatasetRepositoryWriter, S3RegistryCache, }; use kamu_accounts::{CurrentAccountSubject, DEFAULT_ACCOUNT_NAME}; -use kamu_core::{DatasetRepository, DependencyGraphService, SystemTimeSourceDefault}; +use kamu_core::{CreateDatasetFromSnapshotUseCase, DatasetRepository}; +use messaging_outbox::{Outbox, OutboxImmediateImpl}; +use time_source::SystemTimeSourceDefault; use super::test_dataset_repository_shared; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// struct S3RepoHarness { - catalog: dill::Catalog, - dataset_repo: Arc, + _catalog: dill::Catalog, + dataset_repo: Arc, + create_dataset_from_snapshot: Arc, } impl S3RepoHarness { - pub async fn create( - s3: &LocalS3Server, - dataset_action_authorizer: TDatasetActionAuthorizer, - multi_tenant: bool, - registry_caching: bool, - ) -> Self { + pub async fn create(s3: &LocalS3Server, multi_tenant: bool, registry_caching: bool) -> Self { let s3_context = S3Context::from_url(&s3.url).await; - let mut catalog = dill::CatalogBuilder::new(); + let mut b = dill::CatalogBuilder::new(); - catalog - .add::() - .add::() - .add::() + b.add::() + .add_builder( + messaging_outbox::OutboxImmediateImpl::builder() + .with_consumer_filter(messaging_outbox::ConsumerFilter::AllConsumers), + ) + .bind::() .add_value(CurrentAccountSubject::new_test()) - .add_value(dataset_action_authorizer) - .bind::() .add_builder( DatasetRepositoryS3::builder() .with_s3_context(s3_context) .with_multi_tenant(multi_tenant), ) - .bind::(); + .bind::() + .bind::() + .add::(); if registry_caching { - catalog.add::(); + b.add::(); } - let catalog = catalog.build(); + let catalog = b.build(); let dataset_repo = catalog.get_one().unwrap(); + let create_dataset_from_snapshot = catalog.get_one().unwrap(); + Self { - catalog, + _catalog: catalog, dataset_repo, + create_dataset_from_snapshot, } } - - pub async fn dependencies_eager_initialization(&self) { - let dependency_graph_service = self - .catalog - .get_one::() - .unwrap(); - dependency_graph_service - .eager_initialization(&DependencyGraphRepositoryInMemory::new( - self.dataset_repo.clone(), - )) - .await - .unwrap(); - } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -91,13 +79,7 @@ impl S3RepoHarness { #[tokio::test] async fn test_create_dataset() { let s3 = LocalS3Server::new().await; - let harness = S3RepoHarness::create( - &s3, - auth::AlwaysHappyDatasetActionAuthorizer::new(), - false, - false, - ) - .await; + let harness = S3RepoHarness::create(&s3, false, false).await; test_dataset_repository_shared::test_create_dataset(harness.dataset_repo.as_ref(), None).await; } @@ -108,13 +90,7 @@ async fn test_create_dataset() { #[tokio::test] async fn test_create_dataset_multi_tenant() { let s3 = LocalS3Server::new().await; - let harness = S3RepoHarness::create( - &s3, - auth::AlwaysHappyDatasetActionAuthorizer::new(), - true, - false, - ) - .await; + let harness = S3RepoHarness::create(&s3, true, false).await; test_dataset_repository_shared::test_create_dataset( harness.dataset_repo.as_ref(), @@ -129,13 +105,7 @@ async fn test_create_dataset_multi_tenant() { #[tokio::test] async fn test_create_dataset_multi_tenant_with_caching() { let s3 = LocalS3Server::new().await; - let harness = S3RepoHarness::create( - &s3, - auth::AlwaysHappyDatasetActionAuthorizer::new(), - true, - true, - ) - .await; + let harness = S3RepoHarness::create(&s3, true, true).await; test_dataset_repository_shared::test_create_dataset( harness.dataset_repo.as_ref(), @@ -150,13 +120,7 @@ async fn test_create_dataset_multi_tenant_with_caching() { #[tokio::test] async fn test_create_dataset_same_name_multiple_tenants() { let s3 = LocalS3Server::new().await; - let harness = S3RepoHarness::create( - &s3, - auth::AlwaysHappyDatasetActionAuthorizer::new(), - true, - false, - ) - .await; + let harness = S3RepoHarness::create(&s3, true, false).await; test_dataset_repository_shared::test_create_dataset_same_name_multiple_tenants( harness.dataset_repo.as_ref(), @@ -171,13 +135,7 @@ async fn test_create_dataset_same_name_multiple_tenants() { #[tokio::test] async fn test_create_dataset_from_snapshot() { let s3 = LocalS3Server::new().await; - let harness = S3RepoHarness::create( - &s3, - auth::AlwaysHappyDatasetActionAuthorizer::new(), - false, - false, - ) - .await; + let harness = S3RepoHarness::create(&s3, false, false).await; test_dataset_repository_shared::test_create_dataset_from_snapshot( harness.dataset_repo.as_ref(), @@ -192,13 +150,7 @@ async fn test_create_dataset_from_snapshot() { #[tokio::test] async fn test_create_dataset_from_snapshot_multi_tenant() { let s3 = LocalS3Server::new().await; - let harness = S3RepoHarness::create( - &s3, - auth::AlwaysHappyDatasetActionAuthorizer::new(), - true, - false, - ) - .await; + let harness = S3RepoHarness::create(&s3, true, false).await; test_dataset_repository_shared::test_create_dataset_from_snapshot( harness.dataset_repo.as_ref(), @@ -213,13 +165,7 @@ async fn test_create_dataset_from_snapshot_multi_tenant() { #[tokio::test] async fn test_rename_dataset() { let s3 = LocalS3Server::new().await; - let harness = S3RepoHarness::create( - &s3, - MockDatasetActionAuthorizer::new().expect_check_write_a_dataset(1), - false, - false, - ) - .await; + let harness = S3RepoHarness::create(&s3, false, false).await; test_dataset_repository_shared::test_rename_dataset(harness.dataset_repo.as_ref(), None).await; } @@ -230,13 +176,7 @@ async fn test_rename_dataset() { #[tokio::test] async fn test_rename_dataset_multi_tenant() { let s3 = LocalS3Server::new().await; - let harness = S3RepoHarness::create( - &s3, - MockDatasetActionAuthorizer::new().expect_check_write_a_dataset(1), - true, - false, - ) - .await; + let harness = S3RepoHarness::create(&s3, true, false).await; test_dataset_repository_shared::test_rename_dataset( harness.dataset_repo.as_ref(), @@ -251,13 +191,7 @@ async fn test_rename_dataset_multi_tenant() { #[tokio::test] async fn test_rename_dataset_multi_tenant_with_caching() { let s3 = LocalS3Server::new().await; - let harness = S3RepoHarness::create( - &s3, - MockDatasetActionAuthorizer::new().expect_check_write_a_dataset(1), - true, - true, - ) - .await; + let harness = S3RepoHarness::create(&s3, true, true).await; test_dataset_repository_shared::test_rename_dataset( harness.dataset_repo.as_ref(), @@ -272,13 +206,7 @@ async fn test_rename_dataset_multi_tenant_with_caching() { #[tokio::test] async fn test_rename_dataset_same_name_multiple_tenants() { let s3 = LocalS3Server::new().await; - let harness = S3RepoHarness::create( - &s3, - MockDatasetActionAuthorizer::new().expect_check_write_a_dataset(1), - true, - false, - ) - .await; + let harness = S3RepoHarness::create(&s3, true, false).await; test_dataset_repository_shared::test_rename_dataset_same_name_multiple_tenants( harness.dataset_repo.as_ref(), @@ -290,13 +218,13 @@ async fn test_rename_dataset_same_name_multiple_tenants() { #[test_group::group(containerized)] #[tokio::test] -async fn test_rename_unauthorized() { +async fn test_delete_dataset() { let s3 = LocalS3Server::new().await; - let harness = - S3RepoHarness::create(&s3, MockDatasetActionAuthorizer::denying(), true, false).await; + let harness = S3RepoHarness::create(&s3, false, false).await; - test_dataset_repository_shared::test_rename_dataset_unauthorized( + test_dataset_repository_shared::test_delete_dataset( harness.dataset_repo.as_ref(), + harness.create_dataset_from_snapshot.as_ref(), None, ) .await; @@ -304,39 +232,15 @@ async fn test_rename_unauthorized() { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -#[test_group::group(containerized)] -#[tokio::test] -async fn test_delete_dataset() { - let s3 = LocalS3Server::new().await; - let harness = S3RepoHarness::create( - &s3, - auth::AlwaysHappyDatasetActionAuthorizer::new(), - false, - false, - ) - .await; - harness.dependencies_eager_initialization().await; - - test_dataset_repository_shared::test_delete_dataset(harness.dataset_repo.as_ref(), None).await; -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - #[test_group::group(containerized)] #[tokio::test] async fn test_delete_dataset_multi_tenant() { let s3 = LocalS3Server::new().await; - let harness = S3RepoHarness::create( - &s3, - auth::AlwaysHappyDatasetActionAuthorizer::new(), - true, - false, - ) - .await; - harness.dependencies_eager_initialization().await; + let harness = S3RepoHarness::create(&s3, true, false).await; test_dataset_repository_shared::test_delete_dataset( harness.dataset_repo.as_ref(), + harness.create_dataset_from_snapshot.as_ref(), Some(DEFAULT_ACCOUNT_NAME.clone()), ) .await; @@ -344,34 +248,11 @@ async fn test_delete_dataset_multi_tenant() { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -#[test_group::group(containerized)] -#[tokio::test] -async fn test_delete_unauthorized() { - let s3: LocalS3Server = LocalS3Server::new().await; - let harness = - S3RepoHarness::create(&s3, MockDatasetActionAuthorizer::denying(), true, false).await; - harness.dependencies_eager_initialization().await; - - test_dataset_repository_shared::test_delete_dataset_unauthorized( - harness.dataset_repo.as_ref(), - None, - ) - .await; -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - #[test_group::group(containerized)] #[tokio::test] async fn test_iterate_datasets() { let s3 = LocalS3Server::new().await; - let harness = S3RepoHarness::create( - &s3, - auth::AlwaysHappyDatasetActionAuthorizer::new(), - false, - false, - ) - .await; + let harness = S3RepoHarness::create(&s3, false, false).await; test_dataset_repository_shared::test_iterate_datasets(harness.dataset_repo.as_ref()).await; } @@ -382,13 +263,7 @@ async fn test_iterate_datasets() { #[tokio::test] async fn test_iterate_datasets_multi_tenant() { let s3 = LocalS3Server::new().await; - let harness = S3RepoHarness::create( - &s3, - auth::AlwaysHappyDatasetActionAuthorizer::new(), - true, - false, - ) - .await; + let harness = S3RepoHarness::create(&s3, true, false).await; test_dataset_repository_shared::test_iterate_datasets_multi_tenant( harness.dataset_repo.as_ref(), @@ -402,13 +277,7 @@ async fn test_iterate_datasets_multi_tenant() { #[tokio::test] async fn test_create_and_get_case_insensetive_dataset() { let s3 = LocalS3Server::new().await; - let harness = S3RepoHarness::create( - &s3, - auth::AlwaysHappyDatasetActionAuthorizer::new(), - false, - false, - ) - .await; + let harness = S3RepoHarness::create(&s3, false, false).await; test_dataset_repository_shared::test_create_and_get_case_insensetive_dataset( harness.dataset_repo.as_ref(), @@ -423,13 +292,7 @@ async fn test_create_and_get_case_insensetive_dataset() { #[tokio::test] async fn test_create_and_get_case_insensetive_dataset_multi_tenant() { let s3 = LocalS3Server::new().await; - let harness = S3RepoHarness::create( - &s3, - auth::AlwaysHappyDatasetActionAuthorizer::new(), - true, - false, - ) - .await; + let harness = S3RepoHarness::create(&s3, true, false).await; test_dataset_repository_shared::test_create_and_get_case_insensetive_dataset( harness.dataset_repo.as_ref(), diff --git a/src/infra/core/tests/tests/repos/test_dataset_repository_shared.rs b/src/infra/core/tests/tests/repos/test_dataset_repository_shared.rs index b056fab286..355225b12c 100644 --- a/src/infra/core/tests/tests/repos/test_dataset_repository_shared.rs +++ b/src/infra/core/tests/tests/repos/test_dataset_repository_shared.rs @@ -12,16 +12,22 @@ use std::assert_matches::assert_matches; use itertools::Itertools; use kamu::domain::*; use kamu::testing::MetadataFactory; +use kamu::DatasetRepositoryWriter; use kamu_accounts::DEFAULT_ACCOUNT_NAME; use opendatafabric::*; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_create_dataset(repo: &dyn DatasetRepository, account_name: Option) { +pub async fn test_create_dataset< + TDatasetRepository: DatasetRepository + DatasetRepositoryWriter, +>( + repo: &TDatasetRepository, + account_name: Option, +) { let dataset_alias = DatasetAlias::new(account_name, DatasetName::new_unchecked("foo")); assert_matches!( - repo.get_dataset(&dataset_alias.as_local_ref()) + repo.find_dataset_by_ref(&dataset_alias.as_local_ref()) .await .err() .unwrap(), @@ -41,7 +47,7 @@ pub async fn test_create_dataset(repo: &dyn DatasetRepository, account_name: Opt // We should see the dataset assert!(repo - .get_dataset(&dataset_alias.as_local_ref()) + .find_dataset_by_ref(&dataset_alias.as_local_ref()) .await .is_ok()); @@ -62,15 +68,17 @@ pub async fn test_create_dataset(repo: &dyn DatasetRepository, account_name: Opt //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_create_and_get_case_insensetive_dataset( - repo: &dyn DatasetRepository, +pub async fn test_create_and_get_case_insensetive_dataset< + TDatasetRepository: DatasetRepository + DatasetRepositoryWriter, +>( + repo: &TDatasetRepository, account_name: Option, ) { let dataset_alias_to_create = DatasetAlias::new(account_name.clone(), DatasetName::new_unchecked("Foo")); assert_matches!( - repo.get_dataset(&dataset_alias_to_create.as_local_ref()) + repo.find_dataset_by_ref(&dataset_alias_to_create.as_local_ref()) .await .err() .unwrap(), @@ -97,7 +105,7 @@ pub async fn test_create_and_get_case_insensetive_dataset( // We should see the dataset assert!(repo - .get_dataset(&dataset_alias_in_another_registry.as_local_ref()) + .find_dataset_by_ref(&dataset_alias_in_another_registry.as_local_ref()) .await .is_ok()); @@ -116,7 +124,11 @@ pub async fn test_create_and_get_case_insensetive_dataset( .push_event(MetadataFactory::set_polling_source().build()) .build(); - let create_result = repo.create_dataset_from_snapshot(snapshot).await.unwrap(); + let create_result = repo + .create_dataset_from_snapshot(snapshot) + .await + .unwrap() + .create_dataset_result; // Assert dataset_name eq to new alias and account_name eq to old existing one assert_eq!( @@ -145,7 +157,11 @@ pub async fn test_create_and_get_case_insensetive_dataset( //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_create_dataset_same_name_multiple_tenants(repo: &dyn DatasetRepository) { +pub async fn test_create_dataset_same_name_multiple_tenants< + TDatasetRepository: DatasetRepository + DatasetRepositoryWriter, +>( + repo: &TDatasetRepository, +) { let dataset_alias_my = DatasetAlias::new( Some(AccountName::new_unchecked("my")), DatasetName::new_unchecked("foo"), @@ -156,7 +172,7 @@ pub async fn test_create_dataset_same_name_multiple_tenants(repo: &dyn DatasetRe ); assert_matches!( - repo.get_dataset(&dataset_alias_my.as_local_ref()) + repo.find_dataset_by_ref(&dataset_alias_my.as_local_ref()) .await .err() .unwrap(), @@ -164,7 +180,7 @@ pub async fn test_create_dataset_same_name_multiple_tenants(repo: &dyn DatasetRe ); assert_matches!( - repo.get_dataset(&dataset_alias_her.as_local_ref()) + repo.find_dataset_by_ref(&dataset_alias_her.as_local_ref()) .await .err() .unwrap(), @@ -186,12 +202,14 @@ pub async fn test_create_dataset_same_name_multiple_tenants(repo: &dyn DatasetRe let create_result_my = repo .create_dataset_from_snapshot(snapshot_my.clone()) .await - .unwrap(); + .unwrap() + .create_dataset_result; let create_result_her = repo .create_dataset_from_snapshot(snapshot_her.clone()) .await - .unwrap(); + .unwrap() + .create_dataset_result; assert_eq!(create_result_her.dataset_handle.alias, dataset_alias_her); assert_eq!(create_result_my.dataset_handle.alias, dataset_alias_my); @@ -199,12 +217,12 @@ pub async fn test_create_dataset_same_name_multiple_tenants(repo: &dyn DatasetRe // We should see the datasets assert!(repo - .get_dataset(&dataset_alias_my.as_local_ref()) + .find_dataset_by_ref(&dataset_alias_my.as_local_ref()) .await .is_ok()); assert!(repo - .get_dataset(&dataset_alias_her.as_local_ref()) + .find_dataset_by_ref(&dataset_alias_her.as_local_ref()) .await .is_ok()); @@ -238,14 +256,16 @@ pub async fn test_create_dataset_same_name_multiple_tenants(repo: &dyn DatasetRe //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_create_dataset_from_snapshot( - repo: &dyn DatasetRepository, +pub async fn test_create_dataset_from_snapshot< + TDatasetRepository: DatasetRepository + DatasetRepositoryWriter, +>( + repo: &TDatasetRepository, account_name: Option, ) { let dataset_alias = DatasetAlias::new(account_name.clone(), DatasetName::new_unchecked("foo")); assert_matches!( - repo.get_dataset(&dataset_alias.as_local_ref()) + repo.find_dataset_by_ref(&dataset_alias.as_local_ref()) .await .err() .unwrap(), @@ -261,10 +281,11 @@ pub async fn test_create_dataset_from_snapshot( let create_result = repo .create_dataset_from_snapshot(snapshot.clone()) .await - .unwrap(); + .unwrap() + .create_dataset_result; let dataset = repo - .get_dataset(&create_result.dataset_handle.into()) + .find_dataset_by_ref(&create_result.dataset_handle.into()) .await .unwrap(); @@ -284,55 +305,72 @@ pub async fn test_create_dataset_from_snapshot( //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_rename_dataset(repo: &dyn DatasetRepository, account_name: Option) { +pub async fn test_rename_dataset< + TDatasetRepository: DatasetRepository + DatasetRepositoryWriter, +>( + repo: &TDatasetRepository, + account_name: Option, +) { let alias_foo = DatasetAlias::new(account_name.clone(), DatasetName::new_unchecked("foo")); let alias_bar = DatasetAlias::new(account_name.clone(), DatasetName::new_unchecked("bar")); let alias_baz = DatasetAlias::new(account_name.clone(), DatasetName::new_unchecked("baz")); - let snapshots = vec![ - MetadataFactory::dataset_snapshot() - .name(alias_foo.clone()) - .kind(DatasetKind::Root) - .push_event(MetadataFactory::set_polling_source().build()) - .build(), - MetadataFactory::dataset_snapshot() - .name(alias_bar.clone()) - .kind(DatasetKind::Derivative) - .push_event( - MetadataFactory::set_transform() - .inputs_from_refs(["foo"]) - .build(), - ) - .build(), - ]; - - repo.create_datasets_from_snapshots(snapshots).await; + let snapshot_foo = MetadataFactory::dataset_snapshot() + .name(alias_foo.clone()) + .kind(DatasetKind::Root) + .push_event(MetadataFactory::set_polling_source().build()) + .build(); + + let snapshot_bar = MetadataFactory::dataset_snapshot() + .name(alias_bar.clone()) + .kind(DatasetKind::Derivative) + .push_event( + MetadataFactory::set_transform() + .inputs_from_refs(["foo"]) + .build(), + ) + .build(); - assert_matches!( - repo.rename_dataset(&alias_baz.as_local_ref(), &alias_foo.dataset_name) - .await, - Err(RenameDatasetError::NotFound(_)) - ); + let create_result_foo = repo + .create_dataset_from_snapshot(snapshot_foo) + .await + .unwrap(); + repo.create_dataset_from_snapshot(snapshot_bar) + .await + .unwrap(); assert_matches!( - repo.rename_dataset(&alias_foo.as_local_ref(), &alias_bar.dataset_name) - .await, + repo.rename_dataset( + &create_result_foo.create_dataset_result.dataset_handle, + &alias_bar.dataset_name + ) + .await, Err(RenameDatasetError::NameCollision(_)) ); - repo.rename_dataset(&alias_foo.as_local_ref(), &alias_baz.dataset_name) + repo.rename_dataset( + &create_result_foo.create_dataset_result.dataset_handle, + &alias_baz.dataset_name, + ) + .await + .unwrap(); + + let baz = repo + .find_dataset_by_ref(&alias_baz.as_local_ref()) .await .unwrap(); - let baz = repo.get_dataset(&alias_baz.as_local_ref()).await.unwrap(); - use futures::StreamExt; assert_eq!(baz.as_metadata_chain().iter_blocks().count().await, 2); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_rename_dataset_same_name_multiple_tenants(repo: &dyn DatasetRepository) { +pub async fn test_rename_dataset_same_name_multiple_tenants< + TDatasetRepository: DatasetRepository + DatasetRepositoryWriter, +>( + repo: &TDatasetRepository, +) { let account_my = AccountName::new_unchecked("my"); let account_her = AccountName::new_unchecked("her"); @@ -352,7 +390,8 @@ pub async fn test_rename_dataset_same_name_multiple_tenants(repo: &dyn DatasetRe .build(), ) .await - .unwrap(); + .unwrap() + .create_dataset_result; let create_result_her_bar = repo .create_dataset_from_snapshot( @@ -363,9 +402,10 @@ pub async fn test_rename_dataset_same_name_multiple_tenants(repo: &dyn DatasetRe .build(), ) .await - .unwrap(); + .unwrap() + .create_dataset_result; - let _create_result_my_baz = repo + let create_result_my_baz = repo .create_dataset_from_snapshot( MetadataFactory::dataset_snapshot() .name(dataset_alias_my_baz.clone()) @@ -377,19 +417,19 @@ pub async fn test_rename_dataset_same_name_multiple_tenants(repo: &dyn DatasetRe .unwrap(); repo.rename_dataset( - &dataset_alias_my_foo.as_local_ref(), + &create_result_my_foo.dataset_handle, &DatasetName::new_unchecked("bar"), ) .await .unwrap(); let my_bar = repo - .get_dataset(&DatasetRef::try_from("my/bar").unwrap()) + .find_dataset_by_ref(&DatasetRef::try_from("my/bar").unwrap()) .await .unwrap(); let her_bar = repo - .get_dataset(&DatasetRef::try_from("her/bar").unwrap()) + .find_dataset_by_ref(&DatasetRef::try_from("her/bar").unwrap()) .await .unwrap(); @@ -412,7 +452,7 @@ pub async fn test_rename_dataset_same_name_multiple_tenants(repo: &dyn DatasetRe assert_matches!( repo.rename_dataset( - &dataset_alias_my_baz.as_local_ref(), + &create_result_my_baz.create_dataset_result.dataset_handle, &DatasetName::new_unchecked("bar") ) .await, @@ -422,12 +462,14 @@ pub async fn test_rename_dataset_same_name_multiple_tenants(repo: &dyn DatasetRe //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_rename_dataset_unauthorized( - repo: &dyn DatasetRepository, +pub async fn test_delete_dataset< + TDatasetRepository: DatasetRepository + DatasetRepositoryWriter, +>( + repo: &TDatasetRepository, + create_dataset_from_snapshot: &dyn CreateDatasetFromSnapshotUseCase, account_name: Option, ) { let alias_foo = DatasetAlias::new(account_name.clone(), DatasetName::new_unchecked("foo")); - let alias_bar = DatasetAlias::new(account_name.clone(), DatasetName::new_unchecked("bar")); let snapshot = MetadataFactory::dataset_snapshot() .name(alias_foo.clone()) @@ -435,64 +477,22 @@ pub async fn test_rename_dataset_unauthorized( .push_event(MetadataFactory::set_polling_source().build()) .build(); - repo.create_dataset_from_snapshot(snapshot).await.unwrap(); - - let result = repo - .rename_dataset(&alias_foo.as_local_ref(), &alias_bar.dataset_name) - .await; - - assert_matches!(result, Err(RenameDatasetError::Access(_))); - assert!(repo.get_dataset(&alias_foo.as_local_ref()).await.is_ok()); - assert!(repo.get_dataset(&alias_bar.as_local_ref()).await.is_err()); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -pub async fn test_delete_dataset(repo: &dyn DatasetRepository, account_name: Option) { - let alias_foo = DatasetAlias::new(account_name.clone(), DatasetName::new_unchecked("foo")); - let alias_bar = DatasetAlias::new(account_name.clone(), DatasetName::new_unchecked("bar")); - - let snapshots = vec![ - MetadataFactory::dataset_snapshot() - .name(alias_foo.clone()) - .kind(DatasetKind::Root) - .push_event(MetadataFactory::set_polling_source().build()) - .build(), - MetadataFactory::dataset_snapshot() - .name(alias_bar.clone()) - .kind(DatasetKind::Derivative) - .push_event( - MetadataFactory::set_transform() - .inputs_from_refs(["foo"]) - .build(), - ) - .build(), - ]; - - let handles: Vec<_> = repo - .create_datasets_from_snapshots(snapshots) + let create_result = create_dataset_from_snapshot + .execute(snapshot) .await - .into_iter() - .map(|(_, r)| r.unwrap().dataset_handle) - .collect(); - - assert_matches!( - repo.delete_dataset(&alias_foo.as_local_ref()).await, - Err(DeleteDatasetError::DanglingReference(e)) if e.children == vec![handles[1].clone()] - ); - - assert!(repo.get_dataset(&alias_foo.as_local_ref()).await.is_ok()); - assert!(repo.get_dataset(&alias_bar.as_local_ref()).await.is_ok()); + .unwrap(); - repo.delete_dataset(&alias_bar.as_local_ref()) + assert!(repo + .find_dataset_by_ref(&alias_foo.as_local_ref()) .await - .unwrap(); - repo.delete_dataset(&alias_foo.as_local_ref()) + .is_ok()); + + repo.delete_dataset(&create_result.dataset_handle) .await .unwrap(); assert_matches!( - repo.get_dataset(&alias_foo.as_local_ref()) + repo.find_dataset_by_ref(&alias_foo.as_local_ref()) .await .err() .unwrap(), @@ -501,51 +501,37 @@ pub async fn test_delete_dataset(repo: &dyn DatasetRepository, account_name: Opt } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_delete_dataset_unauthorized( - repo: &dyn DatasetRepository, - account_name: Option, + +pub async fn test_iterate_datasets< + TDatasetRepository: DatasetRepository + DatasetRepositoryWriter, +>( + repo: &TDatasetRepository, ) { - let alias_foo = DatasetAlias::new(account_name.clone(), DatasetName::new_unchecked("foo")); + let alias_foo = DatasetAlias::new(None, DatasetName::new_unchecked("foo")); + let alias_bar = DatasetAlias::new(None, DatasetName::new_unchecked("bar")); - let snapshot = MetadataFactory::dataset_snapshot() - .name(alias_foo.clone()) + let snapshot_foo = MetadataFactory::dataset_snapshot() + .name("foo") .kind(DatasetKind::Root) .push_event(MetadataFactory::set_polling_source().build()) .build(); - repo.create_dataset_from_snapshot(snapshot).await.unwrap(); - - assert_matches!( - repo.delete_dataset(&alias_foo.as_local_ref()).await, - Err(DeleteDatasetError::Access(_)) - ); - - assert!(repo.get_dataset(&alias_foo.as_local_ref()).await.is_ok()); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -pub async fn test_iterate_datasets(repo: &dyn DatasetRepository) { - let alias_foo = DatasetAlias::new(None, DatasetName::new_unchecked("foo")); - let alias_bar = DatasetAlias::new(None, DatasetName::new_unchecked("bar")); + let snapshot_bar = MetadataFactory::dataset_snapshot() + .name("bar") + .kind(DatasetKind::Derivative) + .push_event( + MetadataFactory::set_transform() + .inputs_from_refs(["foo"]) + .build(), + ) + .build(); - let snapshots = vec![ - MetadataFactory::dataset_snapshot() - .name("foo") - .kind(DatasetKind::Root) - .push_event(MetadataFactory::set_polling_source().build()) - .build(), - MetadataFactory::dataset_snapshot() - .name("bar") - .kind(DatasetKind::Derivative) - .push_event( - MetadataFactory::set_transform() - .inputs_from_refs(["foo"]) - .build(), - ) - .build(), - ]; - let _: Vec<_> = repo.create_datasets_from_snapshots(snapshots).await; + repo.create_dataset_from_snapshot(snapshot_foo) + .await + .unwrap(); + repo.create_dataset_from_snapshot(snapshot_bar) + .await + .unwrap(); // All check_expected_datasets( @@ -571,7 +557,11 @@ pub async fn test_iterate_datasets(repo: &dyn DatasetRepository) { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_iterate_datasets_multi_tenant(repo: &dyn DatasetRepository) { +pub async fn test_iterate_datasets_multi_tenant< + TDatasetRepository: DatasetRepository + DatasetRepositoryWriter, +>( + repo: &TDatasetRepository, +) { let account_my = AccountName::new_unchecked("my"); let account_her = AccountName::new_unchecked("her"); @@ -584,41 +574,49 @@ pub async fn test_iterate_datasets_multi_tenant(repo: &dyn DatasetRepository) { let alias_my_baz = DatasetAlias::new(Some(account_my.clone()), DatasetName::new_unchecked("baz")); - let my_snapshots = vec![ - MetadataFactory::dataset_snapshot() - .name("my/foo") - .kind(DatasetKind::Root) - .push_event(MetadataFactory::set_polling_source().build()) - .build(), - MetadataFactory::dataset_snapshot() - .name("my/baz") - .kind(DatasetKind::Derivative) - .push_event( - MetadataFactory::set_transform() - .inputs_from_refs_and_aliases([("my/foo", "foo")]) - .build(), - ) - .build(), - ]; - let her_snapshots: Vec = vec![ - MetadataFactory::dataset_snapshot() - .name("her/foo") - .kind(DatasetKind::Root) - .push_event(MetadataFactory::set_polling_source().build()) - .build(), - MetadataFactory::dataset_snapshot() - .name("her/bar") - .kind(DatasetKind::Derivative) - .push_event( - MetadataFactory::set_transform() - .inputs_from_refs_and_aliases([("her/foo", "foo")]) - .build(), - ) - .build(), - ]; - - let _: Vec<_> = repo.create_datasets_from_snapshots(my_snapshots).await; - let _: Vec<_> = repo.create_datasets_from_snapshots(her_snapshots).await; + let snapshot_my_foo = MetadataFactory::dataset_snapshot() + .name("my/foo") + .kind(DatasetKind::Root) + .push_event(MetadataFactory::set_polling_source().build()) + .build(); + let snapshot_my_baz = MetadataFactory::dataset_snapshot() + .name("my/baz") + .kind(DatasetKind::Derivative) + .push_event( + MetadataFactory::set_transform() + .inputs_from_refs_and_aliases([("my/foo", "foo")]) + .build(), + ) + .build(); + + let snapshot_her_foo = MetadataFactory::dataset_snapshot() + .name("her/foo") + .kind(DatasetKind::Root) + .push_event(MetadataFactory::set_polling_source().build()) + .build(); + let snapshot_her_bar = MetadataFactory::dataset_snapshot() + .name("her/bar") + .kind(DatasetKind::Derivative) + .push_event( + MetadataFactory::set_transform() + .inputs_from_refs_and_aliases([("her/foo", "foo")]) + .build(), + ) + .build(); + + repo.create_dataset_from_snapshot(snapshot_my_foo) + .await + .unwrap(); + repo.create_dataset_from_snapshot(snapshot_my_baz) + .await + .unwrap(); + + repo.create_dataset_from_snapshot(snapshot_her_foo) + .await + .unwrap(); + repo.create_dataset_from_snapshot(snapshot_her_bar) + .await + .unwrap(); check_expected_datasets( vec![ diff --git a/src/infra/core/tests/tests/repos/test_metadata_chain_impl.rs b/src/infra/core/tests/tests/repos/test_metadata_chain_impl.rs index 06851e4df5..c48b765047 100644 --- a/src/infra/core/tests/tests/repos/test_metadata_chain_impl.rs +++ b/src/infra/core/tests/tests/repos/test_metadata_chain_impl.rs @@ -12,6 +12,7 @@ use std::path::Path; use std::sync::{Arc, Mutex}; use chrono::{TimeZone, Utc}; +use internal_error::InternalError; use kamu::domain::*; use kamu::testing::*; use kamu::*; diff --git a/src/infra/core/tests/tests/test_compact_service_impl.rs b/src/infra/core/tests/tests/test_compact_service_impl.rs index 33999ad461..bb76bba9fb 100644 --- a/src/infra/core/tests/tests/test_compact_service_impl.rs +++ b/src/infra/core/tests/tests/test_compact_service_impl.rs @@ -15,7 +15,6 @@ use datafusion::execution::config::SessionConfig; use datafusion::execution::context::SessionContext; use dill::Component; use domain::{CompactionError, CompactionOptions, CompactionResult, CompactionService}; -use event_bus::EventBus; use futures::TryStreamExt; use indoc::{formatdoc, indoc}; use kamu::domain::*; @@ -25,6 +24,7 @@ use kamu::*; use kamu_accounts::CurrentAccountSubject; use kamu_core::auth; use opendatafabric::*; +use time_source::{SystemTimeSource, SystemTimeSourceStub}; use super::test_pull_service_impl::TestTransformService; use crate::mock_engine_provisioner; @@ -1057,6 +1057,7 @@ async fn test_dataset_keep_metadata_only_compact() { struct CompactTestHarness { _temp_dir: tempfile::TempDir, dataset_repo: Arc, + dataset_repo_writer: Arc, compaction_svc: Arc, push_ingest_svc: Arc, verification_svc: Arc, @@ -1080,8 +1081,6 @@ impl CompactTestHarness { let catalog = dill::CatalogBuilder::new() .add_value(RunInfoDir::new(run_info_dir)) - .add::() - .add::() .add_value(CurrentAccountSubject::new_test()) .add_builder( DatasetRepositoryLocalFs::builder() @@ -1089,6 +1088,7 @@ impl CompactTestHarness { .with_multi_tenant(false), ) .bind::() + .bind::() .add::() .add_value(SystemTimeSourceStub::new_set(current_date_time)) .bind::() @@ -1106,6 +1106,7 @@ impl CompactTestHarness { .build(); let dataset_repo = catalog.get_one::().unwrap(); + let dataset_repo_writer = catalog.get_one::().unwrap(); let compaction_svc = catalog.get_one::().unwrap(); let push_ingest_svc = catalog.get_one::().unwrap(); let transform_svc = catalog.get_one::().unwrap(); @@ -1114,6 +1115,7 @@ impl CompactTestHarness { Self { _temp_dir: temp_dir, dataset_repo, + dataset_repo_writer, compaction_svc, push_ingest_svc, transform_svc, @@ -1132,14 +1134,13 @@ impl CompactTestHarness { let catalog = dill::CatalogBuilder::new() .add_builder(run_info_dir.clone()) - .add::() - .add::() .add_builder( DatasetRepositoryS3::builder() .with_s3_context(s3_context.clone()) .with_multi_tenant(false), ) .bind::() + .bind::() .add::() .add_value(SystemTimeSourceStub::new_set(current_date_time)) .bind::() @@ -1162,6 +1163,7 @@ impl CompactTestHarness { Self { _temp_dir: temp_dir, dataset_repo: catalog.get_one().unwrap(), + dataset_repo_writer: catalog.get_one().unwrap(), compaction_svc: catalog.get_one().unwrap(), push_ingest_svc: catalog.get_one().unwrap(), verification_svc: catalog.get_one().unwrap(), @@ -1172,7 +1174,11 @@ impl CompactTestHarness { } async fn get_dataset_head(&self, dataset_ref: &DatasetRef) -> Multihash { - let dataset = self.dataset_repo.get_dataset(dataset_ref).await.unwrap(); + let dataset = self + .dataset_repo + .find_dataset_by_ref(dataset_ref) + .await + .unwrap(); dataset .as_metadata_chain() @@ -1182,7 +1188,11 @@ impl CompactTestHarness { } async fn get_dataset_blocks(&self, dataset_ref: &DatasetRef) -> Vec { - let dataset = self.dataset_repo.get_dataset(dataset_ref).await.unwrap(); + let dataset = self + .dataset_repo + .find_dataset_by_ref(dataset_ref) + .await + .unwrap(); let head = self.get_dataset_head(dataset_ref).await; dataset @@ -1195,10 +1205,11 @@ impl CompactTestHarness { } async fn create_dataset(&self, dataset_snapshot: DatasetSnapshot) -> CreateDatasetResult { - self.dataset_repo + self.dataset_repo_writer .create_dataset_from_snapshot(dataset_snapshot) .await .unwrap() + .create_dataset_result } async fn create_test_root_dataset(&self) -> CreateDatasetResult { @@ -1261,7 +1272,11 @@ impl CompactTestHarness { } async fn dataset_data_helper(&self, dataset_ref: &DatasetRef) -> DatasetDataHelper { - let dataset = self.dataset_repo.get_dataset(dataset_ref).await.unwrap(); + let dataset = self + .dataset_repo + .find_dataset_by_ref(dataset_ref) + .await + .unwrap(); DatasetDataHelper::new_with_context(dataset, self.ctx.clone()) } @@ -1302,7 +1317,11 @@ impl CompactTestHarness { } async fn commit_set_licence_block(&self, dataset_ref: &DatasetRef, head: &Multihash) { - let dataset = self.dataset_repo.get_dataset(dataset_ref).await.unwrap(); + let dataset = self + .dataset_repo + .find_dataset_by_ref(dataset_ref) + .await + .unwrap(); let event = SetLicense { short_name: "sl1".to_owned(), name: "set_license1".to_owned(), diff --git a/src/infra/core/tests/tests/test_dataset_changes_service_impl.rs b/src/infra/core/tests/tests/test_dataset_changes_service_impl.rs index b2423ace52..1da97c75cc 100644 --- a/src/infra/core/tests/tests/test_dataset_changes_service_impl.rs +++ b/src/infra/core/tests/tests/test_dataset_changes_service_impl.rs @@ -11,18 +11,15 @@ use std::sync::Arc; use chrono::Utc; use dill::Component; -use event_bus::EventBus; use kamu::testing::MetadataFactory; -use kamu::{DatasetChangesServiceImpl, DatasetRepositoryLocalFs, DependencyGraphServiceInMemory}; +use kamu::{DatasetChangesServiceImpl, DatasetRepositoryLocalFs, DatasetRepositoryWriter}; use kamu_accounts::CurrentAccountSubject; use kamu_core::{ - auth, CommitOpts, CreateDatasetResult, DatasetChangesService, DatasetIntervalIncrement, DatasetRepository, - SystemTimeSourceDefault, }; use opendatafabric::{ Checkpoint, @@ -34,6 +31,7 @@ use opendatafabric::{ Multihash, }; use tempfile::TempDir; +use time_source::SystemTimeSourceDefault; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -108,9 +106,7 @@ async fn test_add_data_differences() { let dataset = harness .dataset_repo - .get_dataset(&foo_result.dataset_handle.as_local_ref()) - .await - .unwrap(); + .get_dataset_by_handle(&foo_result.dataset_handle); // Commit SetDataSchema and 2 data nodes @@ -297,9 +293,7 @@ async fn test_execute_transform_differences() { let bar_dataset = harness .dataset_repo - .get_dataset(&bar_result.dataset_handle.as_local_ref()) - .await - .unwrap(); + .get_dataset_by_handle(&bar_result.dataset_handle); // Commit SetDataSchema and 2 trasnform data nodes @@ -485,9 +479,7 @@ async fn test_multiple_watermarks_within_interval() { let dataset = harness .dataset_repo - .get_dataset(&foo_result.dataset_handle.as_local_ref()) - .await - .unwrap(); + .get_dataset_by_handle(&foo_result.dataset_handle); // Commit SetDataSchema and 2 data nodes each having a watermark @@ -628,9 +620,7 @@ async fn test_older_watermark_before_interval() { let dataset = harness .dataset_repo - .get_dataset(&foo_result.dataset_handle.as_local_ref()) - .await - .unwrap(); + .get_dataset_by_handle(&foo_result.dataset_handle); // Commit SetDataSchema and 3 data nodes, with #1,3 containing watermark @@ -800,6 +790,7 @@ struct DatasetChangesHarness { _workdir: TempDir, _catalog: dill::Catalog, dataset_repo: Arc, + dataset_repo_writer: Arc, dataset_changes_service: Arc, } @@ -811,20 +802,19 @@ impl DatasetChangesHarness { let catalog = dill::CatalogBuilder::new() .add::() - .add::() .add_builder( DatasetRepositoryLocalFs::builder() .with_root(datasets_dir) .with_multi_tenant(false), ) .bind::() + .bind::() .add_value(CurrentAccountSubject::new_test()) - .add::() .add::() - .add::() .build(); let dataset_repo = catalog.get_one::().unwrap(); + let dataset_repo_writer = catalog.get_one::().unwrap(); let dataset_changes_service = catalog.get_one::().unwrap(); @@ -832,6 +822,7 @@ impl DatasetChangesHarness { _workdir: workdir, _catalog: catalog, dataset_repo, + dataset_repo_writer, dataset_changes_service, } } @@ -839,7 +830,7 @@ impl DatasetChangesHarness { async fn create_root_dataset(&self, dataset_name: &str) -> CreateDatasetResult { let alias = DatasetAlias::new(None, DatasetName::new_unchecked(dataset_name)); let create_result = self - .dataset_repo + .dataset_repo_writer .create_dataset( &alias, MetadataFactory::metadata_block( @@ -873,7 +864,7 @@ impl DatasetChangesHarness { dataset_name: &str, input_dataset_names: Vec<&str>, ) -> CreateDatasetResult { - self.dataset_repo + self.dataset_repo_writer .create_dataset_from_snapshot( MetadataFactory::dataset_snapshot() .name(DatasetAlias::new( @@ -890,6 +881,7 @@ impl DatasetChangesHarness { ) .await .unwrap() + .create_dataset_result } async fn check_between_cases( diff --git a/src/infra/core/tests/tests/test_dataset_ownership_service_inmem.rs b/src/infra/core/tests/tests/test_dataset_ownership_service_inmem.rs index 7faa0ded89..f8367905e0 100644 --- a/src/infra/core/tests/tests/test_dataset_ownership_service_inmem.rs +++ b/src/infra/core/tests/tests/test_dataset_ownership_service_inmem.rs @@ -12,13 +12,12 @@ use std::sync::Arc; use database_common::{DatabaseTransactionRunner, NoOpDatabasePlugin}; use dill::Component; -use event_bus::EventBus; use kamu::testing::MetadataFactory; use kamu::{ DatasetOwnershipServiceInMemory, DatasetOwnershipServiceInMemoryStateInitializer, DatasetRepositoryLocalFs, - DependencyGraphServiceInMemory, + DatasetRepositoryWriter, }; use kamu_accounts::{ AccountConfig, @@ -28,16 +27,17 @@ use kamu_accounts::{ PredefinedAccountsConfig, DEFAULT_ACCOUNT_ID, }; -use kamu_accounts_inmem::{AccessTokenRepositoryInMemory, AccountRepositoryInMemory}; +use kamu_accounts_inmem::{InMemoryAccessTokenRepository, InMemoryAccountRepository}; use kamu_accounts_services::{ AccessTokenServiceImpl, AuthenticationServiceImpl, LoginPasswordAuthProvider, PredefinedAccountsRegistrator, }; -use kamu_core::{auth, DatasetOwnershipService, DatasetRepository, SystemTimeSourceDefault}; +use kamu_core::{DatasetOwnershipService, DatasetRepository}; use opendatafabric::{AccountID, AccountName, DatasetAlias, DatasetID, DatasetKind, DatasetName}; use tempfile::TempDir; +use time_source::SystemTimeSourceDefault; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -87,7 +87,7 @@ async fn test_multi_tenant_dataset_owners() { struct DatasetOwnershipHarness { _workdir: TempDir, catalog: dill::Catalog, - dataset_repo: Arc, + dataset_repo_writer: Arc, dataset_ownership_service: Arc, auth_svc: Arc, account_datasets: HashMap>, @@ -114,24 +114,22 @@ impl DatasetOwnershipHarness { let mut b = dill::CatalogBuilder::new(); b.add::() - .add::() .add_builder( DatasetRepositoryLocalFs::builder() .with_root(datasets_dir) .with_multi_tenant(multi_tenant), ) .bind::() + .bind::() .add_value(CurrentAccountSubject::new_test()) .add::() .add::() .add_value(predefined_accounts_config.clone()) .add_value(JwtAuthenticationConfig::default()) - .add::() - .add::() + .add::() + .add::() .add::() .add::() - .add::() - .add::() .add::() .add::() .add::(); @@ -154,7 +152,7 @@ impl DatasetOwnershipHarness { .await .unwrap(); - let dataset_repo = catalog.get_one::().unwrap(); + let dataset_repo_writer = catalog.get_one::().unwrap(); let dataset_ownership_service = catalog.get_one::().unwrap(); let auth_svc = catalog.get_one::().unwrap(); @@ -162,7 +160,7 @@ impl DatasetOwnershipHarness { Self { _workdir: workdir, catalog, - dataset_repo, + dataset_repo_writer, dataset_ownership_service, auth_svc, account_datasets: HashMap::new(), @@ -244,8 +242,9 @@ impl DatasetOwnershipHarness { .await .unwrap() .unwrap(); + let created_dataset = self - .dataset_repo + .dataset_repo_writer .create_dataset_from_snapshot( MetadataFactory::dataset_snapshot() .name(DatasetAlias::new( @@ -257,7 +256,9 @@ impl DatasetOwnershipHarness { .build(), ) .await - .unwrap(); + .unwrap() + .create_dataset_result; + self.account_datasets .entry(account_id.clone()) .and_modify(|e| { @@ -278,8 +279,9 @@ impl DatasetOwnershipHarness { .await .unwrap() .unwrap(); + let created_dataset = self - .dataset_repo + .dataset_repo_writer .create_dataset_from_snapshot( MetadataFactory::dataset_snapshot() .name(DatasetAlias::new( @@ -295,7 +297,8 @@ impl DatasetOwnershipHarness { .build(), ) .await - .unwrap(); + .unwrap() + .create_dataset_result; self.account_datasets .entry(account_id.clone()) diff --git a/src/infra/core/tests/tests/test_datasets_filtering.rs b/src/infra/core/tests/tests/test_datasets_filtering.rs index 62fdcb0338..0e3ae07378 100644 --- a/src/infra/core/tests/tests/test_datasets_filtering.rs +++ b/src/infra/core/tests/tests/test_datasets_filtering.rs @@ -11,7 +11,6 @@ use std::str::FromStr; use std::sync::Arc; use dill::Component; -use event_bus::EventBus; use futures::TryStreamExt; use kamu::testing::MetadataFactory; use kamu::utils::datasets_filtering::{ @@ -19,9 +18,9 @@ use kamu::utils::datasets_filtering::{ matches_local_ref_pattern, matches_remote_ref_pattern, }; -use kamu::{DatasetRepositoryLocalFs, DependencyGraphServiceInMemory}; +use kamu::{DatasetRepositoryLocalFs, DatasetRepositoryWriter}; use kamu_accounts::{CurrentAccountSubject, DEFAULT_ACCOUNT_NAME}; -use kamu_core::{auth, DatasetRepository, SystemTimeSourceDefault}; +use kamu_core::DatasetRepository; use opendatafabric::{ AccountName, DatasetAlias, @@ -35,6 +34,7 @@ use opendatafabric::{ RepoName, }; use tempfile::TempDir; +use time_source::SystemTimeSourceDefault; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -264,6 +264,7 @@ struct DatasetFilteringHarness { _workdir: TempDir, _catalog: dill::Catalog, dataset_repo: Arc, + dataset_repo_writer: Arc, } impl DatasetFilteringHarness { @@ -274,24 +275,24 @@ impl DatasetFilteringHarness { let catalog = dill::CatalogBuilder::new() .add::() - .add::() .add_builder( DatasetRepositoryLocalFs::builder() .with_root(datasets_dir) .with_multi_tenant(is_multi_tenant), ) .bind::() + .bind::() .add_value(CurrentAccountSubject::new_test()) - .add::() - .add::() .build(); let dataset_repo = catalog.get_one::().unwrap(); + let dataset_repo_writer = catalog.get_one::().unwrap(); Self { _workdir: workdir, _catalog: catalog, dataset_repo, + dataset_repo_writer, } } @@ -300,7 +301,7 @@ impl DatasetFilteringHarness { account_name: Option, dataset_name: &str, ) -> DatasetHandle { - self.dataset_repo + self.dataset_repo_writer .create_dataset_from_snapshot( MetadataFactory::dataset_snapshot() .name(DatasetAlias::new( @@ -313,6 +314,7 @@ impl DatasetFilteringHarness { ) .await .unwrap() + .create_dataset_result .dataset_handle } diff --git a/src/infra/core/tests/tests/test_dependency_graph_inmem.rs b/src/infra/core/tests/tests/test_dependency_graph_inmem.rs index 6f69fd0605..3fa6db2d87 100644 --- a/src/infra/core/tests/tests/test_dependency_graph_inmem.rs +++ b/src/infra/core/tests/tests/test_dependency_graph_inmem.rs @@ -11,33 +11,16 @@ use std::collections::HashMap; use std::sync::Arc; use dill::Component; -use event_bus::EventBus; use futures::{future, StreamExt, TryStreamExt}; use internal_error::ResultIntoInternal; use kamu::testing::MetadataFactory; -use kamu::{ - DatasetRepositoryLocalFs, - DependencyGraphRepositoryInMemory, - DependencyGraphServiceInMemory, -}; +use kamu::*; use kamu_accounts::CurrentAccountSubject; -use kamu_core::{ - auth, - DatasetDependencies, - DatasetRepository, - DependencyGraphRepository, - DependencyGraphService, - SystemTimeSourceDefault, -}; -use opendatafabric::{ - AccountName, - DatasetAlias, - DatasetID, - DatasetKind, - DatasetName, - MetadataEvent, -}; +use kamu_core::*; +use messaging_outbox::{register_message_dispatcher, Outbox, OutboxImmediateImpl}; +use opendatafabric::*; use tempfile::TempDir; +use time_source::SystemTimeSourceDefault; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -293,9 +276,12 @@ async fn test_service_dataset_deleted() { "[foo-bar, foo-baz] -> foo-bar-foo-baz -> []" ); - harness - .dataset_repo - .delete_dataset( + let delete_dataset = harness + .catalog + .get_one::() + .unwrap(); + delete_dataset + .execute_via_ref( &DatasetAlias::new(None, DatasetName::new_unchecked("foo-bar-foo-baz")).as_local_ref(), ) .await @@ -575,7 +561,7 @@ async fn test_get_recursive_upstream_dependencies() { struct DependencyGraphHarness { _workdir: TempDir, - _catalog: dill::Catalog, + catalog: dill::Catalog, dataset_repo: Arc, dependency_graph_service: Arc, dependency_graph_repository: Arc, @@ -587,19 +573,33 @@ impl DependencyGraphHarness { let datasets_dir = workdir.path().join("datasets"); std::fs::create_dir(&datasets_dir).unwrap(); - let catalog = dill::CatalogBuilder::new() - .add::() - .add::() + let mut b = dill::CatalogBuilder::new(); + b.add::() + .add_builder( + messaging_outbox::OutboxImmediateImpl::builder() + .with_consumer_filter(messaging_outbox::ConsumerFilter::AllConsumers), + ) + .bind::() .add_builder( DatasetRepositoryLocalFs::builder() .with_root(datasets_dir) .with_multi_tenant(multi_tenant), ) .bind::() + .bind::() .add_value(CurrentAccountSubject::new_test()) .add::() .add::() - .build(); + .add::() + .add::() + .add::(); + + register_message_dispatcher::( + &mut b, + MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE, + ); + + let catalog = b.build(); let dataset_repo = catalog.get_one::().unwrap(); @@ -611,7 +611,7 @@ impl DependencyGraphHarness { Self { _workdir: workdir, - _catalog: catalog, + catalog, dataset_repo, dependency_graph_service, dependency_graph_repository, @@ -864,8 +864,13 @@ impl DependencyGraphHarness { } async fn create_root_dataset(&self, account_name: Option, dataset_name: &str) { - self.dataset_repo - .create_dataset_from_snapshot( + let create_dataset_from_snapshot = self + .catalog + .get_one::() + .unwrap(); + + create_dataset_from_snapshot + .execute( MetadataFactory::dataset_snapshot() .name(DatasetAlias::new( account_name, @@ -885,8 +890,13 @@ impl DependencyGraphHarness { dataset_name: &str, input_aliases: Vec, ) { - self.dataset_repo - .create_dataset_from_snapshot( + let create_dataset_from_snapshot = self + .catalog + .get_one::() + .unwrap(); + + create_dataset_from_snapshot + .execute( MetadataFactory::dataset_snapshot() .name(DatasetAlias::new( account_name, @@ -913,9 +923,9 @@ impl DependencyGraphHarness { let dataset_alias = DatasetAlias::new(account_name, DatasetName::new_unchecked(dataset_name)); - let dataset = self + let dataset_handle = self .dataset_repo - .get_dataset(&dataset_alias.as_local_ref()) + .resolve_dataset_ref(&dataset_alias.as_local_ref()) .await .unwrap(); @@ -928,8 +938,13 @@ impl DependencyGraphHarness { )); } - dataset - .commit_event( + let commit_dataset_event = self + .catalog + .get_one::() + .unwrap(); + commit_dataset_event + .execute( + &dataset_handle, MetadataEvent::SetTransform( MetadataFactory::set_transform() .inputs_from_refs_and_aliases(id_aliases) @@ -1116,3 +1131,5 @@ async fn create_large_dataset_graph() -> DependencyGraphHarness { dependency_harness } + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/core/tests/tests/test_pull_service_impl.rs b/src/infra/core/tests/tests/test_pull_service_impl.rs index b7770bb1f7..6c889eec65 100644 --- a/src/infra/core/tests/tests/test_pull_service_impl.rs +++ b/src/infra/core/tests/tests/test_pull_service_impl.rs @@ -15,12 +15,12 @@ use std::sync::{Arc, Mutex}; use chrono::prelude::*; use dill::*; use domain::auth::AlwaysHappyDatasetActionAuthorizer; -use event_bus::EventBus; use kamu::domain::*; use kamu::testing::*; use kamu::*; use kamu_accounts::{CurrentAccountSubject, DEFAULT_ACCOUNT_NAME_STR}; use opendatafabric::*; +use time_source::SystemTimeSourceDefault; macro_rules! n { ($s:expr) => { @@ -140,8 +140,8 @@ async fn create_graph( // dir and syncing it into the main workspace. TODO: Add simpler way to import // remote dataset async fn create_graph_remote( - event_bus: Arc, dataset_repo: Arc, + dataset_repo_writer: Arc, reg: Arc, datasets: Vec<(DatasetAlias, Vec)>, to_import: Vec, @@ -151,9 +151,6 @@ async fn create_graph_remote( let remote_dataset_repo = DatasetRepositoryLocalFs::new( tmp_repo_dir.path().to_owned(), Arc::new(CurrentAccountSubject::new_test()), - Arc::new(auth::AlwaysHappyDatasetActionAuthorizer::new()), - Arc::new(DependencyGraphServiceInMemory::new(None)), - Arc::new(EventBus::new(Arc::new(CatalogBuilder::new().build()))), false, Arc::new(SystemTimeSourceDefault), ); @@ -171,11 +168,11 @@ async fn create_graph_remote( let sync_service = SyncServiceImpl::new( reg.clone(), dataset_repo, + dataset_repo_writer, Arc::new(auth::AlwaysHappyDatasetActionAuthorizer::new()), Arc::new(DatasetFactoryImpl::new( IpfsGateway::default(), Arc::new(auth::DummyOdfServerAccessTokenResolver::new()), - event_bus, )), Arc::new(DummySmartTransferProtocolClient::new()), Arc::new(kamu::utils::ipfs_wrapper::IpfsClient::default()), @@ -363,7 +360,7 @@ async fn test_pull_batching_complex_with_remote() { // C --------/ / // D -----------/ create_graph_remote( - harness.event_bus.clone(), + harness.dataset_repo.clone(), harness.dataset_repo.clone(), harness.remote_repo_reg.clone(), vec![ @@ -719,8 +716,9 @@ async fn test_set_watermark() { let harness = PullTestHarness::new_with_authorizer( tmp_dir.path(), MockDatasetActionAuthorizer::new().expect_check_write_dataset( - DatasetAlias::new(None, DatasetName::new_unchecked("foo")), + &DatasetAlias::new(None, DatasetName::new_unchecked("foo")), 4, + true, ), false, ); @@ -848,7 +846,6 @@ struct PullTestHarness { remote_repo_reg: Arc, remote_alias_reg: Arc, pull_svc: Arc, - event_bus: Arc, } impl PullTestHarness { @@ -872,8 +869,6 @@ impl PullTestHarness { let catalog = dill::CatalogBuilder::new() .add::() - .add::() - .add::() .add_value(CurrentAccountSubject::new_test()) .add_value(dataset_action_authorizer) .bind::() @@ -883,6 +878,7 @@ impl PullTestHarness { .with_multi_tenant(multi_tenant), ) .bind::() + .bind::() .add_value(RemoteRepositoryRegistryImpl::create(tmp_path.join("repos")).unwrap()) .bind::() .add::() @@ -899,7 +895,6 @@ impl PullTestHarness { let remote_repo_reg = catalog.get_one::().unwrap(); let remote_alias_reg = catalog.get_one::().unwrap(); let pull_svc = catalog.get_one::().unwrap(); - let event_bus = catalog.get_one::().unwrap(); Self { calls, @@ -907,7 +902,6 @@ impl PullTestHarness { remote_repo_reg, remote_alias_reg, pull_svc, - event_bus, } } @@ -925,7 +919,7 @@ impl PullTestHarness { async fn num_blocks(&self, dataset_alias: &DatasetAlias) -> usize { let ds = self .dataset_repo - .get_dataset(&dataset_alias.as_local_ref()) + .find_dataset_by_ref(&dataset_alias.as_local_ref()) .await .unwrap(); @@ -1151,14 +1145,20 @@ impl TransformService for TestTransformService { struct TestSyncService { calls: Arc>>, dataset_repo: Arc, + dataset_repo_writer: Arc, } #[dill::component(pub)] impl TestSyncService { - fn new(calls: Arc>>, dataset_repo: Arc) -> Self { + fn new( + calls: Arc>>, + dataset_repo: Arc, + dataset_repo_writer: Arc, + ) -> Self { Self { calls, dataset_repo, + dataset_repo_writer, } } } @@ -1195,7 +1195,7 @@ impl SyncService for TestSyncService { .unwrap() { None => { - self.dataset_repo + self.dataset_repo_writer .create_dataset_from_snapshot( MetadataFactory::dataset_snapshot() .name(local_ref.alias().unwrap().clone()) diff --git a/src/infra/core/tests/tests/test_query_service_impl.rs b/src/infra/core/tests/tests/test_query_service_impl.rs index ebe7593b44..6df7759850 100644 --- a/src/infra/core/tests/tests/test_query_service_impl.rs +++ b/src/infra/core/tests/tests/test_query_service_impl.rs @@ -17,7 +17,6 @@ use datafusion::arrow::array::*; use datafusion::arrow::datatypes::{DataType, Field, Schema}; use datafusion::arrow::record_batch::RecordBatch; use dill::{Catalog, Component}; -use event_bus::EventBus; use kamu::domain::*; use kamu::testing::{ LocalS3Server, @@ -31,14 +30,15 @@ use kamu_accounts::CurrentAccountSubject; use kamu_ingest_datafusion::DataWriterDataFusion; use opendatafabric::*; use tempfile::TempDir; +use time_source::SystemTimeSourceDefault; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// async fn create_test_dataset(catalog: &dill::Catalog, tempdir: &Path) -> CreateDatasetResult { - let dataset_repo = catalog.get_one::().unwrap(); + let dataset_repo_writer = catalog.get_one::().unwrap(); let dataset_alias = DatasetAlias::new(None, DatasetName::new_unchecked("foo")); - let create_result = dataset_repo + let create_result = dataset_repo_writer .create_dataset( &dataset_alias, MetadataFactory::metadata_block(MetadataFactory::seed(DatasetKind::Root).build()) @@ -125,14 +125,13 @@ fn create_catalog_with_local_workspace( dill::CatalogBuilder::new() .add::() - .add::() - .add::() .add_builder( DatasetRepositoryLocalFs::builder() .with_root(datasets_dir) .with_multi_tenant(false), ) .bind::() + .bind::() .add::() .add::() .add::() @@ -153,14 +152,13 @@ async fn create_catalog_with_s3_workspace( dill::CatalogBuilder::new() .add::() - .add::() - .add::() .add_builder( DatasetRepositoryS3::builder() .with_s3_context(s3_context.clone()) .with_multi_tenant(false), ) .bind::() + .bind::() .add::() .add::() .add::() @@ -261,7 +259,7 @@ fn prepare_test_catalog() -> (TempDir, Catalog) { let tempdir = tempfile::tempdir().unwrap(); let catalog = create_catalog_with_local_workspace( tempdir.path(), - MockDatasetActionAuthorizer::new().expect_check_read_a_dataset(1), + MockDatasetActionAuthorizer::new().expect_check_read_a_dataset(1, true), ); (tempdir, catalog) } @@ -270,7 +268,7 @@ async fn prepare_test_s3_catalog() -> (LocalS3Server, Catalog) { let s3 = LocalS3Server::new().await; let catalog = create_catalog_with_s3_workspace( &s3, - MockDatasetActionAuthorizer::new().expect_check_read_a_dataset(1), + MockDatasetActionAuthorizer::new().expect_check_read_a_dataset(1, true), ) .await; (s3, catalog) @@ -388,7 +386,7 @@ async fn test_dataset_tail_local_fs() { let tempdir = tempfile::tempdir().unwrap(); let catalog = create_catalog_with_local_workspace( tempdir.path(), - MockDatasetActionAuthorizer::new().expect_check_read_a_dataset(4), + MockDatasetActionAuthorizer::new().expect_check_read_a_dataset(4, true), ); test_dataset_tail_common(catalog, &tempdir).await; } @@ -399,7 +397,7 @@ async fn test_dataset_tail_s3() { let s3 = LocalS3Server::new().await; let catalog = create_catalog_with_s3_workspace( &s3, - MockDatasetActionAuthorizer::new().expect_check_read_a_dataset(4), + MockDatasetActionAuthorizer::new().expect_check_read_a_dataset(4, true), ) .await; test_dataset_tail_common(catalog, &s3.tmp_dir).await; @@ -412,14 +410,19 @@ async fn test_dataset_tail_empty_dataset() { let tempdir = tempfile::tempdir().unwrap(); let catalog = create_catalog_with_local_workspace( tempdir.path(), - MockDatasetActionAuthorizer::new().expect_check_read_a_dataset(1), + MockDatasetActionAuthorizer::new().expect_check_read_a_dataset(1, true), ); - let dataset_repo = catalog.get_one::().unwrap(); - dataset_repo - .create_dataset_from_seed( + let dataset_repo_writer = catalog.get_one::().unwrap(); + dataset_repo_writer + .create_dataset( &"foo".try_into().unwrap(), - MetadataFactory::seed(DatasetKind::Root).build(), + MetadataBlockTyped { + system_time: Utc::now(), + prev_block_hash: None, + event: MetadataFactory::seed(DatasetKind::Root).build(), + sequence_number: 0, + }, ) .await .unwrap(); @@ -496,7 +499,7 @@ async fn test_dataset_sql_authorized_local_fs() { let tempdir = tempfile::tempdir().unwrap(); let catalog = create_catalog_with_local_workspace( tempdir.path(), - MockDatasetActionAuthorizer::new().expect_check_read_a_dataset(1), + MockDatasetActionAuthorizer::new().expect_check_read_a_dataset(1, true), ); test_dataset_sql_authorized_common(catalog, &tempdir).await; } @@ -507,7 +510,7 @@ async fn test_dataset_sql_authorized_s3() { let s3 = LocalS3Server::new().await; let catalog = create_catalog_with_s3_workspace( &s3, - MockDatasetActionAuthorizer::new().expect_check_read_a_dataset(1), + MockDatasetActionAuthorizer::new().expect_check_read_a_dataset(1, true), ) .await; test_dataset_sql_authorized_common(catalog, &s3.tmp_dir).await; @@ -679,12 +682,12 @@ async fn test_sql_statement_with_state_simple() { MockDatasetActionAuthorizer::allowing(), ); - let dataset_repo = catalog.get_one::().unwrap(); + let dataset_repo_writer = catalog.get_one::().unwrap(); let ctx = SessionContext::new(); // Dataset init let foo_alias = DatasetAlias::new(None, DatasetName::new_unchecked("foo")); - let foo_create = dataset_repo + let foo_create = dataset_repo_writer .create_dataset( &foo_alias, MetadataFactory::metadata_block(MetadataFactory::seed(DatasetKind::Root).build()) @@ -889,12 +892,12 @@ async fn test_sql_statement_with_state_cte() { MockDatasetActionAuthorizer::allowing(), ); - let dataset_repo = catalog.get_one::().unwrap(); + let dataset_repo_writer = catalog.get_one::().unwrap(); let ctx = SessionContext::new(); // Dataset `foo` let foo_alias = DatasetAlias::new(None, DatasetName::new_unchecked("foo")); - let foo_create = dataset_repo + let foo_create = dataset_repo_writer .create_dataset( &foo_alias, MetadataFactory::metadata_block( @@ -944,7 +947,7 @@ async fn test_sql_statement_with_state_cte() { // Dataset `bar` let bar_alias = DatasetAlias::new(None, DatasetName::new_unchecked("bar")); - let bar_create = dataset_repo + let bar_create = dataset_repo_writer .create_dataset( &bar_alias, MetadataFactory::metadata_block( diff --git a/src/infra/core/tests/tests/test_reset_service_impl.rs b/src/infra/core/tests/tests/test_reset_service_impl.rs index c4d92501d4..55394860a7 100644 --- a/src/infra/core/tests/tests/test_reset_service_impl.rs +++ b/src/infra/core/tests/tests/test_reset_service_impl.rs @@ -11,13 +11,13 @@ use std::assert_matches::assert_matches; use std::sync::Arc; use dill::Component; -use event_bus::EventBus; use kamu::domain::*; use kamu::testing::*; use kamu::*; use kamu_accounts::CurrentAccountSubject; use opendatafabric::*; use tempfile::TempDir; +use time_source::SystemTimeSourceDefault; #[test_log::test(tokio::test)] async fn test_reset_dataset_with_2revisions_drop_last() { @@ -156,6 +156,7 @@ impl ChainWith2BlocksTestCase { struct ResetTestHarness { _temp_dir: TempDir, dataset_repo: Arc, + dataset_repo_writer: Arc, reset_svc: Arc, } @@ -167,10 +168,8 @@ impl ResetTestHarness { let catalog = dill::CatalogBuilder::new() .add::() - .add::() - .add::() .add_value(CurrentAccountSubject::new_test()) - .add_value(MockDatasetActionAuthorizer::new().expect_check_write_a_dataset(1)) + .add_value(MockDatasetActionAuthorizer::new().expect_check_write_a_dataset(1, true)) .bind::() .add_builder( DatasetRepositoryLocalFs::builder() @@ -178,15 +177,18 @@ impl ResetTestHarness { .with_multi_tenant(false), ) .bind::() + .bind::() .add::() .build(); let dataset_repo = catalog.get_one::().unwrap(); + let dataset_repo_writer = catalog.get_one::().unwrap(); let reset_svc = catalog.get_one::().unwrap(); Self { _temp_dir: tempdir, dataset_repo, + dataset_repo_writer, reset_svc, } } @@ -202,7 +204,7 @@ impl ResetTestHarness { .build_typed(); let create_result = self - .dataset_repo + .dataset_repo_writer .create_dataset(&DatasetAlias::new(None, dataset_name.clone()), seed_block) .await .unwrap(); @@ -223,7 +225,7 @@ impl ResetTestHarness { } async fn get_dataset_head(&self, dataset_handle: &DatasetHandle) -> Multihash { - let dataset = self.resolve_dataset(dataset_handle).await; + let dataset = self.resolve_dataset(dataset_handle); dataset .as_metadata_chain() .resolve_ref(&BlockRef::Head) @@ -232,17 +234,14 @@ impl ResetTestHarness { } async fn get_dataset_summary(&self, dataset_handle: &DatasetHandle) -> DatasetSummary { - let dataset = self.resolve_dataset(dataset_handle).await; + let dataset = self.resolve_dataset(dataset_handle); dataset .get_summary(GetSummaryOpts::default()) .await .unwrap() } - async fn resolve_dataset(&self, dataset_handle: &DatasetHandle) -> Arc { - self.dataset_repo - .get_dataset(&dataset_handle.as_local_ref()) - .await - .unwrap() + fn resolve_dataset(&self, dataset_handle: &DatasetHandle) -> Arc { + self.dataset_repo.get_dataset_by_handle(dataset_handle) } } diff --git a/src/infra/core/tests/tests/test_search_service_impl.rs b/src/infra/core/tests/tests/test_search_service_impl.rs index 802133c23f..3ba17e3162 100644 --- a/src/infra/core/tests/tests/test_search_service_impl.rs +++ b/src/infra/core/tests/tests/test_search_service_impl.rs @@ -10,12 +10,12 @@ use std::path::Path; use dill::Component; -use event_bus::EventBus; use kamu::domain::*; use kamu::testing::*; use kamu::*; use kamu_accounts::CurrentAccountSubject; use opendatafabric::*; +use time_source::SystemTimeSourceDefault; use url::Url; // Create repo/bar dataset in a repo and check it appears in searches @@ -28,8 +28,6 @@ async fn do_test_search(tmp_workspace_dir: &Path, repo_url: Url) { let catalog = dill::CatalogBuilder::new() .add::() - .add::() - .add::() .add_value(CurrentAccountSubject::new_test()) .add_builder( DatasetRepositoryLocalFs::builder() @@ -37,6 +35,7 @@ async fn do_test_search(tmp_workspace_dir: &Path, repo_url: Url) { .with_multi_tenant(false), ) .bind::() + .bind::() .add::() .add_value(RemoteRepositoryRegistryImpl::create(tmp_workspace_dir.join("repos")).unwrap()) .bind::() @@ -50,7 +49,7 @@ async fn do_test_search(tmp_workspace_dir: &Path, repo_url: Url) { .build(); let remote_repo_reg = catalog.get_one::().unwrap(); - let dataset_repo = catalog.get_one::().unwrap(); + let dataset_repo_writer = catalog.get_one::().unwrap(); let sync_svc = catalog.get_one::().unwrap(); let search_svc = catalog.get_one::().unwrap(); @@ -60,7 +59,7 @@ async fn do_test_search(tmp_workspace_dir: &Path, repo_url: Url) { .unwrap(); // Add and sync dataset - dataset_repo + dataset_repo_writer .create_dataset_from_snapshot( MetadataFactory::dataset_snapshot() .name(dataset_local_alias.clone()) diff --git a/src/infra/core/tests/tests/test_sync_service_impl.rs b/src/infra/core/tests/tests/test_sync_service_impl.rs index 8aa72a45c9..8bc39041fd 100644 --- a/src/infra/core/tests/tests/test_sync_service_impl.rs +++ b/src/infra/core/tests/tests/test_sync_service_impl.rs @@ -12,13 +12,13 @@ use std::path::Path; use std::str::FromStr; use dill::Component; -use event_bus::EventBus; use kamu::domain::*; use kamu::testing::*; use kamu::utils::ipfs_wrapper::IpfsClient; use kamu::*; use kamu_accounts::CurrentAccountSubject; use opendatafabric::*; +use time_source::SystemTimeSourceDefault; use url::Url; use crate::utils::IpfsDaemon; @@ -63,10 +63,10 @@ fn construct_authorizer( d2_alias: &DatasetAlias, ) -> impl auth::DatasetActionAuthorizer { MockDatasetActionAuthorizer::new() - .expect_check_read_dataset(d1_alias.clone(), authorization_expectations.d1_reads) - .expect_check_read_dataset(d2_alias.clone(), authorization_expectations.d2_reads) - .expect_check_write_dataset(d1_alias.clone(), authorization_expectations.d1_writes) - .expect_check_write_dataset(d2_alias.clone(), authorization_expectations.d2_writes) + .expect_check_read_dataset(d1_alias, authorization_expectations.d1_reads, true) + .expect_check_read_dataset(d2_alias, authorization_expectations.d2_reads, true) + .expect_check_write_dataset(d1_alias, authorization_expectations.d1_writes, true) + .expect_check_write_dataset(d2_alias, authorization_expectations.d2_writes, true) } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -93,8 +93,6 @@ async fn do_test_sync( let catalog = dill::CatalogBuilder::new() .add::() - .add::() - .add::() .add_value(ipfs_gateway) .add_value(ipfs_client) .add_value(CurrentAccountSubject::new_test()) @@ -106,6 +104,7 @@ async fn do_test_sync( .with_multi_tenant(false), ) .bind::() + .bind::() .add_value(RemoteReposDir::new(tmp_workspace_dir.join("repos"))) .add::() .add::() @@ -153,6 +152,7 @@ async fn do_test_sync( .create_dataset_from_snapshot(snapshot) .await .unwrap() + .create_dataset_result .head; // Initial sync /////////////////////////////////////////////////////////// diff --git a/src/infra/core/tests/tests/test_transform_service_impl.rs b/src/infra/core/tests/tests/test_transform_service_impl.rs index 6974149af2..4df272969e 100644 --- a/src/infra/core/tests/tests/test_transform_service_impl.rs +++ b/src/infra/core/tests/tests/test_transform_service_impl.rs @@ -12,7 +12,6 @@ use std::sync::Arc; use chrono::{TimeZone, Utc}; use dill::Component; -use event_bus::EventBus; use futures::TryStreamExt; use indoc::indoc; use kamu::domain::engine::*; @@ -22,6 +21,7 @@ use kamu::*; use kamu_accounts::CurrentAccountSubject; use opendatafabric::*; use tempfile::TempDir; +use time_source::SystemTimeSourceDefault; use crate::mock_engine_provisioner; @@ -30,6 +30,7 @@ use crate::mock_engine_provisioner; struct TransformTestHarness { _tempdir: TempDir, dataset_repo: Arc, + dataset_repo_writer: Arc, transform_service: Arc, compaction_service: Arc, push_ingest_svc: Arc, @@ -51,8 +52,6 @@ impl TransformTestHarness { let catalog = dill::CatalogBuilder::new() .add_value(RunInfoDir::new(run_info_dir)) - .add::() - .add::() .add_value(CurrentAccountSubject::new_test()) .add_builder( DatasetRepositoryLocalFs::builder() @@ -60,6 +59,7 @@ impl TransformTestHarness { .with_multi_tenant(false), ) .bind::() + .bind::() .add_value(dataset_action_authorizer) .bind::() .add::() @@ -78,6 +78,7 @@ impl TransformTestHarness { Self { _tempdir: tempdir, dataset_repo: catalog.get_one().unwrap(), + dataset_repo_writer: catalog.get_one().unwrap(), transform_service: catalog.get_one().unwrap(), compaction_service: catalog.get_one().unwrap(), push_ingest_svc: catalog.get_one().unwrap(), @@ -99,10 +100,11 @@ impl TransformTestHarness { .build(); let create_result = self - .dataset_repo + .dataset_repo_writer .create_dataset_from_snapshot(snap) .await - .unwrap(); + .unwrap() + .create_dataset_result; create_result.dataset_handle } @@ -122,10 +124,11 @@ impl TransformTestHarness { .build(); let create_result = self - .dataset_repo + .dataset_repo_writer .create_dataset_from_snapshot(snap) .await - .unwrap(); + .unwrap() + .create_dataset_result; (create_result.dataset_handle, transform) } @@ -136,7 +139,7 @@ impl TransformTestHarness { ) -> Multihash { let ds = self .dataset_repo - .get_dataset(&dataset_ref.into()) + .find_dataset_by_ref(&dataset_ref.into()) .await .unwrap(); ds.as_metadata_chain() @@ -152,7 +155,7 @@ impl TransformTestHarness { ) -> (Multihash, MetadataBlockTyped) { let ds = self .dataset_repo - .get_dataset(&alias.as_local_ref()) + .find_dataset_by_ref(&alias.as_local_ref()) .await .unwrap(); let chain = ds.as_metadata_chain(); @@ -250,12 +253,14 @@ async fn test_get_next_operation() { async fn test_transform_enforces_authorization() { let mock_dataset_action_authorizer = MockDatasetActionAuthorizer::new() .expect_check_read_dataset( - DatasetAlias::new(None, DatasetName::new_unchecked("foo")), + &DatasetAlias::new(None, DatasetName::new_unchecked("foo")), 1, + true, ) .expect_check_write_dataset( - DatasetAlias::new(None, DatasetName::new_unchecked("bar")), + &DatasetAlias::new(None, DatasetName::new_unchecked("bar")), 1, + true, ); let harness = TransformTestHarness::new_custom( @@ -309,7 +314,7 @@ async fn test_get_verification_plan_one_to_one() { let t0 = Utc.with_ymd_and_hms(2020, 1, 1, 11, 0, 0).unwrap(); let root_alias = DatasetAlias::new(None, DatasetName::new_unchecked("foo")); let root_create_result = harness - .dataset_repo + .dataset_repo_writer .create_dataset( &root_alias, MetadataFactory::metadata_block( @@ -342,7 +347,7 @@ async fn test_get_verification_plan_one_to_one() { // Create derivative let deriv_alias = DatasetAlias::new(None, DatasetName::new_unchecked("bar")); let deriv_create_result = harness - .dataset_repo + .dataset_repo_writer .create_dataset( &deriv_alias, MetadataFactory::metadata_block( @@ -596,11 +601,7 @@ async fn test_get_verification_plan_one_to_one() { .await .unwrap(); - let deriv_ds = harness - .dataset_repo - .get_dataset(&deriv_hdl.as_local_ref()) - .await - .unwrap(); + let deriv_ds = harness.dataset_repo.get_dataset_by_handle(&deriv_hdl); let deriv_chain = deriv_ds.as_metadata_chain(); assert_eq!(plan.len(), 3); @@ -637,7 +638,7 @@ async fn test_transform_with_compaction_retry() { let root_alias = DatasetAlias::new(None, DatasetName::new_unchecked("foo")); let foo_created_result = harness - .dataset_repo + .dataset_repo_writer .create_dataset_from_snapshot( MetadataFactory::dataset_snapshot() .name(root_alias) @@ -666,7 +667,8 @@ async fn test_transform_with_compaction_retry() { .build(), ) .await - .unwrap(); + .unwrap() + .create_dataset_result; let data_str = indoc!( " diff --git a/src/infra/core/tests/tests/test_verification_service_impl.rs b/src/infra/core/tests/tests/test_verification_service_impl.rs index 0babb574c6..025bec3c37 100644 --- a/src/infra/core/tests/tests/test_verification_service_impl.rs +++ b/src/infra/core/tests/tests/test_verification_service_impl.rs @@ -14,15 +14,17 @@ use datafusion::arrow::array::{Array, Int32Array, StringArray}; use datafusion::arrow::datatypes::{DataType, Field, Schema}; use datafusion::arrow::record_batch::RecordBatch; use dill::Component; -use event_bus::EventBus; use kamu::domain::*; use kamu::testing::{MetadataFactory, MockDatasetActionAuthorizer, ParquetWriterHelper}; use kamu::*; use kamu_accounts::CurrentAccountSubject; use opendatafabric::*; +use time_source::SystemTimeSourceDefault; use super::test_pull_service_impl::TestTransformService; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + #[tokio::test] async fn test_verify_data_consistency() { let tempdir = tempfile::tempdir().unwrap(); @@ -33,11 +35,9 @@ async fn test_verify_data_consistency() { let catalog = dill::CatalogBuilder::new() .add::() - .add::() - .add::() .add_value(CurrentAccountSubject::new_test()) .add_value( - MockDatasetActionAuthorizer::new().expect_check_read_dataset(dataset_alias.clone(), 3), + MockDatasetActionAuthorizer::new().expect_check_read_dataset(&dataset_alias, 3, true), ) .bind::() .add_builder( @@ -46,6 +46,7 @@ async fn test_verify_data_consistency() { .with_multi_tenant(false), ) .bind::() + .bind::() .add_value(TestTransformService::new(Arc::new(Mutex::new(Vec::new())))) .bind::() .add::() @@ -53,8 +54,9 @@ async fn test_verify_data_consistency() { let verification_svc = catalog.get_one::().unwrap(); let dataset_repo = catalog.get_one::().unwrap(); + let dataset_repo_writer = catalog.get_one::().unwrap(); - dataset_repo + dataset_repo_writer .create_dataset_from_snapshot( MetadataFactory::dataset_snapshot() .name("foo") @@ -66,7 +68,7 @@ async fn test_verify_data_consistency() { .await .unwrap(); - dataset_repo + dataset_repo_writer .create_dataset_from_snapshot( MetadataFactory::dataset_snapshot() .name(dataset_alias.clone()) @@ -120,7 +122,7 @@ async fn test_verify_data_consistency() { // Commit data let dataset = dataset_repo - .get_dataset(&dataset_alias.as_local_ref()) + .find_dataset_by_ref(&dataset_alias.as_local_ref()) .await .unwrap(); @@ -209,3 +211,5 @@ async fn test_verify_data_consistency() { } if block_hash == head && expected == data_logical_hash, ); } + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/core/tests/tests/use_cases/mod.rs b/src/infra/core/tests/tests/use_cases/mod.rs new file mode 100644 index 0000000000..9a2a97d454 --- /dev/null +++ b/src/infra/core/tests/tests/use_cases/mod.rs @@ -0,0 +1,15 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +mod test_append_dataset_metadata_batch_use_case; +mod test_commit_dataset_event_use_case; +mod test_create_dataset_from_snapshot_use_case; +mod test_create_dataset_use_case; +mod test_delete_dataset_use_case; +mod test_rename_dataset_use_case; diff --git a/src/infra/core/tests/tests/use_cases/test_append_dataset_metadata_batch_use_case.rs b/src/infra/core/tests/tests/use_cases/test_append_dataset_metadata_batch_use_case.rs new file mode 100644 index 0000000000..3f3a1d47a7 --- /dev/null +++ b/src/infra/core/tests/tests/use_cases/test_append_dataset_metadata_batch_use_case.rs @@ -0,0 +1,230 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::assert_matches::assert_matches; +use std::collections::VecDeque; +use std::sync::Arc; + +use chrono::Utc; +use dill::{Catalog, Component}; +use kamu::testing::MetadataFactory; +use kamu::{ + AppendDatasetMetadataBatchUseCaseImpl, + DatasetRepositoryLocalFs, + DatasetRepositoryWriter, +}; +use kamu_accounts::CurrentAccountSubject; +use kamu_core::{ + AppendDatasetMetadataBatchUseCase, + CreateDatasetResult, + DatasetLifecycleMessage, + DatasetRepository, + MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE, +}; +use messaging_outbox::{MockOutbox, Outbox}; +use mockall::predicate::{eq, function}; +use opendatafabric::serde::flatbuffers::FlatbuffersMetadataBlockSerializer; +use opendatafabric::serde::MetadataBlockSerializer; +use opendatafabric::{ + DatasetAlias, + DatasetKind, + DatasetName, + MetadataBlock, + MetadataEvent, + Multicodec, + Multihash, +}; +use time_source::SystemTimeSourceDefault; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[tokio::test] +async fn test_append_dataset_metadata_batch() { + let alias_foo = DatasetAlias::new(None, DatasetName::new_unchecked("foo")); + + let mock_outbox = MockOutbox::new(); + + let harness = AppendDatasetMetadataBatchUseCaseHarness::new(mock_outbox); + let create_result_foo = harness.create_dataset(&alias_foo, DatasetKind::Root).await; + + let foo_dataset = harness + .dataset_repo + .get_dataset_by_handle(&create_result_foo.dataset_handle); + + let set_info_block = MetadataBlock { + system_time: Utc::now(), + prev_block_hash: Some(create_result_foo.head.clone()), + sequence_number: 1, + event: MetadataEvent::SetInfo(MetadataFactory::set_info().description("test").build()), + }; + let hash_set_info_block = + AppendDatasetMetadataBatchUseCaseHarness::hash_from_block(&set_info_block); + + let set_license_block = MetadataBlock { + system_time: Utc::now(), + prev_block_hash: Some(hash_set_info_block.clone()), + sequence_number: 2, + event: MetadataEvent::SetLicense(MetadataFactory::set_license().build()), + }; + let hash_set_license_block = + AppendDatasetMetadataBatchUseCaseHarness::hash_from_block(&set_license_block); + + let new_blocks = VecDeque::from([ + (hash_set_info_block, set_info_block), + (hash_set_license_block, set_license_block), + ]); + + let res = harness + .use_case + .execute(foo_dataset.as_ref(), new_blocks, false) + .await; + assert_matches!(res, Ok(_)); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[tokio::test] +async fn test_append_dataset_metadata_batch_with_new_dependencies() { + let alias_foo = DatasetAlias::new(None, DatasetName::new_unchecked("foo")); + let alias_bar = DatasetAlias::new(None, DatasetName::new_unchecked("bar")); + + let mut mock_outbox = MockOutbox::new(); + AppendDatasetMetadataBatchUseCaseHarness::add_outbox_dataset_dependencies_updated_expectation( + &mut mock_outbox, + 1, + ); + + let harness = AppendDatasetMetadataBatchUseCaseHarness::new(mock_outbox); + let create_result_foo = harness.create_dataset(&alias_foo, DatasetKind::Root).await; + let create_result_bar = harness + .create_dataset(&alias_bar, DatasetKind::Derivative) + .await; + + let bar_dataset = harness + .dataset_repo + .get_dataset_by_handle(&create_result_bar.dataset_handle); + + let set_transform_block = MetadataBlock { + system_time: Utc::now(), + prev_block_hash: Some(create_result_bar.head.clone()), + sequence_number: 1, + event: MetadataEvent::SetTransform( + MetadataFactory::set_transform() + .inputs_from_refs_and_aliases(vec![( + create_result_foo.dataset_handle.id, + alias_foo.to_string(), + )]) + .build(), + ), + }; + let hash_set_transform_block = + AppendDatasetMetadataBatchUseCaseHarness::hash_from_block(&set_transform_block); + + let new_blocks = VecDeque::from([(hash_set_transform_block, set_transform_block)]); + + let res = harness + .use_case + .execute(bar_dataset.as_ref(), new_blocks, false) + .await; + assert_matches!(res, Ok(_)); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +struct AppendDatasetMetadataBatchUseCaseHarness { + _temp_dir: tempfile::TempDir, + catalog: Catalog, + dataset_repo: Arc, + use_case: Arc, +} + +impl AppendDatasetMetadataBatchUseCaseHarness { + fn new(mock_outbox: MockOutbox) -> Self { + let tempdir = tempfile::tempdir().unwrap(); + + let datasets_dir = tempdir.path().join("datasets"); + std::fs::create_dir(&datasets_dir).unwrap(); + + let catalog = dill::CatalogBuilder::new() + .add::() + .add_builder( + DatasetRepositoryLocalFs::builder() + .with_root(datasets_dir) + .with_multi_tenant(false), + ) + .bind::() + .bind::() + .add_value(CurrentAccountSubject::new_test()) + .add::() + .add_value(mock_outbox) + .bind::() + .build(); + + let use_case = catalog + .get_one::() + .unwrap(); + + let dataset_repo = catalog.get_one::().unwrap(); + + Self { + _temp_dir: tempdir, + catalog, + use_case, + dataset_repo, + } + } + + async fn create_dataset(&self, alias: &DatasetAlias, kind: DatasetKind) -> CreateDatasetResult { + let snapshot = MetadataFactory::dataset_snapshot() + .name(alias.clone()) + .kind(kind) + .build(); + + let dataset_repo_writer = self + .catalog + .get_one::() + .unwrap(); + + let result = dataset_repo_writer + .create_dataset_from_snapshot(snapshot) + .await + .unwrap(); + + result.create_dataset_result + } + + fn hash_from_block(block: &MetadataBlock) -> Multihash { + let block_data = FlatbuffersMetadataBlockSerializer + .write_manifest(block) + .unwrap(); + + Multihash::from_digest::(Multicodec::Sha3_256, &block_data) + } + + fn add_outbox_dataset_dependencies_updated_expectation( + mock_outbox: &mut MockOutbox, + times: usize, + ) { + mock_outbox + .expect_post_message_as_json() + .with( + eq(MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE), + function(|message_as_json: &serde_json::Value| { + matches!( + serde_json::from_value::(message_as_json.clone()), + Ok(DatasetLifecycleMessage::DependenciesUpdated(_)) + ) + }), + ) + .times(times) + .returning(|_, _| Ok(())); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/core/tests/tests/use_cases/test_commit_dataset_event_use_case.rs b/src/infra/core/tests/tests/use_cases/test_commit_dataset_event_use_case.rs new file mode 100644 index 0000000000..c30179b102 --- /dev/null +++ b/src/infra/core/tests/tests/use_cases/test_commit_dataset_event_use_case.rs @@ -0,0 +1,205 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::assert_matches::assert_matches; +use std::sync::Arc; + +use dill::{Catalog, Component}; +use kamu::testing::{MetadataFactory, MockDatasetActionAuthorizer}; +use kamu::{CommitDatasetEventUseCaseImpl, DatasetRepositoryLocalFs, DatasetRepositoryWriter}; +use kamu_accounts::CurrentAccountSubject; +use kamu_core::auth::DatasetActionAuthorizer; +use kamu_core::{ + CommitDatasetEventUseCase, + CommitError, + CommitOpts, + CreateDatasetResult, + DatasetLifecycleMessage, + DatasetRepository, + MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE, +}; +use messaging_outbox::{MockOutbox, Outbox}; +use mockall::predicate::{eq, function}; +use opendatafabric::{DatasetAlias, DatasetKind, DatasetName, MetadataEvent}; +use time_source::SystemTimeSourceDefault; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[tokio::test] +async fn test_commit_dataset_event() { + let alias_foo = DatasetAlias::new(None, DatasetName::new_unchecked("foo")); + + let mock_authorizer = + MockDatasetActionAuthorizer::new().expect_check_write_dataset(&alias_foo, 1, true); + + let mock_outbox = MockOutbox::new(); + + let harness = CommitDatasetEventUseCaseHarness::new(mock_authorizer, mock_outbox); + let create_result_foo = harness.create_dataset(&alias_foo, DatasetKind::Root).await; + + let res = harness + .use_case + .execute( + &create_result_foo.dataset_handle, + MetadataEvent::SetInfo(MetadataFactory::set_info().description("test").build()), + CommitOpts::default(), + ) + .await; + assert_matches!(res, Ok(_)); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[tokio::test] +async fn test_commit_event_unauthorized() { + let alias_foo = DatasetAlias::new(None, DatasetName::new_unchecked("foo")); + + let mock_authorizer = + MockDatasetActionAuthorizer::new().expect_check_write_dataset(&alias_foo, 1, false); + + let mock_outbox = MockOutbox::new(); + + let harness = CommitDatasetEventUseCaseHarness::new(mock_authorizer, mock_outbox); + let create_result_foo = harness.create_dataset(&alias_foo, DatasetKind::Root).await; + + let res = harness + .use_case + .execute( + &create_result_foo.dataset_handle, + MetadataEvent::SetInfo(MetadataFactory::set_info().description("test").build()), + CommitOpts::default(), + ) + .await; + assert_matches!(res, Err(CommitError::Access(_))); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[tokio::test] +async fn test_commit_event_with_new_dependencies() { + let alias_foo = DatasetAlias::new(None, DatasetName::new_unchecked("foo")); + let alias_bar = DatasetAlias::new(None, DatasetName::new_unchecked("bar")); + + let mock_authorizer = + MockDatasetActionAuthorizer::new().expect_check_write_dataset(&alias_bar, 1, true); + + let mut mock_outbox = MockOutbox::new(); + CommitDatasetEventUseCaseHarness::add_outbox_dataset_dependencies_updated_expectation( + &mut mock_outbox, + 1, + ); + + let harness = CommitDatasetEventUseCaseHarness::new(mock_authorizer, mock_outbox); + let create_result_foo = harness.create_dataset(&alias_foo, DatasetKind::Root).await; + let create_result_bar = harness + .create_dataset(&alias_bar, DatasetKind::Derivative) + .await; + + let res = harness + .use_case + .execute( + &create_result_bar.dataset_handle, + MetadataEvent::SetTransform( + MetadataFactory::set_transform() + .inputs_from_refs_and_aliases(vec![( + create_result_foo.dataset_handle.id, + alias_foo.to_string(), + )]) + .build(), + ), + CommitOpts::default(), + ) + .await; + assert_matches!(res, Ok(_)); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +struct CommitDatasetEventUseCaseHarness { + _temp_dir: tempfile::TempDir, + catalog: Catalog, + use_case: Arc, +} + +impl CommitDatasetEventUseCaseHarness { + fn new( + mock_dataset_action_authorizer: MockDatasetActionAuthorizer, + mock_outbox: MockOutbox, + ) -> Self { + let tempdir = tempfile::tempdir().unwrap(); + + let datasets_dir = tempdir.path().join("datasets"); + std::fs::create_dir(&datasets_dir).unwrap(); + + let catalog = dill::CatalogBuilder::new() + .add::() + .add_builder( + DatasetRepositoryLocalFs::builder() + .with_root(datasets_dir) + .with_multi_tenant(false), + ) + .bind::() + .bind::() + .add_value(CurrentAccountSubject::new_test()) + .add_value(mock_dataset_action_authorizer) + .bind::() + .add::() + .add_value(mock_outbox) + .bind::() + .build(); + + let use_case = catalog.get_one::().unwrap(); + + Self { + _temp_dir: tempdir, + catalog, + use_case, + } + } + + async fn create_dataset(&self, alias: &DatasetAlias, kind: DatasetKind) -> CreateDatasetResult { + let snapshot = MetadataFactory::dataset_snapshot() + .name(alias.clone()) + .kind(kind) + .build(); + + let dataset_repo_writer = self + .catalog + .get_one::() + .unwrap(); + + let result = dataset_repo_writer + .create_dataset_from_snapshot(snapshot) + .await + .unwrap(); + + result.create_dataset_result + } + + fn add_outbox_dataset_dependencies_updated_expectation( + mock_outbox: &mut MockOutbox, + times: usize, + ) { + mock_outbox + .expect_post_message_as_json() + .with( + eq(MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE), + function(|message_as_json: &serde_json::Value| { + matches!( + serde_json::from_value::(message_as_json.clone()), + Ok(DatasetLifecycleMessage::DependenciesUpdated(_)) + ) + }), + ) + .times(times) + .returning(|_, _| Ok(())); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/core/tests/tests/use_cases/test_create_dataset_from_snapshot_use_case.rs b/src/infra/core/tests/tests/use_cases/test_create_dataset_from_snapshot_use_case.rs new file mode 100644 index 0000000000..6244c4c786 --- /dev/null +++ b/src/infra/core/tests/tests/use_cases/test_create_dataset_from_snapshot_use_case.rs @@ -0,0 +1,182 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::assert_matches::assert_matches; +use std::sync::Arc; + +use dill::{Catalog, Component}; +use kamu::testing::MetadataFactory; +use kamu::{ + CreateDatasetFromSnapshotUseCaseImpl, + DatasetRepositoryLocalFs, + DatasetRepositoryWriter, +}; +use kamu_accounts::CurrentAccountSubject; +use kamu_core::{ + CreateDatasetFromSnapshotUseCase, + DatasetLifecycleMessage, + DatasetRepository, + GetDatasetError, + MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE, +}; +use messaging_outbox::{MockOutbox, Outbox}; +use mockall::predicate::{eq, function}; +use opendatafabric::{DatasetAlias, DatasetKind, DatasetName}; +use time_source::SystemTimeSourceDefault; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[tokio::test] +async fn test_create_root_dataset_fron_snapshot() { + let alias_foo = DatasetAlias::new(None, DatasetName::new_unchecked("foo")); + + // Expact only DatasetCreated message for "foo" + let mut mock_outbox = MockOutbox::new(); + CreateFromSnapshotUseCaseHarness::add_outbox_dataset_created_expectation(&mut mock_outbox, 1); + + let harness = CreateFromSnapshotUseCaseHarness::new(mock_outbox); + + let snapshot = MetadataFactory::dataset_snapshot() + .name(alias_foo.clone()) + .kind(DatasetKind::Root) + .push_event(MetadataFactory::set_polling_source().build()) + .build(); + + harness.use_case.execute(snapshot).await.unwrap(); + + assert_matches!(harness.check_dataset_exists(&alias_foo).await, Ok(_)); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[tokio::test] +async fn test_create_derived_dataset_fron_snapshot() { + let alias_foo = DatasetAlias::new(None, DatasetName::new_unchecked("foo")); + let alias_bar = DatasetAlias::new(None, DatasetName::new_unchecked("bar")); + + // Expact DatasetCreated messages for "foo" and "bar". + // Expect DatasetDependenciesUpdated message for "bar" + let mut mock_outbox = MockOutbox::new(); + CreateFromSnapshotUseCaseHarness::add_outbox_dataset_created_expectation(&mut mock_outbox, 2); + CreateFromSnapshotUseCaseHarness::add_outbox_dataset_dependencies_updated_expectation( + &mut mock_outbox, + 1, + ); + + let harness = CreateFromSnapshotUseCaseHarness::new(mock_outbox); + + let snapshot_root = MetadataFactory::dataset_snapshot() + .name(alias_foo.clone()) + .kind(DatasetKind::Root) + .push_event(MetadataFactory::set_polling_source().build()) + .build(); + + let snapshot_derived = MetadataFactory::dataset_snapshot() + .name(alias_bar.clone()) + .kind(DatasetKind::Derivative) + .push_event( + MetadataFactory::set_transform() + .inputs_from_refs(vec![alias_foo.as_local_ref()]) + .build(), + ) + .build(); + + harness.use_case.execute(snapshot_root).await.unwrap(); + harness.use_case.execute(snapshot_derived).await.unwrap(); + + assert_matches!(harness.check_dataset_exists(&alias_foo).await, Ok(_)); + assert_matches!(harness.check_dataset_exists(&alias_bar).await, Ok(_)); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +struct CreateFromSnapshotUseCaseHarness { + _temp_dir: tempfile::TempDir, + catalog: Catalog, + use_case: Arc, +} + +impl CreateFromSnapshotUseCaseHarness { + fn new(mock_outbox: MockOutbox) -> Self { + let tempdir = tempfile::tempdir().unwrap(); + + let datasets_dir = tempdir.path().join("datasets"); + std::fs::create_dir(&datasets_dir).unwrap(); + + let catalog = dill::CatalogBuilder::new() + .add::() + .add_builder( + DatasetRepositoryLocalFs::builder() + .with_root(datasets_dir) + .with_multi_tenant(false), + ) + .bind::() + .bind::() + .add_value(CurrentAccountSubject::new_test()) + .add::() + .add_value(mock_outbox) + .bind::() + .build(); + + let use_case = catalog + .get_one::() + .unwrap(); + + Self { + _temp_dir: tempdir, + catalog, + use_case, + } + } + + async fn check_dataset_exists(&self, alias: &DatasetAlias) -> Result<(), GetDatasetError> { + let dataset_repo = self.catalog.get_one::().unwrap(); + dataset_repo + .find_dataset_by_ref(&alias.as_local_ref()) + .await?; + Ok(()) + } + + fn add_outbox_dataset_created_expectation(mock_outbox: &mut MockOutbox, times: usize) { + mock_outbox + .expect_post_message_as_json() + .with( + eq(MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE), + function(|message_as_json: &serde_json::Value| { + matches!( + serde_json::from_value::(message_as_json.clone()), + Ok(DatasetLifecycleMessage::Created(_)) + ) + }), + ) + .times(times) + .returning(|_, _| Ok(())); + } + + fn add_outbox_dataset_dependencies_updated_expectation( + mock_outbox: &mut MockOutbox, + times: usize, + ) { + mock_outbox + .expect_post_message_as_json() + .with( + eq(MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE), + function(|message_as_json: &serde_json::Value| { + matches!( + serde_json::from_value::(message_as_json.clone()), + Ok(DatasetLifecycleMessage::DependenciesUpdated(_)) + ) + }), + ) + .times(times) + .returning(|_, _| Ok(())); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/core/tests/tests/use_cases/test_create_dataset_use_case.rs b/src/infra/core/tests/tests/use_cases/test_create_dataset_use_case.rs new file mode 100644 index 0000000000..dea6caf053 --- /dev/null +++ b/src/infra/core/tests/tests/use_cases/test_create_dataset_use_case.rs @@ -0,0 +1,117 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::assert_matches::assert_matches; +use std::sync::Arc; + +use dill::{Catalog, Component}; +use kamu::testing::MetadataFactory; +use kamu::{CreateDatasetUseCaseImpl, DatasetRepositoryLocalFs, DatasetRepositoryWriter}; +use kamu_accounts::CurrentAccountSubject; +use kamu_core::{ + CreateDatasetUseCase, + DatasetLifecycleMessage, + DatasetRepository, + GetDatasetError, + MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE, +}; +use messaging_outbox::{MockOutbox, Outbox}; +use mockall::predicate::{eq, function}; +use opendatafabric::{DatasetAlias, DatasetKind, DatasetName}; +use time_source::SystemTimeSourceDefault; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[tokio::test] +async fn test_create_root_dataset() { + let alias_foo = DatasetAlias::new(None, DatasetName::new_unchecked("foo")); + + let mut mock_outbox = MockOutbox::new(); + CreateUseCaseHarness::add_outbox_dataset_created_expectation(&mut mock_outbox, 1); + + let harness = CreateUseCaseHarness::new(mock_outbox); + + harness + .use_case + .execute( + &alias_foo, + MetadataFactory::metadata_block(MetadataFactory::seed(DatasetKind::Root).build()) + .build_typed(), + ) + .await + .unwrap(); + + assert_matches!(harness.check_dataset_exists(&alias_foo).await, Ok(_)); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +struct CreateUseCaseHarness { + _temp_dir: tempfile::TempDir, + catalog: Catalog, + use_case: Arc, +} + +impl CreateUseCaseHarness { + fn new(mock_outbox: MockOutbox) -> Self { + let tempdir = tempfile::tempdir().unwrap(); + + let datasets_dir = tempdir.path().join("datasets"); + std::fs::create_dir(&datasets_dir).unwrap(); + + let catalog = dill::CatalogBuilder::new() + .add::() + .add_builder( + DatasetRepositoryLocalFs::builder() + .with_root(datasets_dir) + .with_multi_tenant(false), + ) + .bind::() + .bind::() + .add_value(CurrentAccountSubject::new_test()) + .add::() + .add_value(mock_outbox) + .bind::() + .build(); + + let use_case = catalog.get_one::().unwrap(); + + Self { + _temp_dir: tempdir, + catalog, + use_case, + } + } + + async fn check_dataset_exists(&self, alias: &DatasetAlias) -> Result<(), GetDatasetError> { + let dataset_repo = self.catalog.get_one::().unwrap(); + dataset_repo + .find_dataset_by_ref(&alias.as_local_ref()) + .await?; + Ok(()) + } + + fn add_outbox_dataset_created_expectation(mock_outbox: &mut MockOutbox, times: usize) { + mock_outbox + .expect_post_message_as_json() + .with( + eq(MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE), + function(|message_as_json: &serde_json::Value| { + matches!( + serde_json::from_value::(message_as_json.clone()), + Ok(DatasetLifecycleMessage::Created(_)) + ) + }), + ) + .times(times) + .returning(|_, _| Ok(())); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/core/tests/tests/use_cases/test_delete_dataset_use_case.rs b/src/infra/core/tests/tests/use_cases/test_delete_dataset_use_case.rs new file mode 100644 index 0000000000..b22141013a --- /dev/null +++ b/src/infra/core/tests/tests/use_cases/test_delete_dataset_use_case.rs @@ -0,0 +1,345 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::assert_matches::assert_matches; +use std::sync::Arc; + +use dill::{Catalog, Component}; +use kamu::testing::{MetadataFactory, MockDatasetActionAuthorizer}; +use kamu::{ + DatasetRepositoryLocalFs, + DatasetRepositoryWriter, + DeleteDatasetUseCaseImpl, + DependencyGraphRepositoryInMemory, + DependencyGraphServiceInMemory, +}; +use kamu_accounts::CurrentAccountSubject; +use kamu_core::auth::DatasetActionAuthorizer; +use kamu_core::{ + CreateDatasetResult, + DatasetLifecycleMessage, + DatasetRepository, + DeleteDatasetError, + DeleteDatasetUseCase, + DependencyGraphService, + GetDatasetError, + MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE, +}; +use messaging_outbox::{consume_deserialized_message, ConsumerFilter, Message, MockOutbox, Outbox}; +use mockall::predicate::{eq, function}; +use opendatafabric::{DatasetAlias, DatasetKind, DatasetName, DatasetRef}; +use time_source::SystemTimeSourceDefault; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[tokio::test] +async fn test_delete_dataset_success_via_ref() { + let alias_foo = DatasetAlias::new(None, DatasetName::new_unchecked("foo")); + + let mut mock_outbox = MockOutbox::new(); + DeleteUseCaseHarness::add_outbox_dataset_deleted_expectation(&mut mock_outbox, 1); + + let mock_authorizer = + MockDatasetActionAuthorizer::new().expect_check_write_dataset(&alias_foo, 1, true); + + let harness = DeleteUseCaseHarness::new(mock_authorizer, mock_outbox); + + harness.create_root_dataset(&alias_foo).await; + harness.dependencies_eager_initialization().await; + + harness + .use_case + .execute_via_ref(&alias_foo.as_local_ref()) + .await + .unwrap(); + + assert_matches!( + harness.check_dataset_exists(&alias_foo).await, + Err(GetDatasetError::NotFound(_)) + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[tokio::test] +async fn test_delete_dataset_success_via_handle() { + let alias_foo = DatasetAlias::new(None, DatasetName::new_unchecked("foo")); + + let mut mock_outbox = MockOutbox::new(); + DeleteUseCaseHarness::add_outbox_dataset_deleted_expectation(&mut mock_outbox, 1); + + let mock_authorizer = + MockDatasetActionAuthorizer::new().expect_check_write_dataset(&alias_foo, 1, true); + + let harness = DeleteUseCaseHarness::new(mock_authorizer, mock_outbox); + + let create_result_foo = harness.create_root_dataset(&alias_foo).await; + harness.dependencies_eager_initialization().await; + + harness + .use_case + .execute_via_handle(&create_result_foo.dataset_handle) + .await + .unwrap(); + + assert_matches!( + harness.check_dataset_exists(&alias_foo).await, + Err(GetDatasetError::NotFound(_)) + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[tokio::test] +async fn test_delete_dataset_not_found() { + let harness = DeleteUseCaseHarness::new(MockDatasetActionAuthorizer::new(), MockOutbox::new()); + + let alias_foo = DatasetAlias::new(None, DatasetName::new_unchecked("foo")); + assert_matches!( + harness + .use_case + .execute_via_ref(&alias_foo.as_local_ref()) + .await, + Err(DeleteDatasetError::NotFound(_)) + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[tokio::test] +async fn test_delete_unauthorized() { + let alias_foo = DatasetAlias::new(None, DatasetName::new_unchecked("foo")); + + let harness = DeleteUseCaseHarness::new( + MockDatasetActionAuthorizer::new().expect_check_write_dataset(&alias_foo, 1, false), + MockOutbox::new(), + ); + + let create_result_foo = harness.create_root_dataset(&alias_foo).await; + harness.dependencies_eager_initialization().await; + + assert_matches!( + harness + .use_case + .execute_via_handle(&create_result_foo.dataset_handle) + .await, + Err(DeleteDatasetError::Access(_)) + ); + + assert_matches!(harness.check_dataset_exists(&alias_foo).await, Ok(_)); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[tokio::test] +async fn test_delete_dataset_respects_dangling_refs() { + let alias_foo = DatasetAlias::new(None, DatasetName::new_unchecked("foo")); + let alias_bar = DatasetAlias::new(None, DatasetName::new_unchecked("bar")); + + let mut mock_outbox = MockOutbox::new(); + DeleteUseCaseHarness::add_outbox_dataset_deleted_expectation(&mut mock_outbox, 2); + + let harness = DeleteUseCaseHarness::new(MockDatasetActionAuthorizer::allowing(), mock_outbox); + + let create_result_root = harness.create_root_dataset(&alias_foo).await; + let create_result_derived = harness + .create_derived_dataset(&alias_bar, vec![alias_foo.as_local_ref()]) + .await; + harness.dependencies_eager_initialization().await; + + assert_matches!( + harness.use_case.execute_via_handle(&create_result_root.dataset_handle).await, + Err(DeleteDatasetError::DanglingReference(e)) if e.children == vec![create_result_derived.dataset_handle.clone()] + ); + + assert_matches!(harness.check_dataset_exists(&alias_foo).await, Ok(_)); + assert_matches!(harness.check_dataset_exists(&alias_bar).await, Ok(_)); + + harness + .use_case + .execute_via_handle(&create_result_derived.dataset_handle) + .await + .unwrap(); + + harness + .consume_message(DatasetLifecycleMessage::deleted( + create_result_derived.dataset_handle.id, + )) + .await; + + assert_matches!(harness.check_dataset_exists(&alias_foo).await, Ok(_)); + assert_matches!( + harness.check_dataset_exists(&alias_bar).await, + Err(GetDatasetError::NotFound(_)) + ); + + harness + .use_case + .execute_via_handle(&create_result_root.dataset_handle) + .await + .unwrap(); + + harness + .consume_message(DatasetLifecycleMessage::deleted( + create_result_root.dataset_handle.id, + )) + .await; + + assert_matches!( + harness.check_dataset_exists(&alias_foo).await, + Err(GetDatasetError::NotFound(_)) + ); + assert_matches!( + harness.check_dataset_exists(&alias_bar).await, + Err(GetDatasetError::NotFound(_)) + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +struct DeleteUseCaseHarness { + _temp_dir: tempfile::TempDir, + catalog: Catalog, + use_case: Arc, +} + +impl DeleteUseCaseHarness { + fn new( + mock_dataset_action_authorizer: MockDatasetActionAuthorizer, + mock_outbox: MockOutbox, + ) -> Self { + let tempdir = tempfile::tempdir().unwrap(); + + let datasets_dir = tempdir.path().join("datasets"); + std::fs::create_dir(&datasets_dir).unwrap(); + + let catalog = dill::CatalogBuilder::new() + .add::() + .add_builder( + DatasetRepositoryLocalFs::builder() + .with_root(datasets_dir) + .with_multi_tenant(false), + ) + .bind::() + .bind::() + .add_value(CurrentAccountSubject::new_test()) + .add::() + .add_value(mock_dataset_action_authorizer) + .bind::() + .add::() + .add_value(mock_outbox) + .bind::() + .build(); + + let use_case = catalog.get_one::().unwrap(); + + Self { + _temp_dir: tempdir, + catalog, + use_case, + } + } + + async fn create_root_dataset(&self, alias: &DatasetAlias) -> CreateDatasetResult { + let snapshot = MetadataFactory::dataset_snapshot() + .name(alias.clone()) + .kind(DatasetKind::Root) + .push_event(MetadataFactory::set_polling_source().build()) + .build(); + + let dataset_repo_writer = self + .catalog + .get_one::() + .unwrap(); + + let result = dataset_repo_writer + .create_dataset_from_snapshot(snapshot) + .await + .unwrap(); + + result.create_dataset_result + } + + async fn create_derived_dataset( + &self, + alias: &DatasetAlias, + input_dataset_refs: Vec, + ) -> CreateDatasetResult { + let dataset_repo_writer = self + .catalog + .get_one::() + .unwrap(); + + dataset_repo_writer + .create_dataset_from_snapshot( + MetadataFactory::dataset_snapshot() + .name(alias.clone()) + .kind(DatasetKind::Derivative) + .push_event( + MetadataFactory::set_transform() + .inputs_from_refs(input_dataset_refs) + .build(), + ) + .build(), + ) + .await + .unwrap() + .create_dataset_result + } + + async fn check_dataset_exists(&self, alias: &DatasetAlias) -> Result<(), GetDatasetError> { + let dataset_repo = self.catalog.get_one::().unwrap(); + dataset_repo + .find_dataset_by_ref(&alias.as_local_ref()) + .await?; + Ok(()) + } + + async fn dependencies_eager_initialization(&self) { + let dependency_graph_service = self + .catalog + .get_one::() + .unwrap(); + let dataset_repo = self.catalog.get_one::().unwrap(); + + dependency_graph_service + .eager_initialization(&DependencyGraphRepositoryInMemory::new(dataset_repo)) + .await + .unwrap(); + } + + async fn consume_message(&self, message: TMessage) { + let content_json = serde_json::to_string(&message).unwrap(); + consume_deserialized_message::( + &self.catalog, + ConsumerFilter::AllConsumers, + &content_json, + ) + .await + .unwrap(); + } + + fn add_outbox_dataset_deleted_expectation(mock_outbox: &mut MockOutbox, times: usize) { + mock_outbox + .expect_post_message_as_json() + .with( + eq(MESSAGE_PRODUCER_KAMU_CORE_DATASET_SERVICE), + function(|message_as_json: &serde_json::Value| { + matches!( + serde_json::from_value::(message_as_json.clone()), + Ok(DatasetLifecycleMessage::Deleted(_)) + ) + }), + ) + .times(times) + .returning(|_, _| Ok(())); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/core/tests/tests/use_cases/test_rename_dataset_use_case.rs b/src/infra/core/tests/tests/use_cases/test_rename_dataset_use_case.rs new file mode 100644 index 0000000000..6661094836 --- /dev/null +++ b/src/infra/core/tests/tests/use_cases/test_rename_dataset_use_case.rs @@ -0,0 +1,173 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::assert_matches::assert_matches; +use std::sync::Arc; + +use dill::{Catalog, Component}; +use kamu::testing::{MetadataFactory, MockDatasetActionAuthorizer}; +use kamu::{DatasetRepositoryLocalFs, DatasetRepositoryWriter, RenameDatasetUseCaseImpl}; +use kamu_accounts::CurrentAccountSubject; +use kamu_core::auth::DatasetActionAuthorizer; +use kamu_core::{ + CreateDatasetResult, + DatasetRepository, + GetDatasetError, + RenameDatasetError, + RenameDatasetUseCase, +}; +use opendatafabric::{DatasetAlias, DatasetKind, DatasetName}; +use time_source::SystemTimeSourceDefault; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[tokio::test] +async fn test_rename_dataset_success_via_ref() { + let alias_foo = DatasetAlias::new(None, DatasetName::new_unchecked("foo")); + let alias_bar = DatasetAlias::new(None, DatasetName::new_unchecked("bar")); + + let mock_authorizer = + MockDatasetActionAuthorizer::new().expect_check_write_dataset(&alias_foo, 1, true); + + let harness = RenameUseCaseHarness::new(mock_authorizer); + harness.create_root_dataset(&alias_foo).await; + + assert_matches!(harness.check_dataset_exists(&alias_foo).await, Ok(_)); + assert_matches!( + harness.check_dataset_exists(&alias_bar).await, + Err(GetDatasetError::NotFound(_)) + ); + + harness + .use_case + .execute(&alias_foo.as_local_ref(), &alias_bar.dataset_name) + .await + .unwrap(); + + assert_matches!( + harness.check_dataset_exists(&alias_foo).await, + Err(GetDatasetError::NotFound(_)) + ); + assert_matches!(harness.check_dataset_exists(&alias_bar).await, Ok(_)); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[tokio::test] +async fn test_rename_dataset_not_found() { + let harness = RenameUseCaseHarness::new(MockDatasetActionAuthorizer::new()); + + let alias_foo = DatasetAlias::new(None, DatasetName::new_unchecked("foo")); + assert_matches!( + harness + .use_case + .execute( + &alias_foo.as_local_ref(), + &DatasetName::new_unchecked("bar") + ) + .await, + Err(RenameDatasetError::NotFound(_)) + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[tokio::test] +async fn test_rename_dataset_unauthorized() { + let alias_foo = DatasetAlias::new(None, DatasetName::new_unchecked("foo")); + + let harness = RenameUseCaseHarness::new( + MockDatasetActionAuthorizer::new().expect_check_write_dataset(&alias_foo, 1, false), + ); + + harness.create_root_dataset(&alias_foo).await; + + assert_matches!( + harness + .use_case + .execute( + &alias_foo.as_local_ref(), + &DatasetName::new_unchecked("bar") + ) + .await, + Err(RenameDatasetError::Access(_)) + ); + + assert_matches!(harness.check_dataset_exists(&alias_foo).await, Ok(_)); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +struct RenameUseCaseHarness { + _temp_dir: tempfile::TempDir, + catalog: Catalog, + use_case: Arc, +} + +impl RenameUseCaseHarness { + fn new(mock_dataset_action_authorizer: MockDatasetActionAuthorizer) -> Self { + let tempdir = tempfile::tempdir().unwrap(); + + let datasets_dir = tempdir.path().join("datasets"); + std::fs::create_dir(&datasets_dir).unwrap(); + + let catalog = dill::CatalogBuilder::new() + .add::() + .add_builder( + DatasetRepositoryLocalFs::builder() + .with_root(datasets_dir) + .with_multi_tenant(false), + ) + .bind::() + .bind::() + .add_value(CurrentAccountSubject::new_test()) + .add_value(mock_dataset_action_authorizer) + .bind::() + .add::() + .build(); + + let use_case = catalog.get_one::().unwrap(); + + Self { + _temp_dir: tempdir, + catalog, + use_case, + } + } + + async fn create_root_dataset(&self, alias: &DatasetAlias) -> CreateDatasetResult { + let snapshot = MetadataFactory::dataset_snapshot() + .name(alias.clone()) + .kind(DatasetKind::Root) + .push_event(MetadataFactory::set_polling_source().build()) + .build(); + + let dataset_repo_writer = self + .catalog + .get_one::() + .unwrap(); + + let result = dataset_repo_writer + .create_dataset_from_snapshot(snapshot) + .await + .unwrap(); + + result.create_dataset_result + } + + async fn check_dataset_exists(&self, alias: &DatasetAlias) -> Result<(), GetDatasetError> { + let dataset_repo = self.catalog.get_one::().unwrap(); + dataset_repo + .find_dataset_by_ref(&alias.as_local_ref()) + .await?; + Ok(()) + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/datasets/inmem/Cargo.toml b/src/infra/datasets/inmem/Cargo.toml index 6b11a2876e..06484f855f 100644 --- a/src/infra/datasets/inmem/Cargo.toml +++ b/src/infra/datasets/inmem/Cargo.toml @@ -29,7 +29,7 @@ internal-error = { workspace = true } async-trait = { version = "0.1", default-features = false } chrono = { version = "0.4", default-features = false } -dill = "0.8" +dill = "0.9" secrecy = "0.8" thiserror = { version = "1", default-features = false } tokio = { version = "1", default-features = false } diff --git a/src/infra/datasets/inmem/src/repos/dataset_env_var_repository_inmem.rs b/src/infra/datasets/inmem/src/repos/inmem_dataset_env_var_repository.rs similarity index 98% rename from src/infra/datasets/inmem/src/repos/dataset_env_var_repository_inmem.rs rename to src/infra/datasets/inmem/src/repos/inmem_dataset_env_var_repository.rs index 97ae426c1a..c7f728d514 100644 --- a/src/infra/datasets/inmem/src/repos/dataset_env_var_repository_inmem.rs +++ b/src/infra/datasets/inmem/src/repos/inmem_dataset_env_var_repository.rs @@ -20,7 +20,7 @@ use crate::domain::*; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub struct DatasetEnvVarRepositoryInMemory { +pub struct InMemoryDatasetEnvVarRepository { state: Arc>, } @@ -48,7 +48,7 @@ impl State { #[component(pub)] #[interface(dyn DatasetEnvVarRepository)] #[scope(Singleton)] -impl DatasetEnvVarRepositoryInMemory { +impl InMemoryDatasetEnvVarRepository { pub fn new() -> Self { Self { state: Arc::new(Mutex::new(State::new())), @@ -59,7 +59,7 @@ impl DatasetEnvVarRepositoryInMemory { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[async_trait::async_trait] -impl DatasetEnvVarRepository for DatasetEnvVarRepositoryInMemory { +impl DatasetEnvVarRepository for InMemoryDatasetEnvVarRepository { async fn save_dataset_env_var( &self, dataset_env_var: &DatasetEnvVar, diff --git a/src/infra/datasets/inmem/src/repos/dateset_entry_repository_inmem.rs b/src/infra/datasets/inmem/src/repos/inmem_dateset_entry_repository.rs similarity index 97% rename from src/infra/datasets/inmem/src/repos/dateset_entry_repository_inmem.rs rename to src/infra/datasets/inmem/src/repos/inmem_dateset_entry_repository.rs index 59920dd5c5..f55264266d 100644 --- a/src/infra/datasets/inmem/src/repos/dateset_entry_repository_inmem.rs +++ b/src/infra/datasets/inmem/src/repos/inmem_dateset_entry_repository.rs @@ -45,14 +45,14 @@ impl State { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub struct DatasetEntryRepositoryInMemory { +pub struct InMemoryDatasetEntryRepository { state: Arc>, } #[component(pub)] #[interface(dyn DatasetEntryRepository)] #[scope(Singleton)] -impl DatasetEntryRepositoryInMemory { +impl InMemoryDatasetEntryRepository { pub fn new() -> Self { Self { state: Arc::new(RwLock::new(State::new())), @@ -63,7 +63,7 @@ impl DatasetEntryRepositoryInMemory { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[async_trait::async_trait] -impl DatasetEntryRepository for DatasetEntryRepositoryInMemory { +impl DatasetEntryRepository for InMemoryDatasetEntryRepository { async fn get_dataset_entry( &self, dataset_id: &DatasetID, diff --git a/src/infra/datasets/inmem/src/repos/mod.rs b/src/infra/datasets/inmem/src/repos/mod.rs index 3addbe5c64..3d6a04f4da 100644 --- a/src/infra/datasets/inmem/src/repos/mod.rs +++ b/src/infra/datasets/inmem/src/repos/mod.rs @@ -7,8 +7,8 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -mod dataset_env_var_repository_inmem; -mod dateset_entry_repository_inmem; +mod inmem_dataset_env_var_repository; +mod inmem_dateset_entry_repository; -pub use dataset_env_var_repository_inmem::*; -pub use dateset_entry_repository_inmem::*; +pub use inmem_dataset_env_var_repository::*; +pub use inmem_dateset_entry_repository::*; diff --git a/src/infra/datasets/inmem/tests/repos/mod.rs b/src/infra/datasets/inmem/tests/repos/mod.rs index 3b48f80725..411ed29120 100644 --- a/src/infra/datasets/inmem/tests/repos/mod.rs +++ b/src/infra/datasets/inmem/tests/repos/mod.rs @@ -7,5 +7,5 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -mod test_dataset_entry_repository_inmem; -mod test_dataset_env_var_repository_inmem; +mod test_inmem_dataset_entry_repository; +mod test_inmem_dataset_env_var_repository; diff --git a/src/infra/datasets/inmem/tests/repos/test_dataset_entry_repository_inmem.rs b/src/infra/datasets/inmem/tests/repos/test_inmem_dataset_entry_repository.rs similarity index 81% rename from src/infra/datasets/inmem/tests/repos/test_dataset_entry_repository_inmem.rs rename to src/infra/datasets/inmem/tests/repos/test_inmem_dataset_entry_repository.rs index 3207a34350..e8ca69b2b1 100644 --- a/src/infra/datasets/inmem/tests/repos/test_dataset_entry_repository_inmem.rs +++ b/src/infra/datasets/inmem/tests/repos/test_inmem_dataset_entry_repository.rs @@ -8,14 +8,14 @@ // by the Apache License, Version 2.0. use dill::{Catalog, CatalogBuilder}; -use kamu_datasets_inmem::DatasetEntryRepositoryInMemory; +use kamu_datasets_inmem::InMemoryDatasetEntryRepository; use kamu_datasets_repo_tests::dataset_entry_repo; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[test_log::test(tokio::test)] async fn test_get_dataset_entry() { - let harness = InmemDatasetEntryRepositoryHarness::new(); + let harness = InMemoryDatasetEntryRepositoryHarness::new(); dataset_entry_repo::test_get_dataset_entry(&harness.catalog).await; } @@ -24,7 +24,7 @@ async fn test_get_dataset_entry() { #[test_log::test(tokio::test)] async fn test_get_dataset_entry_by_name() { - let harness = InmemDatasetEntryRepositoryHarness::new(); + let harness = InMemoryDatasetEntryRepositoryHarness::new(); dataset_entry_repo::test_get_dataset_entry_by_name(&harness.catalog).await; } @@ -33,7 +33,7 @@ async fn test_get_dataset_entry_by_name() { #[test_log::test(tokio::test)] async fn test_get_dataset_entries_by_owner_id() { - let harness = InmemDatasetEntryRepositoryHarness::new(); + let harness = InMemoryDatasetEntryRepositoryHarness::new(); dataset_entry_repo::test_get_dataset_entries_by_owner_id(&harness.catalog).await; } @@ -42,7 +42,7 @@ async fn test_get_dataset_entries_by_owner_id() { #[test_log::test(tokio::test)] async fn test_try_save_duplicate_dataset_entry() { - let harness = InmemDatasetEntryRepositoryHarness::new(); + let harness = InMemoryDatasetEntryRepositoryHarness::new(); dataset_entry_repo::test_try_save_duplicate_dataset_entry(&harness.catalog).await; } @@ -51,7 +51,7 @@ async fn test_try_save_duplicate_dataset_entry() { #[test_log::test(tokio::test)] async fn test_try_set_same_dataset_name() { - let harness = InmemDatasetEntryRepositoryHarness::new(); + let harness = InMemoryDatasetEntryRepositoryHarness::new(); dataset_entry_repo::test_try_set_same_dataset_name(&harness.catalog).await; } @@ -60,7 +60,7 @@ async fn test_try_set_same_dataset_name() { #[test_log::test(tokio::test)] async fn test_update_dataset_entry_name() { - let harness = InmemDatasetEntryRepositoryHarness::new(); + let harness = InMemoryDatasetEntryRepositoryHarness::new(); dataset_entry_repo::test_update_dataset_entry_name(&harness.catalog).await; } @@ -69,22 +69,22 @@ async fn test_update_dataset_entry_name() { #[test_log::test(tokio::test)] async fn test_delete_dataset_entry() { - let harness = InmemDatasetEntryRepositoryHarness::new(); + let harness = InMemoryDatasetEntryRepositoryHarness::new(); dataset_entry_repo::test_delete_dataset_entry(&harness.catalog).await; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -struct InmemDatasetEntryRepositoryHarness { +struct InMemoryDatasetEntryRepositoryHarness { catalog: Catalog, } -impl InmemDatasetEntryRepositoryHarness { +impl InMemoryDatasetEntryRepositoryHarness { pub fn new() -> Self { let mut catalog_builder = CatalogBuilder::new(); - catalog_builder.add::(); + catalog_builder.add::(); Self { catalog: catalog_builder.build(), diff --git a/src/infra/datasets/inmem/tests/repos/test_dataset_env_var_repository_inmem.rs b/src/infra/datasets/inmem/tests/repos/test_inmem_dataset_env_var_repository.rs similarity index 80% rename from src/infra/datasets/inmem/tests/repos/test_dataset_env_var_repository_inmem.rs rename to src/infra/datasets/inmem/tests/repos/test_inmem_dataset_env_var_repository.rs index 6718854154..a00eadda83 100644 --- a/src/infra/datasets/inmem/tests/repos/test_dataset_env_var_repository_inmem.rs +++ b/src/infra/datasets/inmem/tests/repos/test_inmem_dataset_env_var_repository.rs @@ -8,14 +8,14 @@ // by the Apache License, Version 2.0. use dill::{Catalog, CatalogBuilder}; -use kamu_datasets_inmem::DatasetEnvVarRepositoryInMemory; +use kamu_datasets_inmem::InMemoryDatasetEnvVarRepository; use kamu_datasets_repo_tests::dataset_env_var_repo; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[test_log::test(tokio::test)] async fn test_missing_dataset_env_var_not_found() { - let harness = InmemDatasetEnvVarRepositoryHarness::new(); + let harness = InMemoryDatasetEnvVarRepositoryHarness::new(); dataset_env_var_repo::test_missing_dataset_env_var_not_found(&harness.catalog).await; } @@ -24,7 +24,7 @@ async fn test_missing_dataset_env_var_not_found() { #[test_log::test(tokio::test)] async fn test_insert_and_get_dataset_env_var() { - let harness = InmemDatasetEnvVarRepositoryHarness::new(); + let harness = InMemoryDatasetEnvVarRepositoryHarness::new(); dataset_env_var_repo::test_insert_and_get_dataset_env_var(&harness.catalog).await; } @@ -33,7 +33,7 @@ async fn test_insert_and_get_dataset_env_var() { #[test_log::test(tokio::test)] async fn test_insert_and_get_multiple_dataset_env_vars() { - let harness = InmemDatasetEnvVarRepositoryHarness::new(); + let harness = InMemoryDatasetEnvVarRepositoryHarness::new(); dataset_env_var_repo::test_insert_and_get_multiple_dataset_env_vars(&harness.catalog).await; } @@ -42,7 +42,7 @@ async fn test_insert_and_get_multiple_dataset_env_vars() { #[test_log::test(tokio::test)] async fn test_delete_dataset_env_vars() { - let harness = InmemDatasetEnvVarRepositoryHarness::new(); + let harness = InMemoryDatasetEnvVarRepositoryHarness::new(); dataset_env_var_repo::test_delete_dataset_env_vars(&harness.catalog).await; } @@ -51,21 +51,21 @@ async fn test_delete_dataset_env_vars() { #[test_log::test(tokio::test)] async fn test_modify_dataset_env_vars() { - let harness = InmemDatasetEnvVarRepositoryHarness::new(); + let harness = InMemoryDatasetEnvVarRepositoryHarness::new(); dataset_env_var_repo::test_modify_dataset_env_vars(&harness.catalog).await; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -struct InmemDatasetEnvVarRepositoryHarness { +struct InMemoryDatasetEnvVarRepositoryHarness { catalog: Catalog, } -impl InmemDatasetEnvVarRepositoryHarness { +impl InMemoryDatasetEnvVarRepositoryHarness { pub fn new() -> Self { let mut catalog_builder = CatalogBuilder::new(); - catalog_builder.add::(); + catalog_builder.add::(); Self { catalog: catalog_builder.build(), diff --git a/src/infra/datasets/postgres/Cargo.toml b/src/infra/datasets/postgres/Cargo.toml index 8438bace6b..8157fdae1f 100644 --- a/src/infra/datasets/postgres/Cargo.toml +++ b/src/infra/datasets/postgres/Cargo.toml @@ -29,7 +29,7 @@ opendatafabric = { workspace = true, features = ["sqlx-postgres"] } async-trait = { version = "0.1", default-features = false } chrono = { version = "0.4", default-features = false } -dill = "0.8" +dill = "0.9" secrecy = "0.8" sqlx = { version = "0.7", default-features = false, features = [ "runtime-tokio-rustls", diff --git a/src/infra/datasets/repo-tests/Cargo.toml b/src/infra/datasets/repo-tests/Cargo.toml index 320acf30b4..2d89477b24 100644 --- a/src/infra/datasets/repo-tests/Cargo.toml +++ b/src/infra/datasets/repo-tests/Cargo.toml @@ -27,7 +27,7 @@ opendatafabric = { workspace = true } kamu-datasets = { workspace = true } chrono = { version = "0.4", default-features = false } -dill = "0.8" +dill = "0.9" secrecy = "0.8" uuid = "1" diff --git a/src/infra/datasets/sqlite/Cargo.toml b/src/infra/datasets/sqlite/Cargo.toml index 6b82853389..f829eee789 100644 --- a/src/infra/datasets/sqlite/Cargo.toml +++ b/src/infra/datasets/sqlite/Cargo.toml @@ -29,7 +29,7 @@ opendatafabric = { workspace = true, features = ["sqlx-sqlite"] } async-trait = { version = "0.1", default-features = false } chrono = { version = "0.4", default-features = false } -dill = "0.8" +dill = "0.9" secrecy = "0.8" sqlx = { version = "0.7", default-features = false, features = [ "runtime-tokio-rustls", diff --git a/src/infra/flow-system/inmem/Cargo.toml b/src/infra/flow-system/inmem/Cargo.toml index 59008fada7..43262e1585 100644 --- a/src/infra/flow-system/inmem/Cargo.toml +++ b/src/infra/flow-system/inmem/Cargo.toml @@ -29,7 +29,7 @@ kamu-flow-system = { workspace = true } async-stream = "0.3" async-trait = { version = "0.1", default-features = false } chrono = { version = "0.4", default-features = false } -dill = "0.8" +dill = "0.9" futures = "0.3" thiserror = { version = "1", default-features = false } tokio = { version = "1", default-features = false, features = [] } diff --git a/src/infra/flow-system/inmem/src/flow/flow_event_store_inmem.rs b/src/infra/flow-system/inmem/src/flow/inmem_flow_event_store.rs similarity index 98% rename from src/infra/flow-system/inmem/src/flow/flow_event_store_inmem.rs rename to src/infra/flow-system/inmem/src/flow/inmem_flow_event_store.rs index 5fbeb35f5d..5e049082f4 100644 --- a/src/infra/flow-system/inmem/src/flow/flow_event_store_inmem.rs +++ b/src/infra/flow-system/inmem/src/flow/inmem_flow_event_store.rs @@ -16,8 +16,8 @@ use opendatafabric::{AccountID, DatasetID}; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub struct FlowEventStoreInMem { - inner: EventStoreInMemory, +pub struct InMemoryFlowEventStore { + inner: InMemoryEventStore, } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -151,10 +151,10 @@ impl FlowIndexEntry { #[component(pub)] #[interface(dyn FlowEventStore)] #[scope(Singleton)] -impl FlowEventStoreInMem { +impl InMemoryFlowEventStore { pub fn new() -> Self { Self { - inner: EventStoreInMemory::new(), + inner: InMemoryEventStore::new(), } } @@ -250,7 +250,7 @@ impl FlowEventStoreInMem { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[async_trait::async_trait] -impl EventStore for FlowEventStoreInMem { +impl EventStore for InMemoryFlowEventStore { #[tracing::instrument(level = "debug", skip_all)] async fn len(&self) -> Result { self.inner.len().await @@ -282,7 +282,7 @@ impl EventStore for FlowEventStoreInMem { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[async_trait::async_trait] -impl FlowEventStore for FlowEventStoreInMem { +impl FlowEventStore for InMemoryFlowEventStore { #[tracing::instrument(level = "debug", skip_all)] fn new_flow_id(&self) -> FlowID { self.inner.as_state().lock().unwrap().next_flow_id() diff --git a/src/infra/flow-system/inmem/src/flow/mod.rs b/src/infra/flow-system/inmem/src/flow/mod.rs index 370c975ab0..a5c11a56f9 100644 --- a/src/infra/flow-system/inmem/src/flow/mod.rs +++ b/src/infra/flow-system/inmem/src/flow/mod.rs @@ -7,6 +7,6 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -mod flow_event_store_inmem; +mod inmem_flow_event_store; -pub use flow_event_store_inmem::*; +pub use inmem_flow_event_store::*; diff --git a/src/infra/flow-system/inmem/src/flow_configuration/flow_configuration_event_store_inmem.rs b/src/infra/flow-system/inmem/src/flow_configuration/inmem_flow_configuration_event_store.rs similarity index 89% rename from src/infra/flow-system/inmem/src/flow_configuration/flow_configuration_event_store_inmem.rs rename to src/infra/flow-system/inmem/src/flow_configuration/inmem_flow_configuration_event_store.rs index b471e610cf..7b87062331 100644 --- a/src/infra/flow-system/inmem/src/flow_configuration/flow_configuration_event_store_inmem.rs +++ b/src/infra/flow-system/inmem/src/flow_configuration/inmem_flow_configuration_event_store.rs @@ -13,8 +13,8 @@ use opendatafabric::DatasetID; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub struct FlowConfigurationEventStoreInMem { - inner: EventStoreInMemory, +pub struct InMemoryFlowConfigurationEventStore { + inner: InMemoryEventStore, } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -44,10 +44,10 @@ impl EventStoreState for State { #[component(pub)] #[interface(dyn FlowConfigurationEventStore)] #[scope(Singleton)] -impl FlowConfigurationEventStoreInMem { +impl InMemoryFlowConfigurationEventStore { pub fn new() -> Self { Self { - inner: EventStoreInMemory::new(), + inner: InMemoryEventStore::new(), } } } @@ -55,7 +55,7 @@ impl FlowConfigurationEventStoreInMem { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[async_trait::async_trait] -impl EventStore for FlowConfigurationEventStoreInMem { +impl EventStore for InMemoryFlowConfigurationEventStore { #[tracing::instrument(level = "debug", skip_all)] async fn len(&self) -> Result { self.inner.len().await @@ -93,7 +93,7 @@ impl EventStore for FlowConfigurationEventStoreInMem { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[async_trait::async_trait] -impl FlowConfigurationEventStore for FlowConfigurationEventStoreInMem { +impl FlowConfigurationEventStore for InMemoryFlowConfigurationEventStore { #[tracing::instrument(level = "debug", skip_all)] async fn list_all_dataset_ids(&self) -> FailableDatasetIDStream { use futures::StreamExt; diff --git a/src/infra/flow-system/inmem/src/flow_configuration/mod.rs b/src/infra/flow-system/inmem/src/flow_configuration/mod.rs index a04b950048..5a3bdae4a3 100644 --- a/src/infra/flow-system/inmem/src/flow_configuration/mod.rs +++ b/src/infra/flow-system/inmem/src/flow_configuration/mod.rs @@ -7,6 +7,6 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -mod flow_configuration_event_store_inmem; +mod inmem_flow_configuration_event_store; -pub use flow_configuration_event_store_inmem::*; +pub use inmem_flow_configuration_event_store::*; diff --git a/src/infra/flow-system/inmem/tests/tests/mod.rs b/src/infra/flow-system/inmem/tests/tests/mod.rs index c1e9e7b06a..dbb55608fd 100644 --- a/src/infra/flow-system/inmem/tests/tests/mod.rs +++ b/src/infra/flow-system/inmem/tests/tests/mod.rs @@ -7,5 +7,5 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -mod test_flow_configuration_event_store_inmem; -mod test_flow_event_store_inmem; +mod test_inmem_flow_configuration_event_store; +mod test_inmem_flow_event_store; diff --git a/src/infra/flow-system/inmem/tests/tests/test_flow_configuration_event_store_inmem.rs b/src/infra/flow-system/inmem/tests/tests/test_inmem_flow_configuration_event_store.rs similarity index 90% rename from src/infra/flow-system/inmem/tests/tests/test_flow_configuration_event_store_inmem.rs rename to src/infra/flow-system/inmem/tests/tests/test_inmem_flow_configuration_event_store.rs index cbd62c836d..a17ec99da4 100644 --- a/src/infra/flow-system/inmem/tests/tests/test_flow_configuration_event_store_inmem.rs +++ b/src/infra/flow-system/inmem/tests/tests/test_inmem_flow_configuration_event_store.rs @@ -15,7 +15,7 @@ use kamu_flow_system_inmem::*; #[test_log::test(tokio::test)] async fn test_event_store_empty() { let catalog = CatalogBuilder::new() - .add::() + .add::() .build(); kamu_flow_system_repo_tests::test_event_store_empty(&catalog).await; @@ -26,7 +26,7 @@ async fn test_event_store_empty() { #[test_log::test(tokio::test)] async fn test_event_store_get_streams() { let catalog = CatalogBuilder::new() - .add::() + .add::() .build(); kamu_flow_system_repo_tests::test_event_store_get_streams(&catalog).await; @@ -37,7 +37,7 @@ async fn test_event_store_get_streams() { #[test_log::test(tokio::test)] async fn test_event_store_get_events_with_windowing() { let catalog = CatalogBuilder::new() - .add::() + .add::() .build(); kamu_flow_system_repo_tests::test_event_store_get_events_with_windowing(&catalog).await; diff --git a/src/infra/flow-system/inmem/tests/tests/test_flow_event_store_inmem.rs b/src/infra/flow-system/inmem/tests/tests/test_inmem_flow_event_store.rs similarity index 99% rename from src/infra/flow-system/inmem/tests/tests/test_flow_event_store_inmem.rs rename to src/infra/flow-system/inmem/tests/tests/test_inmem_flow_event_store.rs index 9f16f9aa84..2316f47ed1 100644 --- a/src/infra/flow-system/inmem/tests/tests/test_flow_event_store_inmem.rs +++ b/src/infra/flow-system/inmem/tests/tests/test_inmem_flow_event_store.rs @@ -13,9 +13,9 @@ use std::sync::Arc; use chrono::{Duration, Utc}; use futures::TryStreamExt; use kamu_flow_system::*; -use kamu_flow_system_inmem::FlowEventStoreInMem; +use kamu_flow_system_inmem::InMemoryFlowEventStore; use kamu_task_system::{TaskOutcome, TaskResult, TaskSystemEventStore}; -use kamu_task_system_inmem::TaskSystemEventStoreInMemory; +use kamu_task_system_inmem::InMemoryTaskSystemEventStore; use opendatafabric::{AccountID, DatasetID}; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -834,8 +834,8 @@ async fn test_system_flow_pagination_with_filters() { fn make_event_stores() -> (Arc, Arc) { ( - Arc::new(FlowEventStoreInMem::new()), - Arc::new(TaskSystemEventStoreInMemory::new()), + Arc::new(InMemoryFlowEventStore::new()), + Arc::new(InMemoryTaskSystemEventStore::new()), ) } diff --git a/src/infra/flow-system/postgres/Cargo.toml b/src/infra/flow-system/postgres/Cargo.toml index b6529e40d7..9eee666aef 100644 --- a/src/infra/flow-system/postgres/Cargo.toml +++ b/src/infra/flow-system/postgres/Cargo.toml @@ -29,7 +29,7 @@ opendatafabric = { workspace = true } async-stream = "0.3" async-trait = { version = "0.1", default-features = false } chrono = { version = "0.4", default-features = false } -dill = "0.8" +dill = "0.9" futures = "0.3" serde_json = "1" sqlx = { version = "0.7", default-features = false, features = [ diff --git a/src/infra/flow-system/postgres/src/lib.rs b/src/infra/flow-system/postgres/src/lib.rs index 2b929f692a..7f1427eb11 100644 --- a/src/infra/flow-system/postgres/src/lib.rs +++ b/src/infra/flow-system/postgres/src/lib.rs @@ -10,6 +10,6 @@ // Re-exports pub use kamu_flow_system as domain; -mod flow_configuration_event_store_postgres; +mod postgres_flow_configuration_event_store; -pub use flow_configuration_event_store_postgres::*; +pub use postgres_flow_configuration_event_store::*; diff --git a/src/infra/flow-system/postgres/src/flow_configuration_event_store_postgres.rs b/src/infra/flow-system/postgres/src/postgres_flow_configuration_event_store.rs similarity index 97% rename from src/infra/flow-system/postgres/src/flow_configuration_event_store_postgres.rs rename to src/infra/flow-system/postgres/src/postgres_flow_configuration_event_store.rs index c2d1b000ae..4bc6ffa579 100644 --- a/src/infra/flow-system/postgres/src/flow_configuration_event_store_postgres.rs +++ b/src/infra/flow-system/postgres/src/postgres_flow_configuration_event_store.rs @@ -16,13 +16,13 @@ use sqlx::{FromRow, Postgres, QueryBuilder}; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub struct FlowConfigurationEventStorePostgres { +pub struct PostgresFlowConfigurationEventStore { transaction: TransactionRefT, } #[component(pub)] #[interface(dyn FlowConfigurationEventStore)] -impl FlowConfigurationEventStorePostgres { +impl PostgresFlowConfigurationEventStore { pub fn new(transaction: TransactionRef) -> Self { Self { transaction: transaction.into(), @@ -113,7 +113,7 @@ impl FlowConfigurationEventStorePostgres { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[async_trait::async_trait] -impl EventStore for FlowConfigurationEventStorePostgres { +impl EventStore for PostgresFlowConfigurationEventStore { async fn get_events( &self, flow_key: &FlowKey, @@ -228,7 +228,7 @@ impl EventStore for FlowConfigurationEventStorePostgres //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[async_trait::async_trait] -impl FlowConfigurationEventStore for FlowConfigurationEventStorePostgres { +impl FlowConfigurationEventStore for PostgresFlowConfigurationEventStore { async fn list_all_dataset_ids(&self) -> FailableDatasetIDStream<'_> { let mut tr = self.transaction.lock().await; diff --git a/src/infra/flow-system/postgres/tests/tests/mod.rs b/src/infra/flow-system/postgres/tests/tests/mod.rs index c07473e5dc..56cd0c800f 100644 --- a/src/infra/flow-system/postgres/tests/tests/mod.rs +++ b/src/infra/flow-system/postgres/tests/tests/mod.rs @@ -7,4 +7,4 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -mod test_flow_configuration_event_store_postgres; +mod test_postgres_flow_configuration_event_store; diff --git a/src/infra/flow-system/postgres/tests/tests/test_flow_configuration_event_store_postgres.rs b/src/infra/flow-system/postgres/tests/tests/test_postgres_flow_configuration_event_store.rs similarity index 86% rename from src/infra/flow-system/postgres/tests/tests/test_flow_configuration_event_store_postgres.rs rename to src/infra/flow-system/postgres/tests/tests/test_postgres_flow_configuration_event_store.rs index a47e49e0e7..09c0dd0b5c 100644 --- a/src/infra/flow-system/postgres/tests/tests/test_flow_configuration_event_store_postgres.rs +++ b/src/infra/flow-system/postgres/tests/tests/test_postgres_flow_configuration_event_store.rs @@ -10,7 +10,7 @@ use database_common::{DatabaseTransactionRunner, PostgresTransactionManager}; use dill::{Catalog, CatalogBuilder}; use internal_error::InternalError; -use kamu_flow_system_postgres::FlowConfigurationEventStorePostgres; +use kamu_flow_system_postgres::PostgresFlowConfigurationEventStore; use sqlx::PgPool; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -18,7 +18,7 @@ use sqlx::PgPool; #[test_group::group(database, postgres)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/postgres"))] async fn test_event_store_empty(pg_pool: PgPool) { - let harness = PostgresAccountRepositoryHarness::new(pg_pool); + let harness = PostgresFlowConfigurationEventStoreHarness::new(pg_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -34,7 +34,7 @@ async fn test_event_store_empty(pg_pool: PgPool) { #[test_group::group(database, postgres)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/postgres"))] async fn test_event_store_get_streams(pg_pool: PgPool) { - let harness = PostgresAccountRepositoryHarness::new(pg_pool); + let harness = PostgresFlowConfigurationEventStoreHarness::new(pg_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -50,7 +50,7 @@ async fn test_event_store_get_streams(pg_pool: PgPool) { #[test_group::group(database, postgres)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/postgres"))] async fn test_event_store_get_events_with_windowing(pg_pool: PgPool) { - let harness = PostgresAccountRepositoryHarness::new(pg_pool); + let harness = PostgresFlowConfigurationEventStoreHarness::new(pg_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -65,17 +65,17 @@ async fn test_event_store_get_events_with_windowing(pg_pool: PgPool) { // Harness //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -struct PostgresAccountRepositoryHarness { +struct PostgresFlowConfigurationEventStoreHarness { catalog: Catalog, } -impl PostgresAccountRepositoryHarness { +impl PostgresFlowConfigurationEventStoreHarness { pub fn new(pg_pool: PgPool) -> Self { // Initialize catalog with predefined Postgres pool let mut catalog_builder = CatalogBuilder::new(); catalog_builder.add_value(pg_pool); catalog_builder.add::(); - catalog_builder.add::(); + catalog_builder.add::(); Self { catalog: catalog_builder.build(), diff --git a/src/infra/flow-system/repo-tests/Cargo.toml b/src/infra/flow-system/repo-tests/Cargo.toml index f3cdc70951..7105ed3168 100644 --- a/src/infra/flow-system/repo-tests/Cargo.toml +++ b/src/infra/flow-system/repo-tests/Cargo.toml @@ -26,5 +26,5 @@ opendatafabric = { workspace = true } kamu-flow-system = { workspace = true } chrono = { version = "0.4", default-features = false } -dill = "0.8" +dill = "0.9" futures = "0.3" diff --git a/src/infra/flow-system/sqlite/Cargo.toml b/src/infra/flow-system/sqlite/Cargo.toml index de9b5c3e8c..2d8a2112ba 100644 --- a/src/infra/flow-system/sqlite/Cargo.toml +++ b/src/infra/flow-system/sqlite/Cargo.toml @@ -29,7 +29,7 @@ opendatafabric = { workspace = true } async-stream = "0.3" async-trait = { version = "0.1", default-features = false } chrono = { version = "0.4", default-features = false } -dill = "0.8" +dill = "0.9" futures = "0.3" serde_json = "1" sqlx = { version = "0.7", default-features = false, features = [ diff --git a/src/infra/flow-system/sqlite/src/lib.rs b/src/infra/flow-system/sqlite/src/lib.rs index ec3c5a8270..8877c5cdf2 100644 --- a/src/infra/flow-system/sqlite/src/lib.rs +++ b/src/infra/flow-system/sqlite/src/lib.rs @@ -10,6 +10,6 @@ // Re-exports pub use kamu_flow_system as domain; -mod flow_system_event_store_sqlite; +mod sqlite_flow_system_event_store; -pub use flow_system_event_store_sqlite::*; +pub use sqlite_flow_system_event_store::*; diff --git a/src/infra/flow-system/sqlite/src/flow_system_event_store_sqlite.rs b/src/infra/flow-system/sqlite/src/sqlite_flow_system_event_store.rs similarity index 97% rename from src/infra/flow-system/sqlite/src/flow_system_event_store_sqlite.rs rename to src/infra/flow-system/sqlite/src/sqlite_flow_system_event_store.rs index 908a0035b2..2e699a3851 100644 --- a/src/infra/flow-system/sqlite/src/flow_system_event_store_sqlite.rs +++ b/src/infra/flow-system/sqlite/src/sqlite_flow_system_event_store.rs @@ -30,13 +30,13 @@ struct ReturningEventModel { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub struct FlowSystemEventStoreSqlite { +pub struct SqliteFlowSystemEventStore { transaction: TransactionRefT, } #[component(pub)] #[interface(dyn FlowConfigurationEventStore)] -impl FlowSystemEventStoreSqlite { +impl SqliteFlowSystemEventStore { pub fn new(transaction: TransactionRef) -> Self { Self { transaction: transaction.into(), @@ -133,7 +133,7 @@ impl FlowSystemEventStoreSqlite { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[async_trait::async_trait] -impl EventStore for FlowSystemEventStoreSqlite { +impl EventStore for SqliteFlowSystemEventStore { async fn get_events( &self, flow_key: &FlowKey, @@ -269,7 +269,7 @@ impl EventStore for FlowSystemEventStoreSqlite { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[async_trait::async_trait] -impl FlowConfigurationEventStore for FlowSystemEventStoreSqlite { +impl FlowConfigurationEventStore for SqliteFlowSystemEventStore { async fn list_all_dataset_ids(&self) -> FailableDatasetIDStream<'_> { let mut tr = self.transaction.lock().await; diff --git a/src/infra/flow-system/sqlite/tests/tests/mod.rs b/src/infra/flow-system/sqlite/tests/tests/mod.rs index d72826188c..e9edb7d9d1 100644 --- a/src/infra/flow-system/sqlite/tests/tests/mod.rs +++ b/src/infra/flow-system/sqlite/tests/tests/mod.rs @@ -7,4 +7,4 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -mod test_flow_configuration_event_store_sqlite; +mod test_sqlite_flow_configuration_event_store; diff --git a/src/infra/flow-system/sqlite/tests/tests/test_flow_configuration_event_store_sqlite.rs b/src/infra/flow-system/sqlite/tests/tests/test_sqlite_flow_configuration_event_store.rs similarity index 87% rename from src/infra/flow-system/sqlite/tests/tests/test_flow_configuration_event_store_sqlite.rs rename to src/infra/flow-system/sqlite/tests/tests/test_sqlite_flow_configuration_event_store.rs index 2cee27ddbd..1e07123e6e 100644 --- a/src/infra/flow-system/sqlite/tests/tests/test_flow_configuration_event_store_sqlite.rs +++ b/src/infra/flow-system/sqlite/tests/tests/test_sqlite_flow_configuration_event_store.rs @@ -10,7 +10,7 @@ use database_common::{DatabaseTransactionRunner, SqliteTransactionManager}; use dill::{Catalog, CatalogBuilder}; use internal_error::InternalError; -use kamu_flow_system_sqlite::FlowSystemEventStoreSqlite; +use kamu_flow_system_sqlite::SqliteFlowSystemEventStore; use sqlx::SqlitePool; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -18,7 +18,7 @@ use sqlx::SqlitePool; #[test_group::group(database, sqlite)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/sqlite"))] async fn test_event_store_empty(sqlite_pool: SqlitePool) { - let harness = SqliteAccountRepositoryHarness::new(sqlite_pool); + let harness = SqliteFlowConfigurationEventStoreHarness::new(sqlite_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -34,7 +34,7 @@ async fn test_event_store_empty(sqlite_pool: SqlitePool) { #[test_group::group(database, sqlite)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/sqlite"))] async fn test_event_store_get_streams(sqlite_pool: SqlitePool) { - let harness = SqliteAccountRepositoryHarness::new(sqlite_pool); + let harness = SqliteFlowConfigurationEventStoreHarness::new(sqlite_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -50,7 +50,7 @@ async fn test_event_store_get_streams(sqlite_pool: SqlitePool) { #[test_group::group(database, sqlite)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/sqlite"))] async fn test_event_store_get_events_with_windowing(sqlite_pool: SqlitePool) { - let harness = SqliteAccountRepositoryHarness::new(sqlite_pool); + let harness = SqliteFlowConfigurationEventStoreHarness::new(sqlite_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -65,17 +65,17 @@ async fn test_event_store_get_events_with_windowing(sqlite_pool: SqlitePool) { // Harness //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -struct SqliteAccountRepositoryHarness { +struct SqliteFlowConfigurationEventStoreHarness { catalog: Catalog, } -impl SqliteAccountRepositoryHarness { +impl SqliteFlowConfigurationEventStoreHarness { pub fn new(sqlite_pool: SqlitePool) -> Self { // Initialize catalog with predefined Postgres pool let mut catalog_builder = CatalogBuilder::new(); catalog_builder.add_value(sqlite_pool); catalog_builder.add::(); - catalog_builder.add::(); + catalog_builder.add::(); Self { catalog: catalog_builder.build(), diff --git a/src/utils/event-bus/Cargo.toml b/src/infra/messaging-outbox/inmem/Cargo.toml similarity index 57% rename from src/utils/event-bus/Cargo.toml rename to src/infra/messaging-outbox/inmem/Cargo.toml index f49d7d6966..79c40b1fcb 100644 --- a/src/utils/event-bus/Cargo.toml +++ b/src/infra/messaging-outbox/inmem/Cargo.toml @@ -1,6 +1,6 @@ [package] -name = "event-bus" -description = "Simple in-memory event bus" +name = "kamu-messaging-outbox-inmem" +description = "In-memory implementation of messaging outbox infrastructure" version = { workspace = true } homepage = { workspace = true } repository = { workspace = true } @@ -22,15 +22,20 @@ doctest = false [dependencies] +messaging-outbox = { workspace = true } internal-error = { workspace = true } -async-trait = "0.1" -dill = "0.8" -futures = "0.3" -tracing = "0.1" +async-trait = { version = "0.1", default-features = false } +chrono = { version = "0.4", default-features = false } +dill = "0.9" +tokio = { version = "1", default-features = false } +tokio-stream = "0.1" +thiserror = { version = "1", default-features = false } +tracing = { version = "0.1", default-features = false } + [dev-dependencies] +kamu-messaging-outbox-repo-tests = { workspace = true } + test-log = { version = "0.2", features = ["trace"] } tokio = { version = "1", default-features = false, features = ["rt", "macros"] } -tracing-subscriber = { version = "0.3", features = ["env-filter"] } -thiserror = { version = "1", default-features = false } diff --git a/src/infra/messaging-outbox/inmem/src/lib.rs b/src/infra/messaging-outbox/inmem/src/lib.rs new file mode 100644 index 0000000000..55468c2685 --- /dev/null +++ b/src/infra/messaging-outbox/inmem/src/lib.rs @@ -0,0 +1,17 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +#![feature(btree_cursors)] + +// Re-exports +pub use messaging_outbox as domain; + +mod repos; + +pub use repos::*; diff --git a/src/infra/messaging-outbox/inmem/src/repos/inmem_outbox_message_consumption_repository.rs b/src/infra/messaging-outbox/inmem/src/repos/inmem_outbox_message_consumption_repository.rs new file mode 100644 index 0000000000..5bad0a1bd1 --- /dev/null +++ b/src/infra/messaging-outbox/inmem/src/repos/inmem_outbox_message_consumption_repository.rs @@ -0,0 +1,119 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::collections::hash_map::Entry; +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use dill::{component, interface, scope, Singleton}; +use internal_error::InternalError; + +use crate::domain::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub struct InMemoryOutboxMessageConsumptionRepository { + state: Arc>, +} + +#[derive(Default)] +struct State { + consumption_boundaries: HashMap, +} + +#[component(pub)] +#[scope(Singleton)] +#[interface(dyn OutboxMessageConsumptionRepository)] +impl InMemoryOutboxMessageConsumptionRepository { + pub fn new() -> Self { + Self { + state: Arc::new(Mutex::new(State::default())), + } + } + + fn make_key(&self, consumer_name: &str, producer_name: &str) -> String { + format!("{consumer_name}<->{producer_name}") + } +} + +#[async_trait::async_trait] +impl OutboxMessageConsumptionRepository for InMemoryOutboxMessageConsumptionRepository { + async fn list_consumption_boundaries( + &self, + ) -> Result { + let boundaries = { + let guard = self.state.lock().unwrap(); + guard + .consumption_boundaries + .values() + .cloned() + .map(Ok) + .collect::>() + }; + + Ok(Box::pin(tokio_stream::iter(boundaries))) + } + + async fn find_consumption_boundary( + &self, + consumer_name: &str, + producer_name: &str, + ) -> Result, InternalError> { + let key = self.make_key(consumer_name, producer_name); + let guard = self.state.lock().unwrap(); + Ok(guard.consumption_boundaries.get(&key).cloned()) + } + + async fn create_consumption_boundary( + &self, + boundary: OutboxMessageConsumptionBoundary, + ) -> Result<(), CreateConsumptionBoundaryError> { + let key = self.make_key(&boundary.consumer_name, &boundary.producer_name); + + let mut guard = self.state.lock().unwrap(); + + if let Entry::Vacant(e) = guard.consumption_boundaries.entry(key) { + e.insert(boundary); + Ok(()) + } else { + Err( + CreateConsumptionBoundaryError::DuplicateConsumptionBoundary( + DuplicateConsumptionBoundaryError { + consumer_name: boundary.consumer_name.to_string(), + producer_name: boundary.producer_name.to_string(), + }, + ), + ) + } + } + + async fn update_consumption_boundary( + &self, + boundary: OutboxMessageConsumptionBoundary, + ) -> Result<(), UpdateConsumptionBoundaryError> { + let key = self.make_key(&boundary.consumer_name, &boundary.producer_name); + + let mut guard = self.state.lock().unwrap(); + + let maybe_boundary = guard.consumption_boundaries.get_mut(&key); + if let Some(existing_boundary) = maybe_boundary { + existing_boundary.last_consumed_message_id = boundary.last_consumed_message_id; + Ok(()) + } else { + Err(UpdateConsumptionBoundaryError::ConsumptionBoundaryNotFound( + ConsumptionBoundaryNotFoundError { + consumer_name: boundary.consumer_name.to_string(), + producer_name: boundary.producer_name.to_string(), + }, + )) + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/messaging-outbox/inmem/src/repos/inmem_outbox_message_repository.rs b/src/infra/messaging-outbox/inmem/src/repos/inmem_outbox_message_repository.rs new file mode 100644 index 0000000000..105c83cb1d --- /dev/null +++ b/src/infra/messaging-outbox/inmem/src/repos/inmem_outbox_message_repository.rs @@ -0,0 +1,123 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::collections::{BTreeMap, HashMap}; +use std::ops::Bound; +use std::sync::{Arc, Mutex}; + +use dill::{component, interface, scope, Singleton}; +use internal_error::InternalError; + +use crate::domain::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub struct InMemoryOutboxMessageRepository { + state: Arc>, +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[derive(Default)] +struct State { + last_message_id: Option, + messages: BTreeMap, + latest_message_id_by_producer: HashMap, +} + +impl State { + fn next_message_id(&mut self) -> OutboxMessageID { + let next_message_id = if let Some(last_message_id) = self.last_message_id { + let id = last_message_id.into_inner(); + OutboxMessageID::new(id + 1) + } else { + OutboxMessageID::new(1) + }; + self.last_message_id = Some(next_message_id); + next_message_id + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[component(pub)] +#[scope(Singleton)] +#[interface(dyn OutboxMessageRepository)] +impl InMemoryOutboxMessageRepository { + pub fn new() -> Self { + Self { + state: Arc::new(Mutex::new(State::default())), + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[async_trait::async_trait] +impl OutboxMessageRepository for InMemoryOutboxMessageRepository { + async fn push_message(&self, new_message: NewOutboxMessage) -> Result<(), InternalError> { + let mut guard = self.state.lock().unwrap(); + let message_id: OutboxMessageID = guard.next_message_id(); + + guard + .latest_message_id_by_producer + .insert(new_message.producer_name.clone(), message_id); + + guard.messages.insert( + message_id, + OutboxMessage { + message_id, + producer_name: new_message.producer_name, + content_json: new_message.content_json, + occurred_on: new_message.occurred_on, + }, + ); + Ok(()) + } + + async fn get_producer_messages( + &self, + producer_name: &str, + above_id: OutboxMessageID, + batch_size: usize, + ) -> Result { + let messages = { + let mut messages = Vec::new(); + + let guard = self.state.lock().unwrap(); + + let mut cursor = guard.messages.lower_bound(Bound::Excluded(&above_id)); + while let Some((_, message)) = cursor.next() { + if message.producer_name == producer_name { + messages.push(Ok(message.clone())); + if messages.len() >= batch_size { + break; + } + } + } + + messages + }; + + Ok(Box::pin(tokio_stream::iter(messages))) + } + + async fn get_latest_message_ids_by_producer( + &self, + ) -> Result, InternalError> { + let guard = self.state.lock().unwrap(); + Ok(guard + .latest_message_id_by_producer + .iter() + .map(|e| (e.0.clone(), *(e.1))) + .collect()) + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/messaging-outbox/inmem/src/repos/mod.rs b/src/infra/messaging-outbox/inmem/src/repos/mod.rs new file mode 100644 index 0000000000..02ad20c350 --- /dev/null +++ b/src/infra/messaging-outbox/inmem/src/repos/mod.rs @@ -0,0 +1,14 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +mod inmem_outbox_message_consumption_repository; +mod inmem_outbox_message_repository; + +pub use inmem_outbox_message_consumption_repository::*; +pub use inmem_outbox_message_repository::*; diff --git a/src/domain/core/tests/tests/mod.rs b/src/infra/messaging-outbox/inmem/tests/mod.rs similarity index 96% rename from src/domain/core/tests/tests/mod.rs rename to src/infra/messaging-outbox/inmem/tests/mod.rs index 7abae2c68d..e3f0a19ada 100644 --- a/src/domain/core/tests/tests/mod.rs +++ b/src/infra/messaging-outbox/inmem/tests/mod.rs @@ -7,4 +7,4 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -mod utils; +mod repos; diff --git a/src/infra/messaging-outbox/inmem/tests/repos/mod.rs b/src/infra/messaging-outbox/inmem/tests/repos/mod.rs new file mode 100644 index 0000000000..811c4637cf --- /dev/null +++ b/src/infra/messaging-outbox/inmem/tests/repos/mod.rs @@ -0,0 +1,11 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +mod test_inmem_outbox_message_consumption_repository; +mod test_inmem_outbox_message_repository; diff --git a/src/infra/messaging-outbox/inmem/tests/repos/test_inmem_outbox_message_consumption_repository.rs b/src/infra/messaging-outbox/inmem/tests/repos/test_inmem_outbox_message_consumption_repository.rs new file mode 100644 index 0000000000..84f20b2f84 --- /dev/null +++ b/src/infra/messaging-outbox/inmem/tests/repos/test_inmem_outbox_message_consumption_repository.rs @@ -0,0 +1,73 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use dill::{Catalog, CatalogBuilder}; +use kamu_messaging_outbox_inmem::InMemoryOutboxMessageConsumptionRepository; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_log::test(tokio::test)] +async fn test_no_outbox_consumptions_initially() { + let harness = InmemOutboxMessageConsumptionRepositoryHarness::new(); + kamu_messaging_outbox_repo_tests::test_no_outbox_consumptions_initially(&harness.catalog).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_log::test(tokio::test)] +async fn test_create_consumption() { + let harness = InmemOutboxMessageConsumptionRepositoryHarness::new(); + kamu_messaging_outbox_repo_tests::test_create_consumption(&harness.catalog).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_log::test(tokio::test)] +async fn test_update_existing_consumption() { + let harness = InmemOutboxMessageConsumptionRepositoryHarness::new(); + kamu_messaging_outbox_repo_tests::test_update_existing_consumption(&harness.catalog).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_log::test(tokio::test)] +async fn test_cannot_update_consumption_before_creation() { + let harness = InmemOutboxMessageConsumptionRepositoryHarness::new(); + kamu_messaging_outbox_repo_tests::test_cannot_update_consumption_before_creation( + &harness.catalog, + ) + .await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_log::test(tokio::test)] +async fn test_multiple_boundaries() { + let harness = InmemOutboxMessageConsumptionRepositoryHarness::new(); + kamu_messaging_outbox_repo_tests::test_multiple_boundaries(&harness.catalog).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +struct InmemOutboxMessageConsumptionRepositoryHarness { + catalog: Catalog, +} + +impl InmemOutboxMessageConsumptionRepositoryHarness { + pub fn new() -> Self { + let mut catalog_builder = CatalogBuilder::new(); + catalog_builder.add::(); + + Self { + catalog: catalog_builder.build(), + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/messaging-outbox/inmem/tests/repos/test_inmem_outbox_message_repository.rs b/src/infra/messaging-outbox/inmem/tests/repos/test_inmem_outbox_message_repository.rs new file mode 100644 index 0000000000..42cf188e07 --- /dev/null +++ b/src/infra/messaging-outbox/inmem/tests/repos/test_inmem_outbox_message_repository.rs @@ -0,0 +1,64 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use dill::{Catalog, CatalogBuilder}; +use kamu_messaging_outbox_inmem::InMemoryOutboxMessageRepository; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_log::test(tokio::test)] +async fn test_no_outbox_messages_initially() { + let harness = InmemOutboxMessageRepositoryHarness::new(); + kamu_messaging_outbox_repo_tests::test_no_outbox_messages_initially(&harness.catalog).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_log::test(tokio::test)] +async fn test_push_messages_from_several_producers() { + let harness = InmemOutboxMessageRepositoryHarness::new(); + kamu_messaging_outbox_repo_tests::test_push_messages_from_several_producers(&harness.catalog) + .await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_log::test(tokio::test)] +async fn test_push_many_messages_and_read_parts() { + let harness = InmemOutboxMessageRepositoryHarness::new(); + kamu_messaging_outbox_repo_tests::test_push_many_messages_and_read_parts(&harness.catalog) + .await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_log::test(tokio::test)] +async fn test_try_reading_above_max() { + let harness = InmemOutboxMessageRepositoryHarness::new(); + kamu_messaging_outbox_repo_tests::test_try_reading_above_max(&harness.catalog).await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +struct InmemOutboxMessageRepositoryHarness { + catalog: Catalog, +} + +impl InmemOutboxMessageRepositoryHarness { + pub fn new() -> Self { + let mut catalog_builder = CatalogBuilder::new(); + catalog_builder.add::(); + + Self { + catalog: catalog_builder.build(), + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/messaging-outbox/postgres/.sqlx/query-6a7bc6be8d4f035137972579bfef2f87019a4ba595fb156f968c235723a6cdaf.json b/src/infra/messaging-outbox/postgres/.sqlx/query-6a7bc6be8d4f035137972579bfef2f87019a4ba595fb156f968c235723a6cdaf.json new file mode 100644 index 0000000000..77043f650e --- /dev/null +++ b/src/infra/messaging-outbox/postgres/.sqlx/query-6a7bc6be8d4f035137972579bfef2f87019a4ba595fb156f968c235723a6cdaf.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n producer_name,\n max(message_id) as max_message_id\n FROM outbox_messages\n GROUP BY producer_name\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "producer_name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "max_message_id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + null + ] + }, + "hash": "6a7bc6be8d4f035137972579bfef2f87019a4ba595fb156f968c235723a6cdaf" +} diff --git a/src/infra/messaging-outbox/postgres/.sqlx/query-8a52d43e1c3a16c91850be1c73d0cc65d1e880159e85aa9008e6f78ef28a006a.json b/src/infra/messaging-outbox/postgres/.sqlx/query-8a52d43e1c3a16c91850be1c73d0cc65d1e880159e85aa9008e6f78ef28a006a.json new file mode 100644 index 0000000000..5feb2f284a --- /dev/null +++ b/src/infra/messaging-outbox/postgres/.sqlx/query-8a52d43e1c3a16c91850be1c73d0cc65d1e880159e85aa9008e6f78ef28a006a.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n consumer_name, producer_name, last_consumed_message_id\n FROM outbox_message_consumptions\n WHERE consumer_name = $1 and producer_name = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "consumer_name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "producer_name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "last_consumed_message_id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "8a52d43e1c3a16c91850be1c73d0cc65d1e880159e85aa9008e6f78ef28a006a" +} diff --git a/src/infra/messaging-outbox/postgres/.sqlx/query-8b363fe5b240687390b2e6f32382fde425e780689dc0117308f9e5047b0b0ab2.json b/src/infra/messaging-outbox/postgres/.sqlx/query-8b363fe5b240687390b2e6f32382fde425e780689dc0117308f9e5047b0b0ab2.json new file mode 100644 index 0000000000..ebba524607 --- /dev/null +++ b/src/infra/messaging-outbox/postgres/.sqlx/query-8b363fe5b240687390b2e6f32382fde425e780689dc0117308f9e5047b0b0ab2.json @@ -0,0 +1,42 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n message_id,\n producer_name,\n content_json,\n occurred_on\n FROM outbox_messages\n WHERE producer_name = $1 and message_id > $2\n ORDER BY message_id\n LIMIT $3\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "message_id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "producer_name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "content_json", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "occurred_on", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "8b363fe5b240687390b2e6f32382fde425e780689dc0117308f9e5047b0b0ab2" +} diff --git a/src/infra/messaging-outbox/postgres/.sqlx/query-a00e0b9831fc78664e606f8f6b2316cbd7ef346b7bbcd9c0e8aaa26fe0a2d95a.json b/src/infra/messaging-outbox/postgres/.sqlx/query-a00e0b9831fc78664e606f8f6b2316cbd7ef346b7bbcd9c0e8aaa26fe0a2d95a.json new file mode 100644 index 0000000000..521232ce2c --- /dev/null +++ b/src/infra/messaging-outbox/postgres/.sqlx/query-a00e0b9831fc78664e606f8f6b2316cbd7ef346b7bbcd9c0e8aaa26fe0a2d95a.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n consumer_name, producer_name, last_consumed_message_id\n FROM outbox_message_consumptions\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "consumer_name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "producer_name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "last_consumed_message_id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "a00e0b9831fc78664e606f8f6b2316cbd7ef346b7bbcd9c0e8aaa26fe0a2d95a" +} diff --git a/src/infra/messaging-outbox/postgres/.sqlx/query-ab6e6a2d4fb596ad3eb2f4188735328885659a631a580bf57072abba62aed5d9.json b/src/infra/messaging-outbox/postgres/.sqlx/query-ab6e6a2d4fb596ad3eb2f4188735328885659a631a580bf57072abba62aed5d9.json new file mode 100644 index 0000000000..0c87227f41 --- /dev/null +++ b/src/infra/messaging-outbox/postgres/.sqlx/query-ab6e6a2d4fb596ad3eb2f4188735328885659a631a580bf57072abba62aed5d9.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE outbox_message_consumptions SET last_consumed_message_id = $3\n WHERE consumer_name = $1 and producer_name = $2 and last_consumed_message_id < $3\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "ab6e6a2d4fb596ad3eb2f4188735328885659a631a580bf57072abba62aed5d9" +} diff --git a/src/infra/messaging-outbox/postgres/.sqlx/query-e03881a854596e143a3999a3b057a53849db3487224c3da9cf90aa9395faa716.json b/src/infra/messaging-outbox/postgres/.sqlx/query-e03881a854596e143a3999a3b057a53849db3487224c3da9cf90aa9395faa716.json new file mode 100644 index 0000000000..f3b2d62dae --- /dev/null +++ b/src/infra/messaging-outbox/postgres/.sqlx/query-e03881a854596e143a3999a3b057a53849db3487224c3da9cf90aa9395faa716.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO outbox_message_consumptions (consumer_name, producer_name, last_consumed_message_id)\n VALUES ($1, $2, $3)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "e03881a854596e143a3999a3b057a53849db3487224c3da9cf90aa9395faa716" +} diff --git a/src/infra/messaging-outbox/postgres/.sqlx/query-ee89ee9f37512d9bb74dac0a142315ffe00a2d822f3def0c2bee5edb0168f666.json b/src/infra/messaging-outbox/postgres/.sqlx/query-ee89ee9f37512d9bb74dac0a142315ffe00a2d822f3def0c2bee5edb0168f666.json new file mode 100644 index 0000000000..64dafb70e3 --- /dev/null +++ b/src/infra/messaging-outbox/postgres/.sqlx/query-ee89ee9f37512d9bb74dac0a142315ffe00a2d822f3def0c2bee5edb0168f666.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO outbox_messages (producer_name, content_json, occurred_on)\n VALUES ($1, $2, $3)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Jsonb", + "Timestamptz" + ] + }, + "nullable": [] + }, + "hash": "ee89ee9f37512d9bb74dac0a142315ffe00a2d822f3def0c2bee5edb0168f666" +} diff --git a/src/infra/messaging-outbox/postgres/Cargo.toml b/src/infra/messaging-outbox/postgres/Cargo.toml new file mode 100644 index 0000000000..81abcaaa2c --- /dev/null +++ b/src/infra/messaging-outbox/postgres/Cargo.toml @@ -0,0 +1,51 @@ +[package] +name = "kamu-messaging-outbox-postgres" +description = "Postgres-specific implementation of messaging outbox infrastructure" +version = { workspace = true } +homepage = { workspace = true } +repository = { workspace = true } +authors = { workspace = true } +readme = { workspace = true } +license-file = { workspace = true } +keywords = { workspace = true } +include = { workspace = true } +edition = { workspace = true } +publish = { workspace = true } + + +[lints] +workspace = true + + +[lib] +doctest = false + + +[dependencies] +database-common = { workspace = true } +messaging-outbox = { workspace = true } +internal-error = { workspace = true } + +async-stream = "0.3" +async-trait = { version = "0.1", default-features = false } +chrono = { version = "0.4", default-features = false } +dill = "0.9" +futures = "0.3" +sqlx = { version = "0.7", default-features = false, features = [ + "runtime-tokio-rustls", + "macros", + "postgres", + "chrono", + "json" +] } +thiserror = { version = "1", default-features = false } +tracing = { version = "0.1", default-features = false } +uuid = "1" + + +[dev-dependencies] +kamu-messaging-outbox-repo-tests = { workspace = true } + +test-group = { version = "1" } +test-log = { version = "0.2", features = ["trace"] } +tokio = { version = "1", default-features = false, features = ["rt", "macros"] } diff --git a/src/utils/event-bus/src/lib.rs b/src/infra/messaging-outbox/postgres/src/lib.rs similarity index 77% rename from src/utils/event-bus/src/lib.rs rename to src/infra/messaging-outbox/postgres/src/lib.rs index 3c60511e6e..9cef276c06 100644 --- a/src/utils/event-bus/src/lib.rs +++ b/src/infra/messaging-outbox/postgres/src/lib.rs @@ -7,10 +7,11 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -#![feature(fn_traits)] +#![feature(let_chains)] -mod event_bus; -mod event_handler; +// Re-exports +pub use messaging_outbox as domain; -pub use event_bus::*; -pub use event_handler::*; +mod repos; + +pub use repos::*; diff --git a/src/infra/messaging-outbox/postgres/src/repos/mod.rs b/src/infra/messaging-outbox/postgres/src/repos/mod.rs new file mode 100644 index 0000000000..ff176d5a60 --- /dev/null +++ b/src/infra/messaging-outbox/postgres/src/repos/mod.rs @@ -0,0 +1,14 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +mod outbox_postgres_message_consumption_repository; +mod outbox_postgres_message_repository; + +pub use outbox_postgres_message_consumption_repository::*; +pub use outbox_postgres_message_repository::*; diff --git a/src/infra/messaging-outbox/postgres/src/repos/outbox_postgres_message_consumption_repository.rs b/src/infra/messaging-outbox/postgres/src/repos/outbox_postgres_message_consumption_repository.rs new file mode 100644 index 0000000000..f9ebb0e738 --- /dev/null +++ b/src/infra/messaging-outbox/postgres/src/repos/outbox_postgres_message_consumption_repository.rs @@ -0,0 +1,164 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use database_common::{TransactionRef, TransactionRefT}; +use dill::{component, interface}; +use internal_error::{ErrorIntoInternal, InternalError}; + +use crate::domain::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub struct PostgresOutboxMessageConsumptionRepository { + transaction: TransactionRefT, +} + +#[component(pub)] +#[interface(dyn OutboxMessageConsumptionRepository)] +impl PostgresOutboxMessageConsumptionRepository { + pub fn new(transaction: TransactionRef) -> Self { + Self { + transaction: transaction.into(), + } + } +} + +#[async_trait::async_trait] +impl OutboxMessageConsumptionRepository for PostgresOutboxMessageConsumptionRepository { + async fn list_consumption_boundaries( + &self, + ) -> Result { + let mut tr = self.transaction.lock().await; + + Ok(Box::pin(async_stream::stream! { + let connection_mut = tr + .connection_mut() + .await?; + + let mut query_stream = sqlx::query_as!( + OutboxMessageConsumptionBoundary, + r#" + SELECT + consumer_name, producer_name, last_consumed_message_id + FROM outbox_message_consumptions + "#, + ) + .fetch(connection_mut) + .map_err(ErrorIntoInternal::int_err); + + use futures::TryStreamExt; + while let Some(consumption) = query_stream.try_next().await? { + yield Ok(consumption); + } + })) + } + + async fn find_consumption_boundary( + &self, + consumer_name: &str, + producer_name: &str, + ) -> Result, InternalError> { + let mut tr = self.transaction.lock().await; + + let connection_mut = tr.connection_mut().await?; + + sqlx::query_as!( + OutboxMessageConsumptionBoundary, + r#" + SELECT + consumer_name, producer_name, last_consumed_message_id + FROM outbox_message_consumptions + WHERE consumer_name = $1 and producer_name = $2 + "#, + consumer_name, + producer_name + ) + .fetch_optional(connection_mut) + .await + .map_err(ErrorIntoInternal::int_err) + } + + async fn create_consumption_boundary( + &self, + boundary: OutboxMessageConsumptionBoundary, + ) -> Result<(), CreateConsumptionBoundaryError> { + let mut tr = self.transaction.lock().await; + + let connection_mut = tr + .connection_mut() + .await + .map_err(|e| CreateConsumptionBoundaryError::Internal(e.int_err()))?; + + sqlx::query!( + r#" + INSERT INTO outbox_message_consumptions (consumer_name, producer_name, last_consumed_message_id) + VALUES ($1, $2, $3) + "#, + boundary.consumer_name, + boundary.producer_name, + boundary.last_consumed_message_id.into_inner(), + ) + .execute(connection_mut) + .await + .map_err(|e| { + if let sqlx::Error::Database(e) = &e + && e.is_unique_violation() + { + CreateConsumptionBoundaryError::DuplicateConsumptionBoundary( + DuplicateConsumptionBoundaryError { + consumer_name: boundary.consumer_name, + producer_name: boundary.producer_name, + }, + ) + } else { + CreateConsumptionBoundaryError::Internal(e.int_err()) + } + })?; + + Ok(()) + } + + async fn update_consumption_boundary( + &self, + boundary: OutboxMessageConsumptionBoundary, + ) -> Result<(), UpdateConsumptionBoundaryError> { + let mut tr = self.transaction.lock().await; + + let connection_mut = tr + .connection_mut() + .await + .map_err(|e| UpdateConsumptionBoundaryError::Internal(e.int_err()))?; + + let res = sqlx::query!( + r#" + UPDATE outbox_message_consumptions SET last_consumed_message_id = $3 + WHERE consumer_name = $1 and producer_name = $2 and last_consumed_message_id < $3 + "#, + boundary.consumer_name, + boundary.producer_name, + boundary.last_consumed_message_id.into_inner(), + ) + .execute(connection_mut) + .await + .map_err(|e| UpdateConsumptionBoundaryError::Internal(e.int_err()))?; + + if res.rows_affected() != 1 { + Err(UpdateConsumptionBoundaryError::ConsumptionBoundaryNotFound( + ConsumptionBoundaryNotFoundError { + consumer_name: boundary.consumer_name, + producer_name: boundary.producer_name, + }, + )) + } else { + Ok(()) + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/messaging-outbox/postgres/src/repos/outbox_postgres_message_repository.rs b/src/infra/messaging-outbox/postgres/src/repos/outbox_postgres_message_repository.rs new file mode 100644 index 0000000000..692fdbd088 --- /dev/null +++ b/src/infra/messaging-outbox/postgres/src/repos/outbox_postgres_message_repository.rs @@ -0,0 +1,128 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use database_common::{TransactionRef, TransactionRefT}; +use dill::{component, interface}; +use futures::TryStreamExt; +use internal_error::{ErrorIntoInternal, InternalError, ResultIntoInternal}; + +use crate::domain::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub struct PostgresOutboxMessageRepository { + transaction: TransactionRefT, +} + +#[component(pub)] +#[interface(dyn OutboxMessageRepository)] +impl PostgresOutboxMessageRepository { + pub fn new(transaction: TransactionRef) -> Self { + Self { + transaction: transaction.into(), + } + } +} + +#[async_trait::async_trait] +impl OutboxMessageRepository for PostgresOutboxMessageRepository { + async fn push_message(&self, message: NewOutboxMessage) -> Result<(), InternalError> { + let mut tr = self.transaction.lock().await; + + let connection_mut = tr.connection_mut().await.int_err()?; + + sqlx::query!( + r#" + INSERT INTO outbox_messages (producer_name, content_json, occurred_on) + VALUES ($1, $2, $3) + "#, + message.producer_name, + &message.content_json, + message.occurred_on + ) + .execute(connection_mut) + .await + .int_err()?; + + Ok(()) + } + + async fn get_producer_messages( + &self, + producer_name: &str, + above_id: OutboxMessageID, + batch_size: usize, + ) -> Result { + let mut tr = self.transaction.lock().await; + + let producer_name = producer_name.to_string(); + + Ok(Box::pin(async_stream::stream! { + let connection_mut = tr + .connection_mut() + .await?; + + let mut query_stream = sqlx::query_as!( + OutboxMessage, + r#" + SELECT + message_id, + producer_name, + content_json, + occurred_on + FROM outbox_messages + WHERE producer_name = $1 and message_id > $2 + ORDER BY message_id + LIMIT $3 + "#, + producer_name, + above_id.into_inner(), + i64::try_from(batch_size).unwrap(), + ) + .fetch(connection_mut) + .map_err(ErrorIntoInternal::int_err); + + while let Some(message) = query_stream.try_next().await? { + yield Ok(message); + } + })) + } + + async fn get_latest_message_ids_by_producer( + &self, + ) -> Result, InternalError> { + let mut tr = self.transaction.lock().await; + let connection_mut = tr.connection_mut().await?; + + let records = sqlx::query!( + r#" + SELECT + producer_name, + max(message_id) as max_message_id + FROM outbox_messages + GROUP BY producer_name + "#, + ) + .fetch_all(connection_mut) + .await + .map_err(ErrorIntoInternal::int_err)?; + + Ok(records + .into_iter() + .map(|r| { + ( + r.producer_name, + OutboxMessageID::new(r.max_message_id.unwrap_or(0)), + ) + }) + .collect()) + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/event-bus/tests/mod.rs b/src/infra/messaging-outbox/postgres/tests/mod.rs similarity index 94% rename from src/utils/event-bus/tests/mod.rs rename to src/infra/messaging-outbox/postgres/tests/mod.rs index 6246597fc9..e3f0a19ada 100644 --- a/src/utils/event-bus/tests/mod.rs +++ b/src/infra/messaging-outbox/postgres/tests/mod.rs @@ -7,4 +7,4 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -mod test_event_bus; +mod repos; diff --git a/src/infra/messaging-outbox/postgres/tests/repos/mod.rs b/src/infra/messaging-outbox/postgres/tests/repos/mod.rs new file mode 100644 index 0000000000..f9c57ebc7d --- /dev/null +++ b/src/infra/messaging-outbox/postgres/tests/repos/mod.rs @@ -0,0 +1,11 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +mod test_postgres_outbox_message_consumption_repository; +mod test_postgres_outbox_message_repository; diff --git a/src/infra/messaging-outbox/postgres/tests/repos/test_postgres_outbox_message_consumption_repository.rs b/src/infra/messaging-outbox/postgres/tests/repos/test_postgres_outbox_message_consumption_repository.rs new file mode 100644 index 0000000000..582871b616 --- /dev/null +++ b/src/infra/messaging-outbox/postgres/tests/repos/test_postgres_outbox_message_consumption_repository.rs @@ -0,0 +1,117 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use database_common::{DatabaseTransactionRunner, PostgresTransactionManager}; +use dill::{Catalog, CatalogBuilder}; +use internal_error::InternalError; +use kamu_messaging_outbox_postgres::PostgresOutboxMessageConsumptionRepository; +use sqlx::PgPool; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_group::group(database, postgres)] +#[test_log::test(sqlx::test(migrations = "../../../../migrations/postgres"))] +async fn test_no_outbox_consumptions_initially(pg_pool: PgPool) { + let harness = PostgresOutboxMessageConsumptionRepositoryHarness::new(pg_pool); + + DatabaseTransactionRunner::new(harness.catalog) + .transactional(|catalog| async move { + kamu_messaging_outbox_repo_tests::test_no_outbox_consumptions_initially(&catalog).await; + Ok::<_, InternalError>(()) + }) + .await + .unwrap(); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_group::group(database, postgres)] +#[test_log::test(sqlx::test(migrations = "../../../../migrations/postgres"))] +async fn test_create_consumption(pg_pool: PgPool) { + let harness = PostgresOutboxMessageConsumptionRepositoryHarness::new(pg_pool); + + DatabaseTransactionRunner::new(harness.catalog) + .transactional(|catalog| async move { + kamu_messaging_outbox_repo_tests::test_create_consumption(&catalog).await; + Ok::<_, InternalError>(()) + }) + .await + .unwrap(); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_group::group(database, postgres)] +#[test_log::test(sqlx::test(migrations = "../../../../migrations/postgres"))] +async fn test_update_existing_consumption(pg_pool: PgPool) { + let harness = PostgresOutboxMessageConsumptionRepositoryHarness::new(pg_pool); + + DatabaseTransactionRunner::new(harness.catalog) + .transactional(|catalog| async move { + kamu_messaging_outbox_repo_tests::test_update_existing_consumption(&catalog).await; + Ok::<_, InternalError>(()) + }) + .await + .unwrap(); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_group::group(database, postgres)] +#[test_log::test(sqlx::test(migrations = "../../../../migrations/postgres"))] +async fn test_cannot_update_consumption_before_creation(pg_pool: PgPool) { + let harness = PostgresOutboxMessageConsumptionRepositoryHarness::new(pg_pool); + + DatabaseTransactionRunner::new(harness.catalog) + .transactional(|catalog| async move { + kamu_messaging_outbox_repo_tests::test_cannot_update_consumption_before_creation( + &catalog, + ) + .await; + Ok::<_, InternalError>(()) + }) + .await + .unwrap(); +} +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_group::group(database, postgres)] +#[test_log::test(sqlx::test(migrations = "../../../../migrations/postgres"))] +async fn test_multiple_boundaries(pg_pool: PgPool) { + let harness = PostgresOutboxMessageConsumptionRepositoryHarness::new(pg_pool); + + DatabaseTransactionRunner::new(harness.catalog) + .transactional(|catalog| async move { + kamu_messaging_outbox_repo_tests::test_multiple_boundaries(&catalog).await; + Ok::<_, InternalError>(()) + }) + .await + .unwrap(); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +struct PostgresOutboxMessageConsumptionRepositoryHarness { + catalog: Catalog, +} + +impl PostgresOutboxMessageConsumptionRepositoryHarness { + pub fn new(pg_pool: PgPool) -> Self { + let mut catalog_builder = CatalogBuilder::new(); + catalog_builder.add_value(pg_pool); + catalog_builder.add::(); + catalog_builder.add::(); + + Self { + catalog: catalog_builder.build(), + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/messaging-outbox/postgres/tests/repos/test_postgres_outbox_message_repository.rs b/src/infra/messaging-outbox/postgres/tests/repos/test_postgres_outbox_message_repository.rs new file mode 100644 index 0000000000..f156101a4d --- /dev/null +++ b/src/infra/messaging-outbox/postgres/tests/repos/test_postgres_outbox_message_repository.rs @@ -0,0 +1,101 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use database_common::{DatabaseTransactionRunner, PostgresTransactionManager}; +use dill::{Catalog, CatalogBuilder}; +use internal_error::InternalError; +use kamu_messaging_outbox_postgres::PostgresOutboxMessageRepository; +use sqlx::PgPool; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_group::group(database, postgres)] +#[test_log::test(sqlx::test(migrations = "../../../../migrations/postgres"))] +async fn test_no_outbox_messages_initially(pg_pool: PgPool) { + let harness = PostgresOutboxMessageRepositoryHarness::new(pg_pool); + + DatabaseTransactionRunner::new(harness.catalog) + .transactional(|catalog| async move { + kamu_messaging_outbox_repo_tests::test_no_outbox_messages_initially(&catalog).await; + Ok::<_, InternalError>(()) + }) + .await + .unwrap(); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_group::group(database, postgres)] +#[test_log::test(sqlx::test(migrations = "../../../../migrations/postgres"))] +async fn test_push_messages_from_several_producers(pg_pool: PgPool) { + let harness = PostgresOutboxMessageRepositoryHarness::new(pg_pool); + + DatabaseTransactionRunner::new(harness.catalog) + .transactional(|catalog| async move { + kamu_messaging_outbox_repo_tests::test_push_messages_from_several_producers(&catalog) + .await; + Ok::<_, InternalError>(()) + }) + .await + .unwrap(); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_group::group(database, postgres)] +#[test_log::test(sqlx::test(migrations = "../../../../migrations/postgres"))] +async fn test_push_many_messages_and_read_parts(pg_pool: PgPool) { + let harness = PostgresOutboxMessageRepositoryHarness::new(pg_pool); + + DatabaseTransactionRunner::new(harness.catalog) + .transactional(|catalog| async move { + kamu_messaging_outbox_repo_tests::test_push_many_messages_and_read_parts(&catalog) + .await; + Ok::<_, InternalError>(()) + }) + .await + .unwrap(); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_group::group(database, postgres)] +#[test_log::test(sqlx::test(migrations = "../../../../migrations/postgres"))] +async fn test_try_reading_above_max(pg_pool: PgPool) { + let harness = PostgresOutboxMessageRepositoryHarness::new(pg_pool); + + DatabaseTransactionRunner::new(harness.catalog) + .transactional(|catalog| async move { + kamu_messaging_outbox_repo_tests::test_try_reading_above_max(&catalog).await; + Ok::<_, InternalError>(()) + }) + .await + .unwrap(); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +struct PostgresOutboxMessageRepositoryHarness { + catalog: Catalog, +} + +impl PostgresOutboxMessageRepositoryHarness { + pub fn new(pg_pool: PgPool) -> Self { + let mut catalog_builder = CatalogBuilder::new(); + catalog_builder.add_value(pg_pool); + catalog_builder.add::(); + catalog_builder.add::(); + + Self { + catalog: catalog_builder.build(), + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/messaging-outbox/repo-tests/Cargo.toml b/src/infra/messaging-outbox/repo-tests/Cargo.toml new file mode 100644 index 0000000000..986fcf97a4 --- /dev/null +++ b/src/infra/messaging-outbox/repo-tests/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "kamu-messaging-outbox-repo-tests" +description = "Shared repository tests for Kamu messaging outbox" +version = { workspace = true } +homepage = { workspace = true } +repository = { workspace = true } +authors = { workspace = true } +readme = { workspace = true } +license-file = { workspace = true } +keywords = { workspace = true } +include = { workspace = true } +edition = { workspace = true } +publish = { workspace = true } + + +[lints] +workspace = true + + +[lib] +doctest = false + + +[dependencies] +database-common = { workspace = true } +messaging-outbox = { workspace = true } + +chrono = { version = "0.4", default-features = false } +dill = "0.9" +futures = "0.3" +rand = "0.8" +serde = { version = "1", features = ["derive"] } +serde_json = "1" diff --git a/src/infra/messaging-outbox/repo-tests/src/lib.rs b/src/infra/messaging-outbox/repo-tests/src/lib.rs new file mode 100644 index 0000000000..9e2a05ddd0 --- /dev/null +++ b/src/infra/messaging-outbox/repo-tests/src/lib.rs @@ -0,0 +1,16 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +#![feature(assert_matches)] + +mod outbox_message_consumption_repository_test_suite; +mod outbox_message_repository_test_suite; + +pub use outbox_message_consumption_repository_test_suite::*; +pub use outbox_message_repository_test_suite::*; diff --git a/src/infra/messaging-outbox/repo-tests/src/outbox_message_consumption_repository_test_suite.rs b/src/infra/messaging-outbox/repo-tests/src/outbox_message_consumption_repository_test_suite.rs new file mode 100644 index 0000000000..d3a6d0cc8d --- /dev/null +++ b/src/infra/messaging-outbox/repo-tests/src/outbox_message_consumption_repository_test_suite.rs @@ -0,0 +1,240 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::assert_matches::assert_matches; + +use dill::Catalog; +use messaging_outbox::{ + ConsumptionBoundaryNotFoundError, + CreateConsumptionBoundaryError, + DuplicateConsumptionBoundaryError, + OutboxMessageConsumptionBoundary, + OutboxMessageConsumptionRepository, + OutboxMessageID, + UpdateConsumptionBoundaryError, +}; +use rand::Rng; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +const TEST_CONSUMER: &str = "test-consumer"; +const TEST_PRODUCER: &str = "test-producer"; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_no_outbox_consumptions_initially(catalog: &Catalog) { + let consumption_repo = catalog + .get_one::() + .unwrap(); + + let boundaries: Vec<_> = read_boundaries(consumption_repo.as_ref()).await; + assert_eq!(0, boundaries.len()); + + let res = consumption_repo + .find_consumption_boundary(TEST_CONSUMER, TEST_PRODUCER) + .await; + assert_matches!(res, Ok(None)); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_create_consumption(catalog: &Catalog) { + let consumption_repo = catalog + .get_one::() + .unwrap(); + + // 1st create should pass + let boundary = OutboxMessageConsumptionBoundary { + consumer_name: TEST_CONSUMER.to_string(), + producer_name: TEST_PRODUCER.to_string(), + last_consumed_message_id: OutboxMessageID::new(5), + }; + let res = consumption_repo + .create_consumption_boundary(boundary.clone()) + .await; + assert_matches!(res, Ok(_)); + + // Try reading created record + + let boundaries: Vec<_> = read_boundaries(consumption_repo.as_ref()).await; + assert_eq!(boundaries, vec![boundary.clone()],); + + let res = consumption_repo + .find_consumption_boundary(TEST_CONSUMER, TEST_PRODUCER) + .await; + assert_matches!(res, Ok(Some(a_boundary)) if a_boundary == boundary); + + // 2nd create attempt should fail + let res = consumption_repo + .create_consumption_boundary(boundary.clone()) + .await; + assert_matches!( + res, + Err( + CreateConsumptionBoundaryError::DuplicateConsumptionBoundary( + DuplicateConsumptionBoundaryError { + consumer_name, + producer_name, + } + ) + ) if consumer_name == TEST_CONSUMER && producer_name == TEST_PRODUCER + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_update_existing_consumption(catalog: &Catalog) { + let consumption_repo = catalog + .get_one::() + .unwrap(); + + let boundary = OutboxMessageConsumptionBoundary { + consumer_name: TEST_CONSUMER.to_string(), + producer_name: TEST_PRODUCER.to_string(), + last_consumed_message_id: OutboxMessageID::new(5), + }; + + // Create + let res = consumption_repo + .create_consumption_boundary(boundary.clone()) + .await; + assert_matches!(res, Ok(_)); + + // Update + let updated_boundary = OutboxMessageConsumptionBoundary { + last_consumed_message_id: OutboxMessageID::new(15), + ..boundary + }; + let res = consumption_repo + .update_consumption_boundary(updated_boundary.clone()) + .await; + assert_matches!(res, Ok(_)); + + // Read + let boundaries: Vec<_> = read_boundaries(consumption_repo.as_ref()).await; + assert_eq!(boundaries, vec![updated_boundary.clone()],); + + let res = consumption_repo + .find_consumption_boundary(TEST_CONSUMER, TEST_PRODUCER) + .await; + assert_matches!(res, Ok(Some(a_boundary)) if a_boundary == updated_boundary); + + // 2nd update should pass + let updated_boundary_2 = OutboxMessageConsumptionBoundary { + last_consumed_message_id: OutboxMessageID::new(25), + ..updated_boundary + }; + let res = consumption_repo + .update_consumption_boundary(updated_boundary_2.clone()) + .await; + assert_matches!(res, Ok(_)); + + // Read + let boundaries: Vec<_> = read_boundaries(consumption_repo.as_ref()).await; + assert_eq!(boundaries, vec![updated_boundary_2.clone()],); + + let res = consumption_repo + .find_consumption_boundary(TEST_CONSUMER, TEST_PRODUCER) + .await; + assert_matches!(res, Ok(Some(a_boundary)) if a_boundary == updated_boundary_2); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_cannot_update_consumption_before_creation(catalog: &Catalog) { + let consumption_repo = catalog + .get_one::() + .unwrap(); + + let boundary = OutboxMessageConsumptionBoundary { + consumer_name: TEST_CONSUMER.to_string(), + producer_name: TEST_PRODUCER.to_string(), + last_consumed_message_id: OutboxMessageID::new(5), + }; + + // Update without create + let updated_boundary = OutboxMessageConsumptionBoundary { + last_consumed_message_id: OutboxMessageID::new(15), + ..boundary + }; + let res = consumption_repo + .update_consumption_boundary(updated_boundary.clone()) + .await; + assert_matches!( + res, + Err( + UpdateConsumptionBoundaryError::ConsumptionBoundaryNotFound( + ConsumptionBoundaryNotFoundError { + consumer_name, + producer_name, + } + ) + ) if consumer_name == TEST_CONSUMER && producer_name == TEST_PRODUCER + ); + + // Read + let boundaries: Vec<_> = read_boundaries(consumption_repo.as_ref()).await; + assert_eq!(boundaries.len(), 0); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_multiple_boundaries(catalog: &Catalog) { + let consumption_repo = catalog + .get_one::() + .unwrap(); + + let mut rng = rand::thread_rng(); + + for producer_suffix in ["A", "B"] { + for consumer_suffix in ["X", "Y"] { + let boundary = OutboxMessageConsumptionBoundary { + consumer_name: format!("{TEST_CONSUMER}_{consumer_suffix}"), + producer_name: format!("{TEST_PRODUCER}_{producer_suffix}"), + last_consumed_message_id: OutboxMessageID::new(rng.gen()), + }; + let res = consumption_repo.create_consumption_boundary(boundary).await; + assert_matches!(res, Ok(_)); + } + } + + // Read + let mut boundaries: Vec<_> = read_boundaries(consumption_repo.as_ref()).await; + boundaries.sort(); + + assert_eq!( + boundaries + .iter() + .map(|b| (b.producer_name.as_str(), b.consumer_name.as_str())) + .collect::>(), + vec![ + ("test-producer_A", "test-consumer_X"), + ("test-producer_A", "test-consumer_Y"), + ("test-producer_B", "test-consumer_X"), + ("test-producer_B", "test-consumer_Y"), + ] + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +async fn read_boundaries( + outbox_message_consumption_repo: &dyn OutboxMessageConsumptionRepository, +) -> Vec { + use futures::TryStreamExt; + outbox_message_consumption_repo + .list_consumption_boundaries() + .await + .unwrap() + .try_collect() + .await + .unwrap() +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/messaging-outbox/repo-tests/src/outbox_message_repository_test_suite.rs b/src/infra/messaging-outbox/repo-tests/src/outbox_message_repository_test_suite.rs new file mode 100644 index 0000000000..9faaa4c463 --- /dev/null +++ b/src/infra/messaging-outbox/repo-tests/src/outbox_message_repository_test_suite.rs @@ -0,0 +1,211 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use chrono::Utc; +use dill::Catalog; +use futures::TryStreamExt; +use messaging_outbox::{NewOutboxMessage, OutboxMessage, OutboxMessageID, OutboxMessageRepository}; +use serde::{Deserialize, Serialize}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_no_outbox_messages_initially(catalog: &Catalog) { + let outbox_message_repo = catalog.get_one::().unwrap(); + + let messages_by_producer = outbox_message_repo + .get_latest_message_ids_by_producer() + .await + .unwrap(); + assert_eq!(0, messages_by_producer.len()); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_push_messages_from_several_producers(catalog: &Catalog) { + let outbox_message_repo = catalog.get_one::().unwrap(); + + let messages = vec![ + NewOutboxMessage { + producer_name: "A".to_string(), + content_json: serde_json::to_value(&MessageA { x: 35, y: 256 }).unwrap(), + occurred_on: Utc::now(), + }, + NewOutboxMessage { + producer_name: "A".to_string(), + content_json: serde_json::to_value(&MessageA { x: 27, y: 315 }).unwrap(), + occurred_on: Utc::now(), + }, + NewOutboxMessage { + producer_name: "B".to_string(), + content_json: serde_json::to_value(&MessageB { + a: "test".to_string(), + b: vec!["foo".to_string(), "bar".to_string()], + }) + .unwrap(), + occurred_on: Utc::now(), + }, + ]; + + for message in messages { + outbox_message_repo.push_message(message).await.unwrap(); + } + + let mut message_ids_by_producer = outbox_message_repo + .get_latest_message_ids_by_producer() + .await + .unwrap(); + message_ids_by_producer.sort_by(|a, b| a.0.cmp(&b.0)); + + assert_eq!( + message_ids_by_producer, + vec![ + ("A".to_string(), OutboxMessageID::new(2),), + ("B".to_string(), OutboxMessageID::new(3),), + ] + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_push_many_messages_and_read_parts(catalog: &Catalog) { + let outbox_message_repo = catalog.get_one::().unwrap(); + + for i in 1..=10 { + outbox_message_repo + .push_message(NewOutboxMessage { + producer_name: "A".to_string(), + content_json: serde_json::to_value(&MessageA { + x: i * 2, + y: u64::try_from(256 + i).unwrap(), + }) + .unwrap(), + occurred_on: Utc::now(), + }) + .await + .unwrap(); + } + + outbox_message_repo + .push_message(NewOutboxMessage { + producer_name: "dummy".to_string(), + content_json: serde_json::to_value("dummy").unwrap(), + occurred_on: Utc::now(), + }) + .await + .unwrap(); + + fn assert_expected_message_a(message: OutboxMessage, i: i32) { + let original_message = serde_json::from_value::(message.content_json).unwrap(); + + assert_eq!(original_message.x, i * 2); + assert_eq!(original_message.y, u64::try_from(256 + i).unwrap()); + } + + let messages: Vec<_> = outbox_message_repo + .get_producer_messages("A", OutboxMessageID::new(0), 3) + .await + .unwrap() + .try_collect() + .await + .unwrap(); + + for i in 1..4 { + let message = messages.get(i - 1).unwrap().clone(); + assert_expected_message_a(message, i32::try_from(i).unwrap()); + } + + let messages: Vec<_> = outbox_message_repo + .get_producer_messages("A", OutboxMessageID::new(5), 4) + .await + .unwrap() + .try_collect() + .await + .unwrap(); + + for i in 6..10 { + let message = messages.get(i - 6).unwrap().clone(); + assert_expected_message_a(message, i32::try_from(i).unwrap()); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub async fn test_try_reading_above_max(catalog: &Catalog) { + let outbox_message_repo = catalog.get_one::().unwrap(); + + for i in 1..=5 { + outbox_message_repo + .push_message(NewOutboxMessage { + producer_name: "A".to_string(), + content_json: serde_json::to_value(&MessageA { + x: i, + y: u64::try_from(i * 2).unwrap(), + }) + .unwrap(), + occurred_on: Utc::now(), + }) + .await + .unwrap(); + } + + outbox_message_repo + .push_message(NewOutboxMessage { + producer_name: "dummy".to_string(), + content_json: serde_json::to_value("dummy").unwrap(), + occurred_on: Utc::now(), + }) + .await + .unwrap(); + + fn assert_expected_message_a(message: OutboxMessage, i: i32) { + let original_message = serde_json::from_value::(message.content_json).unwrap(); + + assert_eq!(original_message.x, i); + assert_eq!(original_message.y, u64::try_from(i * 2).unwrap()); + } + + let messages: Vec<_> = outbox_message_repo + .get_producer_messages("A", OutboxMessageID::new(5), 3) + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(messages.len(), 0); + + let messages: Vec<_> = outbox_message_repo + .get_producer_messages("A", OutboxMessageID::new(3), 6) + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(messages.len(), 2); // 4, 5 + + for i in 4..=5 { + let message = messages.get(i - 4).unwrap().clone(); + assert_expected_message_a(message, i32::try_from(i).unwrap()); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[derive(Debug, Serialize, Deserialize)] +struct MessageA { + x: i32, + y: u64, +} + +#[derive(Debug, Serialize, Deserialize)] +struct MessageB { + a: String, + b: Vec, +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/messaging-outbox/sqlite/.sqlx/query-738d1c4230062bd4a670e924ef842930c873cfedd1b8d864745ed7c26fede9ac.json b/src/infra/messaging-outbox/sqlite/.sqlx/query-738d1c4230062bd4a670e924ef842930c873cfedd1b8d864745ed7c26fede9ac.json new file mode 100644 index 0000000000..59e794efdb --- /dev/null +++ b/src/infra/messaging-outbox/sqlite/.sqlx/query-738d1c4230062bd4a670e924ef842930c873cfedd1b8d864745ed7c26fede9ac.json @@ -0,0 +1,38 @@ +{ + "db_name": "SQLite", + "query": "\n SELECT\n message_id,\n producer_name,\n content_json as \"content_json: _\",\n occurred_on as \"occurred_on: _\"\n FROM outbox_messages\n WHERE producer_name = $1 and message_id > $2\n ORDER BY message_id\n LIMIT $3\n ", + "describe": { + "columns": [ + { + "name": "message_id", + "ordinal": 0, + "type_info": "Int64" + }, + { + "name": "producer_name", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "content_json: _", + "ordinal": 2, + "type_info": "Null" + }, + { + "name": "occurred_on: _", + "ordinal": 3, + "type_info": "Null" + } + ], + "parameters": { + "Right": 3 + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "738d1c4230062bd4a670e924ef842930c873cfedd1b8d864745ed7c26fede9ac" +} diff --git a/src/infra/messaging-outbox/sqlite/.sqlx/query-8a52d43e1c3a16c91850be1c73d0cc65d1e880159e85aa9008e6f78ef28a006a.json b/src/infra/messaging-outbox/sqlite/.sqlx/query-8a52d43e1c3a16c91850be1c73d0cc65d1e880159e85aa9008e6f78ef28a006a.json new file mode 100644 index 0000000000..49c03744e4 --- /dev/null +++ b/src/infra/messaging-outbox/sqlite/.sqlx/query-8a52d43e1c3a16c91850be1c73d0cc65d1e880159e85aa9008e6f78ef28a006a.json @@ -0,0 +1,32 @@ +{ + "db_name": "SQLite", + "query": "\n SELECT\n consumer_name, producer_name, last_consumed_message_id\n FROM outbox_message_consumptions\n WHERE consumer_name = $1 and producer_name = $2\n ", + "describe": { + "columns": [ + { + "name": "consumer_name", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "producer_name", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "last_consumed_message_id", + "ordinal": 2, + "type_info": "Int64" + } + ], + "parameters": { + "Right": 2 + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "8a52d43e1c3a16c91850be1c73d0cc65d1e880159e85aa9008e6f78ef28a006a" +} diff --git a/src/infra/messaging-outbox/sqlite/.sqlx/query-a00e0b9831fc78664e606f8f6b2316cbd7ef346b7bbcd9c0e8aaa26fe0a2d95a.json b/src/infra/messaging-outbox/sqlite/.sqlx/query-a00e0b9831fc78664e606f8f6b2316cbd7ef346b7bbcd9c0e8aaa26fe0a2d95a.json new file mode 100644 index 0000000000..8918deda38 --- /dev/null +++ b/src/infra/messaging-outbox/sqlite/.sqlx/query-a00e0b9831fc78664e606f8f6b2316cbd7ef346b7bbcd9c0e8aaa26fe0a2d95a.json @@ -0,0 +1,32 @@ +{ + "db_name": "SQLite", + "query": "\n SELECT\n consumer_name, producer_name, last_consumed_message_id\n FROM outbox_message_consumptions\n ", + "describe": { + "columns": [ + { + "name": "consumer_name", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "producer_name", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "last_consumed_message_id", + "ordinal": 2, + "type_info": "Int64" + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "a00e0b9831fc78664e606f8f6b2316cbd7ef346b7bbcd9c0e8aaa26fe0a2d95a" +} diff --git a/src/infra/messaging-outbox/sqlite/.sqlx/query-ab6e6a2d4fb596ad3eb2f4188735328885659a631a580bf57072abba62aed5d9.json b/src/infra/messaging-outbox/sqlite/.sqlx/query-ab6e6a2d4fb596ad3eb2f4188735328885659a631a580bf57072abba62aed5d9.json new file mode 100644 index 0000000000..f4be24419e --- /dev/null +++ b/src/infra/messaging-outbox/sqlite/.sqlx/query-ab6e6a2d4fb596ad3eb2f4188735328885659a631a580bf57072abba62aed5d9.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "\n UPDATE outbox_message_consumptions SET last_consumed_message_id = $3\n WHERE consumer_name = $1 and producer_name = $2 and last_consumed_message_id < $3\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + }, + "hash": "ab6e6a2d4fb596ad3eb2f4188735328885659a631a580bf57072abba62aed5d9" +} diff --git a/src/infra/messaging-outbox/sqlite/.sqlx/query-d1d9fdcec93cbf10079386a1f8aaee30cde75a4a9afcb4c2485825e0fec0eb7b.json b/src/infra/messaging-outbox/sqlite/.sqlx/query-d1d9fdcec93cbf10079386a1f8aaee30cde75a4a9afcb4c2485825e0fec0eb7b.json new file mode 100644 index 0000000000..1b8ca498bc --- /dev/null +++ b/src/infra/messaging-outbox/sqlite/.sqlx/query-d1d9fdcec93cbf10079386a1f8aaee30cde75a4a9afcb4c2485825e0fec0eb7b.json @@ -0,0 +1,26 @@ +{ + "db_name": "SQLite", + "query": "\n SELECT\n producer_name,\n IFNULL(MAX(message_id), 0) as max_message_id\n FROM outbox_messages\n GROUP BY producer_name\n ", + "describe": { + "columns": [ + { + "name": "producer_name", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "max_message_id", + "ordinal": 1, + "type_info": "Int" + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + true, + false + ] + }, + "hash": "d1d9fdcec93cbf10079386a1f8aaee30cde75a4a9afcb4c2485825e0fec0eb7b" +} diff --git a/src/infra/messaging-outbox/sqlite/.sqlx/query-e03881a854596e143a3999a3b057a53849db3487224c3da9cf90aa9395faa716.json b/src/infra/messaging-outbox/sqlite/.sqlx/query-e03881a854596e143a3999a3b057a53849db3487224c3da9cf90aa9395faa716.json new file mode 100644 index 0000000000..4c56329ec5 --- /dev/null +++ b/src/infra/messaging-outbox/sqlite/.sqlx/query-e03881a854596e143a3999a3b057a53849db3487224c3da9cf90aa9395faa716.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "\n INSERT INTO outbox_message_consumptions (consumer_name, producer_name, last_consumed_message_id)\n VALUES ($1, $2, $3)\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + }, + "hash": "e03881a854596e143a3999a3b057a53849db3487224c3da9cf90aa9395faa716" +} diff --git a/src/infra/messaging-outbox/sqlite/.sqlx/query-ee89ee9f37512d9bb74dac0a142315ffe00a2d822f3def0c2bee5edb0168f666.json b/src/infra/messaging-outbox/sqlite/.sqlx/query-ee89ee9f37512d9bb74dac0a142315ffe00a2d822f3def0c2bee5edb0168f666.json new file mode 100644 index 0000000000..ab377c11ab --- /dev/null +++ b/src/infra/messaging-outbox/sqlite/.sqlx/query-ee89ee9f37512d9bb74dac0a142315ffe00a2d822f3def0c2bee5edb0168f666.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "\n INSERT INTO outbox_messages (producer_name, content_json, occurred_on)\n VALUES ($1, $2, $3)\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + }, + "hash": "ee89ee9f37512d9bb74dac0a142315ffe00a2d822f3def0c2bee5edb0168f666" +} diff --git a/src/infra/messaging-outbox/sqlite/Cargo.toml b/src/infra/messaging-outbox/sqlite/Cargo.toml new file mode 100644 index 0000000000..afa67a8fb0 --- /dev/null +++ b/src/infra/messaging-outbox/sqlite/Cargo.toml @@ -0,0 +1,50 @@ +[package] +name = "kamu-messaging-outbox-sqlite" +description = "Sqlite-specific implementation of messaging outbox infrastructure" +version = { workspace = true } +homepage = { workspace = true } +repository = { workspace = true } +authors = { workspace = true } +readme = { workspace = true } +license-file = { workspace = true } +keywords = { workspace = true } +include = { workspace = true } +edition = { workspace = true } +publish = { workspace = true } + + +[lints] +workspace = true + + +[lib] +doctest = false + + +[dependencies] +database-common = { workspace = true } +messaging-outbox = { workspace = true } +internal-error = { workspace = true } + +async-stream = "0.3" +async-trait = { version = "0.1", default-features = false } +chrono = { version = "0.4", default-features = false } +dill = "0.9" +futures = "0.3" +sqlx = { version = "0.7", default-features = false, features = [ + "runtime-tokio-rustls", + "macros", + "sqlite", + "chrono", + "json" +] } +thiserror = { version = "1", default-features = false } +tracing = { version = "0.1", default-features = false } + + +[dev-dependencies] +kamu-messaging-outbox-repo-tests = { workspace = true } + +test-group = { version = "1" } +test-log = { version = "0.2", features = ["trace"] } +tokio = { version = "1", default-features = false, features = ["rt", "macros"] } diff --git a/src/infra/messaging-outbox/sqlite/src/lib.rs b/src/infra/messaging-outbox/sqlite/src/lib.rs new file mode 100644 index 0000000000..9cef276c06 --- /dev/null +++ b/src/infra/messaging-outbox/sqlite/src/lib.rs @@ -0,0 +1,17 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +#![feature(let_chains)] + +// Re-exports +pub use messaging_outbox as domain; + +mod repos; + +pub use repos::*; diff --git a/src/infra/messaging-outbox/sqlite/src/repos/mod.rs b/src/infra/messaging-outbox/sqlite/src/repos/mod.rs new file mode 100644 index 0000000000..6c569613c6 --- /dev/null +++ b/src/infra/messaging-outbox/sqlite/src/repos/mod.rs @@ -0,0 +1,14 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +mod sqlite_outbox_message_consumption_repository; +mod sqlite_outbox_message_repository; + +pub use sqlite_outbox_message_consumption_repository::*; +pub use sqlite_outbox_message_repository::*; diff --git a/src/infra/messaging-outbox/sqlite/src/repos/sqlite_outbox_message_consumption_repository.rs b/src/infra/messaging-outbox/sqlite/src/repos/sqlite_outbox_message_consumption_repository.rs new file mode 100644 index 0000000000..f9060c0b76 --- /dev/null +++ b/src/infra/messaging-outbox/sqlite/src/repos/sqlite_outbox_message_consumption_repository.rs @@ -0,0 +1,168 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use database_common::{TransactionRef, TransactionRefT}; +use dill::{component, interface}; +use internal_error::{ErrorIntoInternal, InternalError}; + +use crate::domain::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub struct SqliteOutboxMessageConsumptionRepository { + transaction: TransactionRefT, +} + +#[component(pub)] +#[interface(dyn OutboxMessageConsumptionRepository)] +impl SqliteOutboxMessageConsumptionRepository { + pub fn new(transaction: TransactionRef) -> Self { + Self { + transaction: transaction.into(), + } + } +} + +#[async_trait::async_trait] +impl OutboxMessageConsumptionRepository for SqliteOutboxMessageConsumptionRepository { + async fn list_consumption_boundaries( + &self, + ) -> Result { + let mut tr = self.transaction.lock().await; + + Ok(Box::pin(async_stream::stream! { + let connection_mut = tr + .connection_mut() + .await?; + + let mut query_stream = sqlx::query_as!( + OutboxMessageConsumptionBoundary, + r#" + SELECT + consumer_name, producer_name, last_consumed_message_id + FROM outbox_message_consumptions + "#, + ) + .fetch(connection_mut) + .map_err(ErrorIntoInternal::int_err); + + use futures::TryStreamExt; + while let Some(consumption) = query_stream.try_next().await? { + yield Ok(consumption); + } + })) + } + + async fn find_consumption_boundary( + &self, + consumer_name: &str, + producer_name: &str, + ) -> Result, InternalError> { + let mut tr = self.transaction.lock().await; + + let connection_mut = tr.connection_mut().await?; + + sqlx::query_as!( + OutboxMessageConsumptionBoundary, + r#" + SELECT + consumer_name, producer_name, last_consumed_message_id + FROM outbox_message_consumptions + WHERE consumer_name = $1 and producer_name = $2 + "#, + consumer_name, + producer_name, + ) + .fetch_optional(connection_mut) + .await + .map_err(ErrorIntoInternal::int_err) + } + + async fn create_consumption_boundary( + &self, + boundary: OutboxMessageConsumptionBoundary, + ) -> Result<(), CreateConsumptionBoundaryError> { + let mut tr = self.transaction.lock().await; + + let connection_mut = tr + .connection_mut() + .await + .map_err(|e| CreateConsumptionBoundaryError::Internal(e.int_err()))?; + + let last_consumed_message_id = boundary.last_consumed_message_id.into_inner(); + + sqlx::query!( + r#" + INSERT INTO outbox_message_consumptions (consumer_name, producer_name, last_consumed_message_id) + VALUES ($1, $2, $3) + "#, + boundary.consumer_name, + boundary.producer_name, + last_consumed_message_id, + ) + .execute(connection_mut) + .await + .map_err(|e| { + if let sqlx::Error::Database(e) = &e + && e.is_unique_violation() + { + CreateConsumptionBoundaryError::DuplicateConsumptionBoundary( + DuplicateConsumptionBoundaryError { + consumer_name: boundary.consumer_name, + producer_name: boundary.producer_name, + }, + ) + } else { + CreateConsumptionBoundaryError::Internal(e.int_err()) + } + })?; + + Ok(()) + } + + async fn update_consumption_boundary( + &self, + boundary: OutboxMessageConsumptionBoundary, + ) -> Result<(), UpdateConsumptionBoundaryError> { + let mut tr = self.transaction.lock().await; + + let connection_mut = tr + .connection_mut() + .await + .map_err(|e| UpdateConsumptionBoundaryError::Internal(e.int_err()))?; + + let last_consumed_message_id = boundary.last_consumed_message_id.into_inner(); + + let res = sqlx::query!( + r#" + UPDATE outbox_message_consumptions SET last_consumed_message_id = $3 + WHERE consumer_name = $1 and producer_name = $2 and last_consumed_message_id < $3 + "#, + boundary.consumer_name, + boundary.producer_name, + last_consumed_message_id, + ) + .execute(connection_mut) + .await + .map_err(|e| UpdateConsumptionBoundaryError::Internal(e.int_err()))?; + + if res.rows_affected() != 1 { + Err(UpdateConsumptionBoundaryError::ConsumptionBoundaryNotFound( + ConsumptionBoundaryNotFoundError { + consumer_name: boundary.consumer_name, + producer_name: boundary.producer_name, + }, + )) + } else { + Ok(()) + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/messaging-outbox/sqlite/src/repos/sqlite_outbox_message_repository.rs b/src/infra/messaging-outbox/sqlite/src/repos/sqlite_outbox_message_repository.rs new file mode 100644 index 0000000000..690359299a --- /dev/null +++ b/src/infra/messaging-outbox/sqlite/src/repos/sqlite_outbox_message_repository.rs @@ -0,0 +1,133 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use database_common::{TransactionRef, TransactionRefT}; +use dill::{component, interface}; +use futures::TryStreamExt; +use internal_error::{ErrorIntoInternal, InternalError, ResultIntoInternal}; + +use crate::domain::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub struct SqliteOutboxMessageRepository { + transaction: TransactionRefT, +} + +#[component(pub)] +#[interface(dyn OutboxMessageRepository)] +impl SqliteOutboxMessageRepository { + pub fn new(transaction: TransactionRef) -> Self { + Self { + transaction: transaction.into(), + } + } +} + +#[async_trait::async_trait] +impl OutboxMessageRepository for SqliteOutboxMessageRepository { + async fn push_message(&self, message: NewOutboxMessage) -> Result<(), InternalError> { + let mut tr = self.transaction.lock().await; + + let connection_mut = tr.connection_mut().await.int_err()?; + + let message_content_json = message.content_json; + + sqlx::query!( + r#" + INSERT INTO outbox_messages (producer_name, content_json, occurred_on) + VALUES ($1, $2, $3) + "#, + message.producer_name, + message_content_json, + message.occurred_on + ) + .execute(connection_mut) + .await + .int_err()?; + + Ok(()) + } + + async fn get_producer_messages( + &self, + producer_name: &str, + above_id: OutboxMessageID, + batch_size: usize, + ) -> Result { + let mut tr = self.transaction.lock().await; + + let producer_name = producer_name.to_string(); + + Ok(Box::pin(async_stream::stream! { + let connection_mut = tr + .connection_mut() + .await?; + + let above_id = above_id.into_inner(); + let batch_size = i64::try_from(batch_size).unwrap(); + + let mut query_stream = sqlx::query_as!( + OutboxMessage, + r#" + SELECT + message_id, + producer_name, + content_json as "content_json: _", + occurred_on as "occurred_on: _" + FROM outbox_messages + WHERE producer_name = $1 and message_id > $2 + ORDER BY message_id + LIMIT $3 + "#, + producer_name, + above_id, + batch_size, + ) + .fetch(connection_mut) + .map_err(ErrorIntoInternal::int_err); + + while let Some(message) = query_stream.try_next().await? { + yield Ok(message); + } + })) + } + + async fn get_latest_message_ids_by_producer( + &self, + ) -> Result, InternalError> { + let mut tr = self.transaction.lock().await; + let connection_mut = tr.connection_mut().await?; + + let records = sqlx::query!( + r#" + SELECT + producer_name, + IFNULL(MAX(message_id), 0) as max_message_id + FROM outbox_messages + GROUP BY producer_name + "#, + ) + .fetch_all(connection_mut) + .await + .map_err(ErrorIntoInternal::int_err)?; + + Ok(records + .into_iter() + .map(|r| { + ( + r.producer_name.unwrap(), + OutboxMessageID::new(i64::from(r.max_message_id)), + ) + }) + .collect()) + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/messaging-outbox/sqlite/tests/mod.rs b/src/infra/messaging-outbox/sqlite/tests/mod.rs new file mode 100644 index 0000000000..e3f0a19ada --- /dev/null +++ b/src/infra/messaging-outbox/sqlite/tests/mod.rs @@ -0,0 +1,10 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +mod repos; diff --git a/src/infra/messaging-outbox/sqlite/tests/repos/mod.rs b/src/infra/messaging-outbox/sqlite/tests/repos/mod.rs new file mode 100644 index 0000000000..4400d600e6 --- /dev/null +++ b/src/infra/messaging-outbox/sqlite/tests/repos/mod.rs @@ -0,0 +1,11 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +mod test_sqlite_outbox_message_consumption_repository; +mod test_sqlite_outbox_message_repository; diff --git a/src/infra/messaging-outbox/sqlite/tests/repos/test_sqlite_outbox_message_consumption_repository.rs b/src/infra/messaging-outbox/sqlite/tests/repos/test_sqlite_outbox_message_consumption_repository.rs new file mode 100644 index 0000000000..e28c0a3140 --- /dev/null +++ b/src/infra/messaging-outbox/sqlite/tests/repos/test_sqlite_outbox_message_consumption_repository.rs @@ -0,0 +1,118 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use database_common::{DatabaseTransactionRunner, SqliteTransactionManager}; +use dill::{Catalog, CatalogBuilder}; +use internal_error::InternalError; +use kamu_messaging_outbox_sqlite::SqliteOutboxMessageConsumptionRepository; +use sqlx::SqlitePool; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_group::group(database, sqlite)] +#[test_log::test(sqlx::test(migrations = "../../../../migrations/sqlite"))] +async fn test_no_outbox_consumptions_initially(sqlite_pool: SqlitePool) { + let harness = SqliteOutboxMessageConsumptionRepositoryHarness::new(sqlite_pool); + + DatabaseTransactionRunner::new(harness.catalog) + .transactional(|catalog| async move { + kamu_messaging_outbox_repo_tests::test_no_outbox_consumptions_initially(&catalog).await; + Ok::<_, InternalError>(()) + }) + .await + .unwrap(); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_group::group(database, sqlite)] +#[test_log::test(sqlx::test(migrations = "../../../../migrations/sqlite"))] +async fn test_create_consumption(sqlite_pool: SqlitePool) { + let harness = SqliteOutboxMessageConsumptionRepositoryHarness::new(sqlite_pool); + + DatabaseTransactionRunner::new(harness.catalog) + .transactional(|catalog| async move { + kamu_messaging_outbox_repo_tests::test_create_consumption(&catalog).await; + Ok::<_, InternalError>(()) + }) + .await + .unwrap(); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_group::group(database, sqlite)] +#[test_log::test(sqlx::test(migrations = "../../../../migrations/sqlite"))] +async fn test_update_existing_consumption(sqlite_pool: SqlitePool) { + let harness = SqliteOutboxMessageConsumptionRepositoryHarness::new(sqlite_pool); + + DatabaseTransactionRunner::new(harness.catalog) + .transactional(|catalog| async move { + kamu_messaging_outbox_repo_tests::test_update_existing_consumption(&catalog).await; + Ok::<_, InternalError>(()) + }) + .await + .unwrap(); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_group::group(database, sqlite)] +#[test_log::test(sqlx::test(migrations = "../../../../migrations/sqlite"))] +async fn test_cannot_update_consumption_before_creation(sqlite_pool: SqlitePool) { + let harness = SqliteOutboxMessageConsumptionRepositoryHarness::new(sqlite_pool); + + DatabaseTransactionRunner::new(harness.catalog) + .transactional(|catalog| async move { + kamu_messaging_outbox_repo_tests::test_cannot_update_consumption_before_creation( + &catalog, + ) + .await; + Ok::<_, InternalError>(()) + }) + .await + .unwrap(); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_group::group(database, sqlite)] +#[test_log::test(sqlx::test(migrations = "../../../../migrations/sqlite"))] +async fn test_multiple_boundaries(sqlite_pool: SqlitePool) { + let harness = SqliteOutboxMessageConsumptionRepositoryHarness::new(sqlite_pool); + + DatabaseTransactionRunner::new(harness.catalog) + .transactional(|catalog| async move { + kamu_messaging_outbox_repo_tests::test_multiple_boundaries(&catalog).await; + Ok::<_, InternalError>(()) + }) + .await + .unwrap(); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +struct SqliteOutboxMessageConsumptionRepositoryHarness { + catalog: Catalog, +} + +impl SqliteOutboxMessageConsumptionRepositoryHarness { + pub fn new(sqlite_pool: SqlitePool) -> Self { + let mut catalog_builder = CatalogBuilder::new(); + catalog_builder.add_value(sqlite_pool); + catalog_builder.add::(); + catalog_builder.add::(); + + Self { + catalog: catalog_builder.build(), + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/messaging-outbox/sqlite/tests/repos/test_sqlite_outbox_message_repository.rs b/src/infra/messaging-outbox/sqlite/tests/repos/test_sqlite_outbox_message_repository.rs new file mode 100644 index 0000000000..f7fba8820e --- /dev/null +++ b/src/infra/messaging-outbox/sqlite/tests/repos/test_sqlite_outbox_message_repository.rs @@ -0,0 +1,101 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use database_common::{DatabaseTransactionRunner, SqliteTransactionManager}; +use dill::{Catalog, CatalogBuilder}; +use internal_error::InternalError; +use kamu_messaging_outbox_sqlite::SqliteOutboxMessageRepository; +use sqlx::SqlitePool; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_group::group(database, sqlite)] +#[test_log::test(sqlx::test(migrations = "../../../../migrations/sqlite"))] +async fn test_no_outbox_messages_initially(sqlite_pool: SqlitePool) { + let harness = SqliteOutboxMessageRepositoryHarness::new(sqlite_pool); + + DatabaseTransactionRunner::new(harness.catalog) + .transactional(|catalog| async move { + kamu_messaging_outbox_repo_tests::test_no_outbox_messages_initially(&catalog).await; + Ok::<_, InternalError>(()) + }) + .await + .unwrap(); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_group::group(database, sqlite)] +#[test_log::test(sqlx::test(migrations = "../../../../migrations/sqlite"))] +async fn test_push_messages_from_several_producers(sqlite_pool: SqlitePool) { + let harness = SqliteOutboxMessageRepositoryHarness::new(sqlite_pool); + + DatabaseTransactionRunner::new(harness.catalog) + .transactional(|catalog| async move { + kamu_messaging_outbox_repo_tests::test_push_messages_from_several_producers(&catalog) + .await; + Ok::<_, InternalError>(()) + }) + .await + .unwrap(); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_group::group(database, sqlite)] +#[test_log::test(sqlx::test(migrations = "../../../../migrations/sqlite"))] +async fn test_push_many_messages_and_read_parts(sqlite_pool: SqlitePool) { + let harness = SqliteOutboxMessageRepositoryHarness::new(sqlite_pool); + + DatabaseTransactionRunner::new(harness.catalog) + .transactional(|catalog| async move { + kamu_messaging_outbox_repo_tests::test_push_many_messages_and_read_parts(&catalog) + .await; + Ok::<_, InternalError>(()) + }) + .await + .unwrap(); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_group::group(database, sqlite)] +#[test_log::test(sqlx::test(migrations = "../../../../migrations/sqlite"))] +async fn test_try_reading_above_max(sqlite_pool: SqlitePool) { + let harness = SqliteOutboxMessageRepositoryHarness::new(sqlite_pool); + + DatabaseTransactionRunner::new(harness.catalog) + .transactional(|catalog| async move { + kamu_messaging_outbox_repo_tests::test_try_reading_above_max(&catalog).await; + Ok::<_, InternalError>(()) + }) + .await + .unwrap(); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +struct SqliteOutboxMessageRepositoryHarness { + catalog: Catalog, +} + +impl SqliteOutboxMessageRepositoryHarness { + pub fn new(sqlite_pool: SqlitePool) -> Self { + let mut catalog_builder = CatalogBuilder::new(); + catalog_builder.add_value(sqlite_pool); + catalog_builder.add::(); + catalog_builder.add::(); + + Self { + catalog: catalog_builder.build(), + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/infra/task-system/inmem/Cargo.toml b/src/infra/task-system/inmem/Cargo.toml index 6e35f6a1c6..393ab0d8eb 100644 --- a/src/infra/task-system/inmem/Cargo.toml +++ b/src/infra/task-system/inmem/Cargo.toml @@ -27,7 +27,7 @@ kamu-task-system = { workspace = true } async-trait = { version = "0.1", default-features = false } chrono = { version = "0.4", default-features = false } -dill = "0.8" +dill = "0.9" futures = "0.3" diff --git a/src/infra/task-system/inmem/src/task_system_event_store_inmem.rs b/src/infra/task-system/inmem/src/inmem_task_system_event_store.rs similarity index 94% rename from src/infra/task-system/inmem/src/task_system_event_store_inmem.rs rename to src/infra/task-system/inmem/src/inmem_task_system_event_store.rs index 449aadef6e..7fe110e545 100644 --- a/src/infra/task-system/inmem/src/task_system_event_store_inmem.rs +++ b/src/infra/task-system/inmem/src/inmem_task_system_event_store.rs @@ -15,8 +15,8 @@ use opendatafabric::DatasetID; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub struct TaskSystemEventStoreInMemory { - inner: EventStoreInMemory, +pub struct InMemoryTaskSystemEventStore { + inner: InMemoryEventStore, } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -60,10 +60,10 @@ impl EventStoreState for State { #[component(pub)] #[interface(dyn TaskSystemEventStore)] #[scope(Singleton)] -impl TaskSystemEventStoreInMemory { +impl InMemoryTaskSystemEventStore { pub fn new() -> Self { Self { - inner: EventStoreInMemory::new(), + inner: InMemoryEventStore::new(), } } @@ -86,7 +86,7 @@ impl TaskSystemEventStoreInMemory { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[async_trait::async_trait] -impl EventStore for TaskSystemEventStoreInMemory { +impl EventStore for InMemoryTaskSystemEventStore { async fn len(&self) -> Result { self.inner.len().await } @@ -119,7 +119,7 @@ impl EventStore for TaskSystemEventStoreInMemory { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[async_trait::async_trait] -impl TaskSystemEventStore for TaskSystemEventStoreInMemory { +impl TaskSystemEventStore for InMemoryTaskSystemEventStore { async fn new_task_id(&self) -> Result { Ok(self.inner.as_state().lock().unwrap().next_task_id()) } diff --git a/src/infra/task-system/inmem/src/lib.rs b/src/infra/task-system/inmem/src/lib.rs index c93416cba2..a191d86538 100644 --- a/src/infra/task-system/inmem/src/lib.rs +++ b/src/infra/task-system/inmem/src/lib.rs @@ -14,6 +14,6 @@ // Re-exports pub use kamu_task_system as domain; -mod task_system_event_store_inmem; +mod inmem_task_system_event_store; -pub use task_system_event_store_inmem::*; +pub use inmem_task_system_event_store::*; diff --git a/src/infra/task-system/inmem/tests/tests/mod.rs b/src/infra/task-system/inmem/tests/tests/mod.rs index 36db29d4a1..52e89c9106 100644 --- a/src/infra/task-system/inmem/tests/tests/mod.rs +++ b/src/infra/task-system/inmem/tests/tests/mod.rs @@ -7,4 +7,4 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -mod test_task_system_event_store_inmem; +mod test_inmem_task_system_event_store; diff --git a/src/infra/task-system/inmem/tests/tests/test_task_system_event_store_inmem.rs b/src/infra/task-system/inmem/tests/tests/test_inmem_task_system_event_store.rs similarity index 90% rename from src/infra/task-system/inmem/tests/tests/test_task_system_event_store_inmem.rs rename to src/infra/task-system/inmem/tests/tests/test_inmem_task_system_event_store.rs index 75727e130b..ca8cd7c218 100644 --- a/src/infra/task-system/inmem/tests/tests/test_task_system_event_store_inmem.rs +++ b/src/infra/task-system/inmem/tests/tests/test_inmem_task_system_event_store.rs @@ -15,7 +15,7 @@ use kamu_task_system_inmem::*; #[test_log::test(tokio::test)] async fn test_event_store_empty() { let catalog = CatalogBuilder::new() - .add::() + .add::() .build(); kamu_task_system_repo_tests::test_event_store_empty(&catalog).await; @@ -26,7 +26,7 @@ async fn test_event_store_empty() { #[test_log::test(tokio::test)] async fn test_event_store_get_streams() { let catalog = CatalogBuilder::new() - .add::() + .add::() .build(); kamu_task_system_repo_tests::test_event_store_get_streams(&catalog).await; @@ -37,7 +37,7 @@ async fn test_event_store_get_streams() { #[test_log::test(tokio::test)] async fn test_event_store_get_events_with_windowing() { let catalog = CatalogBuilder::new() - .add::() + .add::() .build(); kamu_task_system_repo_tests::test_event_store_get_events_with_windowing(&catalog).await; @@ -48,7 +48,7 @@ async fn test_event_store_get_events_with_windowing() { #[test_log::test(tokio::test)] async fn test_event_store_get_events_by_tasks() { let catalog = CatalogBuilder::new() - .add::() + .add::() .build(); kamu_task_system_repo_tests::test_event_store_get_events_by_tasks(&catalog).await; @@ -59,7 +59,7 @@ async fn test_event_store_get_events_by_tasks() { #[test_log::test(tokio::test)] async fn test_event_store_get_dataset_tasks() { let catalog = CatalogBuilder::new() - .add::() + .add::() .build(); kamu_task_system_repo_tests::test_event_store_get_dataset_tasks(&catalog).await; diff --git a/src/infra/task-system/postgres/Cargo.toml b/src/infra/task-system/postgres/Cargo.toml index 82e50f585b..1d1103eb16 100644 --- a/src/infra/task-system/postgres/Cargo.toml +++ b/src/infra/task-system/postgres/Cargo.toml @@ -29,7 +29,7 @@ kamu-task-system = { workspace = true } async-stream = "0.3" async-trait = { version = "0.1", default-features = false } chrono = { version = "0.4", default-features = false } -dill = "0.8" +dill = "0.9" futures = "0.3" serde_json = "1" sqlx = { version = "0.7", default-features = false, features = [ diff --git a/src/infra/task-system/postgres/src/lib.rs b/src/infra/task-system/postgres/src/lib.rs index 8c33330ee0..5f5ce876eb 100644 --- a/src/infra/task-system/postgres/src/lib.rs +++ b/src/infra/task-system/postgres/src/lib.rs @@ -10,6 +10,6 @@ // Re-exports pub use kamu_task_system as domain; -mod task_system_event_store_postgres; +mod postgres_task_system_event_store; -pub use task_system_event_store_postgres::*; +pub use postgres_task_system_event_store::*; diff --git a/src/infra/task-system/postgres/src/task_system_event_store_postgres.rs b/src/infra/task-system/postgres/src/postgres_task_system_event_store.rs similarity index 97% rename from src/infra/task-system/postgres/src/task_system_event_store_postgres.rs rename to src/infra/task-system/postgres/src/postgres_task_system_event_store.rs index 800dc823fc..fe6ec65ed0 100644 --- a/src/infra/task-system/postgres/src/task_system_event_store_postgres.rs +++ b/src/infra/task-system/postgres/src/postgres_task_system_event_store.rs @@ -16,13 +16,13 @@ use sqlx::{FromRow, QueryBuilder}; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub struct TaskSystemEventStorePostgres { +pub struct PostgresTaskSystemEventStore { transaction: TransactionRefT, } #[component(pub)] #[interface(dyn TaskSystemEventStore)] -impl TaskSystemEventStorePostgres { +impl PostgresTaskSystemEventStore { pub fn new(transaction: TransactionRef) -> Self { Self { transaction: transaction.into(), @@ -33,7 +33,7 @@ impl TaskSystemEventStorePostgres { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[async_trait::async_trait] -impl EventStore for TaskSystemEventStorePostgres { +impl EventStore for PostgresTaskSystemEventStore { async fn get_events(&self, task_id: &TaskID, opts: GetEventsOpts) -> EventStream { let mut tr = self.transaction.lock().await; @@ -137,7 +137,7 @@ impl EventStore for TaskSystemEventStorePostgres { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[async_trait::async_trait] -impl TaskSystemEventStore for TaskSystemEventStorePostgres { +impl TaskSystemEventStore for PostgresTaskSystemEventStore { /// Generates new unique task identifier async fn new_task_id(&self) -> Result { let mut tr = self.transaction.lock().await; diff --git a/src/infra/task-system/postgres/tests/tests/mod.rs b/src/infra/task-system/postgres/tests/tests/mod.rs index 7ed4b473c5..a8d28e2a87 100644 --- a/src/infra/task-system/postgres/tests/tests/mod.rs +++ b/src/infra/task-system/postgres/tests/tests/mod.rs @@ -7,4 +7,4 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -mod test_task_system_event_store_postgres; +mod test_postgres_task_system_event_store; diff --git a/src/infra/task-system/postgres/tests/tests/test_task_system_event_store_postgres.rs b/src/infra/task-system/postgres/tests/tests/test_postgres_task_system_event_store.rs similarity index 87% rename from src/infra/task-system/postgres/tests/tests/test_task_system_event_store_postgres.rs rename to src/infra/task-system/postgres/tests/tests/test_postgres_task_system_event_store.rs index bb7abca730..e1b1f9b7e2 100644 --- a/src/infra/task-system/postgres/tests/tests/test_task_system_event_store_postgres.rs +++ b/src/infra/task-system/postgres/tests/tests/test_postgres_task_system_event_store.rs @@ -10,7 +10,7 @@ use database_common::{DatabaseTransactionRunner, PostgresTransactionManager}; use dill::{Catalog, CatalogBuilder}; use internal_error::InternalError; -use kamu_task_system_postgres::TaskSystemEventStorePostgres; +use kamu_task_system_postgres::PostgresTaskSystemEventStore; use sqlx::PgPool; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -18,7 +18,7 @@ use sqlx::PgPool; #[test_group::group(database, postgres)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/postgres"))] async fn test_event_store_empty(pg_pool: PgPool) { - let harness = PostgresAccountRepositoryHarness::new(pg_pool); + let harness = PostgresTaskSystemEventStoreHarness::new(pg_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -34,7 +34,7 @@ async fn test_event_store_empty(pg_pool: PgPool) { #[test_group::group(database, postgres)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/postgres"))] async fn test_event_store_get_streams(pg_pool: PgPool) { - let harness = PostgresAccountRepositoryHarness::new(pg_pool); + let harness = PostgresTaskSystemEventStoreHarness::new(pg_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -50,7 +50,7 @@ async fn test_event_store_get_streams(pg_pool: PgPool) { #[test_group::group(database, postgres)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/postgres"))] async fn test_event_store_get_events_with_windowing(pg_pool: PgPool) { - let harness = PostgresAccountRepositoryHarness::new(pg_pool); + let harness = PostgresTaskSystemEventStoreHarness::new(pg_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -66,7 +66,7 @@ async fn test_event_store_get_events_with_windowing(pg_pool: PgPool) { #[test_group::group(database, postgres)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/postgres"))] async fn test_event_store_get_events_by_tasks(pg_pool: PgPool) { - let harness = PostgresAccountRepositoryHarness::new(pg_pool); + let harness = PostgresTaskSystemEventStoreHarness::new(pg_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -82,7 +82,7 @@ async fn test_event_store_get_events_by_tasks(pg_pool: PgPool) { #[test_group::group(database, postgres)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/postgres"))] async fn test_event_store_get_dataset_tasks(pg_pool: PgPool) { - let harness = PostgresAccountRepositoryHarness::new(pg_pool); + let harness = PostgresTaskSystemEventStoreHarness::new(pg_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -95,17 +95,17 @@ async fn test_event_store_get_dataset_tasks(pg_pool: PgPool) { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -struct PostgresAccountRepositoryHarness { +struct PostgresTaskSystemEventStoreHarness { catalog: Catalog, } -impl PostgresAccountRepositoryHarness { +impl PostgresTaskSystemEventStoreHarness { pub fn new(pg_pool: PgPool) -> Self { // Initialize catalog with predefined Postgres pool let mut catalog_builder = CatalogBuilder::new(); catalog_builder.add_value(pg_pool); catalog_builder.add::(); - catalog_builder.add::(); + catalog_builder.add::(); Self { catalog: catalog_builder.build(), diff --git a/src/infra/task-system/repo-tests/Cargo.toml b/src/infra/task-system/repo-tests/Cargo.toml index 8823e9d032..723515c29b 100644 --- a/src/infra/task-system/repo-tests/Cargo.toml +++ b/src/infra/task-system/repo-tests/Cargo.toml @@ -26,5 +26,5 @@ kamu-task-system = { workspace = true } opendatafabric = { workspace = true } chrono = { version = "0.4", default-features = false } -dill = "0.8" +dill = "0.9" futures = "0.3" diff --git a/src/infra/task-system/sqlite/Cargo.toml b/src/infra/task-system/sqlite/Cargo.toml index 473620f261..02505131e0 100644 --- a/src/infra/task-system/sqlite/Cargo.toml +++ b/src/infra/task-system/sqlite/Cargo.toml @@ -29,7 +29,7 @@ kamu-task-system = { workspace = true } async-stream = "0.3" async-trait = { version = "0.1", default-features = false } chrono = { version = "0.4", default-features = false } -dill = "0.8" +dill = "0.9" futures = "0.3" serde_json = "1" sqlx = { version = "0.7", default-features = false, features = [ diff --git a/src/infra/task-system/sqlite/src/lib.rs b/src/infra/task-system/sqlite/src/lib.rs index 63765c837f..521faa0119 100644 --- a/src/infra/task-system/sqlite/src/lib.rs +++ b/src/infra/task-system/sqlite/src/lib.rs @@ -14,6 +14,6 @@ // Re-exports pub use kamu_task_system as domain; -mod task_system_event_store_sqlite; +mod sqlite_task_system_event_store; -pub use task_system_event_store_sqlite::*; +pub use sqlite_task_system_event_store::*; diff --git a/src/infra/task-system/sqlite/src/task_system_event_store_sqlite.rs b/src/infra/task-system/sqlite/src/sqlite_task_system_event_store.rs similarity index 97% rename from src/infra/task-system/sqlite/src/task_system_event_store_sqlite.rs rename to src/infra/task-system/sqlite/src/sqlite_task_system_event_store.rs index e607b8589a..a8a04381c4 100644 --- a/src/infra/task-system/sqlite/src/task_system_event_store_sqlite.rs +++ b/src/infra/task-system/sqlite/src/sqlite_task_system_event_store.rs @@ -17,13 +17,13 @@ use sqlx::{FromRow, QueryBuilder}; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub struct TaskSystemEventStoreSqlite { +pub struct SqliteTaskSystemEventStore { transaction: TransactionRefT, } #[component(pub)] #[interface(dyn TaskSystemEventStore)] -impl TaskSystemEventStoreSqlite { +impl SqliteTaskSystemEventStore { pub fn new(transaction: TransactionRef) -> Self { Self { transaction: transaction.into(), @@ -34,7 +34,7 @@ impl TaskSystemEventStoreSqlite { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[async_trait::async_trait] -impl EventStore for TaskSystemEventStoreSqlite { +impl EventStore for SqliteTaskSystemEventStore { async fn get_events(&self, task_id: &TaskID, opts: GetEventsOpts) -> EventStream { let mut tr = self.transaction.lock().await; @@ -146,7 +146,7 @@ impl EventStore for TaskSystemEventStoreSqlite { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #[async_trait::async_trait] -impl TaskSystemEventStore for TaskSystemEventStoreSqlite { +impl TaskSystemEventStore for SqliteTaskSystemEventStore { /// Generates new unique task identifier async fn new_task_id(&self) -> Result { let mut tr = self.transaction.lock().await; diff --git a/src/infra/task-system/sqlite/tests/tests/mod.rs b/src/infra/task-system/sqlite/tests/tests/mod.rs index b1dfc8c733..6fbd0e36f8 100644 --- a/src/infra/task-system/sqlite/tests/tests/mod.rs +++ b/src/infra/task-system/sqlite/tests/tests/mod.rs @@ -7,4 +7,4 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -mod test_task_system_event_store_sqlite; +mod test_sqlite_task_system_event_store; diff --git a/src/infra/task-system/sqlite/tests/tests/test_task_system_event_store_sqlite.rs b/src/infra/task-system/sqlite/tests/tests/test_sqlite_task_system_event_store.rs similarity index 87% rename from src/infra/task-system/sqlite/tests/tests/test_task_system_event_store_sqlite.rs rename to src/infra/task-system/sqlite/tests/tests/test_sqlite_task_system_event_store.rs index a30ecf7791..8ae1114aea 100644 --- a/src/infra/task-system/sqlite/tests/tests/test_task_system_event_store_sqlite.rs +++ b/src/infra/task-system/sqlite/tests/tests/test_sqlite_task_system_event_store.rs @@ -10,7 +10,7 @@ use database_common::{DatabaseTransactionRunner, SqliteTransactionManager}; use dill::{Catalog, CatalogBuilder}; use kamu_task_system::InternalError; -use kamu_task_system_sqlite::TaskSystemEventStoreSqlite; +use kamu_task_system_sqlite::SqliteTaskSystemEventStore; use sqlx::SqlitePool; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -18,7 +18,7 @@ use sqlx::SqlitePool; #[test_group::group(database, sqlite)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/sqlite"))] async fn test_event_store_empty(sqlite_pool: SqlitePool) { - let harness = SqliteAccountRepositoryHarness::new(sqlite_pool); + let harness = SqliteTaskSystemEventStoreHarness::new(sqlite_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -34,7 +34,7 @@ async fn test_event_store_empty(sqlite_pool: SqlitePool) { #[test_group::group(database, sqlite)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/sqlite"))] async fn test_event_store_get_streams(sqlite_pool: SqlitePool) { - let harness = SqliteAccountRepositoryHarness::new(sqlite_pool); + let harness = SqliteTaskSystemEventStoreHarness::new(sqlite_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -50,7 +50,7 @@ async fn test_event_store_get_streams(sqlite_pool: SqlitePool) { #[test_group::group(database, sqlite)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/sqlite"))] async fn test_event_store_get_events_with_windowing(sqlite_pool: SqlitePool) { - let harness = SqliteAccountRepositoryHarness::new(sqlite_pool); + let harness = SqliteTaskSystemEventStoreHarness::new(sqlite_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -66,7 +66,7 @@ async fn test_event_store_get_events_with_windowing(sqlite_pool: SqlitePool) { #[test_group::group(database, sqlite)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/sqlite"))] async fn test_event_store_get_events_by_tasks(sqlite_pool: SqlitePool) { - let harness = SqliteAccountRepositoryHarness::new(sqlite_pool); + let harness = SqliteTaskSystemEventStoreHarness::new(sqlite_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -82,7 +82,7 @@ async fn test_event_store_get_events_by_tasks(sqlite_pool: SqlitePool) { #[test_group::group(database, sqlite)] #[test_log::test(sqlx::test(migrations = "../../../../migrations/sqlite"))] async fn test_event_store_get_dataset_tasks(sqlite_pool: SqlitePool) { - let harness = SqliteAccountRepositoryHarness::new(sqlite_pool); + let harness = SqliteTaskSystemEventStoreHarness::new(sqlite_pool); DatabaseTransactionRunner::new(harness.catalog) .transactional(|catalog| async move { @@ -95,17 +95,17 @@ async fn test_event_store_get_dataset_tasks(sqlite_pool: SqlitePool) { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -struct SqliteAccountRepositoryHarness { +struct SqliteTaskSystemEventStoreHarness { catalog: Catalog, } -impl SqliteAccountRepositoryHarness { +impl SqliteTaskSystemEventStoreHarness { pub fn new(sqlite_pool: SqlitePool) -> Self { // Initialize catalog with predefined SQLite pool let mut catalog_builder = CatalogBuilder::new(); catalog_builder.add_value(sqlite_pool); catalog_builder.add::(); - catalog_builder.add::(); + catalog_builder.add::(); Self { catalog: catalog_builder.build(), diff --git a/src/utils/container-runtime/Cargo.toml b/src/utils/container-runtime/Cargo.toml index 5a5ad642ef..7a0a8c101a 100644 --- a/src/utils/container-runtime/Cargo.toml +++ b/src/utils/container-runtime/Cargo.toml @@ -26,7 +26,7 @@ random-names = { workspace = true } async-trait = "0.1" cfg-if = "1" -dill = "0.8" +dill = "0.9" libc = "0.2" regex = "1" serde = { version = "1", features = ["derive"] } diff --git a/src/utils/database-common/Cargo.toml b/src/utils/database-common/Cargo.toml index 474e1ebf5e..95b9b865d3 100644 --- a/src/utils/database-common/Cargo.toml +++ b/src/utils/database-common/Cargo.toml @@ -27,7 +27,7 @@ aws-sdk-secretsmanager = "0.35" aws-credential-types = "0.57" async-trait = "0.1" chrono = { version = "0.4", default-features = false } -dill = "0.8" +dill = "0.9" hex = "0.4" hmac = "0.12" internal-error = { workspace = true } diff --git a/src/utils/database-common/src/plugins/sqlite_plugin.rs b/src/utils/database-common/src/plugins/sqlite_plugin.rs index ad90f3c72e..102785efc9 100644 --- a/src/utils/database-common/src/plugins/sqlite_plugin.rs +++ b/src/utils/database-common/src/plugins/sqlite_plugin.rs @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0. use dill::*; -use sqlx::sqlite::SqliteConnectOptions; +use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions}; use sqlx::SqlitePool; use crate::*; @@ -42,7 +42,9 @@ impl SqlitePlugin { #[tracing::instrument(level = "info", skip_all)] fn open_sqlite_pool(db_connection_settings: &DatabaseConnectionSettings) -> SqlitePool { let sqlite_options = SqliteConnectOptions::new().filename(&db_connection_settings.host); - SqlitePool::connect_lazy_with(sqlite_options) + SqlitePoolOptions::new() + .max_connections(1) + .connect_lazy_with(sqlite_options) } } diff --git a/src/utils/database-common/src/transactions/db_transaction_manager.rs b/src/utils/database-common/src/transactions/db_transaction_manager.rs index 2cc33c5f02..539fc7366b 100644 --- a/src/utils/database-common/src/transactions/db_transaction_manager.rs +++ b/src/utils/database-common/src/transactions/db_transaction_manager.rs @@ -110,6 +110,26 @@ impl DatabaseTransactionRunner { }) .await } + + pub async fn transactional_with2( + &self, + callback: H, + ) -> Result + where + Iface1: 'static + ?Sized + Send + Sync, + Iface2: 'static + ?Sized + Send + Sync, + H: FnOnce(Arc, Arc) -> HFut, + HFut: std::future::Future>, + HFutResultE: From, + { + self.transactional(|transactional_catalog| async move { + let catalog_item1 = transactional_catalog.get_one().int_err()?; + let catalog_item2 = transactional_catalog.get_one().int_err()?; + + callback(catalog_item1, catalog_item2).await + }) + .await + } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/event-bus/src/event_bus.rs b/src/utils/event-bus/src/event_bus.rs deleted file mode 100644 index f85453c390..0000000000 --- a/src/utils/event-bus/src/event_bus.rs +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright Kamu Data, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -use std::sync::Arc; - -use dill::{Builder, Catalog}; -use internal_error::InternalError; - -use crate::{AsyncEventHandler, EventHandler}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -pub struct EventBus { - catalog: Arc, -} - -#[dill::component(pub)] -impl EventBus { - pub fn new(catalog: Arc) -> EventBus { - Self { catalog } - } - - pub async fn dispatch_event(&self, event: TEvent) -> Result<(), InternalError> - where - TEvent: 'static + Clone, - { - self.sync_dispatch(&event)?; - self.async_dispatch(&event).await?; - - Ok(()) - } - - fn sync_dispatch(&self, event: &TEvent) -> Result<(), InternalError> { - let builders = self.catalog.builders_for::>(); - - for b in builders { - tracing::debug!( - handler = b.instance_type_name(), - event = std::any::type_name::(), - "Dispatching event to sync handler" - ); - let inst = b.get(&self.catalog).unwrap(); - inst.handle(event)?; - } - - Ok(()) - } - - async fn async_dispatch( - &self, - event: &TEvent, - ) -> Result<(), InternalError> { - let builders = self.catalog.builders_for::>(); - - let mut handlers = Vec::new(); - for b in builders { - tracing::debug!( - handler = b.instance_type_name(), - event = std::any::type_name::(), - "Dispatching event to async handler" - ); - let handler = b.get(&self.catalog).unwrap(); - handlers.push(handler); - } - - let futures: Vec<_> = handlers - .iter() - .map(|handler| handler.handle(event)) - .collect(); - - let results = futures::future::join_all(futures).await; - results.into_iter().try_for_each(|res| res)?; - - Ok(()) - } -} diff --git a/src/utils/event-bus/tests/test_event_bus.rs b/src/utils/event-bus/tests/test_event_bus.rs deleted file mode 100644 index 0ada8128df..0000000000 --- a/src/utils/event-bus/tests/test_event_bus.rs +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright Kamu Data, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -use std::sync::{Arc, Mutex}; - -use dill::*; -use event_bus::{AsyncEventHandler, EventBus, EventHandler}; -use internal_error::InternalError; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -#[derive(Copy, Clone)] -struct Event {} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -struct TestSyncHandler { - invoked: Arc>, -} - -#[component(pub)] -#[interface(dyn EventHandler)] -#[scope(Singleton)] -impl TestSyncHandler { - fn new() -> Self { - Self { - invoked: Arc::new(Mutex::new(false)), - } - } - - fn was_invoked(&self) -> bool { - *self.invoked.lock().unwrap() - } -} - -impl EventHandler for TestSyncHandler { - fn handle(&self, _: &Event) -> Result<(), InternalError> { - let mut invoked = self.invoked.lock().unwrap(); - *invoked = true; - Ok(()) - } -} - -#[test_log::test(tokio::test)] -async fn test_bus_sync_handler() { - let catalog = dill::CatalogBuilder::new() - .add::() - .add::() - .build(); - let event_bus = catalog.get_one::().unwrap(); - - event_bus.dispatch_event(Event {}).await.unwrap(); - - let handler = catalog.get_one::().unwrap(); - assert!(handler.was_invoked()); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -struct TestAsyncHandler { - invoked: Arc>, -} - -#[component(pub)] -#[interface(dyn AsyncEventHandler)] -#[scope(Singleton)] -impl TestAsyncHandler { - fn new() -> Self { - Self { - invoked: Arc::new(Mutex::new(false)), - } - } - - fn was_invoked(&self) -> bool { - *self.invoked.lock().unwrap() - } -} - -#[async_trait::async_trait] -impl AsyncEventHandler for TestAsyncHandler { - async fn handle(&self, _: &Event) -> Result<(), InternalError> { - let mut invoked = self.invoked.lock().unwrap(); - *invoked = true; - Ok(()) - } -} - -#[test_log::test(tokio::test)] -async fn test_bus_async_handler() { - let catalog = dill::CatalogBuilder::new() - .add::() - .add::() - .build(); - let event_bus = catalog.get_one::().unwrap(); - - event_bus.dispatch_event(Event {}).await.unwrap(); - - let handler = catalog.get_one::().unwrap(); - assert!(handler.was_invoked()); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/event-sourcing/Cargo.toml b/src/utils/event-sourcing/Cargo.toml index d53eb1b5e9..ef959151be 100644 --- a/src/utils/event-sourcing/Cargo.toml +++ b/src/utils/event-sourcing/Cargo.toml @@ -27,6 +27,7 @@ internal-error = { workspace = true } async-stream = "0.3" async-trait = { version = "0.1", default-features = false } +serde = { version = "1", features = ["derive"] } thiserror = { version = "1", default-features = false } tokio-stream = { version = "0.1", default-features = false } tracing = { version = "0.1", default-features = false, features = ["attributes"] } diff --git a/src/utils/event-sourcing/src/event_id.rs b/src/utils/event-sourcing/src/event_id.rs index 6a473a5bad..d95a3490cc 100644 --- a/src/utils/event-sourcing/src/event_id.rs +++ b/src/utils/event-sourcing/src/event_id.rs @@ -9,6 +9,9 @@ /// A monotonically increasing identifier /// assigned by event stores to events + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct EventID(i64); @@ -39,3 +42,5 @@ impl std::fmt::Display for EventID { write!(f, "{}", self.0) } } + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/event-sourcing/src/event_store_inmem.rs b/src/utils/event-sourcing/src/inmem_event_store.rs similarity index 95% rename from src/utils/event-sourcing/src/event_store_inmem.rs rename to src/utils/event-sourcing/src/inmem_event_store.rs index 49aa2d730d..af7dc2f0c5 100644 --- a/src/utils/event-sourcing/src/event_store_inmem.rs +++ b/src/utils/event-sourcing/src/inmem_event_store.rs @@ -17,12 +17,12 @@ use crate::*; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub struct EventStoreInMemory> { +pub struct InMemoryEventStore> { state: Arc>, _proj: PhantomData, } -impl> EventStoreInMemory { +impl> InMemoryEventStore { pub fn new() -> Self { Self { state: Arc::new(Mutex::new(State::default())), @@ -37,7 +37,7 @@ impl> EventStoreInMemory> EventStore - for EventStoreInMemory + for InMemoryEventStore { async fn len(&self) -> Result { Ok(self.state.lock().unwrap().events_count()) diff --git a/src/utils/event-sourcing/src/lib.rs b/src/utils/event-sourcing/src/lib.rs index a4916375bf..00baac178d 100644 --- a/src/utils/event-sourcing/src/lib.rs +++ b/src/utils/event-sourcing/src/lib.rs @@ -14,13 +14,13 @@ pub use internal_error::*; mod aggregate; mod event_id; mod event_store; -mod event_store_inmem; +mod inmem_event_store; mod projection; mod projection_event; pub use aggregate::*; pub use event_id::*; pub use event_store::*; -pub use event_store_inmem::*; +pub use inmem_event_store::*; pub use projection::*; pub use projection_event::*; diff --git a/src/utils/http-common/Cargo.toml b/src/utils/http-common/Cargo.toml index 2f6398e74e..475358f068 100644 --- a/src/utils/http-common/Cargo.toml +++ b/src/utils/http-common/Cargo.toml @@ -22,6 +22,7 @@ doctest = false [dependencies] +internal-error = { workspace = true } kamu-core = { workspace = true } axum = "0.6" diff --git a/src/utils/http-common/src/api_error.rs b/src/utils/http-common/src/api_error.rs index f5e7b1c161..fcf4ddecbe 100644 --- a/src/utils/http-common/src/api_error.rs +++ b/src/utils/http-common/src/api_error.rs @@ -16,6 +16,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +use internal_error::InternalError; use kamu_core::*; use thiserror::Error; diff --git a/src/utils/messaging-outbox/Cargo.toml b/src/utils/messaging-outbox/Cargo.toml new file mode 100644 index 0000000000..4bafd7971e --- /dev/null +++ b/src/utils/messaging-outbox/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "messaging-outbox" +description = "Code organizing reliable message exchange between modules" +version = { workspace = true } +homepage = { workspace = true } +repository = { workspace = true } +authors = { workspace = true } +readme = { workspace = true } +license-file = { workspace = true } +keywords = { workspace = true } +include = { workspace = true } +edition = { workspace = true } +publish = { workspace = true } + + +[lints] +workspace = true + + +[lib] +doctest = false + + +[dependencies] +database-common = { workspace = true } +internal-error = { workspace = true } +time-source = { workspace = true } + +async-trait = "0.1" +chrono = { version = "0.4" } +dill = "0.9" +futures = "0.3" +mockall = "0.11" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +thiserror = "1" +tokio = { version = "1", default-features = false } +tokio-stream = { version = "0.1", default-features = false } +tracing = "0.1" + +[dev-dependencies] +kamu-messaging-outbox-inmem = { workspace = true } + +paste = "1" +serde = { version = "1", features = ["derive"] } +test-log = { version = "0.2", features = ["trace"] } +tokio = { version = "1", default-features = false, features = ["rt", "macros"] } \ No newline at end of file diff --git a/src/domain/core/src/entities/events.rs b/src/utils/messaging-outbox/src/consumers/message_consumer.rs similarity index 58% rename from src/domain/core/src/entities/events.rs rename to src/utils/messaging-outbox/src/consumers/message_consumer.rs index 90860fa714..85a1c298d5 100644 --- a/src/domain/core/src/entities/events.rs +++ b/src/utils/messaging-outbox/src/consumers/message_consumer.rs @@ -7,38 +7,42 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use opendatafabric::{AccountID, DatasetID}; +use dill::Catalog; +use internal_error::InternalError; + +use crate::Message; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum DatasetEvent { - Created(DatasetEventCreated), - Deleted(DatasetEventDeleted), - DependenciesUpdated(DatasetEventDependenciesUpdated), -} +#[async_trait::async_trait] +pub trait MessageConsumer: Send + Sync {} //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct DatasetEventCreated { - pub dataset_id: DatasetID, - pub owner_account_id: AccountID, +#[async_trait::async_trait] +pub trait MessageConsumerT: MessageConsumer { + async fn consume_message( + &self, + target_catalog: &Catalog, + message: &TMessage, + ) -> Result<(), InternalError>; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct DatasetEventDeleted { - pub dataset_id: DatasetID, +#[derive(Debug)] +pub struct MessageConsumerMeta { + pub consumer_name: &'static str, + pub feeding_producers: &'static [&'static str], + pub durability: MessageConsumptionDurability, } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct DatasetEventDependenciesUpdated { - pub dataset_id: DatasetID, - pub new_upstream_ids: Vec, +#[derive(Debug, Eq, PartialEq, Copy, Clone)] +pub enum MessageConsumptionDurability { + Durable, + BestEffort, } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/messaging-outbox/src/consumers/message_consumers_utils.rs b/src/utils/messaging-outbox/src/consumers/message_consumers_utils.rs new file mode 100644 index 0000000000..66c0675d1c --- /dev/null +++ b/src/utils/messaging-outbox/src/consumers/message_consumers_utils.rs @@ -0,0 +1,183 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use dill::{Builder, BuilderExt, Catalog, TypecastBuilder}; +use futures::{StreamExt, TryStreamExt}; +use internal_error::{InternalError, ResultIntoInternal}; + +use super::{ + ConsumerFilter, + MessageConsumer, + MessageConsumerMeta, + MessageConsumptionDurability, + MessageDispatcher, +}; +use crate::{Message, MessageConsumerT, MessageSubscription}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[tracing::instrument(level = "debug", skip_all, fields(%content_json))] +pub async fn consume_deserialized_message<'a, TMessage: Message + 'static>( + catalog: &Catalog, + consumer_filter: ConsumerFilter<'a>, + content_json: &str, +) -> Result<(), InternalError> { + let message = serde_json::from_str::(content_json).int_err()?; + + let consumers = match consumer_filter { + ConsumerFilter::AllConsumers => all_consumers_for::(catalog), + ConsumerFilter::BestEffortConsumers => best_effort_consumers_for::(catalog), + ConsumerFilter::SelectedConsumer(consumer_name) => { + particular_consumers_for::(catalog, consumer_name) + } + }; + + let consumption_tasks = consumers.into_iter().map(|consumer| (consumer, &message)); + + futures::stream::iter(consumption_tasks) + .map(Ok) + .try_for_each_concurrent(/* limit */ None, |(consumer, message)| async move { + consumer.consume_message(catalog, message).await + }) + .await?; + + Ok(()) +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +fn all_consumers_for( + catalog: &Catalog, +) -> Vec>> { + consumers_from_builders( + catalog, + catalog.builders_for::>(), + ) +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +fn best_effort_consumers_for( + catalog: &Catalog, +) -> Vec>> { + consumers_from_builders( + catalog, + catalog.builders_for_with_meta::, _>( + |meta: &MessageConsumerMeta| { + meta.durability == MessageConsumptionDurability::BestEffort + }, + ), + ) +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +fn particular_consumers_for( + catalog: &Catalog, + consumer_name: &str, +) -> Vec>> { + consumers_from_builders( + catalog, + catalog.builders_for_with_meta::, _>( + |meta: &MessageConsumerMeta| meta.consumer_name == consumer_name, + ), + ) +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +fn consumers_from_builders<'a, TMessage: Message + 'static>( + catalog: &'a Catalog, + builders: Box>> + 'a>, +) -> Vec>> { + let mut consumers = Vec::new(); + for b in builders { + let consumer = b.get(catalog).unwrap(); + consumers.push(consumer); + } + + consumers +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub(crate) fn group_message_dispatchers_by_producer( + dispatchers: &[Arc], +) -> HashMap> { + let mut dispatchers_by_producers = HashMap::new(); + for dispatcher in dispatchers { + let producer_name = dispatcher.get_producer_name(); + assert!( + !dispatchers_by_producers.contains_key(producer_name), + "Duplicate dispatcher for producer '{producer_name}'" + ); + dispatchers_by_producers.insert((*producer_name).to_string(), dispatcher.clone()); + } + + dispatchers_by_producers +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub(crate) fn enumerate_messaging_routes( + catalog: &Catalog, + durability: MessageConsumptionDurability, +) -> Vec { + let mut res = Vec::new(); + + let all_consumer_builders = catalog.builders_for::(); + for consumer_builder in all_consumer_builders { + let all_metadata: Vec<&MessageConsumerMeta> = consumer_builder.metadata_get_all(); + assert!( + all_metadata.len() <= 1, + "Multiple consumer metadata records unexpected for {}", + consumer_builder.instance_type_name() + ); + for metadata in all_metadata { + if metadata.durability == durability { + for producer_name in metadata.feeding_producers { + res.push(MessageSubscription::new( + producer_name, + metadata.consumer_name, + )); + } + } + } + } + + res +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub(crate) fn group_consumers_by_producers( + meta_info_rows: &[MessageSubscription], +) -> HashMap> { + let mut unique_consumers_by_producer: HashMap> = HashMap::new(); + for row in meta_info_rows { + unique_consumers_by_producer + .entry(row.producer_name.to_string()) + .and_modify(|v| { + v.insert(row.consumer_name.to_string()); + }) + .or_insert_with(|| HashSet::from([row.consumer_name.to_string()])); + } + + let mut res = HashMap::new(); + for (producer_name, consumer_names) in unique_consumers_by_producer { + res.insert(producer_name, consumer_names.into_iter().collect()); + } + + res +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/messaging-outbox/src/consumers/message_dispatcher.rs b/src/utils/messaging-outbox/src/consumers/message_dispatcher.rs new file mode 100644 index 0000000000..4bb21dbe98 --- /dev/null +++ b/src/utils/messaging-outbox/src/consumers/message_dispatcher.rs @@ -0,0 +1,84 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::marker::PhantomData; + +use dill::Catalog; +use internal_error::InternalError; + +use super::consume_deserialized_message; +use crate::Message; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[derive(Debug, Copy, Clone)] +pub enum ConsumerFilter<'a> { + AllConsumers, + BestEffortConsumers, + SelectedConsumer(&'a str), +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[async_trait::async_trait] +pub trait MessageDispatcher: Send + Sync { + fn get_producer_name(&self) -> &'static str; + + async fn dispatch_message<'a>( + &self, + catalog: &Catalog, + consumer_filter: ConsumerFilter<'a>, + content_json: &str, + ) -> Result<(), InternalError>; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub fn register_message_dispatcher( + catalog_builder: &mut dill::CatalogBuilder, + producer_name: &'static str, +) { + let dispatcher = MessageDispatcherT::::new(producer_name); + catalog_builder.add_value(dispatcher); + catalog_builder.bind::>(); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub struct MessageDispatcherT { + producer_name: &'static str, + _phantom: PhantomData, +} + +impl MessageDispatcherT { + pub fn new(producer_name: &'static str) -> Self { + Self { + producer_name, + _phantom: PhantomData, + } + } +} + +#[async_trait::async_trait] +impl MessageDispatcher for MessageDispatcherT { + fn get_producer_name(&self) -> &'static str { + self.producer_name + } + + async fn dispatch_message<'a>( + &self, + catalog: &Catalog, + consumer_filter: ConsumerFilter<'a>, + content_json: &str, + ) -> Result<(), InternalError> { + consume_deserialized_message::(catalog, consumer_filter, content_json).await + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/messaging-outbox/src/consumers/message_subscription.rs b/src/utils/messaging-outbox/src/consumers/message_subscription.rs new file mode 100644 index 0000000000..fb0103c953 --- /dev/null +++ b/src/utils/messaging-outbox/src/consumers/message_subscription.rs @@ -0,0 +1,32 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub(crate) struct MessageSubscription { + pub producer_name: &'static str, + pub consumer_name: &'static str, +} + +impl MessageSubscription { + pub(crate) fn new(producer_name: &'static str, consumer_name: &'static str) -> Self { + Self { + producer_name, + consumer_name, + } + } +} + +impl std::fmt::Display for MessageSubscription { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} ----> {}", self.producer_name, self.consumer_name) + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/messaging-outbox/src/consumers/mod.rs b/src/utils/messaging-outbox/src/consumers/mod.rs new file mode 100644 index 0000000000..7ef5ff7a52 --- /dev/null +++ b/src/utils/messaging-outbox/src/consumers/mod.rs @@ -0,0 +1,18 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +mod message_consumer; +mod message_consumers_utils; +mod message_dispatcher; +mod message_subscription; + +pub use message_consumer::*; +pub use message_consumers_utils::*; +pub use message_dispatcher::*; +pub(crate) use message_subscription::*; diff --git a/src/utils/messaging-outbox/src/entities/mod.rs b/src/utils/messaging-outbox/src/entities/mod.rs new file mode 100644 index 0000000000..229465e2cf --- /dev/null +++ b/src/utils/messaging-outbox/src/entities/mod.rs @@ -0,0 +1,14 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +mod outbox_message; +mod outbox_message_id; + +pub use outbox_message::*; +pub use outbox_message_id::*; diff --git a/src/utils/event-bus/src/event_handler.rs b/src/utils/messaging-outbox/src/entities/outbox_message.rs similarity index 63% rename from src/utils/event-bus/src/event_handler.rs rename to src/utils/messaging-outbox/src/entities/outbox_message.rs index 287068787c..e1e5ad22f5 100644 --- a/src/utils/event-bus/src/event_handler.rs +++ b/src/utils/messaging-outbox/src/entities/outbox_message.rs @@ -7,19 +7,27 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use internal_error::InternalError; +use chrono::{DateTime, Utc}; + +use crate::OutboxMessageID; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub trait EventHandler: Sync + Send { - fn handle(&self, event: &TEvent) -> Result<(), InternalError>; +#[derive(Debug, Clone)] +pub struct OutboxMessage { + pub message_id: OutboxMessageID, + pub producer_name: String, + pub content_json: serde_json::Value, + pub occurred_on: DateTime, } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -#[async_trait::async_trait] -pub trait AsyncEventHandler: Sync + Send { - async fn handle(&self, event: &TEvent) -> Result<(), InternalError>; +#[derive(Debug)] +pub struct NewOutboxMessage { + pub producer_name: String, + pub content_json: serde_json::Value, + pub occurred_on: DateTime, } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/messaging-outbox/src/entities/outbox_message_id.rs b/src/utils/messaging-outbox/src/entities/outbox_message_id.rs new file mode 100644 index 0000000000..e1211a8148 --- /dev/null +++ b/src/utils/messaging-outbox/src/entities/outbox_message_id.rs @@ -0,0 +1,52 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +/// A monotonically increasing identifier +/// assigned by event stores to events + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct OutboxMessageID(i64); + +impl OutboxMessageID { + pub fn new(v: i64) -> Self { + Self(v) + } + + pub fn into_inner(self) -> i64 { + self.0 + } +} + +impl From for i64 { + fn from(val: OutboxMessageID) -> Self { + val.0 + } +} + +impl From for OutboxMessageID { + fn from(val: i64) -> Self { + OutboxMessageID::new(val) + } +} + +impl std::fmt::Debug for OutboxMessageID { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl std::fmt::Display for OutboxMessageID { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/messaging-outbox/src/lib.rs b/src/utils/messaging-outbox/src/lib.rs new file mode 100644 index 0000000000..c19e0e2adb --- /dev/null +++ b/src/utils/messaging-outbox/src/lib.rs @@ -0,0 +1,22 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +#![feature(let_chains)] + +mod consumers; +mod entities; +mod message; +mod repos; +mod services; + +pub use consumers::*; +pub use entities::*; +pub use message::*; +pub use repos::*; +pub use services::*; diff --git a/src/utils/messaging-outbox/src/message.rs b/src/utils/messaging-outbox/src/message.rs new file mode 100644 index 0000000000..b3ecac873b --- /dev/null +++ b/src/utils/messaging-outbox/src/message.rs @@ -0,0 +1,16 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use serde::{Deserialize, Serialize}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub trait Message: Clone + Serialize + for<'a> Deserialize<'a> + Send + Sync {} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/messaging-outbox/src/repos/mod.rs b/src/utils/messaging-outbox/src/repos/mod.rs new file mode 100644 index 0000000000..0621df794c --- /dev/null +++ b/src/utils/messaging-outbox/src/repos/mod.rs @@ -0,0 +1,14 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +mod outbox_message_consumption_repository; +mod outbox_message_repository; + +pub use outbox_message_consumption_repository::*; +pub use outbox_message_repository::*; diff --git a/src/utils/messaging-outbox/src/repos/outbox_message_consumption_repository.rs b/src/utils/messaging-outbox/src/repos/outbox_message_consumption_repository.rs new file mode 100644 index 0000000000..603e88ea35 --- /dev/null +++ b/src/utils/messaging-outbox/src/repos/outbox_message_consumption_repository.rs @@ -0,0 +1,100 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use internal_error::InternalError; +use thiserror::Error; + +use crate::OutboxMessageID; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[async_trait::async_trait] +pub trait OutboxMessageConsumptionRepository: Send + Sync { + async fn list_consumption_boundaries( + &self, + ) -> Result; + + async fn find_consumption_boundary( + &self, + consumer_name: &str, + producer_name: &str, + ) -> Result, InternalError>; + + async fn create_consumption_boundary( + &self, + boundary: OutboxMessageConsumptionBoundary, + ) -> Result<(), CreateConsumptionBoundaryError>; + + async fn update_consumption_boundary( + &self, + boundary: OutboxMessageConsumptionBoundary, + ) -> Result<(), UpdateConsumptionBoundaryError>; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)] +pub struct OutboxMessageConsumptionBoundary { + pub producer_name: String, + pub consumer_name: String, + pub last_consumed_message_id: OutboxMessageID, +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[derive(Error, Debug)] +pub enum CreateConsumptionBoundaryError { + #[error(transparent)] + DuplicateConsumptionBoundary(DuplicateConsumptionBoundaryError), + + #[error(transparent)] + Internal(InternalError), +} + +#[derive(Error, Debug)] +#[error( + "A boundary for consumer '{consumer_name}' and producer '{producer_name}' is already \ + registered" +)] +pub struct DuplicateConsumptionBoundaryError { + pub consumer_name: String, + pub producer_name: String, +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[derive(Error, Debug)] +pub enum UpdateConsumptionBoundaryError { + #[error(transparent)] + ConsumptionBoundaryNotFound(ConsumptionBoundaryNotFoundError), + + #[error(transparent)] + Internal(InternalError), +} + +#[derive(Error, Debug)] +#[error( + "A boundary for consumer '{consumer_name}' and producer '{producer_name}' is not registered" +)] +pub struct ConsumptionBoundaryNotFoundError { + pub consumer_name: String, + pub producer_name: String, +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub type OutboxMessageConsumptionBoundariesStream<'a> = std::pin::Pin< + Box< + dyn tokio_stream::Stream> + + Send + + 'a, + >, +>; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/messaging-outbox/src/repos/outbox_message_repository.rs b/src/utils/messaging-outbox/src/repos/outbox_message_repository.rs new file mode 100644 index 0000000000..77442b7c1a --- /dev/null +++ b/src/utils/messaging-outbox/src/repos/outbox_message_repository.rs @@ -0,0 +1,38 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use internal_error::InternalError; + +use crate::{NewOutboxMessage, OutboxMessage, OutboxMessageID}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[async_trait::async_trait] +pub trait OutboxMessageRepository: Send + Sync { + async fn push_message(&self, message: NewOutboxMessage) -> Result<(), InternalError>; + + async fn get_producer_messages( + &self, + producer_name: &str, + above_id: OutboxMessageID, + batch_size: usize, + ) -> Result; + + async fn get_latest_message_ids_by_producer( + &self, + ) -> Result, InternalError>; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub type OutboxMessageStream<'a> = std::pin::Pin< + Box> + Send + 'a>, +>; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/messaging-outbox/src/services/implementation/mod.rs b/src/utils/messaging-outbox/src/services/implementation/mod.rs new file mode 100644 index 0000000000..68bd6bf1d3 --- /dev/null +++ b/src/utils/messaging-outbox/src/services/implementation/mod.rs @@ -0,0 +1,16 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +mod outbox_dispatching_impl; +mod outbox_immediate_impl; +mod outbox_transactional_impl; + +pub use outbox_dispatching_impl::*; +pub use outbox_immediate_impl::*; +pub use outbox_transactional_impl::*; diff --git a/src/utils/messaging-outbox/src/services/implementation/outbox_dispatching_impl.rs b/src/utils/messaging-outbox/src/services/implementation/outbox_dispatching_impl.rs new file mode 100644 index 0000000000..83ab54a006 --- /dev/null +++ b/src/utils/messaging-outbox/src/services/implementation/outbox_dispatching_impl.rs @@ -0,0 +1,102 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::collections::HashSet; +use std::sync::Arc; + +use dill::*; +use internal_error::InternalError; + +use super::{OutboxImmediateImpl, OutboxTransactionalImpl}; +use crate::{MessageConsumer, MessageConsumerMeta, MessageConsumptionDurability, Outbox}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub struct OutboxDispatchingImpl { + immediate_outbox: Arc, + transactional_outbox: Arc, + durable_producers: HashSet, + best_effort_producers: HashSet, +} + +#[component(pub)] +impl OutboxDispatchingImpl { + #[allow(clippy::needless_pass_by_value)] + pub fn new( + catalog: Catalog, + immediate_outbox: Arc, + transactional_outbox: Arc, + ) -> Self { + let (durable_producers, best_effort_producers) = Self::classify_message_routes(&catalog); + + Self { + immediate_outbox, + transactional_outbox, + durable_producers, + best_effort_producers, + } + } + + fn classify_message_routes(catalog: &Catalog) -> (HashSet, HashSet) { + let mut durable_producers = HashSet::new(); + let mut best_effort_producers = HashSet::new(); + + let all_consumer_builders = catalog.builders_for::(); + for consumer_builder in all_consumer_builders { + let all_metadata: Vec<&MessageConsumerMeta> = consumer_builder.metadata_get_all(); + assert!( + all_metadata.len() <= 1, + "Multiple consumer metadata records unexpected for {}", + consumer_builder.instance_type_name() + ); + for metadata in all_metadata { + for producer_name in metadata.feeding_producers { + match metadata.durability { + MessageConsumptionDurability::Durable => { + durable_producers.insert((*producer_name).to_string()); + } + MessageConsumptionDurability::BestEffort => { + best_effort_producers.insert((*producer_name).to_string()); + } + } + } + } + } + + (durable_producers, best_effort_producers) + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[async_trait::async_trait] +impl Outbox for OutboxDispatchingImpl { + #[tracing::instrument(level = "debug", skip_all, fields(producer_name, content_json))] + async fn post_message_as_json( + &self, + producer_name: &str, + content_json: &serde_json::Value, + ) -> Result<(), InternalError> { + if self.durable_producers.contains(producer_name) { + self.transactional_outbox + .post_message_as_json(producer_name, content_json) + .await?; + } + + if self.best_effort_producers.contains(producer_name) { + self.immediate_outbox + .post_message_as_json(producer_name, content_json) + .await?; + } + + Ok(()) + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/messaging-outbox/src/services/implementation/outbox_immediate_impl.rs b/src/utils/messaging-outbox/src/services/implementation/outbox_immediate_impl.rs new file mode 100644 index 0000000000..f425e5380f --- /dev/null +++ b/src/utils/messaging-outbox/src/services/implementation/outbox_immediate_impl.rs @@ -0,0 +1,65 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::collections::HashMap; +use std::sync::Arc; + +use dill::{component, Catalog}; +use internal_error::InternalError; + +use crate::{group_message_dispatchers_by_producer, ConsumerFilter, MessageDispatcher, Outbox}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub struct OutboxImmediateImpl { + catalog: Catalog, + message_dispatchers_by_producers: HashMap>, + consumer_filter: ConsumerFilter<'static>, +} + +#[component(pub)] +impl OutboxImmediateImpl { + #[allow(clippy::needless_pass_by_value)] + pub fn new( + catalog: Catalog, + message_dispatchers: Vec>, + consumer_filter: ConsumerFilter<'static>, + ) -> Self { + Self { + catalog, + message_dispatchers_by_producers: group_message_dispatchers_by_producer( + &message_dispatchers, + ), + consumer_filter, + } + } +} + +#[async_trait::async_trait] +impl Outbox for OutboxImmediateImpl { + #[tracing::instrument(level = "debug", skip_all, fields(producer_name, content_json))] + async fn post_message_as_json( + &self, + producer_name: &str, + content_json: &serde_json::Value, + ) -> Result<(), InternalError> { + let maybe_dispatcher = self.message_dispatchers_by_producers.get(producer_name); + if let Some(dispatcher) = maybe_dispatcher { + let content_json = content_json.to_string(); + + dispatcher + .dispatch_message(&self.catalog, self.consumer_filter, &content_json) + .await?; + } + + Ok(()) + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/messaging-outbox/src/services/implementation/outbox_transactional_impl.rs b/src/utils/messaging-outbox/src/services/implementation/outbox_transactional_impl.rs new file mode 100644 index 0000000000..cc5aec6e7e --- /dev/null +++ b/src/utils/messaging-outbox/src/services/implementation/outbox_transactional_impl.rs @@ -0,0 +1,58 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::sync::Arc; + +use dill::component; +use internal_error::InternalError; +use time_source::SystemTimeSource; + +use crate::{NewOutboxMessage, Outbox, OutboxMessageRepository}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub struct OutboxTransactionalImpl { + outbox_message_repository: Arc, + time_source: Arc, +} + +#[component(pub)] +impl OutboxTransactionalImpl { + pub fn new( + outbox_message_repository: Arc, + time_source: Arc, + ) -> Self { + Self { + outbox_message_repository, + time_source, + } + } +} + +#[async_trait::async_trait] +impl Outbox for OutboxTransactionalImpl { + #[tracing::instrument(level = "debug", skip_all, fields(producer_name, content_json))] + async fn post_message_as_json( + &self, + producer_name: &str, + content_json: &serde_json::Value, + ) -> Result<(), InternalError> { + let new_outbox_message = NewOutboxMessage { + content_json: content_json.clone(), + producer_name: producer_name.to_string(), + occurred_on: self.time_source.now(), + }; + + self.outbox_message_repository + .push_message(new_outbox_message) + .await + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/messaging-outbox/src/services/mod.rs b/src/utils/messaging-outbox/src/services/mod.rs new file mode 100644 index 0000000000..39fe6efefb --- /dev/null +++ b/src/utils/messaging-outbox/src/services/mod.rs @@ -0,0 +1,21 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +mod implementation; +mod testing; + +mod outbox; +mod outbox_config; +mod outbox_transactional_processor; + +pub use implementation::*; +pub use outbox::*; +pub use outbox_config::*; +pub use outbox_transactional_processor::*; +pub use testing::*; diff --git a/src/utils/messaging-outbox/src/services/outbox.rs b/src/utils/messaging-outbox/src/services/outbox.rs new file mode 100644 index 0000000000..87f97d9f20 --- /dev/null +++ b/src/utils/messaging-outbox/src/services/outbox.rs @@ -0,0 +1,50 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use internal_error::InternalError; + +use crate::Message; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[async_trait::async_trait] +pub trait Outbox: Send + Sync { + async fn post_message_as_json( + &self, + producer_name: &str, + content_json: &serde_json::Value, + ) -> Result<(), InternalError>; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[async_trait::async_trait] +pub trait OutboxExt { + async fn post_message( + &self, + producer_name: &str, + message: M, + ) -> Result<(), InternalError>; +} + +#[async_trait::async_trait] +impl OutboxExt for T { + #[inline] + async fn post_message( + &self, + producer_name: &str, + message: M, + ) -> Result<(), InternalError> { + let message_as_json = serde_json::to_value(&message).unwrap(); + self.post_message_as_json(producer_name, &message_as_json) + .await + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/messaging-outbox/src/services/outbox_config.rs b/src/utils/messaging-outbox/src/services/outbox_config.rs new file mode 100644 index 0000000000..3a780c4606 --- /dev/null +++ b/src/utils/messaging-outbox/src/services/outbox_config.rs @@ -0,0 +1,41 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use chrono::Duration; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[derive(Debug)] +pub struct OutboxConfig { + /// Defines discretization of the main scheduling loop: how often new data + /// is checked and processed + pub awaiting_step: Duration, + /// Defines maximum number of messages attempted to read in 1 step + pub batch_size: i64, +} + +impl OutboxConfig { + pub fn new(awaiting_step: Duration, batch_size: i64) -> Self { + Self { + awaiting_step, + batch_size, + } + } +} + +impl Default for OutboxConfig { + fn default() -> Self { + Self { + awaiting_step: Duration::seconds(1), + batch_size: 20, + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/messaging-outbox/src/services/outbox_transactional_processor.rs b/src/utils/messaging-outbox/src/services/outbox_transactional_processor.rs new file mode 100644 index 0000000000..9c75d18ea6 --- /dev/null +++ b/src/utils/messaging-outbox/src/services/outbox_transactional_processor.rs @@ -0,0 +1,466 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use database_common::DatabaseTransactionRunner; +use dill::{component, scope, Catalog, Singleton}; +use internal_error::{InternalError, ResultIntoInternal}; + +use crate::*; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pub struct OutboxTransactionalProcessor { + catalog: Catalog, + config: Arc, + routes_static_info: Arc, + producer_relay_jobs: Vec, +} + +#[component(pub)] +#[scope(Singleton)] +impl OutboxTransactionalProcessor { + pub fn new( + catalog: Catalog, + config: Arc, + message_dispatchers_by_producers: Vec>, + ) -> Self { + let routes_static_info = Arc::new(Self::make_static_routes_info( + &catalog, + message_dispatchers_by_producers, + )); + + let mut producer_relay_jobs = Vec::new(); + for (producer_name, consumer_names) in &routes_static_info.consumers_by_producers { + producer_relay_jobs.push(ProducerRelayJob::new( + catalog.clone(), + config.clone(), + routes_static_info.clone(), + producer_name.clone(), + consumer_names.clone(), + )); + } + + Self { + catalog, + config, + routes_static_info, + producer_relay_jobs, + } + } + + #[allow(clippy::needless_pass_by_value)] + fn make_static_routes_info( + catalog: &Catalog, + message_dispatchers: Vec>, + ) -> RoutesStaticInfo { + let all_durable_messaging_routes = + enumerate_messaging_routes(catalog, MessageConsumptionDurability::Durable); + let consumers_by_producers = group_consumers_by_producers(&all_durable_messaging_routes); + let message_dispatchers_by_producers = + group_message_dispatchers_by_producer(&message_dispatchers); + + RoutesStaticInfo::new( + message_dispatchers_by_producers, + all_durable_messaging_routes, + consumers_by_producers, + ) + } + + #[tracing::instrument(level = "debug", skip_all)] + pub async fn run(&self) -> Result<(), InternalError> { + // Trace current routes + self.debug_message_routes(); + + // Make sure consumption records represent the routes + self.init_consumption_records().await?; + + // Main relay loop + loop { + self.run_relay_iteration().await?; + + tracing::debug!("Awaiting next iteration"); + tokio::time::sleep(self.config.awaiting_step.to_std().unwrap()).await; + } + } + + // To be used by tests only! + #[tracing::instrument(level = "debug", skip_all)] + pub async fn run_single_iteration_only(&self) -> Result<(), InternalError> { + // Trace current routes + self.debug_message_routes(); + + // Make sure consumption records represent the routes + self.init_consumption_records().await?; + + // Run single iteration instead of a loop + self.run_relay_iteration().await?; + Ok(()) + } + + fn debug_message_routes(&self) { + for messaging_route in &self.routes_static_info.all_durable_messaging_routes { + tracing::debug!("{messaging_route}"); + } + } + + #[tracing::instrument(level = "debug", skip_all)] + async fn init_consumption_records(&self) -> Result<(), InternalError> { + DatabaseTransactionRunner::new(self.catalog.clone()) + .transactional_with( + |outbox_consumption_repository: Arc| async move { + // Load existing consumption records + use futures::TryStreamExt; + let consumptions = outbox_consumption_repository + .list_consumption_boundaries() + .await? + .try_collect::>().await?; + + // Build a set of producer-consumer pairs that already exist in the database + let mut matched_consumptions = HashSet::new(); + for consumption in &consumptions { + matched_consumptions.insert((&consumption.producer_name, &consumption.consumer_name)); + } + + // Detect new routes, which are not associated with a consumption record yet + for (producer_name, consumer_names) in &self.routes_static_info.consumers_by_producers { + for consumer_name in consumer_names { + if !matched_consumptions.contains(&(producer_name, consumer_name)) { + // Create a new consumption boundary + outbox_consumption_repository.create_consumption_boundary(OutboxMessageConsumptionBoundary { + consumer_name: consumer_name.clone(), + producer_name: producer_name.clone(), + last_consumed_message_id: OutboxMessageID::new(0) + }).await.int_err()?; + } + } + } + + Ok(()) + }, + ) + .await + } + + #[tracing::instrument(level = "debug", skip_all)] + async fn run_relay_iteration(&self) -> Result<(), InternalError> { + // producer A - message 17 + // producer B - message 19 + let latest_message_ids_by_producer = self.select_latest_message_ids_by_producers().await?; + + // producer A -> + // consumer X -> message 15 + // consumer Y -> message 14 + // producer B -> + // consumer X -> message 12 + let mut consumption_boundaries_by_producer = self + .select_latest_consumption_boundaries_by_producers() + .await?; + + // Prepare iteration for each producer + let mut producer_tasks = Vec::new(); + for producer_relay_job in &self.producer_relay_jobs { + // Extract latest message ID by producer + let Some(latest_produced_message_id) = + latest_message_ids_by_producer.get(&producer_relay_job.producer_name) + else { + continue; + }; + + // Take consumption boundaries for this producer + let Some(consumption_boundaries) = + consumption_boundaries_by_producer.remove(&producer_relay_job.producer_name) + else { + continue; + }; + + producer_tasks.push(( + producer_relay_job, + latest_produced_message_id, + consumption_boundaries, + )); + } + + // Run relay jobs of each producer concurrently + use futures::{StreamExt, TryStreamExt}; + futures::stream::iter(producer_tasks) + .map(Ok) + .try_for_each_concurrent(/* limit */ None, |producer_task| async move { + producer_task + .0 + .run_iteration(*producer_task.1, producer_task.2) + .await + }) + .await?; + + Ok(()) + } + + async fn select_latest_message_ids_by_producers( + &self, + ) -> Result, InternalError> { + // Extract latest (producer, max message id) relation + let latest_message_ids_by_producer = DatabaseTransactionRunner::new(self.catalog.clone()) + .transactional_with( + |outbox_message_repository: Arc| async move { + outbox_message_repository + .get_latest_message_ids_by_producer() + .await + }, + ) + .await + .unwrap(); + + // Convert into map + Ok(latest_message_ids_by_producer + .into_iter() + .collect::>()) + } + + async fn select_latest_consumption_boundaries_by_producers( + &self, + ) -> Result>, InternalError> { + // Extract consumption boundaries for all routes + let all_boundaries = DatabaseTransactionRunner::new(self.catalog.clone()) + .transactional_with( + |outbox_consumption_repository: Arc| async move { + let consumptions_stream = outbox_consumption_repository + .list_consumption_boundaries() + .await?; + + use futures::TryStreamExt; + consumptions_stream.try_collect::>().await + }, + ) + .await + .unwrap(); + + // Organize by producer->consumer hierarchically + let mut boundaries_by_producer = HashMap::new(); + for boundary in all_boundaries { + boundaries_by_producer + .entry(boundary.producer_name) + .and_modify(|by_consumer: &mut HashMap| { + by_consumer.insert( + boundary.consumer_name.clone(), + boundary.last_consumed_message_id, + ); + }) + .or_insert_with(|| { + HashMap::from([(boundary.consumer_name, boundary.last_consumed_message_id)]) + }); + } + + Ok(boundaries_by_producer) + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +struct RoutesStaticInfo { + message_dispatchers_by_producers: HashMap>, + all_durable_messaging_routes: Vec, + consumers_by_producers: HashMap>, +} + +impl RoutesStaticInfo { + fn new( + message_dispatchers_by_producers: HashMap>, + all_durable_messaging_routes: Vec, + consumers_by_producers: HashMap>, + ) -> Self { + Self { + message_dispatchers_by_producers, + all_durable_messaging_routes, + consumers_by_producers, + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +struct ProducerRelayJob { + catalog: Catalog, + config: Arc, + relay_routes_static_info: Arc, + producer_name: String, + consumer_names: Vec, +} + +impl ProducerRelayJob { + fn new( + catalog: Catalog, + config: Arc, + relay_routes_static_info: Arc, + producer_name: String, + consumer_names: Vec, + ) -> Self { + Self { + catalog, + config, + relay_routes_static_info, + producer_name, + consumer_names, + } + } + + #[tracing::instrument(level = "debug", skip_all, fields(latest_produced_message_id))] + async fn run_iteration( + &self, + latest_produced_message_id: OutboxMessageID, + consumption_boundaries: HashMap, + ) -> Result<(), InternalError> { + // Decide on the earliest message that was processed by all of the consumers + let maybe_processed_boundary_id = + self.determine_processed_boundary_id(&consumption_boundaries); + + tracing::debug!( + "Processed boundary for producer '{}' is {:?}", + self.producer_name, + maybe_processed_boundary_id + ); + + // Was there an advancement? + if let Some(processed_boundary_id) = maybe_processed_boundary_id + && processed_boundary_id < latest_produced_message_id + { + // Load all messages after the earliest + let unprocessed_messages = self + .load_messages_above( + processed_boundary_id, + usize::try_from(self.config.batch_size).unwrap(), + ) + .await?; + + // Feed consumers if they are behind this message + // We must respect the sequential order of messages, + // but individual consumers may process each message concurrently + for message in unprocessed_messages { + // Prepare consumer invocation tasks + let mut consumer_tasks = Vec::new(); + for consumer_name in &self.consumer_names { + let boundary_id = consumption_boundaries + .get(consumer_name) + .copied() + .unwrap_or_else(|| OutboxMessageID::new(0)); + if boundary_id < message.message_id { + consumer_tasks.push((consumer_name.as_str(), &message)); + } + } + + // Consume concurrently + use futures::{StreamExt, TryStreamExt}; + futures::stream::iter(consumer_tasks) + .map(Ok) + .try_for_each_concurrent( + /* limit */ None, + |(consumer_name, message)| async move { + self.invoke_consumer(consumer_name, message).await?; + Ok(()) + }, + ) + .await?; + } + } + + Ok(()) + } + + fn determine_processed_boundary_id( + &self, + consumption_boundaries: &HashMap, + ) -> Option { + let mut earliest_seen_id: Option = None; + + for consumer in &self.consumer_names { + if let Some(boundary_id) = consumption_boundaries.get(consumer) { + match earliest_seen_id { + Some(id) => { + if *boundary_id < id { + earliest_seen_id = Some(*boundary_id); + } + } + None => earliest_seen_id = Some(*boundary_id), + } + } else { + // We are seeing a new consumer, a full synchronization is a must + return Some(OutboxMessageID::new(0)); + } + } + + earliest_seen_id + } + + #[tracing::instrument(level = "debug", skip_all, fields(above_id, batch_size))] + async fn load_messages_above( + &self, + above_id: OutboxMessageID, + batch_size: usize, + ) -> Result, InternalError> { + DatabaseTransactionRunner::new(self.catalog.clone()) + .transactional_with( + |outbox_message_repository: Arc| async move { + let messages_stream = outbox_message_repository + .get_producer_messages(&self.producer_name, above_id, batch_size) + .await?; + + use futures::TryStreamExt; + messages_stream.try_collect::>().await + }, + ) + .await + } + + #[tracing::instrument(level = "debug", skip_all, fields(consumer_name, ?message, is_first_consumption))] + async fn invoke_consumer( + &self, + consumer_name: &str, + message: &OutboxMessage, + ) -> Result<(), InternalError> { + DatabaseTransactionRunner::new(self.catalog.clone()) + .transactional(|transaction_catalog| async move { + let dispatcher = self + .relay_routes_static_info + .message_dispatchers_by_producers + .get(&message.producer_name) + .expect("No dispatcher for producer"); + + let content_json = message.content_json.to_string(); + + dispatcher + .dispatch_message( + &transaction_catalog, + ConsumerFilter::SelectedConsumer(consumer_name), + &content_json, + ) + .await?; + + // Shift consumption record regardless of whether the consumer was interested in + // the message + let consumption_repository = transaction_catalog + .get_one::() + .unwrap(); + consumption_repository + .update_consumption_boundary(OutboxMessageConsumptionBoundary { + consumer_name: consumer_name.to_string(), + producer_name: message.producer_name.to_string(), + last_consumed_message_id: message.message_id, + }) + .await + .int_err()?; + + Ok(()) + }) + .await + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/messaging-outbox/src/services/testing/dummy_outbox_impl.rs b/src/utils/messaging-outbox/src/services/testing/dummy_outbox_impl.rs new file mode 100644 index 0000000000..1463ced337 --- /dev/null +++ b/src/utils/messaging-outbox/src/services/testing/dummy_outbox_impl.rs @@ -0,0 +1,33 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use dill::{component, interface}; +use internal_error::InternalError; + +use crate::Outbox; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[component(pub)] +#[interface(dyn Outbox)] +pub struct DummyOutboxImpl {} + +#[async_trait::async_trait] +impl Outbox for DummyOutboxImpl { + async fn post_message_as_json( + &self, + _producer_name: &str, + _content_json: &serde_json::Value, + ) -> Result<(), InternalError> { + // We are happy + Ok(()) + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/messaging-outbox/src/services/testing/mock_outbox_impl.rs b/src/utils/messaging-outbox/src/services/testing/mock_outbox_impl.rs new file mode 100644 index 0000000000..1068fa4e81 --- /dev/null +++ b/src/utils/messaging-outbox/src/services/testing/mock_outbox_impl.rs @@ -0,0 +1,29 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use internal_error::InternalError; + +use crate::Outbox; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +mockall::mock! { + pub Outbox {} + + #[async_trait::async_trait] + impl Outbox for Outbox { + async fn post_message_as_json( + &self, + producer_name: &str, + content_json: &serde_json::Value, + ) -> Result<(), InternalError>; + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/messaging-outbox/src/services/testing/mod.rs b/src/utils/messaging-outbox/src/services/testing/mod.rs new file mode 100644 index 0000000000..e1f0ec0e3d --- /dev/null +++ b/src/utils/messaging-outbox/src/services/testing/mod.rs @@ -0,0 +1,14 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +mod dummy_outbox_impl; +mod mock_outbox_impl; + +pub use dummy_outbox_impl::*; +pub use mock_outbox_impl::*; diff --git a/src/utils/messaging-outbox/tests/mod.rs b/src/utils/messaging-outbox/tests/mod.rs new file mode 100644 index 0000000000..071c0eb613 --- /dev/null +++ b/src/utils/messaging-outbox/tests/mod.rs @@ -0,0 +1,91 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +#![feature(assert_matches)] + +mod tests; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +macro_rules! test_message_type { + ($message_type_suffix: ident) => { + paste::paste! { + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] + pub(crate) struct [] { + pub(crate) body: String, + } + + impl Message for [] {} + } + }; +} + +macro_rules! test_message_consumer { + ($message_type_suffix: ident, $message_consumer_suffix: ident, $producer_name: ident, $durability: ident) => { + paste::paste! { + struct [<"TestMessageConsumer" $message_consumer_suffix>] { + state: Arc]>>, + } + + struct [<"State" $message_consumer_suffix>] { + captured_messages: Vec<[]>, + } + + impl Default for [<"State" $message_consumer_suffix>] { + fn default() -> Self { + Self { + captured_messages: vec![], + } + } + } + + #[component(pub)] + #[scope(Singleton)] + #[interface(dyn MessageConsumer)] + #[interface(dyn MessageConsumerT<[]>)] + #[meta(MessageConsumerMeta { + consumer_name: concat!("TestMessageConsumer", stringify!($message_consumer_suffix)), + feeding_producers: &[$producer_name], + durability: MessageConsumptionDurability::$durability, + })] + impl [<"TestMessageConsumer" $message_consumer_suffix>] { + fn new() -> Self { + Self { + state: Arc::new(Mutex::new(Default::default())), + } + } + + #[allow(dead_code)] + fn get_messages(&self) -> Vec<[]> { + let guard = self.state.lock().unwrap(); + guard.captured_messages.clone() + } + } + + impl MessageConsumer for [<"TestMessageConsumer" $message_consumer_suffix>] {} + + #[async_trait::async_trait] + impl MessageConsumerT<[]> for [<"TestMessageConsumer" $message_consumer_suffix>] { + async fn consume_message( + &self, + _: &Catalog, + message: &[], + ) -> Result<(), InternalError> { + let mut guard = self.state.lock().unwrap(); + guard.captured_messages.push(message.clone()); + Ok(()) + } + } + } + }; +} + +pub(crate) use {test_message_consumer, test_message_type}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/messaging-outbox/tests/tests/mod.rs b/src/utils/messaging-outbox/tests/tests/mod.rs new file mode 100644 index 0000000000..93e1850171 --- /dev/null +++ b/src/utils/messaging-outbox/tests/tests/mod.rs @@ -0,0 +1,13 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +mod test_dispatching_outbox_impl; +mod test_immediate_outbox_impl; +mod test_outbox_transactional_processor; +mod test_transactional_outbox_impl; diff --git a/src/utils/messaging-outbox/tests/tests/test_dispatching_outbox_impl.rs b/src/utils/messaging-outbox/tests/tests/test_dispatching_outbox_impl.rs new file mode 100644 index 0000000000..a3892e9846 --- /dev/null +++ b/src/utils/messaging-outbox/tests/tests/test_dispatching_outbox_impl.rs @@ -0,0 +1,276 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::assert_matches::assert_matches; +use std::sync::{Arc, Mutex}; + +use dill::*; +use internal_error::InternalError; +use kamu_messaging_outbox_inmem::InMemoryOutboxMessageRepository; +use messaging_outbox::*; +use serde::{Deserialize, Serialize}; +use time_source::SystemTimeSourceDefault; + +use crate::{test_message_consumer, test_message_type}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +const TEST_PRODUCER_A: &str = "TEST-PRODUCER-A"; +const TEST_PRODUCER_B: &str = "TEST-PRODUCER-B"; +const TEST_PRODUCER_C: &str = "TEST-PRODUCER-C"; + +test_message_type!(A); +test_message_type!(B); +test_message_type!(C); + +test_message_consumer!(A, A, TEST_PRODUCER_A, BestEffort); +test_message_consumer!(B, B, TEST_PRODUCER_B, Durable); +test_message_consumer!(C, CB, TEST_PRODUCER_C, BestEffort); +test_message_consumer!(C, CD, TEST_PRODUCER_C, Durable); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_log::test(tokio::test)] +async fn test_best_effort_only_messages() { + let message_1 = TestMessageA { + body: "foo".to_string(), + }; + let message_2 = TestMessageA { + body: "bar".to_string(), + }; + + let harness = DispatchingOutboxHarness::new(); + + let res = harness + .outbox + .post_message(TEST_PRODUCER_A, message_1.clone()) + .await; + assert_matches!(res, Ok(_)); + + let res = harness + .outbox + .post_message(TEST_PRODUCER_A, message_2.clone()) + .await; + assert_matches!(res, Ok(_)); + + assert_eq!( + harness.test_consumer_a.get_messages(), + vec![message_1, message_2] + ); + assert_eq!(harness.test_consumer_b.get_messages(), vec![]); + assert_eq!(harness.test_consumer_cb.get_messages(), vec![]); + assert_eq!(harness.test_consumer_cd.get_messages(), vec![]); + + assert_eq!( + harness + .get_saved_messages::(TEST_PRODUCER_A) + .await + .len(), + 0 + ); + assert_eq!( + harness + .get_saved_messages::(TEST_PRODUCER_B) + .await + .len(), + 0 + ); + assert_eq!( + harness + .get_saved_messages::(TEST_PRODUCER_C) + .await + .len(), + 0 + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_log::test(tokio::test)] +async fn test_durable_only_messages() { + let message_1 = TestMessageB { + body: "foo".to_string(), + }; + let message_2 = TestMessageB { + body: "bar".to_string(), + }; + + let harness = DispatchingOutboxHarness::new(); + + let res = harness + .outbox + .post_message(TEST_PRODUCER_B, message_1.clone()) + .await; + assert_matches!(res, Ok(_)); + + let res = harness + .outbox + .post_message(TEST_PRODUCER_B, message_2.clone()) + .await; + assert_matches!(res, Ok(_)); + + assert_eq!(harness.test_consumer_a.get_messages(), vec![]); + assert_eq!(harness.test_consumer_b.get_messages(), vec![]); + assert_eq!(harness.test_consumer_cb.get_messages(), vec![]); + assert_eq!(harness.test_consumer_cd.get_messages(), vec![]); + + assert_eq!( + harness + .get_saved_messages::(TEST_PRODUCER_A) + .await + .len(), + 0 + ); + assert_eq!( + harness + .get_saved_messages::(TEST_PRODUCER_B) + .await, + vec![message_1, message_2] + ); + assert_eq!( + harness + .get_saved_messages::(TEST_PRODUCER_C) + .await + .len(), + 0 + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_log::test(tokio::test)] +async fn test_messages_mixed_durability() { + let message_1 = TestMessageC { + body: "foo".to_string(), + }; + let message_2 = TestMessageC { + body: "bar".to_string(), + }; + + let harness = DispatchingOutboxHarness::new(); + + let res = harness + .outbox + .post_message(TEST_PRODUCER_C, message_1.clone()) + .await; + assert_matches!(res, Ok(_)); + + let res = harness + .outbox + .post_message(TEST_PRODUCER_C, message_2.clone()) + .await; + assert_matches!(res, Ok(_)); + + assert_eq!(harness.test_consumer_a.get_messages(), vec![]); + assert_eq!(harness.test_consumer_b.get_messages(), vec![]); + assert_eq!( + harness.test_consumer_cb.get_messages(), + vec![message_1.clone(), message_2.clone()] + ); + assert_eq!(harness.test_consumer_cd.get_messages(), vec![]); + + assert_eq!( + harness + .get_saved_messages::(TEST_PRODUCER_A) + .await + .len(), + 0 + ); + assert_eq!( + harness + .get_saved_messages::(TEST_PRODUCER_B) + .await + .len(), + 0 + ); + assert_eq!( + harness + .get_saved_messages::(TEST_PRODUCER_C) + .await, + vec![message_1, message_2] + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +struct DispatchingOutboxHarness { + _catalog: Catalog, + outbox: Arc, + outbox_message_repository: Arc, + test_consumer_a: Arc, + test_consumer_b: Arc, + test_consumer_cb: Arc, + test_consumer_cd: Arc, +} + +impl DispatchingOutboxHarness { + fn new() -> Self { + let mut b = CatalogBuilder::new(); + b.add_builder( + messaging_outbox::OutboxImmediateImpl::builder() + .with_consumer_filter(messaging_outbox::ConsumerFilter::BestEffortConsumers), + ); + b.add::(); + b.add::(); + b.bind::(); + b.add::(); + b.add::(); + b.add::(); + b.add::(); + b.add::(); + b.add::(); + + register_message_dispatcher::(&mut b, TEST_PRODUCER_A); + register_message_dispatcher::(&mut b, TEST_PRODUCER_B); + register_message_dispatcher::(&mut b, TEST_PRODUCER_C); + + let catalog = b.build(); + + let outbox = catalog.get_one::().unwrap(); + let outbox_message_repository = catalog.get_one::().unwrap(); + let test_consumer_a = catalog.get_one::().unwrap(); + let test_consumer_b = catalog.get_one::().unwrap(); + let test_consumer_cb = catalog.get_one::().unwrap(); + let test_consumer_cd = catalog.get_one::().unwrap(); + + Self { + _catalog: catalog, + outbox, + outbox_message_repository, + test_consumer_a, + test_consumer_b, + test_consumer_cb, + test_consumer_cd, + } + } + + async fn get_saved_messages(&self, producer_name: &str) -> Vec { + use futures::TryStreamExt; + let outbox_messages: Vec<_> = self + .outbox_message_repository + .get_producer_messages(producer_name, OutboxMessageID::new(0), 10) + .await + .unwrap() + .try_collect() + .await + .unwrap(); + + let decoded_messages: Vec<_> = outbox_messages + .into_iter() + .map(|om| { + assert_eq!(om.producer_name, producer_name); + serde_json::from_value::(om.content_json).unwrap() + }) + .collect(); + + decoded_messages + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/messaging-outbox/tests/tests/test_immediate_outbox_impl.rs b/src/utils/messaging-outbox/tests/tests/test_immediate_outbox_impl.rs new file mode 100644 index 0000000000..6148f39f5b --- /dev/null +++ b/src/utils/messaging-outbox/tests/tests/test_immediate_outbox_impl.rs @@ -0,0 +1,210 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::assert_matches::assert_matches; +use std::sync::{Arc, Mutex}; + +use dill::*; +use internal_error::InternalError; +use messaging_outbox::*; +use serde::{Deserialize, Serialize}; + +use crate::{test_message_consumer, test_message_type}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +const TEST_PRODUCER_A: &str = "TEST-PRODUCER-A"; +const TEST_PRODUCER_B: &str = "TEST-PRODUCER-B"; +const TEST_PRODUCER_C: &str = "TEST-PRODUCER-C"; +const TEST_PRODUCER_D: &str = "TEST-PRODUCER-D"; + +test_message_type!(A); +test_message_type!(B); +test_message_type!(C); // No consumers +test_message_type!(D); + +test_message_consumer!(A, A, TEST_PRODUCER_A, BestEffort); +test_message_consumer!(B, B, TEST_PRODUCER_B, BestEffort); +test_message_consumer!(D, D1, TEST_PRODUCER_D, BestEffort); +test_message_consumer!(D, D2, TEST_PRODUCER_D, BestEffort); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_log::test(tokio::test)] +async fn test_immediate_outbox_messages_of_one_type() { + let message_1 = TestMessageA { + body: "foo".to_string(), + }; + let message_2 = TestMessageA { + body: "bar".to_string(), + }; + + let harness = ImmediateOutboxHarness::new(); + + let res = harness + .outbox + .post_message(TEST_PRODUCER_A, message_1.clone()) + .await; + assert_matches!(res, Ok(_)); + + let res = harness + .outbox + .post_message(TEST_PRODUCER_A, message_2.clone()) + .await; + assert_matches!(res, Ok(_)); + + assert_eq!( + harness.test_message_consumer_a.get_messages(), + vec![message_1, message_2] + ); + assert_eq!(harness.test_message_consumer_b.get_messages(), vec![]); + assert_eq!(harness.test_message_consumer_d1.get_messages(), vec![]); + assert_eq!(harness.test_message_consumer_d2.get_messages(), vec![]); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_log::test(tokio::test)] +async fn test_immediate_outbox_messages_of_two_types() { + let message_1 = TestMessageA { + body: "foo".to_string(), + }; + let message_2 = TestMessageB { + body: "bar".to_string(), + }; + + let harness = ImmediateOutboxHarness::new(); + + let res = harness + .outbox + .post_message(TEST_PRODUCER_A, message_1.clone()) + .await; + assert_matches!(res, Ok(_)); + + let res = harness + .outbox + .post_message(TEST_PRODUCER_B, message_2.clone()) + .await; + assert_matches!(res, Ok(_)); + + assert_eq!( + harness.test_message_consumer_a.get_messages(), + vec![message_1] + ); + assert_eq!( + harness.test_message_consumer_b.get_messages(), + vec![message_2] + ); + assert_eq!(harness.test_message_consumer_d1.get_messages(), vec![]); + assert_eq!(harness.test_message_consumer_d2.get_messages(), vec![]); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_log::test(tokio::test)] +async fn test_immediate_outbox_message_without_consumers() { + let message = TestMessageC { + body: "foo".to_string(), + }; + + let harness = ImmediateOutboxHarness::new(); + + let res = harness.outbox.post_message(TEST_PRODUCER_C, message).await; + assert_matches!(res, Ok(_)); + + assert_eq!(harness.test_message_consumer_a.get_messages(), vec![]); + assert_eq!(harness.test_message_consumer_b.get_messages(), vec![]); + assert_eq!(harness.test_message_consumer_d1.get_messages(), vec![]); + assert_eq!(harness.test_message_consumer_d2.get_messages(), vec![]); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_log::test(tokio::test)] +async fn test_immediate_outbox_messages_two_handlers_for_same() { + let message_1 = TestMessageD { + body: "foo".to_string(), + }; + let message_2 = TestMessageD { + body: "bar".to_string(), + }; + + let harness = ImmediateOutboxHarness::new(); + + let res = harness + .outbox + .post_message(TEST_PRODUCER_D, message_1.clone()) + .await; + assert_matches!(res, Ok(_)); + + let res = harness + .outbox + .post_message(TEST_PRODUCER_D, message_2.clone()) + .await; + assert_matches!(res, Ok(_)); + + assert_eq!(harness.test_message_consumer_a.get_messages(), vec![]); + assert_eq!(harness.test_message_consumer_b.get_messages(), vec![]); + assert_eq!( + harness.test_message_consumer_d1.get_messages(), + vec![message_1.clone(), message_2.clone()] + ); + assert_eq!( + harness.test_message_consumer_d2.get_messages(), + vec![message_1, message_2] + ); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +struct ImmediateOutboxHarness { + _catalog: Catalog, + outbox: Arc, + test_message_consumer_a: Arc, + test_message_consumer_b: Arc, + test_message_consumer_d1: Arc, + test_message_consumer_d2: Arc, +} + +impl ImmediateOutboxHarness { + fn new() -> Self { + let mut b = CatalogBuilder::new(); + b.add_builder( + messaging_outbox::OutboxImmediateImpl::builder() + .with_consumer_filter(messaging_outbox::ConsumerFilter::AllConsumers), + ); + b.bind::(); + b.add::(); + b.add::(); + b.add::(); + b.add::(); + register_message_dispatcher::(&mut b, TEST_PRODUCER_A); + register_message_dispatcher::(&mut b, TEST_PRODUCER_B); + register_message_dispatcher::(&mut b, TEST_PRODUCER_D); + + let catalog = b.build(); + + let outbox = catalog.get_one::().unwrap(); + let test_message_consumer_a = catalog.get_one::().unwrap(); + let test_message_consumer_b = catalog.get_one::().unwrap(); + let test_message_consumer_d1 = catalog.get_one::().unwrap(); + let test_message_consumer_d2 = catalog.get_one::().unwrap(); + + Self { + _catalog: catalog, + outbox, + test_message_consumer_a, + test_message_consumer_b, + test_message_consumer_d1, + test_message_consumer_d2, + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/messaging-outbox/tests/tests/test_outbox_transactional_processor.rs b/src/utils/messaging-outbox/tests/tests/test_outbox_transactional_processor.rs new file mode 100644 index 0000000000..f7f8af8fbb --- /dev/null +++ b/src/utils/messaging-outbox/tests/tests/test_outbox_transactional_processor.rs @@ -0,0 +1,366 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::sync::{Arc, Mutex}; + +use database_common::NoOpDatabasePlugin; +use dill::*; +use internal_error::InternalError; +use kamu_messaging_outbox_inmem::{ + InMemoryOutboxMessageConsumptionRepository, + InMemoryOutboxMessageRepository, +}; +use messaging_outbox::*; +use serde::{Deserialize, Serialize}; +use time_source::SystemTimeSourceDefault; + +use crate::{test_message_consumer, test_message_type}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +const TEST_PRODUCER_A: &str = "TEST-PRODUCER-A"; +const TEST_PRODUCER_B: &str = "TEST-PRODUCER-B"; +const TEST_PRODUCER_C: &str = "TEST-PRODUCER-C"; + +test_message_type!(A); +test_message_type!(B); +test_message_type!(C); + +test_message_consumer!(A, A, TEST_PRODUCER_A, Durable); +test_message_consumer!(B, B, TEST_PRODUCER_B, Durable); +test_message_consumer!(C, C1, TEST_PRODUCER_C, Durable); +test_message_consumer!(C, C2, TEST_PRODUCER_C, Durable); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_log::test(tokio::test)] +async fn test_deliver_messages_of_one_type() { + let message_1 = TestMessageA { + body: "foo".to_string(), + }; + let message_2 = TestMessageA { + body: "bar".to_string(), + }; + + let harness = TransactionalOutboxProcessorHarness::new(); + + harness + .outbox + .post_message(TEST_PRODUCER_A, message_1.clone()) + .await + .unwrap(); + harness + .outbox + .post_message(TEST_PRODUCER_A, message_2.clone()) + .await + .unwrap(); + + // Posted, but not delivered yet! + harness.check_delivered_messages(&[], &[], &[], &[]); + harness.check_consumption_boundaries(&[]).await; + + // Run relay iteration + harness + .outbox_processor + .run_single_iteration_only() + .await + .unwrap(); + + // Should be delivered now + harness.check_delivered_messages(&[message_1, message_2], &[], &[], &[]); + harness + .check_consumption_boundaries(&[ + (TEST_PRODUCER_A, "TestMessageConsumerA", 2), + (TEST_PRODUCER_B, "TestMessageConsumerB", 0), + (TEST_PRODUCER_C, "TestMessageConsumerC1", 0), + (TEST_PRODUCER_C, "TestMessageConsumerC2", 0), + ]) + .await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_log::test(tokio::test)] +async fn test_deliver_messages_of_two_types() { + let message_1 = TestMessageA { + body: "foo".to_string(), + }; + let message_2 = TestMessageB { + body: "bar".to_string(), + }; + + let harness = TransactionalOutboxProcessorHarness::new(); + + harness + .outbox + .post_message(TEST_PRODUCER_A, message_1.clone()) + .await + .unwrap(); + harness + .outbox + .post_message(TEST_PRODUCER_B, message_2.clone()) + .await + .unwrap(); + + // Posted, but not delivered yet! + harness.check_delivered_messages(&[], &[], &[], &[]); + harness.check_consumption_boundaries(&[]).await; + + // Run relay iteration + harness + .outbox_processor + .run_single_iteration_only() + .await + .unwrap(); + + // Should be delivered now + harness.check_delivered_messages(&[message_1], &[message_2], &[], &[]); + + harness + .check_consumption_boundaries(&[ + (TEST_PRODUCER_A, "TestMessageConsumerA", 1), + (TEST_PRODUCER_B, "TestMessageConsumerB", 2), + (TEST_PRODUCER_C, "TestMessageConsumerC1", 0), + (TEST_PRODUCER_C, "TestMessageConsumerC2", 0), + ]) + .await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_log::test(tokio::test)] +async fn test_deliver_messages_multiple_consumers() { + let message_1 = TestMessageC { + body: "foo".to_string(), + }; + let message_2 = TestMessageC { + body: "bar".to_string(), + }; + + let harness = TransactionalOutboxProcessorHarness::new(); + + harness + .outbox + .post_message(TEST_PRODUCER_C, message_1.clone()) + .await + .unwrap(); + harness + .outbox + .post_message(TEST_PRODUCER_C, message_2.clone()) + .await + .unwrap(); + + // Posted, but not delivered yet! + harness.check_delivered_messages(&[], &[], &[], &[]); + harness.check_consumption_boundaries(&[]).await; + + // Run relay iteration + harness + .outbox_processor + .run_single_iteration_only() + .await + .unwrap(); + + // Should be delivered now + harness.check_delivered_messages( + &[], + &[], + &[message_1.clone(), message_2.clone()], + &[message_1, message_2], + ); + + harness + .check_consumption_boundaries(&[ + (TEST_PRODUCER_A, "TestMessageConsumerA", 0), + (TEST_PRODUCER_B, "TestMessageConsumerB", 0), + (TEST_PRODUCER_C, "TestMessageConsumerC1", 2), + (TEST_PRODUCER_C, "TestMessageConsumerC2", 2), + ]) + .await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_log::test(tokio::test)] +async fn test_deliver_messages_with_partial_consumption() { + let harness = TransactionalOutboxProcessorHarness::new(); + + let message_texts = ["foo", "bar", "baz", "super", "duper"]; + for message_text in message_texts { + harness + .outbox + .post_message( + TEST_PRODUCER_C, + TestMessageC { + body: message_text.to_string(), + }, + ) + .await + .unwrap(); + } + + // Let's assume some initial partial boundaries + harness + .outbox_consumption_repository + .create_consumption_boundary(OutboxMessageConsumptionBoundary { + producer_name: TEST_PRODUCER_C.to_string(), + consumer_name: "TestMessageConsumerC1".to_string(), + last_consumed_message_id: OutboxMessageID::new(2), + }) + .await + .unwrap(); + harness + .outbox_consumption_repository + .create_consumption_boundary(OutboxMessageConsumptionBoundary { + producer_name: TEST_PRODUCER_C.to_string(), + consumer_name: "TestMessageConsumerC2".to_string(), + last_consumed_message_id: OutboxMessageID::new(4), + }) + .await + .unwrap(); + + // Posted, but not delivered yet! + harness.check_delivered_messages(&[], &[], &[], &[]); + harness + .check_consumption_boundaries(&[ + (TEST_PRODUCER_C, "TestMessageConsumerC1", 2), + (TEST_PRODUCER_C, "TestMessageConsumerC2", 4), + ]) + .await; + + // Run relay iteration + harness + .outbox_processor + .run_single_iteration_only() + .await + .unwrap(); + + harness.check_delivered_messages( + &[], + &[], + &message_texts[2..] + .iter() + .map(|text| TestMessageC { + body: (*text).to_string(), + }) + .collect::>(), + &message_texts[4..] + .iter() + .map(|text| TestMessageC { + body: (*text).to_string(), + }) + .collect::>(), + ); + harness + .check_consumption_boundaries(&[ + (TEST_PRODUCER_A, "TestMessageConsumerA", 0), + (TEST_PRODUCER_B, "TestMessageConsumerB", 0), + (TEST_PRODUCER_C, "TestMessageConsumerC1", 5), + (TEST_PRODUCER_C, "TestMessageConsumerC2", 5), + ]) + .await; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +struct TransactionalOutboxProcessorHarness { + catalog: Catalog, + outbox_processor: Arc, + outbox: Arc, + outbox_consumption_repository: Arc, +} + +impl TransactionalOutboxProcessorHarness { + fn new() -> Self { + let mut b = CatalogBuilder::new(); + b.add::(); + b.add_value(OutboxConfig::default()); + b.add::(); + b.add::(); + b.add::(); + b.bind::(); + b.add::(); + + b.add::(); + b.add::(); + b.add::(); + b.add::(); + + register_message_dispatcher::(&mut b, TEST_PRODUCER_A); + register_message_dispatcher::(&mut b, TEST_PRODUCER_B); + register_message_dispatcher::(&mut b, TEST_PRODUCER_C); + + NoOpDatabasePlugin::init_database_components(&mut b); + + let catalog = b.build(); + + let outbox_processor = catalog.get_one::().unwrap(); + let outbox = catalog.get_one::().unwrap(); + let outbox_consumption_repository = catalog + .get_one::() + .unwrap(); + + Self { + catalog, + outbox_processor, + outbox, + outbox_consumption_repository, + } + } + + fn check_delivered_messages( + &self, + a_messages: &[TestMessageA], + b_messages: &[TestMessageB], + c1_messages: &[TestMessageC], + c2_messages: &[TestMessageC], + ) { + let test_message_consumer_a = self.catalog.get_one::().unwrap(); + let test_message_consumer_b = self.catalog.get_one::().unwrap(); + let test_message_consumer_c1 = self.catalog.get_one::().unwrap(); + let test_message_consumer_c2 = self.catalog.get_one::().unwrap(); + + assert_eq!(test_message_consumer_a.get_messages(), a_messages); + assert_eq!(test_message_consumer_b.get_messages(), b_messages); + assert_eq!(test_message_consumer_c1.get_messages(), c1_messages); + assert_eq!(test_message_consumer_c2.get_messages(), c2_messages); + } + + async fn check_consumption_boundaries(&self, patterns: &[(&str, &str, i64)]) { + let boundaries = self.read_consumption_boundaries().await; + assert_eq!( + boundaries, + patterns + .iter() + .map(|pattern| OutboxMessageConsumptionBoundary { + producer_name: pattern.0.to_string(), + consumer_name: pattern.1.to_string(), + last_consumed_message_id: OutboxMessageID::new(pattern.2), + }) + .collect::>() + ); + } + + async fn read_consumption_boundaries(&self) -> Vec { + use futures::TryStreamExt; + let mut boundaries: Vec<_> = self + .outbox_consumption_repository + .list_consumption_boundaries() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + + boundaries.sort(); + boundaries + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/messaging-outbox/tests/tests/test_transactional_outbox_impl.rs b/src/utils/messaging-outbox/tests/tests/test_transactional_outbox_impl.rs new file mode 100644 index 0000000000..28f07fb2da --- /dev/null +++ b/src/utils/messaging-outbox/tests/tests/test_transactional_outbox_impl.rs @@ -0,0 +1,154 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::assert_matches::assert_matches; +use std::sync::Arc; + +use dill::{Catalog, CatalogBuilder}; +use kamu_messaging_outbox_inmem::InMemoryOutboxMessageRepository; +use messaging_outbox::{ + Message, + Outbox, + OutboxExt, + OutboxMessageID, + OutboxMessageRepository, + OutboxTransactionalImpl, +}; +use serde::{Deserialize, Serialize}; +use time_source::SystemTimeSourceDefault; + +use crate::test_message_type; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +const TEST_PRODUCER_A: &str = "TEST-PRODUCER-A"; +const TEST_PRODUCER_B: &str = "TEST-PRODUCER-B"; + +test_message_type!(A); +test_message_type!(B); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_log::test(tokio::test)] +async fn test_transactional_outbox_messages_of_one_type() { + let message_1 = TestMessageA { + body: "foo".to_string(), + }; + let message_2 = TestMessageA { + body: "bar".to_string(), + }; + + let harness = TransactionalOutboxHarness::new(); + + let res = harness + .outbox + .post_message(TEST_PRODUCER_A, message_1.clone()) + .await; + assert_matches!(res, Ok(_)); + + let res = harness + .outbox + .post_message(TEST_PRODUCER_A, message_2.clone()) + .await; + assert_matches!(res, Ok(_)); + + let messages: Vec<_> = harness + .get_saved_messages::(TEST_PRODUCER_A) + .await; + assert_eq!(messages, vec![message_1, message_2]); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#[test_log::test(tokio::test)] +async fn test_transactional_outbox_messages_of_two_types() { + let message_1 = TestMessageA { + body: "foo".to_string(), + }; + let message_2 = TestMessageB { + body: "bar".to_string(), + }; + + let harness = TransactionalOutboxHarness::new(); + + let res = harness + .outbox + .post_message(TEST_PRODUCER_A, message_1.clone()) + .await; + assert_matches!(res, Ok(_)); + + let res = harness + .outbox + .post_message(TEST_PRODUCER_B, message_2.clone()) + .await; + assert_matches!(res, Ok(_)); + + let messages: Vec<_> = harness + .get_saved_messages::(TEST_PRODUCER_A) + .await; + assert_eq!(messages, vec![message_1]); + + let messages: Vec<_> = harness + .get_saved_messages::(TEST_PRODUCER_B) + .await; + assert_eq!(messages, vec![message_2]); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +struct TransactionalOutboxHarness { + _catalog: Catalog, + outbox: Arc, + outbox_message_repository: Arc, +} + +impl TransactionalOutboxHarness { + fn new() -> Self { + let mut b = CatalogBuilder::new(); + b.add::(); + b.bind::(); + b.add::(); + b.add::(); + + let catalog = b.build(); + + let outbox: Arc = catalog.get_one::().unwrap(); + let outbox_message_repository = catalog.get_one::().unwrap(); + + Self { + _catalog: catalog, + outbox, + outbox_message_repository, + } + } + + async fn get_saved_messages(&self, producer_name: &str) -> Vec { + use futures::TryStreamExt; + let outbox_messages: Vec<_> = self + .outbox_message_repository + .get_producer_messages(producer_name, OutboxMessageID::new(0), 10) + .await + .unwrap() + .try_collect() + .await + .unwrap(); + + let decoded_messages: Vec<_> = outbox_messages + .into_iter() + .map(|om| { + assert_eq!(om.producer_name, producer_name); + serde_json::from_value::(om.content_json).unwrap() + }) + .collect(); + + decoded_messages + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/utils/time-source/Cargo.toml b/src/utils/time-source/Cargo.toml new file mode 100644 index 0000000000..d2dd073849 --- /dev/null +++ b/src/utils/time-source/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "time-source" +description = "A helper encapsulating access to time (real or faked)" +version = { workspace = true } +homepage = { workspace = true } +repository = { workspace = true } +authors = { workspace = true } +readme = { workspace = true } +license-file = { workspace = true } +keywords = { workspace = true } +include = { workspace = true } +edition = { workspace = true } +publish = { workspace = true } + + +[lints] +workspace = true + + +[lib] +doctest = false + + +[dependencies] +async-trait = { version = "0.1", default-features = false } +chrono = { version = "0.4", default-features = false } +dill = "0.9" +tokio = { version = "1", default-features = false } + + +[dev-dependencies] +futures = { version = "0.3", default-features = false } \ No newline at end of file diff --git a/src/utils/time-source/src/lib.rs b/src/utils/time-source/src/lib.rs new file mode 100644 index 0000000000..233409861e --- /dev/null +++ b/src/utils/time-source/src/lib.rs @@ -0,0 +1,12 @@ +// Copyright Kamu Data, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +mod time_source; + +pub use time_source::*; diff --git a/src/domain/core/src/utils/time_source.rs b/src/utils/time-source/src/time_source.rs similarity index 100% rename from src/domain/core/src/utils/time_source.rs rename to src/utils/time-source/src/time_source.rs diff --git a/src/domain/core/tests/mod.rs b/src/utils/time-source/tests/mod.rs similarity index 100% rename from src/domain/core/tests/mod.rs rename to src/utils/time-source/tests/mod.rs diff --git a/src/domain/core/tests/tests/utils/mod.rs b/src/utils/time-source/tests/tests/mod.rs similarity index 100% rename from src/domain/core/tests/tests/utils/mod.rs rename to src/utils/time-source/tests/tests/mod.rs diff --git a/src/domain/core/tests/tests/utils/test_time_source.rs b/src/utils/time-source/tests/tests/test_time_source.rs similarity index 99% rename from src/domain/core/tests/tests/utils/test_time_source.rs rename to src/utils/time-source/tests/tests/test_time_source.rs index dbb2c9a230..15c1db701a 100644 --- a/src/domain/core/tests/tests/utils/test_time_source.rs +++ b/src/utils/time-source/tests/tests/test_time_source.rs @@ -14,7 +14,7 @@ use std::time::Duration as StdDuration; use chrono::{DateTime, Duration, TimeZone, Utc}; use futures::FutureExt; -use kamu_core::{FakeSystemTimeSource, SystemTimeSource}; +use time_source::{FakeSystemTimeSource, SystemTimeSource}; use tokio::time::timeout; #[tokio::test]