From 351b678e65d4f02be16cc99e7184b34e857d5c6c Mon Sep 17 00:00:00 2001 From: m-paz Date: Fri, 29 Oct 2021 17:40:51 +0200 Subject: [PATCH 1/3] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bumped=20version?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_viadot.py | 2 +- viadot/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_viadot.py b/tests/test_viadot.py index b88a264f9..1a7dc9444 100644 --- a/tests/test_viadot.py +++ b/tests/test_viadot.py @@ -2,4 +2,4 @@ def test_version(): - assert __version__ == "0.2.10" + assert __version__ == "0.2.11" diff --git a/viadot/__init__.py b/viadot/__init__.py index 6232f7ab1..5635676f6 100644 --- a/viadot/__init__.py +++ b/viadot/__init__.py @@ -1 +1 @@ -__version__ = "0.2.10" +__version__ = "0.2.11" From 9c994bdc35bced011448fc1a2b840d432e47c76c Mon Sep 17 00:00:00 2001 From: m-paz Date: Sat, 30 Oct 2021 19:43:11 +0200 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=90=9B=20Fixed=20to=5Fcsv=20in=20adls?= =?UTF-8?q?=5Fto=5Fazure=5Fsql=20and=20local=20json=20path=20in=20supermet?= =?UTF-8?q?rics=5Fto=5Fadls?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/flows/adls_to_azure_sql.py | 32 ++++++---------------------- viadot/flows/supermetrics_to_adls.py | 2 -- 2 files changed, 6 insertions(+), 28 deletions(-) diff --git a/viadot/flows/adls_to_azure_sql.py b/viadot/flows/adls_to_azure_sql.py index e9e88d489..05fe340b6 100644 --- a/viadot/flows/adls_to_azure_sql.py +++ b/viadot/flows/adls_to_azure_sql.py @@ -74,11 +74,6 @@ def df_to_csv_task(df, path: str, sep: str = "\t"): df.to_csv(path, sep=sep, index=False) -@task -def df_to_parquet_task(df, path: str): - df.to_parquet(path) - - class ADLSToAzureSQL(Flow): def __init__( self, @@ -200,23 +195,6 @@ def get_promoted_path(self, env: str) -> str: return promoted_path - def create_to_file_task(self, df, file_type): - df_to_type = None - if file_type == "csv": - df_to_type = df_to_csv_task.bind( - df=df, - path=self.local_file_path, - sep=self.write_sep, - flow=self, - ) - else: - df_to_type = df_to_parquet_task.bind( - df=df, - path=self.local_file_path, - flow=self, - ) - return df_to_type - def gen_flow(self) -> Flow: df = lake_to_df_task.bind( path=self.adls_path, @@ -237,8 +215,9 @@ def gen_flow(self) -> Flow: else: dtypes = self.dtypes - adls_file_type = self.adls_path.split(".")[-1] - df_to_type = self.create_to_file_task(df, adls_file_type) + df_to_csv = df_to_csv_task.bind( + df=df, path=self.local_file_path, sep=self.write_sep, flow=self + ) promote_to_conformed_task.bind( from_path=self.local_file_path, @@ -274,8 +253,9 @@ def gen_flow(self) -> Flow: ) # dtypes.set_upstream(download_json_file_task, flow=self) - promote_to_conformed_task.set_upstream(df_to_type, flow=self) + promote_to_conformed_task.set_upstream(df_to_csv, flow=self) + promote_to_conformed_task.set_upstream(df_to_csv, flow=self) # map_data_types_task.set_upstream(download_json_file_task, flow=self) - create_table_task.set_upstream(df_to_type, flow=self) + create_table_task.set_upstream(df_to_csv, flow=self) promote_to_operations_task.set_upstream(promote_to_conformed_task, flow=self) bulk_insert_task.set_upstream(create_table_task, flow=self) diff --git a/viadot/flows/supermetrics_to_adls.py b/viadot/flows/supermetrics_to_adls.py index a9e051c59..794d258be 100644 --- a/viadot/flows/supermetrics_to_adls.py +++ b/viadot/flows/supermetrics_to_adls.py @@ -346,5 +346,3 @@ def gen_flow(self) -> Flow: file_to_adls_task.set_upstream(df_to_file, flow=self) json_to_adls_task.set_upstream(dtypes_to_json_task, flow=self) set_key_value(key=self.adls_dir_path, value=self.adls_file_path) - - shutil.rmtree(self.local_json_path) From c1b15c9f685e86d7dec324fe5e8d86fbda5bcf35 Mon Sep 17 00:00:00 2001 From: m-paz Date: Sat, 30 Oct 2021 19:44:59 +0200 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=93=9D=20Updated=20changelog?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 899e3cdab..9a68d95ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.11] +### Fixed +- ADLSToAzureSQL - fixed path to csv issue. +- SupermetricsToADLS - fixed local json path issue. + +## [0.2.10] - 2021-10-29 +### Release due to CI/CD error + +## [0.2.9] - 2021-10-29 +### Release due to CI/CD error + ## [0.2.8] - 2021-10-29 ### Changed - CI/CD: `dev` image is now only published on push to the `dev` branch