Merge pull request #217 from dyvenia/dev

Release 0.2.13 PR

CHANGELOG.md

@@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.2.13]
+### Added
+- Added support for file path to `CloudForCustomersReportToADLS` flow
+
+### Fixed
+- `Supermetrics` source: `to_df()` now correctly handles `if_empty` in case of empty results
+
+### Changed
+- `Sharepoint` and `CloudForCustomers` sources will now provide an informative `CredentialError` which is also raised early. This will make issues with input credenials immediately clear to the user.
+- Removed set_key_value from `CloudForCustomersReportToADLS` flow
+
 ## [0.2.12]
 ### Added
 - Added `Sharepoint` source

tests/integration/test_cloud_for_customers.py

@@ -1,47 +1,9 @@
-import os
-import numpy
-import pytest
-
 from viadot.sources import CloudForCustomers
-from viadot.config import local_config
-
-LOCAL_TESTS_PATH = "/home/viadot/tests"
-TEST_FILE_1 = os.path.join(LOCAL_TESTS_PATH, "tests_out.csv")
-
-
-@pytest.fixture(scope="session")
-def cloud_for_customers():
-    url = "http://services.odata.org/V2/Northwind/Northwind.svc/"
-    endpoint = "Employees"
-    cloud_for_customers = CloudForCustomers(
-        url=url, endpoint=endpoint, params={"$top": "2"}
-    )
-    yield cloud_for_customers
-    os.remove(TEST_FILE_1)
-
-
-def test_to_records(cloud_for_customers):
-    data = cloud_for_customers.to_records()
-    assert "EmployeeID" in data[0].keys()
-
-
-def test_to_df(cloud_for_customers):
-    df = cloud_for_customers.to_df(fields=["EmployeeID", "FirstName", "LastName"])
-    assert type(df["EmployeeID"][0]) == numpy.int64
-
-
-def test_csv(cloud_for_customers):
-    csv = cloud_for_customers.to_csv(
-        path=TEST_FILE_1, fields=["EmployeeID", "FirstName", "LastName"]
-    )
-    assert os.path.isfile(TEST_FILE_1) == True
 
 
 def test_credentials():
-    qa_credentials = local_config.get("CLOUD_FOR_CUSTOMERS")["QA"]
-    url = qa_credentials["server"]
     endpoint = "ServiceRequestCollection"
-    c4c = CloudForCustomers(url=url, endpoint=endpoint, params={"$top": "2"})
+    c4c = CloudForCustomers(endpoint=endpoint, params={"$top": "2"})
     df = c4c.to_df(
         fields=["ProductRecipientPartyName", "CreationDateTime", "CreatedBy"]
     )

tests/integration/test_sharepoint.py

@@ -2,60 +2,67 @@
 import os
 import pathlib
 import pandas as pd
-from typing import List
+from viadot.exceptions import CredentialError
 
 from viadot.sources import Sharepoint
-from viadot.flows import SharepointToADLS as s_flow
 from viadot.config import local_config
 from viadot.task_utils import df_get_data_types_task
 
-s = Sharepoint()
 
-FILE_NAME = "EUL Data.xlsm"
-s.download_file(download_to_path=FILE_NAME)
-DF = pd.read_excel(FILE_NAME, sheet_name=0)
+@pytest.fixture(scope="session")
+def sharepoint():
+    s = Sharepoint()
+    yield s
+
+
+@pytest.fixture(scope="session")
+def FILE_NAME(sharepoint):
+    path = "EUL Data.xlsm"
+    sharepoint.download_file(download_to_path=path)
+    yield path
+    os.remove(path)
 
 
 def test_credentials():
     credentials = {"site": "tenant.sharepoint.com", "username": "User"}
     s = Sharepoint(credentials=credentials)
-    with pytest.raises(ValueError, match="Missing credentials."):
+    with pytest.raises(CredentialError, match="Missing credentials."):
         s.get_connection()
 
 
-def test_connection():
+def test_connection(sharepoint):
     credentials = local_config.get("SHAREPOINT")
     site = f'https://{credentials["site"]}'
-    conn = s.get_connection()
+    conn = sharepoint.get_connection()
     response = conn.get(site)
     assert response.status_code == 200
 
 
-def test_file_extension():
-    file_ext = [".xlsm", ".xlsx"]
-    assert pathlib.Path(s.download_from_path).suffix in file_ext
-
-
-def test_file_name():
-    assert os.path.basename(s.download_from_path) == FILE_NAME
-
-
-def test_file_download():
-    s.download_file(download_to_path=FILE_NAME)
+def test_file_download(FILE_NAME):
     files = []
     for file in os.listdir():
         if os.path.isfile(os.path.join(file)):
             files.append(file)
     assert FILE_NAME in files
-    os.remove(FILE_NAME)
 
 
-def test_file_to_df():
+def test_autopopulating_download_from(FILE_NAME):
+    assert os.path.basename(sharepoint.download_from_path) == FILE_NAME
+
+
+def test_file_extension(sharepoint):
+    file_ext = [".xlsm", ".xlsx"]
+    assert pathlib.Path(sharepoint.download_from_path).suffix in file_ext
+
+
+def test_file_to_df(FILE_NAME):
+    df = pd.read_excel(FILE_NAME, sheet_name=0)
     df_test = pd.DataFrame(data={"col1": [1, 2]})
-    assert type(DF) == type(df_test)
+    assert type(df) == type(df_test)
 
 
-def test_get_data_types():
-    dtypes_map = df_get_data_types_task.run(DF)
+def test_get_data_types(FILE_NAME):
+    df = pd.read_excel(FILE_NAME, sheet_name=0)
+    dtypes_map = df_get_data_types_task.run(df)
     dtypes = [v for k, v in dtypes_map.items()]
     assert "String" in dtypes

tests/integration/test_supermetrics.py

@@ -9,17 +9,13 @@ def test_connection():
         "ds_id": "AW",
         "ds_accounts": ["1007802423"],
         "ds_user": credentials["USER"],
-        "date_range_type": "last_year_inc",
+        "date_range_type": "last_month",
         "fields": [
             "Date",
-            "profile",
             "Campaignname",
-            "Impressions",
             "Clicks",
-            "Cost_eur",
-            "SearchImpressionShare",
         ],
-        "max_rows": 1000000,
+        "max_rows": 1,
     }
     df = s.query(google_ads_params).to_df()
     assert df.count()[0] > 0

tests/integration/test_uk_carbon_intensity.py

@@ -1,7 +1,6 @@
 import datetime
 import os
 
-import numpy
 import pyarrow
 import pytest
 
@@ -29,7 +28,7 @@ def test_to_json(carbon):
 def test_to_df(carbon):
     carbon.query("/intensity")
     df = carbon.to_df()
-    assert type(df["actual"][0]) == numpy.int64
+    assert not df.empty
 
 
 def test_to_arrow(carbon):
@@ -46,7 +45,7 @@ def test_to_csv(carbon):
 
 def test_stats_to_csv(carbon):
     now = datetime.datetime.now()
-    for i in range(10):
+    for i in range(3):
         from_delta = datetime.timedelta(days=i + 1)
         to_delta = datetime.timedelta(days=i)
         to = now - to_delta

tests/test_viadot.py

@@ -2,4 +2,4 @@
 
 
 def test_version():
-    assert __version__ == "0.2.12"
+    assert __version__ == "0.2.13"

tests/unit/test_cloud_for_customers.py

@@ -0,0 +1,36 @@
+from viadot.sources import CloudForCustomers
+
+import os
+import numpy
+import pytest
+
+
+TEST_FILE_1 = "tests_out.csv"
+
+
+@pytest.fixture(scope="session")
+def cloud_for_customers():
+    url = "http://services.odata.org/V2/Northwind/Northwind.svc/"
+    endpoint = "Employees"
+    cloud_for_customers = CloudForCustomers(
+        url=url, endpoint=endpoint, params={"$top": "2"}
+    )
+    yield cloud_for_customers
+    os.remove(TEST_FILE_1)
+
+
+def test_to_records(cloud_for_customers):
+    data = cloud_for_customers.to_records()
+    assert "EmployeeID" in data[0].keys()
+
+
+def test_to_df(cloud_for_customers):
+    df = cloud_for_customers.to_df(fields=["EmployeeID", "FirstName", "LastName"])
+    assert type(df["EmployeeID"][0]) == numpy.int64
+
+
+def test_csv(cloud_for_customers):
+    csv = cloud_for_customers.to_csv(
+        path=TEST_FILE_1, fields=["EmployeeID", "FirstName", "LastName"]
+    )
+    assert os.path.isfile(TEST_FILE_1) == True

tests/unit/test_supermetrics.py

@@ -0,0 +1,121 @@
+import pytest
+from viadot.sources import Supermetrics
+
+RESPONSE_PIVOTED = {
+    "meta": {
+        "query": {
+            "fields": [
+                {
+                    "id": "Date",
+                    "field_id": "Date",
+                    "field_name": "Date",
+                    "field_type": "dim",
+                    "field_split": "row",
+                },
+                {
+                    "id": "profile",
+                    "field_id": "profile",
+                    "field_name": "View",
+                    "field_type": "dim",
+                    "field_split": "row",
+                },
+                {
+                    "id": "segment",
+                    "field_id": "segment",
+                    "field_name": "Segment",
+                    "field_type": "dim",
+                    "field_split": "column",
+                },
+                {
+                    "id": "Sessions",
+                    "field_id": "Sessions",
+                    "field_name": "Sessions",
+                    "field_type": "met",
+                    "field_split": "row",
+                },
+            ]
+        },
+        "result": {"total_columns": 6, "total_rows": 700},
+    },
+    "data": [
+        [
+            "Date",
+            "View",
+            "M-site_TOTAL: Bounces Landing",
+            "M-site_TOTAL: Click to EDSP",
+            "M-site_TOTAL: MQL Conversion Page Sessions",
+            "M-site_TOTAL: Click to RWS",
+        ],
+        ["2020-01-01", "REDACTED", 123, 456, 78, 9],
+    ],
+}
+
+RESPONSE_PIVOTED_NO_DATA = {
+    "meta": {
+        "query": {
+            "fields": [
+                {
+                    "id": "Date",
+                    "field_id": "Date",
+                    "field_name": "Date",
+                    "field_type": "dim",
+                    "field_split": "row",
+                },
+                {
+                    "id": "profileID",
+                    "field_id": "profileID",
+                    "field_name": "View ID",
+                    "field_type": "dim",
+                    "field_split": "row",
+                },
+                {
+                    "id": "Hostname",
+                    "field_id": "Hostname",
+                    "field_name": "Hostname",
+                    "field_type": "dim",
+                    "field_split": "row",
+                },
+                {
+                    "id": "profile",
+                    "field_id": "profile",
+                    "field_name": "View",
+                    "field_type": "dim",
+                    "field_split": "row",
+                },
+                {
+                    "id": "segment",
+                    "field_id": "segment",
+                    "field_name": "Segment",
+                    "field_type": "dim",
+                    "field_split": "column",
+                },
+                {
+                    "id": "Sessions",
+                    "field_id": "Sessions",
+                    "field_name": "Sessions",
+                    "field_type": "met",
+                    "field_split": "row",
+                },
+            ]
+        },
+        "result": {"total_columns": 0, "total_rows": 0},
+    },
+    "data": [],
+}
+
+
+def test___get_col_names_google_analytics_pivoted():
+    columns = Supermetrics._get_col_names_google_analytics(response=RESPONSE_PIVOTED)
+    assert columns == [
+        "Date",
+        "View",
+        "M-site_TOTAL: Bounces Landing",
+        "M-site_TOTAL: Click to EDSP",
+        "M-site_TOTAL: MQL Conversion Page Sessions",
+        "M-site_TOTAL: Click to RWS",
+    ]
+
+
+def test___get_col_names_google_analytics_pivoted_no_data():
+    with pytest.raises(ValueError):
+        Supermetrics._get_col_names_google_analytics(response=RESPONSE_PIVOTED_NO_DATA)

viadot/__init__.py

@@ -1 +1 @@
-__version__ = "0.2.12"
+__version__ = "0.2.13"

viadot/exceptions.py

@@ -4,3 +4,7 @@ class ValidationError(Exception):
 
 class APIError(Exception):
     pass
+
+
+class CredentialError(Exception):
+    pass