From b4ecc5c2ac95834e2af01b147de8f58272d4bb1e Mon Sep 17 00:00:00 2001 From: Yuhui Date: Sun, 29 Dec 2024 23:18:55 +0800 Subject: [PATCH 1/3] fix: don't convert response's keys to snake_case, leave them as camelCase --- singstat/client.py | 12 ++-- singstat/singstat.py | 16 ++--- singstat/types.py | 83 +++++++++++++------------- tests/mocks/api_response_empty_data.py | 4 +- tests/test_singstat.py | 4 +- 5 files changed, 57 insertions(+), 62 deletions(-) diff --git a/singstat/client.py b/singstat/client.py index 4d13b39..4e68740 100644 --- a/singstat/client.py +++ b/singstat/client.py @@ -57,8 +57,8 @@ def metadata(self, resource_id: str) -> MetadataDict: metadata_endpoint = f'{METADATA_ENDPOINT}/{resource_id}' metadata = self.send_request(metadata_endpoint) - data_count = metadata['data_count'] - records = metadata['data']['records'] + data_count = metadata['DataCount'] + records = metadata['Data']['records'] if data_count == 1 and len(records) == 0: warn('Empty data set returned', RuntimeWarning) @@ -102,8 +102,8 @@ def resource_id(self, **kwargs: Any) -> ResourceIdDict: resources = self.send_request(RESOURCE_ID_ENDPOINT, params) - data_count = resources['data_count'] - total = resources['data']['total'] + data_count = resources['DataCount'] + total = resources['Data']['total'] if data_count == 1 and total == 0: warn('Empty data set returned', RuntimeWarning) @@ -189,8 +189,8 @@ def tabledata(self, resource_id: str, **kwargs: Any) -> TabledataDict: tabledata_endpoint = f'{TABLEDATA_ENDPOINT}/{resource_id}' tabledata = self.send_request(tabledata_endpoint, params) - data_count = tabledata['data_count'] - rows = tabledata['data']['row'] + data_count = tabledata['DataCount'] + rows = tabledata['Data']['row'] if data_count == 1 and len(rows) == 0: warn('Empty data set returned', RuntimeWarning) diff --git a/singstat/singstat.py b/singstat/singstat.py index 4722e33..a510682 100644 --- a/singstat/singstat.py +++ b/singstat/singstat.py @@ -140,7 +140,6 @@ def sanitise_data( 2-value tuple. The ``dict`` keys are: "between", \ "dataLastUpdated", "dateGenerated", "limit", "offset", "rowNo", \ "total" - - ``dict`` keys: convert to use Python's snake_case. :param value: Value to sanitise. :type value: Any @@ -163,16 +162,10 @@ def sanitise_data( elif iterate and isinstance(value, dict): sanitised_value = {} for k, v in value.items(): - # Convert dict key to snake_case. - # Ref: https://www.geeksforgeeks.org/python-program-to-convert-camel-case-string-to-snake-case/ - key = ''.join( - ['_' + i.lower() if i.isupper() else i for i in k] - ).lstrip('_') - if k in DATA_KEYS_TO_SANITISE or isinstance(v, (dict, list)): - sanitised_value[key] = self.sanitise_data(v, iterate=iterate) + sanitised_value[k] = self.sanitise_data(v, iterate=iterate) else: - sanitised_value[key] = v + sanitised_value[k] = v elif isinstance(value, str): try: # pylint: disable=broad-exception-caught @@ -255,11 +248,10 @@ def send_request( except ValueError: pass - data_count = response_json['DataCount'] \ - if 'DataCount' in response_json else 0 - data = self.sanitise_data(response_json) if sanitise else response_json + data_count = response_json['DataCount'] \ + if 'DataCount' in response_json else 0 if data_count == 0: raise APIError('No data records returned', data=response_json) diff --git a/singstat/types.py b/singstat/types.py index 4c1f9b7..c47922e 100644 --- a/singstat/types.py +++ b/singstat/types.py @@ -25,6 +25,7 @@ Url: TypeAlias = str """URL of link.""" +# Resource ID class _ResourceIdDataRecordDict(TypedDict): """Type definition for _ResourceIdDataDict @@ -34,16 +35,16 @@ class _ResourceIdDataRecordDict(TypedDict): """ id: NotRequired[str] """ID""" - table_type: NotRequired[str] + tableType: NotRequired[str] """Table type""" title: NotRequired[str] """Title""" class _ResourceIdDataDict(TypedDict): """Type definition for ResourceIdDict""" - generated_by: str + generatedBy: str """Generated by""" - date_generated: date + dateGenerated: date """Data generated""" total: int """Total""" @@ -52,23 +53,24 @@ class _ResourceIdDataDict(TypedDict): class ResourceIdDict(TypedDict): """Type definition for resource_id()""" - data: _ResourceIdDataDict + Data: _ResourceIdDataDict """Data""" - data_count: int + DataCount: int """Data count""" - status_code: int + StatusCode: int """Status code""" - message: str + Message: str """Message""" +# Metadata class _MetadataDataRecordsRowDict(TypedDict): """Type definition for _MetadataDataRecordsDict""" - series_no: str + seriesNo: str """Series number""" - row_text: str + rowText: str """Row text""" - uo_m: str + uoM: str """Unit of measurement""" footnote: str """Footnote""" @@ -85,15 +87,15 @@ class _MetadataDataRecordsDict(TypedDict): """Title""" frequency: NotRequired[str] """Frequency""" - data_source: NotRequired[str] + dataSource: NotRequired[str] """Data source""" footnote: NotRequired[str] """Footnote""" - data_last_updated: NotRequired[date] + dataLastUpdated: NotRequired[date] """Data last updated""" - start_period: NotRequired[str] + startPeriod: NotRequired[str] """Start period""" - end_period: NotRequired[str] + endPeriod: NotRequired[str] """End period""" total: NotRequired[int] """Total""" @@ -102,24 +104,25 @@ class _MetadataDataRecordsDict(TypedDict): class _MetadataDataDict(TypedDict): """Type definition for MetadataDict""" - generated_by: str + generatedBy: str """Generated by""" - date_generated: date + dateGenerated: date """Date generated""" records: _MetadataDataRecordsDict """Records""" class MetadataDict(TypedDict): """Type definition for metadata()""" - data: _MetadataDataDict + Data: _MetadataDataDict """Data""" - data_count: int + DataCount: int """Data count""" - status_code: int + StatusCode: int """Status code""" - message: str + Message: str """Message""" +# Tabledata class _TabledataDataRowColumnDict(TypedDict): """Type definition for _TabledataDataRowColumnColumnDict, \ @@ -142,11 +145,11 @@ class _TabledataDataRowColumnColumnDict(TypedDict): class _TabledataDataTimeseriesRowDict(TypedDict): """Type definition for _TabledataDataTimeseriesDict""" - series_no: str + seriesNo: str """Series number""" - row_text: str + rowText: str """Row text""" - uo_m: str + uoM: str """Unit of measurement""" footnote: str """Footnote""" @@ -157,11 +160,11 @@ class _TabledataDataCrossSectionalMultiDimensionalCubeRowDict(TypedDict): """Type definition for \ _TabledataDataCrossSectionalMultiDimensionalCubeDict """ - row_no: int + rowNo: int """Row number""" - row_text: str + rowText: str """Row text""" - uo_m: str + uoM: str """Unit of measurement""" footnote: str """Footnote""" @@ -186,19 +189,19 @@ class _TabledataDataTimeseriesDict(TypedDict): """Frequency""" datasource: NotRequired[str] """Data source""" - generated_by: NotRequired[str] + generatedBy: NotRequired[str] """Generated by""" - data_last_updated: NotRequired[date] + dataLastUpdated: NotRequired[date] """Data last updated""" - date_generated: NotRequired[date] + dateGenerated: NotRequired[date] """Date generated""" offset: NotRequired[int | None] """Offset""" limit: NotRequired[int] """Limit""" - sort_by: NotRequired[str | None] + sortBy: NotRequired[str | None] """Sort by""" - time_filter: NotRequired[str | None] + timeFilter: NotRequired[str | None] """Time filter""" between: NotRequired[tuple[int, ...] | None] """Between""" @@ -215,7 +218,7 @@ class _TabledataDataCrossSectionalMultiDimensionalCubeDict(TypedDict): """ id: NotRequired[str] """ID""" - table_type: NotRequired[str] + tableType: NotRequired[str] """Table type""" title: NotRequired[str] """Title""" @@ -223,13 +226,13 @@ class _TabledataDataCrossSectionalMultiDimensionalCubeDict(TypedDict): """Footnote""" frequency: NotRequired[str] """Frequency""" - data_source: NotRequired[str] + dataSource: NotRequired[str] """Data source""" - generated_by: NotRequired[str] + generatedBy: NotRequired[str] """Generated by""" - data_last_updated: NotRequired[date] + dataLastUpdated: NotRequired[date] """Data last updated""" - date_generated: NotRequired[date] + dateGenerated: NotRequired[date] """Date generated""" offset: NotRequired[int | None] """Offset""" @@ -244,13 +247,13 @@ class _TabledataDataCrossSectionalMultiDimensionalCubeDict(TypedDict): class TabledataDict(TypedDict): """Type definition for tabledata()""" - data: _TabledataDataTimeseriesDict | _TabledataDataCrossSectionalMultiDimensionalCubeDict + Data: _TabledataDataTimeseriesDict | _TabledataDataCrossSectionalMultiDimensionalCubeDict """Data""" - data_count: int + DataCount: int """Data count""" - status_code: int + StatusCode: int """Status code""" - message: str + Message: str """Message""" __all__ = [ diff --git a/tests/mocks/api_response_empty_data.py b/tests/mocks/api_response_empty_data.py index 115ab6d..b602657 100644 --- a/tests/mocks/api_response_empty_data.py +++ b/tests/mocks/api_response_empty_data.py @@ -24,7 +24,7 @@ class APIResponseEmptyMetadata: @staticmethod def json(): return { - 'data': { + 'Data': { 'generatedBy': 'SingStat Table Builder', 'dateGenerated': date(2024, 12, 1), 'records': {}, @@ -40,7 +40,7 @@ class APIResponseEmptyTabledata: @staticmethod def json(): return { - 'data': { + 'Data': { 'generatedBy': 'SingStat Table Builder', 'dateGenerated': date(2024, 12, 1), 'row': [], diff --git a/tests/test_singstat.py b/tests/test_singstat.py index 459d17b..a03dd3a 100644 --- a/tests/test_singstat.py +++ b/tests/test_singstat.py @@ -132,12 +132,12 @@ def test_build_params_with_bad_inputs(client, original_params, default_params): 'value_date': '1/7/2019', 'value_list': [1, '2', { 'between': 'foo,77', - 'data_last_updated': date(2021, 3, 12), + 'dataLastUpdated': date(2021, 3, 12), }], 'value_dict': { 'key1': '316', 'between': (45, 89), - 'date_generated': date(2024, 12, 1), + 'dateGenerated': date(2024, 12, 1), }, }, ), From 918841aa9342da7f1557e2e30fa9c42c49a12a5c Mon Sep 17 00:00:00 2001 From: Yuhui Date: Sun, 29 Dec 2024 23:19:11 +0800 Subject: [PATCH 2/3] feat: allow cache backend to be specified --- singstat/singstat.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/singstat/singstat.py b/singstat/singstat.py index a510682..234c81b 100644 --- a/singstat/singstat.py +++ b/singstat/singstat.py @@ -18,7 +18,7 @@ from requests import codes as requests_codes from requests.adapters import HTTPAdapter, Retry -from requests_cache import CachedSession +from requests_cache import BaseCache, CachedSession from typeguard import check_type, typechecked from .constants import ( @@ -44,6 +44,11 @@ class SingStat: - Cache to expire after 12 hours. - User-agent header. + :param cache_backend: Cache backend name or instance to use. Refer to \ + https://requests-cache.readthedocs.io/en/stable/user_guide/backends.html \ + for more information and allowed values. Defaults to "sqlite". + :type cache_backend: str | BaseCache + :param is_test_api: Whether to use SingStat's test API. If this is set to \ True, then ``isTestApi=true`` is added to the parameters when calling \ ``send_request()``. Defaults to False. @@ -53,19 +58,24 @@ class SingStat: is_test_api: bool @typechecked - def __init__(self, is_test_api: bool=False) -> None: + def __init__( + self, + cache_backend: str | BaseCache='sqlite', + is_test_api: bool=False, + ) -> None: """Constructor method""" self.is_test_api = is_test_api - expire_after = CACHE_TWELVE_HOURS retries = Retry( total=5, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504] ) + expire_after = CACHE_TWELVE_HOURS self.session = CachedSession( CACHE_NAME, + backend=cache_backend, expire_after=expire_after, stale_if_error=False, ) From 74abbe7b23ba6c8b0d7e891b01ec52c56899d9d5 Mon Sep 17 00:00:00 2001 From: Yuhui Date: Sun, 29 Dec 2024 23:19:25 +0800 Subject: [PATCH 3/3] chore: miscellaneous updates --- singstat/__init__.py | 2 +- tests/test_client.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/singstat/__init__.py b/singstat/__init__.py index e6cbc0a..96210d2 100644 --- a/singstat/__init__.py +++ b/singstat/__init__.py @@ -18,7 +18,7 @@ from .types import Url NAME = 'singstat' -VERSION = '2.0.1' # Production +VERSION = '2.0.2' # Production # VERSION = f'{VERSION}.{datetime.now().strftime("%Y%m%d%H%M")}' # Development AUTHOR = 'Yuhui' AUTHOR_EMAIL = 'yuhuibc@gmail.com' diff --git a/tests/test_client.py b/tests/test_client.py index e07618c..3a83910 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -24,7 +24,6 @@ from typeguard import check_type from singstat import Client -from singstat.exceptions import APIError from singstat.types import MetadataDict, ResourceIdDict, TabledataDict from .mocks.api_response_empty_data import (