IO Package refactor

pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
 
 [tool.poetry.dependencies]
 python = "^3.9"
-httpx = {version = "0.*", optional = true}
+httpx = "^0.27.0"
 msgspec = "0.*"
 lxml = {version = "5.*", optional = true}
 xmltodict = {version = "0.*", optional = true}
@@ -34,7 +34,6 @@ pandas = {version = "^2.2.2", optional = true}
 
 [tool.poetry.extras]
 dc = ["python-dateutil"]
-fmr = ["httpx"]
 xml = ["lxml", "xmltodict", "sdmxschemas"]
 data = ["pandas"]
 

src/pysdmx/io/__init__.py

@@ -1 +1,5 @@
 """IO module for SDMX data."""
+
+from pysdmx.io.reader import get_datasets, read_sdmx
+
+__all__ = ["read_sdmx", "get_datasets"]

src/pysdmx/io/csv/sdmx10/reader/__init__.py

@@ -1,7 +1,7 @@
 """SDMX 1.0 CSV reader module."""
 
 from io import StringIO
-from typing import Dict
+from typing import Sequence
 
 import pandas as pd
 
@@ -14,10 +14,7 @@ def __generate_dataset_from_sdmx_csv(data: pd.DataFrame) -> PandasDataset:
     structure_id = data["DATAFLOW"].iloc[0]
     # Drop 'DATAFLOW' column from DataFrame
     df_csv = data.drop(["DATAFLOW"], axis=1)
-    urn = (
-        f"urn:sdmx:org.sdmx.infomodel.datastructure."
-        f"DataFlow={structure_id}"
-    )
+    urn = f"Dataflow={structure_id}"
 
     # Extract dataset attributes from sdmx-csv (all values are the same)
     attributes = {
@@ -36,11 +33,11 @@ def __generate_dataset_from_sdmx_csv(data: pd.DataFrame) -> PandasDataset:
     )
 
 
-def read(infile: str) -> Dict[str, PandasDataset]:
+def read(input_str: str) -> Sequence[PandasDataset]:
     """Reads csv file and returns a payload dictionary.
 
     Args:
-        infile: Path to file, str.
+        input_str: Path to file, str.
 
     Returns:
         payload: dict.
@@ -49,7 +46,7 @@ def read(infile: str) -> Dict[str, PandasDataset]:
         Invalid: If it is an invalid CSV file.
     """
     # Get Dataframe from CSV file
-    df_csv = pd.read_csv(StringIO(infile))
+    df_csv = pd.read_csv(StringIO(input_str))
     # Drop empty columns
     df_csv = df_csv.dropna(axis=1, how="all")
 
@@ -88,13 +85,13 @@ def read(infile: str) -> Dict[str, PandasDataset]:
 
     # Create a payload dictionary to store datasets with the
     # different unique_ids as keys
-    payload = {}
+    payload = []
     for df in list_df:
         # Generate a dataset from each subset of the DataFrame
         dataset = __generate_dataset_from_sdmx_csv(data=df)
 
         # Add the dataset to the payload dictionary
-        payload[dataset.short_urn] = dataset
+        payload.append(dataset)
 
     # Return the payload generated
     return payload

src/pysdmx/io/csv/sdmx10/writer/__init__.py

@@ -1,35 +1,42 @@
 """SDMX 1.0 CSV writer module."""
 
 from copy import copy
-from typing import Optional
+from typing import Optional, Sequence
 
 import pandas as pd
 
 from pysdmx.io.pd import PandasDataset
 
 
 def writer(
-    dataset: PandasDataset, output_path: Optional[str] = None
+    datasets: Sequence[PandasDataset], output_path: Optional[str] = None
 ) -> Optional[str]:
-    """Converts a dataset to an SDMX CSV format.
+    """Write data to SDMX-CSV 1.0 format.
 
     Args:
-        dataset: dataset
-        output_path: output_path
+        datasets: List of datasets to write.
+          Must have the same components.
+        output_path: Path to write the data to.
+          If None, the data is returned as a string.
 
     Returns:
-        SDMX CSV data as a string
+        SDMX CSV data as a string, if output_path is None.
     """
     # Link to pandas.to_csv documentation on sphinx:
     # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_csv.html
 
     # Create a copy of the dataset
-    df: pd.DataFrame = copy(dataset.data)
-    df.insert(0, "DATAFLOW", dataset.short_urn.split("=")[1])
-
-    # Add additional attributes to the dataset
-    for k, v in dataset.attributes.items():
-        df[k] = v
-
+    dataframes = []
+    for dataset in datasets:
+        df: pd.DataFrame = copy(dataset.data)
+        df.insert(0, "DATAFLOW", dataset.short_urn.split("=")[1])
+
+        # Add additional attributes to the dataset
+        for k, v in dataset.attributes.items():
+            df[k] = v
+        dataframes.append(df)
+
+    # Concatenate the dataframes
+    all_data = pd.concat(dataframes, ignore_index=True, axis=0)
     # Return the SDMX CSV data as a string
-    return df.to_csv(output_path, index=False, header=True)
+    return all_data.to_csv(output_path, index=False, header=True)

src/pysdmx/io/csv/sdmx20/__init__.py

@@ -1,6 +1,6 @@
 """SDMX 2.0 CSV reader and writer."""
 
-from pysdmx.model.message import ActionType
+from pysdmx.model.dataset import ActionType
 
 SDMX_CSV_ACTION_MAPPER = {
     ActionType.Append: "A",

src/pysdmx/io/csv/sdmx20/reader/__init__.py

@@ -1,13 +1,13 @@
 """SDMX 2.0 CSV reader module."""
 
 from io import StringIO
-from typing import Dict
+from typing import Sequence
 
 import pandas as pd
 
 from pysdmx.errors import Invalid
 from pysdmx.io.pd import PandasDataset
-from pysdmx.model.message import ActionType
+from pysdmx.model.dataset import ActionType
 
 ACTION_SDMX_CSV_MAPPER_READING = {
     "A": ActionType.Append,
@@ -49,20 +49,11 @@ def __generate_dataset_from_sdmx_csv(data: pd.DataFrame) -> PandasDataset:
     df_csv = data.drop(["STRUCTURE", "STRUCTURE_ID"], axis=1)
 
     if structure_type == "DataStructure".lower():
-        urn = (
-            "urn:sdmx:org.sdmx.infomodel.datastructure."
-            f"DataStructure={structure_id}"
-        )
-    elif structure_type == "DataFlow".lower():
-        urn = (
-            "urn:sdmx:org.sdmx.infomodel.datastructure."
-            f"DataFlow={structure_id}"
-        )
+        urn = f"DataStructure={structure_id}"
+    elif structure_type == "Dataflow".lower():
+        urn = f"Dataflow={structure_id}"
     elif structure_type == "dataprovision":
-        urn = (
-            f"urn:sdmx:org.sdmx.infomodel.registry."
-            f"ProvisionAgreement={structure_id}"
-        )
+        urn = f"ProvisionAgreement={structure_id}"
     else:
         raise Invalid(
             "Invalid value on STRUCTURE column",
@@ -87,11 +78,11 @@ def __generate_dataset_from_sdmx_csv(data: pd.DataFrame) -> PandasDataset:
     )
 
 
-def read(infile: str) -> Dict[str, PandasDataset]:
+def read(input_str: str) -> Sequence[PandasDataset]:
     """Reads csv file and returns a payload dictionary.
 
     Args:
-        infile: Path to file, str.
+        input_str: Path to file, str.
 
     Returns:
         payload: dict.
@@ -100,7 +91,7 @@ def read(infile: str) -> Dict[str, PandasDataset]:
         Invalid: If it is an invalid CSV file.
     """
     # Get Dataframe from CSV file
-    df_csv = pd.read_csv(StringIO(infile))
+    df_csv = pd.read_csv(StringIO(input_str))
     # Drop empty columns
     df_csv = df_csv.dropna(axis=1, how="all")
 
@@ -142,13 +133,13 @@ def read(infile: str) -> Dict[str, PandasDataset]:
 
     # Create a payload dictionary to store datasets with the
     # different unique_ids as keys
-    payload = {}
+    payload = []
     for df in list_df:
         # Generate a dataset from each subset of the DataFrame
         dataset = __generate_dataset_from_sdmx_csv(data=df)
 
         # Add the dataset to the payload dictionary
-        payload[dataset.short_urn] = dataset
+        payload.append(dataset)
 
     # Return the payload generated
     return payload

src/pysdmx/io/csv/sdmx20/writer/__init__.py

@@ -1,7 +1,7 @@
 """SDMX 2.0 CSV writer module."""
 
 from copy import copy
-from typing import Optional
+from typing import Optional, Sequence
 
 import pandas as pd
 
@@ -10,38 +10,46 @@
 
 
 def writer(
-    dataset: PandasDataset, output_path: Optional[str] = None
+    datasets: Sequence[PandasDataset], output_path: Optional[str] = None
 ) -> Optional[str]:
-    """Converts a dataset to an SDMX CSV format.
+    """Write data to SDMX-CSV 2.0 format.
 
     Args:
-        dataset: dataset
-        output_path: output_path
+        datasets: List of datasets to write.
+          Must have the same components.
+        output_path: Path to write the data to.
+          If None, the data is returned as a string.
 
     Returns:
-        SDMX CSV data as a string
+        SDMX CSV data as a string, if output_path is None.
     """
     # Link to pandas.to_csv documentation on sphinx:
     # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_csv.html
 
-    # Create a copy of the dataset
-    df: pd.DataFrame = copy(dataset.data)
+    dataframes = []
+    for dataset in datasets:
+        # Create a copy of the dataset
+        df: pd.DataFrame = copy(dataset.data)
 
-    # Add additional attributes to the dataset
-    for k, v in dataset.attributes.items():
-        df[k] = v
+        # Add additional attributes to the dataset
+        for k, v in dataset.attributes.items():
+            df[k] = v
 
-    structure_ref, unique_id = dataset.short_urn.split("=", maxsplit=1)
-    if structure_ref in ["DataStructure", "DataFlow"]:
-        structure_ref = structure_ref.lower()
-    else:
-        structure_ref = "dataprovision"
+        structure_ref, unique_id = dataset.short_urn.split("=", maxsplit=1)
+        if structure_ref in ["DataStructure", "Dataflow"]:
+            structure_ref = structure_ref.lower()
+        else:
+            structure_ref = "dataprovision"
 
-    # Insert two columns at the beginning of the data set
-    df.insert(0, "STRUCTURE", structure_ref)
-    df.insert(1, "STRUCTURE_ID", unique_id)
-    action_value = SDMX_CSV_ACTION_MAPPER[dataset.action]
-    df.insert(2, "ACTION", action_value)
+        # Insert two columns at the beginning of the data set
+        df.insert(0, "STRUCTURE", structure_ref)
+        df.insert(1, "STRUCTURE_ID", unique_id)
+        action_value = SDMX_CSV_ACTION_MAPPER[dataset.action]
+        df.insert(2, "ACTION", action_value)
+
+        dataframes.append(df)
+
+    all_data = pd.concat(dataframes, ignore_index=True, axis=0)
 
     # Convert the dataset into a csv file
-    return df.to_csv(output_path, index=False, header=True)
+    return all_data.to_csv(output_path, index=False, header=True)

src/pysdmx/io/enums.py

@@ -0,0 +1,21 @@
+"""IO Enumerations for SDMX files."""
+
+from enum import Enum
+
+
+class SDMXFormat(Enum):
+    """Enumeration of supported SDMX read formats."""
+
+    SDMX_ML_2_1_STRUCTURE = "SDMX-ML 2.1 Structure"
+    SDMX_ML_2_1_DATA_STRUCTURE_SPECIFIC = "SDMX-ML 2.1 StructureSpecific"
+    SDMX_ML_2_1_DATA_GENERIC = "SDMX-ML 2.1 Generic"
+    SDMX_ML_2_1_REGISTRY_INTERFACE = "SDMX-ML 2.1 Registry Interface"
+    SDMX_ML_2_1_ERROR = "SDMX-ML 2.1 Error"
+    SDMX_JSON_2 = "SDMX-JSON 2.0.0"
+    FUSION_JSON = "FusionJSON"
+    SDMX_CSV_1_0 = "SDMX-CSV 1.0"
+    SDMX_CSV_2_0 = "SDMX-CSV 2.0"
+
+    def __str__(self) -> str:
+        """Return the string representation of the format."""
+        return self.value