From 50af79bfc0337c5a381fb21a9b6b621f74e0e4d3 Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Thu, 16 Jan 2025 11:45:50 +0100 Subject: [PATCH 1/6] Correct annotations --- .../fmr/samples/df/tmp_dataflow_with_has.json | 54 +++++++++---------- .../fmr/samples/pra/hierarchy_hca.fusion.json | 4 +- tests/api/fmr/samples/pra/no_hca.fusion.json | 4 +- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/tests/api/fmr/samples/df/tmp_dataflow_with_has.json b/tests/api/fmr/samples/df/tmp_dataflow_with_has.json index 631ef168..db8372ae 100644 --- a/tests/api/fmr/samples/df/tmp_dataflow_with_has.json +++ b/tests/api/fmr/samples/df/tmp_dataflow_with_has.json @@ -40,7 +40,7 @@ "urn": "urn:sdmx:org.sdmx.infomodel.base.Agency=BIS:AGENCIES(1.0).MDD", "annotations": [ { - "title": "cemejuma", + "title": "xx000000", "type": "contact_uid" } ], @@ -82,7 +82,7 @@ "urn": "urn:sdmx:org.sdmx.infomodel.base.Agency=BIS:AGENCIES(1.0).SFT", "annotations": [ { - "title": "cbmemcpa", + "title": "xx000000", "type": "contact_uid" } ], @@ -100,7 +100,7 @@ ], "contacts": [ { - "id": "cbmemcpa", + "id": "xx000000", "roles": [ { "locale": "en", @@ -158,7 +158,7 @@ "urn": "urn:sdmx:org.sdmx.infomodel.base.Agency=BIS:AGENCIES(1.0).CBS", "annotations": [ { - "title": "cbmemcpa", + "title": "xx000000", "type": "contact_uid" } ], @@ -176,7 +176,7 @@ ], "contacts": [ { - "id": "cbmemcpa", + "id": "xx000000", "roles": [ { "locale": "en", @@ -268,7 +268,7 @@ "urn": "urn:sdmx:org.sdmx.infomodel.base.Agency=BIS:AGENCIES(1.0).FIDA", "annotations": [ { - "title": "cbmemcpa", + "title": "xx000000", "type": "contact_uid" } ], @@ -286,7 +286,7 @@ ], "contacts": [ { - "id": "cbmemcpa", + "id": "xx000000", "roles": [ { "locale": "en", @@ -310,7 +310,7 @@ "urn": "urn:sdmx:org.sdmx.infomodel.base.Agency=BIS:AGENCIES(1.0).DER", "annotations": [ { - "title": "cbmemcpa", + "title": "xx000000", "type": "contact_uid" } ], @@ -328,7 +328,7 @@ ], "contacts": [ { - "id": "cbmemcpa", + "id": "xx000000", "roles": [ { "locale": "en", @@ -403,7 +403,7 @@ "urn": "urn:sdmx:org.sdmx.infomodel.base.Agency=BIS:AGENCIES(1.0).EXR", "annotations": [ { - "title": "cbmemcpa", + "title": "xx000000", "type": "contact_uid" } ], @@ -421,7 +421,7 @@ ], "contacts": [ { - "id": "cbmemcpa", + "id": "xx000000", "roles": [ { "locale": "en", @@ -445,7 +445,7 @@ "urn": "urn:sdmx:org.sdmx.infomodel.base.Agency=BIS:AGENCIES(1.0).LBS", "annotations": [ { - "title": "cbmemcpa", + "title": "xx000000", "type": "contact_uid" } ], @@ -463,7 +463,7 @@ ], "contacts": [ { - "id": "cbmemcpa", + "id": "xx000000", "roles": [ { "locale": "en", @@ -600,7 +600,7 @@ "urn": "urn:sdmx:org.sdmx.infomodel.base.Agency=BIS:AGENCIES(1.0).XTD", "annotations": [ { - "title": "cbmemcpa", + "title": "xx000000", "type": "contact_uid" } ], @@ -618,7 +618,7 @@ ], "contacts": [ { - "id": "cbmemcpa", + "id": "xx000000", "roles": [ { "locale": "en", @@ -1174,7 +1174,7 @@ "urn": "urn:sdmx:org.sdmx.infomodel.base.Agency=BIS:AGENCIES(1.0).DEBT_SEC", "annotations": [ { - "title": "cbmemcpa", + "title": "xx000000", "type": "contact_uid" } ], @@ -1192,7 +1192,7 @@ ], "contacts": [ { - "id": "cbmemcpa", + "id": "xx000000", "roles": [ { "locale": "en", @@ -1343,7 +1343,7 @@ "urn": "urn:sdmx:org.sdmx.infomodel.base.Agency=BIS:AGENCIES(1.0).GLI", "annotations": [ { - "title": "cbmemcpa", + "title": "xx000000", "type": "contact_uid" } ], @@ -1361,7 +1361,7 @@ ], "contacts": [ { - "id": "cbmemcpa", + "id": "xx000000", "roles": [ { "locale": "en", @@ -2203,7 +2203,7 @@ "urn": "urn:sdmx:org.sdmx.infomodel.base.Agency=BIS:AGENCIES(1.0).TFFS", "annotations": [ { - "title": "cbmemcpa", + "title": "xx000000", "type": "contact_uid" } ], @@ -2221,7 +2221,7 @@ ], "contacts": [ { - "id": "cbmemcpa", + "id": "xx000000", "roles": [ { "locale": "en", @@ -3686,7 +3686,7 @@ "urn": "urn:sdmx:org.sdmx.infomodel.base.Agency=BIS:AGENCIES(1.0).SEC", "annotations": [ { - "title": "cbmemcpa", + "title": "xx000000", "type": "contact_uid" } ], @@ -3704,7 +3704,7 @@ ], "contacts": [ { - "id": "cbmemcpa", + "id": "xx000000", "roles": [ { "locale": "en", @@ -4074,7 +4074,7 @@ "urn": "urn:sdmx:org.sdmx.infomodel.base.Agency=BIS:AGENCIES(1.0).IDS", "annotations": [ { - "title": "cbmemcpa", + "title": "xx000000", "type": "contact_uid" } ], @@ -4092,7 +4092,7 @@ ], "contacts": [ { - "id": "cbmemcpa", + "id": "xx000000", "roles": [ { "locale": "en", @@ -4179,7 +4179,7 @@ "urn": "urn:sdmx:org.sdmx.infomodel.base.Agency=BIS:AGENCIES(1.0).DEALOGIC", "annotations": [ { - "title": "cbmemcpa", + "title": "xx000000", "type": "contact_uid" } ], @@ -4197,7 +4197,7 @@ ], "contacts": [ { - "id": "cbmemcpa", + "id": "xx000000", "roles": [ { "locale": "en", diff --git a/tests/api/fmr/samples/pra/hierarchy_hca.fusion.json b/tests/api/fmr/samples/pra/hierarchy_hca.fusion.json index d74e4dc1..480c3f5d 100644 --- a/tests/api/fmr/samples/pra/hierarchy_hca.fusion.json +++ b/tests/api/fmr/samples/pra/hierarchy_hca.fusion.json @@ -30,7 +30,7 @@ "urn": "urn:sdmx:org.sdmx.infomodel.base.Agency=BIS:AGENCIES(1.0).CBS", "annotations": [ { - "title": "cbmemcpa", + "title": "xx000000", "type": "contact_uid" } ], @@ -48,7 +48,7 @@ ], "contacts": [ { - "id": "cbmemcpa", + "id": "xx000000", "roles": [ { "locale": "en", diff --git a/tests/api/fmr/samples/pra/no_hca.fusion.json b/tests/api/fmr/samples/pra/no_hca.fusion.json index ba46bee7..1c5a16fd 100644 --- a/tests/api/fmr/samples/pra/no_hca.fusion.json +++ b/tests/api/fmr/samples/pra/no_hca.fusion.json @@ -30,7 +30,7 @@ "urn": "urn:sdmx:org.sdmx.infomodel.base.Agency=BIS:AGENCIES(1.0).CBS", "annotations": [ { - "title": "cbmemcpa", + "title": "xx000000", "type": "contact_uid" } ], @@ -48,7 +48,7 @@ ], "contacts": [ { - "id": "cbmemcpa", + "id": "xx000000", "roles": [ { "locale": "en", From effd6b476dd6fa593480eb14a8a029af8372071d Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Thu, 16 Jan 2025 11:48:26 +0100 Subject: [PATCH 2/6] Add test reproducing the issue --- tests/api/fmr/samples/code/freq.fusion.json | 8 ++++++++ tests/api/fmr/samples/code/freq.json | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/tests/api/fmr/samples/code/freq.fusion.json b/tests/api/fmr/samples/code/freq.fusion.json index a6663dad..a306570c 100644 --- a/tests/api/fmr/samples/code/freq.fusion.json +++ b/tests/api/fmr/samples/code/freq.fusion.json @@ -34,6 +34,14 @@ "value": "N:Annual" } ] + }, + { + "text": [ + { + "locale": "en", + "value": "Empty" + } + ] } ], "names": [ diff --git a/tests/api/fmr/samples/code/freq.json b/tests/api/fmr/samples/code/freq.json index f0725904..00587a31 100644 --- a/tests/api/fmr/samples/code/freq.json +++ b/tests/api/fmr/samples/code/freq.json @@ -55,6 +55,12 @@ "texts": { "en": "N:Annual" } + }, + { + "text": "Empty", + "texts": { + "en": "Empty" + } } ], "name": "Annual", From 4f2a010996b00e853441b8b0f916218d3bfa238b Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Thu, 16 Jan 2025 11:53:26 +0100 Subject: [PATCH 3/6] Align FusionAnnotation with SDMX Information Model --- src/pysdmx/io/json/fusion/messages/core.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/pysdmx/io/json/fusion/messages/core.py b/src/pysdmx/io/json/fusion/messages/core.py index 087b6c2e..b1917ae9 100644 --- a/src/pysdmx/io/json/fusion/messages/core.py +++ b/src/pysdmx/io/json/fusion/messages/core.py @@ -10,13 +10,6 @@ from pysdmx.util import find_by_urn -class FusionAnnotation(msgspec.Struct, frozen=True): - """Fusion-JSON payload for annotations.""" - - title: str - type: str - - class FusionString(msgspec.Struct, frozen=True): """Fusion-JSON payload for an international string.""" @@ -24,6 +17,16 @@ class FusionString(msgspec.Struct, frozen=True): value: str +class FusionAnnotation(msgspec.Struct, frozen=True): + """Fusion-JSON payload for annotations.""" + + id: Optional[str] = None + title: Optional[str] = None + type: Optional[str] = None + value: Optional[str] = None + text: Sequence[FusionString] = () + + class FusionLink(msgspec.Struct, frozen=True): """Fusion-JSON payload for link objects.""" From 1b96a9a21b1a1c6cf591fd7b70606a6e167cb723 Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Thu, 16 Jan 2025 11:55:54 +0100 Subject: [PATCH 4/6] Add check that title exists --- src/pysdmx/io/json/fusion/messages/code.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/pysdmx/io/json/fusion/messages/code.py b/src/pysdmx/io/json/fusion/messages/code.py index c10dcb9f..a787701a 100644 --- a/src/pysdmx/io/json/fusion/messages/code.py +++ b/src/pysdmx/io/json/fusion/messages/code.py @@ -41,13 +41,16 @@ def __handle_date(self, datestr: str) -> datetime: def __get_val( self, a: FusionAnnotation ) -> Tuple[Optional[datetime], Optional[datetime]]: - vals = a.title.split("/") - if a.title.startswith("/"): - return (None, self.__handle_date(vals[1])) + if a.title: + vals = a.title.split("/") + if a.title.startswith("/"): + return (None, self.__handle_date(vals[1])) + else: + valid_from = self.__handle_date(vals[0]) + valid_to = self.__handle_date(vals[1]) if vals[1] else None + return (valid_from, valid_to) else: - valid_from = self.__handle_date(vals[0]) - valid_to = self.__handle_date(vals[1]) if vals[1] else None - return (valid_from, valid_to) + return (None, None) def to_model(self) -> Code: """Converts a FusionCode to a standard code.""" From a5f2b2ee0c5b9341e79cae75e8c889903ecaf63a Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Thu, 16 Jan 2025 11:57:41 +0100 Subject: [PATCH 5/6] Improve handling of code annotations --- src/pysdmx/io/json/sdmxjson2/messages/code.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/pysdmx/io/json/sdmxjson2/messages/code.py b/src/pysdmx/io/json/sdmxjson2/messages/code.py index 27c0d5f9..e99cd3a9 100644 --- a/src/pysdmx/io/json/sdmxjson2/messages/code.py +++ b/src/pysdmx/io/json/sdmxjson2/messages/code.py @@ -32,13 +32,16 @@ def __handle_date(self, datestr: str) -> datetime: def __get_val( self, a: JsonAnnotation ) -> Tuple[Optional[datetime], Optional[datetime]]: - vals = a.title.split("/") # type: ignore[union-attr] - if a.title.startswith("/"): # type: ignore[union-attr] - return (None, self.__handle_date(vals[1])) + if a.title: + vals = a.title.split("/") + if a.title.startswith("/"): + return (None, self.__handle_date(vals[1])) + else: + valid_from = self.__handle_date(vals[0]) + valid_to = self.__handle_date(vals[1]) if vals[1] else None + return (valid_from, valid_to) else: - valid_from = self.__handle_date(vals[0]) - valid_to = self.__handle_date(vals[1]) if vals[1] else None - return (valid_from, valid_to) + return (None, None) def to_model(self) -> Code: """Converts a JsonCode to a standard code.""" From efceddff9e224b8392a3891bdc9aa23d5cbadbb9 Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Thu, 16 Jan 2025 12:01:17 +0100 Subject: [PATCH 6/6] Remove unreachable part of the code --- src/pysdmx/io/json/fusion/messages/code.py | 15 ++++++--------- src/pysdmx/io/json/sdmxjson2/messages/code.py | 15 ++++++--------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/src/pysdmx/io/json/fusion/messages/code.py b/src/pysdmx/io/json/fusion/messages/code.py index a787701a..32340dbc 100644 --- a/src/pysdmx/io/json/fusion/messages/code.py +++ b/src/pysdmx/io/json/fusion/messages/code.py @@ -41,16 +41,13 @@ def __handle_date(self, datestr: str) -> datetime: def __get_val( self, a: FusionAnnotation ) -> Tuple[Optional[datetime], Optional[datetime]]: - if a.title: - vals = a.title.split("/") - if a.title.startswith("/"): - return (None, self.__handle_date(vals[1])) - else: - valid_from = self.__handle_date(vals[0]) - valid_to = self.__handle_date(vals[1]) if vals[1] else None - return (valid_from, valid_to) + vals = a.title.split("/") # type: ignore[union-attr] + if a.title.startswith("/"): # type: ignore[union-attr] + return (None, self.__handle_date(vals[1])) else: - return (None, None) + valid_from = self.__handle_date(vals[0]) + valid_to = self.__handle_date(vals[1]) if vals[1] else None + return (valid_from, valid_to) def to_model(self) -> Code: """Converts a FusionCode to a standard code.""" diff --git a/src/pysdmx/io/json/sdmxjson2/messages/code.py b/src/pysdmx/io/json/sdmxjson2/messages/code.py index e99cd3a9..27c0d5f9 100644 --- a/src/pysdmx/io/json/sdmxjson2/messages/code.py +++ b/src/pysdmx/io/json/sdmxjson2/messages/code.py @@ -32,16 +32,13 @@ def __handle_date(self, datestr: str) -> datetime: def __get_val( self, a: JsonAnnotation ) -> Tuple[Optional[datetime], Optional[datetime]]: - if a.title: - vals = a.title.split("/") - if a.title.startswith("/"): - return (None, self.__handle_date(vals[1])) - else: - valid_from = self.__handle_date(vals[0]) - valid_to = self.__handle_date(vals[1]) if vals[1] else None - return (valid_from, valid_to) + vals = a.title.split("/") # type: ignore[union-attr] + if a.title.startswith("/"): # type: ignore[union-attr] + return (None, self.__handle_date(vals[1])) else: - return (None, None) + valid_from = self.__handle_date(vals[0]) + valid_to = self.__handle_date(vals[1]) if vals[1] else None + return (valid_from, valid_to) def to_model(self) -> Code: """Converts a JsonCode to a standard code."""