From df35a6381a527b071f73a3f8413c1769d8e9a925 Mon Sep 17 00:00:00 2001 From: "andres.sole" Date: Wed, 6 Sep 2023 10:18:10 +0200 Subject: [PATCH 01/26] feat sdmx: initial commit --- .gitignore | 3 + docker-compose-non-dev.yml | 5 + docker-compose.yml | 1 + docker/.env | 4 +- example.yaml | 156 ++++++++++ superset-frontend/package-lock.json | 2 +- .../ImportDashboardSDMXModal/Modal.css | 24 ++ .../ImportDashboardSDMXModal/Modal.tsx | 68 +++++ .../ImportDashboardSDMXModal/index.ts | 20 ++ .../src/components/ImportSDMXModal/Modal.css | 24 ++ .../src/components/ImportSDMXModal/Modal.tsx | 60 ++++ .../src/components/ImportSDMXModal/index.ts | 20 ++ .../src/pages/DashboardList/index.tsx | 18 +- .../src/pages/DatasetList/index.tsx | 23 +- superset/charts/data/api.py | 27 +- superset/config.py | 65 +++- superset/connectors/sqla/models.py | 3 + superset/datasets/schemas.py | 3 +- ...32_0456266a4e03_add_is_sdmx_to_sqltable.py | 210 +++++++++++++ ..._d697a7f4f020_add_sdmx_url_to_sqlatable.py | 42 +++ ...-09-04_20-23_449e43460c4e_add_smdx_uuid.py | 42 +++ superset/views/api.py | 283 ++++++++++++++++++ 22 files changed, 1083 insertions(+), 20 deletions(-) create mode 100644 example.yaml create mode 100644 superset-frontend/src/components/ImportDashboardSDMXModal/Modal.css create mode 100644 superset-frontend/src/components/ImportDashboardSDMXModal/Modal.tsx create mode 100644 superset-frontend/src/components/ImportDashboardSDMXModal/index.ts create mode 100644 superset-frontend/src/components/ImportSDMXModal/Modal.css create mode 100644 superset-frontend/src/components/ImportSDMXModal/Modal.tsx create mode 100644 superset-frontend/src/components/ImportSDMXModal/index.ts create mode 100644 superset/migrations/versions/2023-09-04_19-32_0456266a4e03_add_is_sdmx_to_sqltable.py create mode 100644 superset/migrations/versions/2023-09-04_19-36_d697a7f4f020_add_sdmx_url_to_sqlatable.py create mode 100644 superset/migrations/versions/2023-09-04_20-23_449e43460c4e_add_smdx_uuid.py diff --git a/.gitignore b/.gitignore index a23cbb9ba5..59a2ff4bf8 100644 --- a/.gitignore +++ b/.gitignore @@ -110,3 +110,6 @@ messages.mo docker/requirements-local.txt cache/ + +# SDMX Insight +dbs/ \ No newline at end of file diff --git a/docker-compose-non-dev.yml b/docker-compose-non-dev.yml index cf36ae833d..d3a006cdd8 100644 --- a/docker-compose-non-dev.yml +++ b/docker-compose-non-dev.yml @@ -22,6 +22,11 @@ x-superset-volumes: &superset-volumes # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container - ./docker:/app/docker - superset_home:/app/superset_home + - ./dbs:/app/dbs + - ./superset:/app/superset + - ./superset-frontend:/app/superset-frontend + + version: "3.7" services: diff --git a/docker-compose.yml b/docker-compose.yml index dc9f9d5589..4c74e5d968 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,6 +26,7 @@ x-superset-volumes: &superset-volumes - ./superset-frontend:/app/superset-frontend - superset_home:/app/superset_home - ./tests:/app/tests + - ./dbs:/app/dbs version: "3.7" services: diff --git a/docker/.env b/docker/.env index 25bdac0ab7..11d8904226 100644 --- a/docker/.env +++ b/docker/.env @@ -47,7 +47,7 @@ REDIS_PORT=6379 FLASK_DEBUG=true SUPERSET_ENV=development -SUPERSET_LOAD_EXAMPLES=yes +SUPERSET_LOAD_EXAMPLES=no CYPRESS_CONFIG=false SUPERSET_PORT=8088 -MAPBOX_API_KEY='' +MAPBOX_API_KEY='' \ No newline at end of file diff --git a/example.yaml b/example.yaml new file mode 100644 index 0000000000..d2457ad83b --- /dev/null +++ b/example.yaml @@ -0,0 +1,156 @@ +--- +DashID: ExampleCL +Rows: +- + Row: 0 + chartType: TITLE + Title: " The Labour Market at a glance" + Subtitle: "Example of a dashboard script with data from Chile" + Unit: + unitLoc: + Decimals: + LabelsYN: + legendConcept: + legendLoc: + xAxisConcept: + yAxisConcept: + downloadYN: + dataLink: + metadataLink: + DATA: +- + Row: 1 + chartType: VALUE + Title: "Unemployment rate" + Subtitle: "{$TIME_PERIOD}" + Unit: "%" + unitLoc: SUFFIX + Decimals: "{$DECIMALS}" + LabelsYN: No + legendConcept: + legendLoc: + xAxisConcept: + yAxisConcept: + downloadYN: + dataLink: + metadataLink: + DATA: "https://www.ilo.org/sdmx/rest/data/ILO,DF_UNE_DEAP_SEX_AGE_RT,1.0/CHL.A..SEX_T.AGE_YTHADULT_YGE15?endPeriod=2022&lastNObservations=1" +- + Row: 1 + chartType: VALUE + Title: "Number of persons with multiple jobs" + Subtitle: "{$TIME_PERIOD}" + Unit: + unitLoc: HIDE + Decimals: 0 + LabelsYN: No + legendConcept: + legendLoc: + xAxisConcept: + yAxisConcept: + downloadYN: + dataLink: + metadataLink: + DATA: "https://www.ilo.org/sdmx/rest/data/ILO,DF_EES_TEES_SEX_MJH_NB,1.0/CHL.A..SEX_T.MJH_AGGREGATE_MULTI?endPeriod=2022&lastNObservations=1 * {UNIT_MULT}" +- + Row: 1 + chartType: VALUE + Title: "NEET - Youth not in employment, education or training" + Subtitle: "{$TIME_PERIOD}" + Unit: "%" + unitLoc: SUFFIX + Decimals: 2 + LabelsYN: No + legendConcept: + legendLoc: + xAxisConcept: + yAxisConcept: + downloadYN: + dataLink: + metadataLink: + DATA: "https://www.ilo.org/sdmx/rest/data/ILO,DF_EIP_NEET_SEX_RT,1.0/CHL.A..SEX_T?endPeriod=2022&lastNObservations=1" +- + Row: 2 + chartType: PIE + Title: "Persons outside the labour force by sex" + Subtitle: "{$TIME_PERIOD}" + Unit: "%" + unitLoc: SUFFIX + Decimals: 0 + LabelsYN: Yes + legendConcept: SEX + legendLoc: RIGHT + xAxisConcept: + yAxisConcept: + downloadYN: + dataLink: + metadataLink: + DATA: "https://www.ilo.org/sdmx/rest/data/ILO,DF_EIP_TEIP_SEX_AGE_NB,1.0/CHL.A..SEX_F+SEX_M.AGE_AGGREGATE_TOTAL?endPeriod=2022&lastNObservations=1&dimensionAtObservation=AllDimensions" +- + Row: 2 + chartType: VALUE + Title: "{$MEASURE}" + Subtitle: "{$TIME_PERIOD}" + Unit: + unitLoc: HIDE + Decimals: "{$DECIMALS}" + LabelsYN: No + legendConcept: + legendLoc: + xAxisConcept: + yAxisConcept: + downloadYN: + dataLink: + metadataLink: + DATA: "https://www.ilo.org/sdmx/rest/data/ILO,DF_LAI_INDE_NOC_RT,1.0/CHL.A.?endPeriod=2021&lastNObservations=1" +- + Row: 2 + chartType: PIE + Title: "Employment by economic activity" + Subtitle: "{$TIME_PERIOD}" + Unit: "%" + unitLoc: SUFFIX + Decimals: 0 + LabelsYN: Yes + legendConcept: ECO + legendLoc: HIDE + xAxisConcept: + yAxisConcept: + downloadYN: + dataLink: + metadataLink: + DATA: "https://www.ilo.org/sdmx/rest/data/ILO,DF_EMP_TEMP_SEX_ECO_NB,1.0/CHL.A..SEX_T.ECO_SECTOR_X+ECO_SECTOR_SER+ECO_SECTOR_IND+ECO_SECTOR_AGR?endPeriod=2022&lastNObservations=1" +- + Row: 3 + chartType: LINES, double + Title: "Labour Force participation rates" + Subtitle: + Unit: + unitLoc: + Decimals: 0 + LabelsYN: No + legendConcept: SEX + legendLoc: BOTTOM + xAxisConcept: TIME_PERIOD + yAxisConcept: OBS_VALUE + downloadYN: + dataLink: + metadataLink: + DATA: "https://www.ilo.org/sdmx/rest/data/ILO,DF_EAP_DWAP_SEX_AGE_RT,1.0/CHL.A..SEX_O+SEX_F+SEX_M+SEX_T.AGE_YTHADULT_YGE15?startPeriod=2010&endPeriod=2022" +- + Row: 3 + chartType: BARS + Title: "Status in employment" + Subtitle: "{$TIME_PERIOD}" + Unit: + unitLoc: HIDE + Decimals: 0 + LabelsYN: Yes + legendConcept: + legendLoc: HIDE + xAxisConcept: STE + yAxisConcept: OBS_VALUE + downloadYN: + dataLink: + metadataLink: + DATA: "https://www.ilo.org/sdmx/rest/data/ILO,DF_EMP_TEMP_SEX_AGE_STE_NB,1.0/CHL.A..SEX_T.AGE_YTHADULT_YGE15.STE_ICSE93_6+STE_ICSE93_5+STE_ICSE93_4+STE_ICSE93_3+STE_ICSE93_2+STE_ICSE93_1?endPeriod=2022&lastNObservations=1" diff --git a/superset-frontend/package-lock.json b/superset-frontend/package-lock.json index 694cfd9193..4c0fa255e1 100644 --- a/superset-frontend/package-lock.json +++ b/superset-frontend/package-lock.json @@ -77984,7 +77984,7 @@ "@mapbox/geojson-extent": "^1.0.1", "@math.gl/web-mercator": "^3.2.2", "@types/d3-array": "^2.0.0", - "@types/mapbox__geojson-extent": "*", + "@types/mapbox__geojson-extent": "^1.0.0", "@types/underscore": "^1.11.6", "@types/urijs": "^1.19.19", "bootstrap-slider": "^10.0.0", diff --git a/superset-frontend/src/components/ImportDashboardSDMXModal/Modal.css b/superset-frontend/src/components/ImportDashboardSDMXModal/Modal.css new file mode 100644 index 0000000000..1e166544e2 --- /dev/null +++ b/superset-frontend/src/components/ImportDashboardSDMXModal/Modal.css @@ -0,0 +1,24 @@ +.modal-overlay { + position: fixed; + top: 0; + left: 0; + right: 0; + bottom: 0; + /* background-color: rgba(0, 0, 0, 0.7); */ + display: flex; + justify-content: center; + align-items: center; + } + + .file-uploader { + padding: 20px; + } + + .modal-content { + background-color: #fff; + padding: 20px; + border-radius: 8px; + max-width: 500px; + width: 90%; + position: relative; + } \ No newline at end of file diff --git a/superset-frontend/src/components/ImportDashboardSDMXModal/Modal.tsx b/superset-frontend/src/components/ImportDashboardSDMXModal/Modal.tsx new file mode 100644 index 0000000000..77ca500681 --- /dev/null +++ b/superset-frontend/src/components/ImportDashboardSDMXModal/Modal.tsx @@ -0,0 +1,68 @@ +import React, { useState } from 'react'; +import PropTypes from 'prop-types'; +import './Modal.css'; // Assuming you place your CSS in this file +import { SupersetClient } from '@superset-ui/core'; +import { Upload, Space } from 'antd'; +import { set } from 'lodash'; +import { Input } from '../Input'; +import Button from '../Button'; + +const Modal = ({ isOpen, onClose }) => { + if (!isOpen) { + return null; + } + + const [sdmxFile, setSdmxFile] = useState([]); + + const onUpload = () => { + const formData = new FormData(); + + formData.append('file', sdmxFile[0]); + + SupersetClient.post({ + endpoint: 'api/v1/sdmx/dashboard', + postPayload: formData, + }).then(res => { + window.location.reload(); + }); + }; + + return ( +
+
e.stopPropagation()}> +

Import SDMX Dashboard YAML definition

+
+ + { + setSdmxFile([value]); + return false; + }} + > + + + + +
+ +
+
+ ); +}; + +Modal.propTypes = { + isOpen: PropTypes.bool.isRequired, + onClose: PropTypes.func, + children: PropTypes.node, +}; + +export default Modal; diff --git a/superset-frontend/src/components/ImportDashboardSDMXModal/index.ts b/superset-frontend/src/components/ImportDashboardSDMXModal/index.ts new file mode 100644 index 0000000000..9466591c7a --- /dev/null +++ b/superset-frontend/src/components/ImportDashboardSDMXModal/index.ts @@ -0,0 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +export * from './Modal'; +export { default } from './Modal'; diff --git a/superset-frontend/src/components/ImportSDMXModal/Modal.css b/superset-frontend/src/components/ImportSDMXModal/Modal.css new file mode 100644 index 0000000000..1e166544e2 --- /dev/null +++ b/superset-frontend/src/components/ImportSDMXModal/Modal.css @@ -0,0 +1,24 @@ +.modal-overlay { + position: fixed; + top: 0; + left: 0; + right: 0; + bottom: 0; + /* background-color: rgba(0, 0, 0, 0.7); */ + display: flex; + justify-content: center; + align-items: center; + } + + .file-uploader { + padding: 20px; + } + + .modal-content { + background-color: #fff; + padding: 20px; + border-radius: 8px; + max-width: 500px; + width: 90%; + position: relative; + } \ No newline at end of file diff --git a/superset-frontend/src/components/ImportSDMXModal/Modal.tsx b/superset-frontend/src/components/ImportSDMXModal/Modal.tsx new file mode 100644 index 0000000000..9c78bd3864 --- /dev/null +++ b/superset-frontend/src/components/ImportSDMXModal/Modal.tsx @@ -0,0 +1,60 @@ +import React, { useState } from 'react'; +import PropTypes from 'prop-types'; +import './Modal.css'; // Assuming you place your CSS in this file +import { SupersetClient } from '@superset-ui/core'; +import { Space } from 'antd'; +import Button from '../Button'; +import { Input } from '../Input'; + +const Modal = ({ isOpen, onClose, children }) => { + if (!isOpen) { + return null; + } + + const [sdmxUrl, setSdmxUrl] = useState(''); + + const onUpload = () => { + SupersetClient.post({ + endpoint: 'api/v1/sdmx/', + jsonPayload: { + sdmxUrl, + }, + }).then(res => { + window.location.reload(); + }); + }; + + return ( +
+
e.stopPropagation()}> +

Import SDMX

+
+ + setSdmxUrl(evt.target.value)} + /> + {children} + + +
+ +
+
+ ); +}; + +Modal.propTypes = { + isOpen: PropTypes.bool.isRequired, + onClose: PropTypes.func, + children: PropTypes.node, +}; + +export default Modal; diff --git a/superset-frontend/src/components/ImportSDMXModal/index.ts b/superset-frontend/src/components/ImportSDMXModal/index.ts new file mode 100644 index 0000000000..9466591c7a --- /dev/null +++ b/superset-frontend/src/components/ImportSDMXModal/index.ts @@ -0,0 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +export * from './Modal'; +export { default } from './Modal'; diff --git a/superset-frontend/src/pages/DashboardList/index.tsx b/superset-frontend/src/pages/DashboardList/index.tsx index 2b6252309d..503e321b9b 100644 --- a/superset-frontend/src/pages/DashboardList/index.tsx +++ b/superset-frontend/src/pages/DashboardList/index.tsx @@ -61,6 +61,7 @@ import CertifiedBadge from 'src/components/CertifiedBadge'; import { loadTags } from 'src/components/Tags/utils'; import DashboardCard from 'src/features/dashboards/DashboardCard'; import { DashboardStatus } from 'src/features/dashboards/types'; +import ImportDashboardSDMXModal from 'src/components/ImportDashboardSDMXModal'; const PAGE_SIZE = 25; const PASSWORDS_NEEDED_MESSAGE = t( @@ -646,6 +647,18 @@ function DashboardList(props: DashboardListProps) { ); const subMenuButtons: SubMenuProps['buttons'] = []; + + const [importYamlModalOpen, handleImportYamlModalOpen] = + useState(false); + + subMenuButtons.push({ + name: 'Import SDMX Dashboard YAML', + buttonStyle: 'tertiary', + onClick: () => { + handleImportYamlModalOpen(true); + }, + }); + if (canDelete || canExport) { subMenuButtons.push({ name: t('Bulk select'), @@ -775,7 +788,10 @@ function DashboardList(props: DashboardListProps) { ); }} - + handleImportYamlModalOpen(false)} + /> = ({ const buttonArr: Array = []; + const [isModalOpen, setIsModalOpen] = useState(false); + + const toggleImportFileModal = () => { + setIsModalOpen(true); + }; + + buttonArr.push({ + name: 'Import SDMX url', + onClick: toggleImportFileModal, + buttonStyle: 'tertiary', + }); + if (canDelete || canExport) { buttonArr.push({ name: t('Bulk select'), @@ -684,7 +697,6 @@ const DatasetList: FunctionComponent = ({ ), ); }; - const handleBulkDatasetDelete = (datasetsToDelete: Dataset[]) => { SupersetClient.delete({ endpoint: `/api/v1/dataset/?q=${rison.encode( @@ -727,9 +739,18 @@ const DatasetList: FunctionComponent = ({ ); }; + const openModal = () => { + setIsModalOpen(true); + }; + + const closeModal = () => { + setIsModalOpen(false); + }; + return ( <> + {datasetCurrentlyDeleting && ( Response: json_body = json.loads(request.form["form_data"]) if json_body is None: return self.response_400(message=_("Request is not JSON")) + if "slice_id" in json_body['form_data']: + slice = db.session.query(Slice).filter_by(id=json_body['form_data']['slice_id']).first() + datasource_id = slice.datasource_id + datasource = db.session.query(SqlaTable).filter_by(id=datasource_id).first() + database = db.session.query(Database).filter_by(id=datasource.database_id).first() + if datasource and datasource.is_sdmx and json_body['form_data']['force']: + message = read_sdmx(datasource.sdmx_url) + datasource.sdmx_uuid = database.database_name.split(" ")[0] + engine = create_engine(f"sqlite:///dbs/{datasource.sdmx_uuid}", echo=False) + + for dataset in message.payload.keys(): + df = message.payload[dataset].data + table_name = str(dataset) + " " + str(datetime.datetime.now()) + df.to_sql(table_name, con=engine) + datasource.table_name = table_name + + db.session.commit() + try: query_context = self._create_query_context_from_form(json_body) diff --git a/superset/config.py b/superset/config.py index 18a6157578..89863e9755 100644 --- a/superset/config.py +++ b/superset/config.py @@ -37,10 +37,10 @@ from typing import Any, Callable, Literal, TYPE_CHECKING, TypedDict import pkg_resources +from cachelib.base import BaseCache from celery.schedules import crontab from flask import Blueprint from flask_appbuilder.security.manager import AUTH_DB -from flask_caching.backends.base import BaseCache from pandas import Series from pandas._libs.parsers import STR_NA_VALUES # pylint: disable=no-name-in-module from sqlalchemy.orm.query import Query @@ -283,11 +283,6 @@ def _try_json_readsha(filepath: str, length: int) -> str | None: # ------------------------------ # GLOBALS FOR APP Builder # ------------------------------ -# Uncomment to setup Your App name -APP_NAME = "Superset" - -# Specify the App icon -APP_ICON = "/static/assets/images/superset-logo-horiz.png" # Specify where clicking the logo would take the user # e.g. setting it to '/' would take the user to '/superset/welcome/' @@ -740,7 +735,6 @@ class D3Format(TypedDict, total=False): # Disabling this option is not recommended for security reasons. If you wish to allow # valid safe elements that are not included in the default sanitization schema, use the # HTML_SANITIZATION_SCHEMA_EXTENSIONS configuration. -HTML_SANITIZATION = True # Use this configuration to extend the HTML sanitization schema. # By default we use the Gihtub schema defined in @@ -916,7 +910,7 @@ class D3Format(TypedDict, total=False): # Default celery config is to use SQLA as a broker, in a production setting # you'll want to use a proper broker as specified here: -# https://docs.celeryq.dev/en/stable/getting-started/backends-and-brokers/index.html +# http://docs.celeryproject.org/en/latest/getting-started/brokers/index.html class CeleryConfig: # pylint: disable=too-few-public-methods @@ -1403,11 +1397,8 @@ def EMAIL_HEADER_MUTATOR( # pylint: disable=invalid-name,unused-argument # one here. TEST_DATABASE_CONNECTION_TIMEOUT = timedelta(seconds=30) -# Enable/disable CSP warning -CONTENT_SECURITY_POLICY_WARNING = True - # Do you want Talisman enabled? -TALISMAN_ENABLED = utils.cast_to_boolean(os.environ.get("TALISMAN_ENABLED", True)) +TALISMAN_ENABLED = False # If you want Talisman, how do you want it configured?? TALISMAN_CONFIG = { @@ -1471,7 +1462,7 @@ def EMAIL_HEADER_MUTATOR( # pylint: disable=invalid-name,unused-argument # Typically these should not be allowed. PREVENT_UNSAFE_DB_CONNECTIONS = True -# If true all default urls on datasets will be handled as relative URLs by the frontend +# Prevents unsafe default endpoints to be registered on datasets. PREVENT_UNSAFE_DEFAULT_URLS_ON_DATASET = True # Define a list of allowed URLs for dataset data imports (v1). @@ -1676,3 +1667,51 @@ class ExtraDynamicQueryFilters(TypedDict, total=False): except Exception: logger.exception("Found but failed to import local superset_config") raise + +PREVENT_UNSAFE_DB_CONNECTIONS = False + +from flask_appbuilder.security.views import expose +from superset.security import SupersetSecurityManager +from flask_appbuilder.security.manager import BaseSecurityManager +from flask_appbuilder.security.manager import AUTH_REMOTE_USER +from flask import redirect +from flask_login import login_user + +# AuthRemoteUserView=BaseSecurityManager.authremoteuserview +# class AuthRemoteUserView(AuthRemoteUserView): +# @expose('/login/') +# def login(self): +# user = self.appbuilder.sm.auth_user_db("admin", "admin") +# login_user(user, remember=False) +# return redirect(self.appbuilder.get_url_for_index) + + +# class CustomSecurityManager(SupersetSecurityManager): +# authremoteuserview = AuthRemoteUserView + +# CUSTOM_SECURITY_MANAGER = CustomSecurityManager + +# AUTH_TYPE = AUTH_REMOTE_USER + +# OVERRIDE_HTTP_HEADERS = {'X-Frame-Options': 'ALLOWALL'} + +HTML_SANITIZATION = False +CONTENT_SECURITY_POLICY_WARNING = False + +APP_ICON="/static/assets/images/logo.png" +APP_NAME="SDMX Insight" + +THEME_OVERRIDES = { + "borderRadius": 4, + "colors": { + "primary": { + "primary": '#3c86ae', + }, + "secondary": { + "base": '#3c86ae', + }, + "grayscale": { + "base": '#525252', + } + } +} diff --git a/superset/connectors/sqla/models.py b/superset/connectors/sqla/models.py index cdebe8724c..52e4e80706 100644 --- a/superset/connectors/sqla/models.py +++ b/superset/connectors/sqla/models.py @@ -534,6 +534,9 @@ class SqlaTable( database_id = Column(Integer, ForeignKey("dbs.id"), nullable=False) fetch_values_predicate = Column(Text) owners = relationship(owner_class, secondary=sqlatable_user, backref="tables") + is_sdmx = Column(Boolean, default=False) + sdmx_url = Column(String(1024), nullable=True) + sdmx_uuid = Column(String(256), nullable=True) database: Database = relationship( "Database", backref=backref("tables", cascade="all, delete-orphan"), diff --git a/superset/datasets/schemas.py b/superset/datasets/schemas.py index 66d80e1842..9dc786a797 100644 --- a/superset/datasets/schemas.py +++ b/superset/datasets/schemas.py @@ -99,7 +99,8 @@ class DatasetPostSchema(Schema): is_managed_externally = fields.Boolean(allow_none=True, dump_default=False) external_url = fields.String(allow_none=True) normalize_columns = fields.Boolean(load_default=False) - + is_sdmx = fields.Boolean(allow_none=True) + sdmx_url = fields.String(allow_none=True) class DatasetPutSchema(Schema): table_name = fields.String(allow_none=True, validate=Length(1, 250)) diff --git a/superset/migrations/versions/2023-09-04_19-32_0456266a4e03_add_is_sdmx_to_sqltable.py b/superset/migrations/versions/2023-09-04_19-32_0456266a4e03_add_is_sdmx_to_sqltable.py new file mode 100644 index 0000000000..ff8b205a85 --- /dev/null +++ b/superset/migrations/versions/2023-09-04_19-32_0456266a4e03_add_is_sdmx_to_sqltable.py @@ -0,0 +1,210 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""add is_sdmx to sqltable + +Revision ID: 0456266a4e03 +Revises: ec54aca4c8a2 +Create Date: 2023-09-04 19:32:10.949475 + +""" + +# revision identifiers, used by Alembic. +revision = '0456266a4e03' +down_revision = 'ec54aca4c8a2' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('annotation', 'layer_id', + existing_type=sa.INTEGER(), + nullable=False) + op.alter_column('dashboard_roles', 'dashboard_id', + existing_type=sa.INTEGER(), + nullable=False) + op.alter_column('dbs', 'allow_file_upload', + existing_type=sa.BOOLEAN(), + nullable=True, + existing_server_default=sa.text('true')) + op.create_unique_constraint(None, 'dynamic_plugin', ['bundle_url']) + op.alter_column('embedded_dashboards', 'uuid', + existing_type=postgresql.UUID(), + nullable=False) + op.create_foreign_key(None, 'embedded_dashboards', 'ab_user', ['changed_by_fk'], ['id']) + op.create_foreign_key(None, 'embedded_dashboards', 'ab_user', ['created_by_fk'], ['id']) + op.alter_column('filter_sets', 'dashboard_id', + existing_type=sa.INTEGER(), + nullable=True) + op.create_unique_constraint(None, 'filter_sets', ['name']) + op.drop_index('ix_key_value_expires_on', table_name='key_value') + op.drop_index('ix_key_value_uuid', table_name='key_value') + op.create_unique_constraint(None, 'key_value', ['uuid']) + op.drop_index('ix_logs_user_id_dttm', table_name='logs') + op.alter_column('report_schedule', 'extra_json', + existing_type=sa.TEXT(), + nullable=True) + op.drop_index('ix_creation_method', table_name='report_schedule') + op.create_unique_constraint(None, 'report_schedule_user', ['user_id', 'report_schedule_id']) + op.drop_index('ix_row_level_security_filters_filter_type', table_name='row_level_security_filters') + op.alter_column('sl_columns', 'is_additive', + existing_type=sa.BOOLEAN(), + nullable=True) + op.alter_column('sl_columns', 'is_aggregation', + existing_type=sa.BOOLEAN(), + nullable=True) + op.alter_column('sl_columns', 'is_increase_desired', + existing_type=sa.BOOLEAN(), + nullable=True) + op.alter_column('sl_columns', 'is_partition', + existing_type=sa.BOOLEAN(), + nullable=True) + op.alter_column('sl_columns', 'is_physical', + existing_type=sa.BOOLEAN(), + nullable=True) + op.alter_column('sl_columns', 'is_spatial', + existing_type=sa.BOOLEAN(), + nullable=True) + op.alter_column('sl_columns', 'is_temporal', + existing_type=sa.BOOLEAN(), + nullable=True) + op.drop_constraint('sl_table_columns_table_id_fkey', 'sl_table_columns', type_='foreignkey') + op.drop_constraint('sl_table_columns_column_id_fkey', 'sl_table_columns', type_='foreignkey') + op.create_foreign_key(None, 'sl_table_columns', 'sl_tables', ['table_id'], ['id'], ondelete='CASCADE') + op.create_foreign_key(None, 'sl_table_columns', 'sl_columns', ['column_id'], ['id'], ondelete='CASCADE') + op.create_unique_constraint(None, 'sl_tables', ['database_id', 'catalog', 'schema', 'name']) + op.alter_column('ssh_tunnels', 'database_id', + existing_type=sa.INTEGER(), + nullable=False) + op.drop_index('ix_ssh_tunnels_database_id', table_name='ssh_tunnels') + op.drop_index('ix_ssh_tunnels_uuid', table_name='ssh_tunnels') + op.create_unique_constraint(None, 'ssh_tunnels', ['uuid']) + op.create_unique_constraint(None, 'ssh_tunnels', ['database_id']) + op.create_foreign_key(None, 'ssh_tunnels', 'ab_user', ['changed_by_fk'], ['id']) + op.create_foreign_key(None, 'ssh_tunnels', 'ab_user', ['created_by_fk'], ['id']) + op.alter_column('tab_state', 'autorun', + existing_type=sa.BOOLEAN(), + nullable=True) + op.alter_column('tab_state', 'hide_left_bar', + existing_type=sa.BOOLEAN(), + nullable=True, + existing_server_default=sa.text('false')) + op.drop_index('ix_tab_state_id', table_name='tab_state') + op.drop_index('ix_table_schema_id', table_name='table_schema') + op.add_column('tables', sa.Column('is_sdmx', sa.Boolean(), nullable=True)) + op.drop_index('ix_tagged_object_object_id', table_name='tagged_object') + op.create_foreign_key(None, 'tagged_object', 'dashboards', ['object_id'], ['id']) + op.create_foreign_key(None, 'tagged_object', 'saved_query', ['object_id'], ['id']) + op.create_foreign_key(None, 'tagged_object', 'slices', ['object_id'], ['id']) + op.alter_column('user_favorite_tag', 'user_id', + existing_type=sa.INTEGER(), + nullable=True) + op.alter_column('user_favorite_tag', 'tag_id', + existing_type=sa.INTEGER(), + nullable=True) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('user_favorite_tag', 'tag_id', + existing_type=sa.INTEGER(), + nullable=False) + op.alter_column('user_favorite_tag', 'user_id', + existing_type=sa.INTEGER(), + nullable=False) + op.drop_constraint(None, 'tagged_object', type_='foreignkey') + op.drop_constraint(None, 'tagged_object', type_='foreignkey') + op.drop_constraint(None, 'tagged_object', type_='foreignkey') + op.create_index('ix_tagged_object_object_id', 'tagged_object', ['object_id'], unique=False) + op.drop_column('tables', 'is_sdmx') + op.create_index('ix_table_schema_id', 'table_schema', ['id'], unique=False) + op.create_index('ix_tab_state_id', 'tab_state', ['id'], unique=False) + op.alter_column('tab_state', 'hide_left_bar', + existing_type=sa.BOOLEAN(), + nullable=False, + existing_server_default=sa.text('false')) + op.alter_column('tab_state', 'autorun', + existing_type=sa.BOOLEAN(), + nullable=False) + op.drop_constraint(None, 'ssh_tunnels', type_='foreignkey') + op.drop_constraint(None, 'ssh_tunnels', type_='foreignkey') + op.drop_constraint(None, 'ssh_tunnels', type_='unique') + op.drop_constraint(None, 'ssh_tunnels', type_='unique') + op.create_index('ix_ssh_tunnels_uuid', 'ssh_tunnels', ['uuid'], unique=False) + op.create_index('ix_ssh_tunnels_database_id', 'ssh_tunnels', ['database_id'], unique=False) + op.alter_column('ssh_tunnels', 'database_id', + existing_type=sa.INTEGER(), + nullable=True) + op.drop_constraint(None, 'sl_tables', type_='unique') + op.drop_constraint(None, 'sl_table_columns', type_='foreignkey') + op.drop_constraint(None, 'sl_table_columns', type_='foreignkey') + op.create_foreign_key('sl_table_columns_column_id_fkey', 'sl_table_columns', 'sl_columns', ['column_id'], ['id']) + op.create_foreign_key('sl_table_columns_table_id_fkey', 'sl_table_columns', 'sl_tables', ['table_id'], ['id']) + op.alter_column('sl_columns', 'is_temporal', + existing_type=sa.BOOLEAN(), + nullable=False) + op.alter_column('sl_columns', 'is_spatial', + existing_type=sa.BOOLEAN(), + nullable=False) + op.alter_column('sl_columns', 'is_physical', + existing_type=sa.BOOLEAN(), + nullable=False) + op.alter_column('sl_columns', 'is_partition', + existing_type=sa.BOOLEAN(), + nullable=False) + op.alter_column('sl_columns', 'is_increase_desired', + existing_type=sa.BOOLEAN(), + nullable=False) + op.alter_column('sl_columns', 'is_aggregation', + existing_type=sa.BOOLEAN(), + nullable=False) + op.alter_column('sl_columns', 'is_additive', + existing_type=sa.BOOLEAN(), + nullable=False) + op.create_index('ix_row_level_security_filters_filter_type', 'row_level_security_filters', ['filter_type'], unique=False) + op.drop_constraint(None, 'report_schedule_user', type_='unique') + op.create_index('ix_creation_method', 'report_schedule', ['creation_method'], unique=False) + op.alter_column('report_schedule', 'extra_json', + existing_type=sa.TEXT(), + nullable=False) + op.create_index('ix_logs_user_id_dttm', 'logs', ['user_id', 'dttm'], unique=False) + op.drop_constraint(None, 'key_value', type_='unique') + op.create_index('ix_key_value_uuid', 'key_value', ['uuid'], unique=False) + op.create_index('ix_key_value_expires_on', 'key_value', ['expires_on'], unique=False) + op.drop_constraint(None, 'filter_sets', type_='unique') + op.alter_column('filter_sets', 'dashboard_id', + existing_type=sa.INTEGER(), + nullable=False) + op.drop_constraint(None, 'embedded_dashboards', type_='foreignkey') + op.drop_constraint(None, 'embedded_dashboards', type_='foreignkey') + op.alter_column('embedded_dashboards', 'uuid', + existing_type=postgresql.UUID(), + nullable=True) + op.drop_constraint(None, 'dynamic_plugin', type_='unique') + op.alter_column('dbs', 'allow_file_upload', + existing_type=sa.BOOLEAN(), + nullable=False, + existing_server_default=sa.text('true')) + op.alter_column('dashboard_roles', 'dashboard_id', + existing_type=sa.INTEGER(), + nullable=True) + op.alter_column('annotation', 'layer_id', + existing_type=sa.INTEGER(), + nullable=True) + # ### end Alembic commands ### diff --git a/superset/migrations/versions/2023-09-04_19-36_d697a7f4f020_add_sdmx_url_to_sqlatable.py b/superset/migrations/versions/2023-09-04_19-36_d697a7f4f020_add_sdmx_url_to_sqlatable.py new file mode 100644 index 0000000000..a50d5a36b1 --- /dev/null +++ b/superset/migrations/versions/2023-09-04_19-36_d697a7f4f020_add_sdmx_url_to_sqlatable.py @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""add sdmx_url to sqlatable + +Revision ID: d697a7f4f020 +Revises: 0456266a4e03 +Create Date: 2023-09-04 19:36:49.726448 + +""" + +# revision identifiers, used by Alembic. +revision = 'd697a7f4f020' +down_revision = '0456266a4e03' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('tables', sa.Column('sdmx_url', sa.String(length=1024), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('tables', 'sdmx_url') + # ### end Alembic commands ### diff --git a/superset/migrations/versions/2023-09-04_20-23_449e43460c4e_add_smdx_uuid.py b/superset/migrations/versions/2023-09-04_20-23_449e43460c4e_add_smdx_uuid.py new file mode 100644 index 0000000000..cca77085d0 --- /dev/null +++ b/superset/migrations/versions/2023-09-04_20-23_449e43460c4e_add_smdx_uuid.py @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""add smdx_uuid + +Revision ID: 449e43460c4e +Revises: d697a7f4f020 +Create Date: 2023-09-04 20:23:15.679873 + +""" + +# revision identifiers, used by Alembic. +revision = '449e43460c4e' +down_revision = 'd697a7f4f020' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('tables', sa.Column('sdmx_uuid', sa.String(length=256), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('tables', 'sdmx_uuid') + # ### end Alembic commands ### diff --git a/superset/views/api.py b/superset/views/api.py index 312efb947e..0ed948b76f 100644 --- a/superset/views/api.py +++ b/superset/views/api.py @@ -36,16 +36,299 @@ from superset.utils import core as utils from superset.utils.date_parser import get_since_until from superset.views.base import api, BaseSupersetView, handle_api_exception +from sqlalchemy import create_engine +from sdmxthon import read_sdmx +import uuid +import datetime +import requests +import yaml if TYPE_CHECKING: from superset.common.query_context_factory import QueryContextFactory get_time_range_schema = {"type": "string"} +def load_databases(spec, base_url, tokens): + result = [] + for row in spec["Rows"]: + if not row["DATA"]: + result.append(None) + continue + + sdmx_url = row["DATA"].split(" ")[0] + + message = read_sdmx(sdmx_url) + dataset_uuid = uuid.uuid4() + + engine = create_engine(f"sqlite:///dbs/{dataset_uuid}", echo=False) + + for dataset in message.payload.keys(): + df = message.payload[dataset].data + df.to_sql(str(dataset) + " " + str(datetime.datetime.now()), con=engine) + + response = requests.post( + base_url + "api/v1/database", + json={ + "sqlalchemy_uri": f"sqlite:///dbs/{dataset_uuid}", + "database_name": f"{dataset_uuid}", + }, + headers={"Authorization": f"Bearer {tokens['access_token']}"}, + ) + + database_id = response.json()["id"] + + tables = requests.get( + base_url + f"api/v1/database/{database_id}/tables", + params={"q": f"(schema_name:main)"}, + headers={"Authorization": f"Bearer {tokens['access_token']}"}, + ).json()["result"] + + for table in tables: + response = requests.post( + base_url + "api/v1/dataset", + json={ + "table_name": f"{table['value']}", + "database": f"{database_id}", + "schema": "main", + "is_sdmx": True, + "sdmx_url": sdmx_url, + }, + headers={"Authorization": f"Bearer {tokens['access_token']}"}, + ) + result.append(response.json()["data"]) + return result + + +def create_dashboard(spec, base_url, tokens): + response = requests.post( + base_url + "api/v1/dashboard/", + json={ + "certification_details": None, + "certified_by": None, + "css": "", + "dashboard_title": spec["DashID"], + "external_url": None, + "is_managed_externally": True, + "json_metadata": "", + "owners": [1], + "position_json": "", + "published": False, + "roles": [1], + "slug": None, + }, + headers={"Authorization": f"Bearer {tokens['access_token']}"}, + ) + + return response.json()["data"] if "data" in response.json() else response.json() + + +def create_charts(spec, datasets, dashboard, base_url, tokens): + + for idx, row in enumerate(spec["Rows"]): + if not row["DATA"]: + continue + + if row["chartType"] == "VALUE": + response = requests.post( + base_url + "api/v1/chart/", + json={ + "cache_timeout": 0, + "certification_details": None, + "certified_by": None, + "dashboards": [ dashboard["id"] ], + "description": None, + "is_managed_externally": True, + "owners": [1], + "datasource_id": datasets[idx]["id"], + "params": "{\"datasource\":\"" + str(datasets[idx]["id"]) + "__table\",\"viz_type\":\"handlebars\",\"query_mode\":\"aggregate\",\"groupby\":[],\"metrics\":[{\"aggregate\":null,\"column\":null,\"datasourceWarning\":false,\"expressionType\":\"SQL\",\"hasCustomLabel\":false,\"label\":\"OBS_VALUE\",\"optionName\":\"\",\"sqlExpression\":\"OBS_VALUE\"}],\"all_columns\":[],\"percent_metrics\":[],\"order_by_cols\":[],\"order_desc\":true,\"row_limit\":10000,\"server_page_length\":10,\"adhoc_filters\":[],\"handlebarsTemplate\":\"

\\n {{#each data}}\\n {{this.OBS_VALUE}}\\n {{/each}}\\n

\",\"styleTemplate\":\"\\n.data-chart {\\n text-align: center;\\n font-size: 6em;\\n}\\n\",\"extra_form_data\":{},\"dashboards\":[" + str(dashboard['id']) +"]}", + "query_context": "{\"datasource\":{\"id\": " + str(datasets[idx]["id"]) + ",\"type\":\"table\"},\"force\":false,\"queries\":[{\"filters\":[],\"extras\":{\"having\":\"\",\"where\":\"\"},\"applied_time_extras\":{},\"columns\":[],\"metrics\":[{\"aggregate\":null,\"column\":null,\"datasourceWarning\":false,\"expressionType\":\"SQL\",\"hasCustomLabel\":false,\"label\":\"OBS_VALUE\",\"optionName\":\"\",\"sqlExpression\":\"OBS_VALUE\"}],\"orderby\":[[{\"aggregate\":null,\"column\":null,\"datasourceWarning\":false,\"expressionType\":\"SQL\",\"hasCustomLabel\":false,\"label\":\"OBS_VALUE\",\"optionName\":\"\",\"sqlExpression\":\"OBS_VALUE\"},false]],\"annotation_layers\":[],\"row_limit\":10000,\"series_limit\":0,\"order_desc\":true,\"url_params\":{},\"custom_params\":{},\"custom_form_data\":{}}],\"form_data\":{\"datasource\":\"" + str(datasets[idx]["id"]) + "__table\",\"viz_type\":\"handlebars\",\"query_mode\":\"aggregate\",\"groupby\":[],\"metrics\":[{\"aggregate\":null,\"column\":null,\"datasourceWarning\":false,\"expressionType\":\"SQL\",\"hasCustomLabel\":false,\"label\":\"OBS_VALUE\",\"optionName\":\"\",\"sqlExpression\":\"OBS_VALUE\"}],\"all_columns\":[],\"percent_metrics\":[],\"order_by_cols\":[],\"order_desc\":true,\"row_limit\":10000,\"server_page_length\":10,\"adhoc_filters\":[],\"handlebarsTemplate\":\"

\\n {{#each data}}\\n {{this.OBS_VALUE}}\\n {{/each}}\\n

\",\"styleTemplate\":\"\\n.data-chart {\\n text-align: center;\\n font-size: 6em;\\n}\\n\",\"extra_form_data\":{},\"dashboards\":[40],\"force\":false,\"result_format\":\"json\",\"result_type\":\"full\"},\"result_format\":\"json\",\"result_type\":\"full\"}", + "slice_name": row["Title"], + "datasource_type": "table", + "viz_type": "handlebars", + }, + headers={"Authorization": f"Bearer {tokens['access_token']}"}, + ) + elif row["chartType"] == "PIE": + response = requests.post( + base_url + "api/v1/chart/", + json={ + "cache_timeout": 0, + "certification_details": None, + "certified_by": None, + "dashboards": [dashboard["id"]], + "description": None, + "is_managed_externally": True, + "owners": [1], + "datasource_id": datasets[idx]["id"], + "params": "{\"viz_type\":\"pie\",\"groupby\":[\"" + row['legendConcept'] + "\"],\"metric\":{\"aggregate\":null,\"column\":null,\"datasourceWarning\":false,\"expressionType\":\"SQL\",\"hasCustomLabel\":false,\"label\":\"OBS_VALUE\",\"optionName\":\"metric_2o7qsp2myck_m08ex1fqyyp\",\"sqlExpression\":\"OBS_VALUE\"},\"adhoc_filters\":[],\"row_limit\":10000,\"sort_by_metric\":true,\"color_scheme\":\"supersetColors\",\"show_labels_threshold\":5,\"show_legend\":true,\"legendType\":\"scroll\",\"legendOrientation\":\"right\",\"label_type\":\"key_value_percent\",\"number_format\":\"SMART_NUMBER\",\"date_format\":\"smart_date\",\"show_labels\":true,\"labels_outside\":true,\"label_line\":true,\"outerRadius\":70,\"innerRadius\":30,\"extra_form_data\":{},\"dashboards\":[" + str(dashboard["id"]) + "]}", + "query_context": "", + "slice_name": row["Title"], + "datasource_type": "table", + "viz_type": "pie", + }, + headers={"Authorization": f"Bearer {tokens['access_token']}"}, + ) + elif "LINES" in row["chartType"]: + response = requests.post( + base_url + "api/v1/chart/", + json={ + "cache_timeout": 0, + "certification_details": None, + "certified_by": None, + "dashboards": [dashboard["id"]], + "description": None, + "is_managed_externally": True, + "owners": [1], + "datasource_id": datasets[idx]["id"], + "params": "{\"datasource\":\"" + str(datasets[idx]['id']) + "__table\",\"viz_type\":\"echarts_timeseries_line\",\"x_axis\":\"" + row["xAxisConcept"] + "\",\"time_grain_sqla\":\"P1D\",\"x_axis_sort_asc\":true,\"x_axis_sort_series\":\"name\",\"x_axis_sort_series_ascending\":true,\"metrics\":[{\"expressionType\":\"SQL\",\"sqlExpression\":\"OBS_VALUE\",\"column\":null,\"aggregate\":null,\"datasourceWarning\":false,\"hasCustomLabel\":false,\"label\":\"OBS_VALUE\",\"optionName\":\"metric_2pzqjh4ddnr_9fsj62hsc9c\"}],\"groupby\":[\"SEX\"],\"adhoc_filters\":[],\"order_desc\":true,\"row_limit\":10000,\"truncate_metric\":true,\"show_empty_columns\":true,\"comparison_type\":\"values\",\"annotation_layers\":[],\"forecastPeriods\":10,\"forecastInterval\":0.8,\"x_axis_title_margin\":15,\"y_axis_title_margin\":15,\"y_axis_title_position\":\"Left\",\"sort_series_type\":\"sum\",\"color_scheme\":\"supersetColors\",\"seriesType\":\"line\",\"show_value\":false,\"only_total\":true,\"opacity\":0.2,\"markerSize\":6,\"zoomable\":false,\"show_legend\":true,\"legendType\":\"scroll\",\"legendOrientation\":\"top\",\"x_axis_time_format\":\"smart_date\",\"rich_tooltip\":true,\"tooltipTimeFormat\":\"smart_date\",\"y_axis_format\":\"SMART_NUMBER\",\"y_axis_bounds\":[null,null],\"extra_form_data\":{},\"dashboards\":[1]}", + "query_context": "{\"datasource\":{\"id\":" + str(datasets[idx]['id']) + ",\"type\":\"table\"},\"force\":false,\"queries\":[{\"filters\":[],\"extras\":{\"having\":\"\",\"where\":\"\"},\"applied_time_extras\":{},\"columns\":[{\"timeGrain\":\"P1D\",\"columnType\":\"BASE_AXIS\",\"sqlExpression\":\"" + row["xAxisConcept"] + "\",\"label\":\"" + row["xAxisConcept"] + "\",\"expressionType\":\"SQL\"},\"SEX\"],\"metrics\":[{\"expressionType\":\"SQL\",\"sqlExpression\":\"OBS_VALUE\",\"column\":null,\"aggregate\":null,\"datasourceWarning\":false,\"hasCustomLabel\":false,\"label\":\"OBS_VALUE\",\"optionName\":\"metric_2pzqjh4ddnr_9fsj62hsc9c\"}],\"orderby\":[[{\"expressionType\":\"SQL\",\"sqlExpression\":\"OBS_VALUE\",\"column\":null,\"aggregate\":null,\"datasourceWarning\":false,\"hasCustomLabel\":false,\"label\":\"OBS_VALUE\",\"optionName\":\"metric_2pzqjh4ddnr_9fsj62hsc9c\"},false]],\"annotation_layers\":[],\"row_limit\":10000,\"series_columns\":[\"SEX\"],\"series_limit\":0,\"order_desc\":true,\"url_params\":{},\"custom_params\":{},\"custom_form_data\":{},\"time_offsets\":[],\"post_processing\":[{\"operation\":\"pivot\",\"options\":{\"index\":[\"" + row["xAxisConcept"] + "\"],\"columns\":[\"SEX\"],\"aggregates\":{\"OBS_VALUE\":{\"operator\":\"mean\"}},\"drop_missing_columns\":false}},{\"operation\":\"rename\",\"options\":{\"columns\":{\"OBS_VALUE\":null},\"level\":0,\"inplace\":true}},{\"operation\":\"flatten\"}]}],\"form_data\":{\"datasource\":\"9__table\",\"viz_type\":\"echarts_timeseries_line\",\"x_axis\":\"" + row["xAxisConcept"] + "\",\"time_grain_sqla\":\"P1D\",\"x_axis_sort_asc\":true,\"x_axis_sort_series\":\"name\",\"x_axis_sort_series_ascending\":true,\"metrics\":[{\"expressionType\":\"SQL\",\"sqlExpression\":\"OBS_VALUE\",\"column\":null,\"aggregate\":null,\"datasourceWarning\":false,\"hasCustomLabel\":false,\"label\":\"OBS_VALUE\",\"optionName\":\"metric_2pzqjh4ddnr_9fsj62hsc9c\"}],\"groupby\":[\"SEX\"],\"adhoc_filters\":[],\"order_desc\":true,\"row_limit\":10000,\"truncate_metric\":true,\"show_empty_columns\":true,\"comparison_type\":\"values\",\"annotation_layers\":[],\"forecastPeriods\":10,\"forecastInterval\":0.8,\"x_axis_title_margin\":15,\"y_axis_title_margin\":15,\"y_axis_title_position\":\"Left\",\"sort_series_type\":\"sum\",\"color_scheme\":\"supersetColors\",\"seriesType\":\"line\",\"show_value\":false,\"only_total\":true,\"opacity\":0.2,\"markerSize\":6,\"zoomable\":false,\"show_legend\":true,\"legendType\":\"scroll\",\"legendOrientation\":\"top\",\"x_axis_time_format\":\"smart_date\",\"rich_tooltip\":true,\"tooltipTimeFormat\":\"smart_date\",\"y_axis_format\":\"SMART_NUMBER\",\"y_axis_bounds\":[null,null],\"extra_form_data\":{},\"dashboards\":[ " + str(dashboard['id']) + " ],\"force\":false,\"result_format\":\"json\",\"result_type\":\"full\"},\"result_format\":\"json\",\"result_type\":\"full\"}", + "query_context": "", + "slice_name": row["Title"], + "datasource_type": "table", + "viz_type": "echarts_timeseries_line", + }, + headers={"Authorization": f"Bearer {tokens['access_token']}"}, + ) + elif "BARS" in row["chartType"]: + response = requests.post( + base_url + "api/v1/chart/", + json={ + "cache_timeout": 0, + "certification_details": None, + "certified_by": None, + "dashboards": [dashboard["id"]], + "description": None, + "is_managed_externally": True, + "owners": [1], + "datasource_id": datasets[idx]["id"], + "params": "{\"datasource\":\"" + str(datasets[idx]["id"]) + "__table\",\"viz_type\":\"echarts_timeseries_bar\",\"x_axis\":\"" + row['xAxisConcept'] + "\",\"time_grain_sqla\":\"P1D\",\"x_axis_sort\":\"OBS_VALUE\",\"x_axis_sort_asc\":false,\"x_axis_sort_series\":\"name\",\"x_axis_sort_series_ascending\":true,\"metrics\":[{\"expressionType\":\"SQL\",\"sqlExpression\":\"OBS_VALUE\",\"column\":null,\"aggregate\":null,\"datasourceWarning\":false,\"hasCustomLabel\":false,\"label\":\"OBS_VALUE\",\"optionName\":\"metric_qll9scn97y_rj89e0zvnw\"}],\"groupby\":[],\"adhoc_filters\":[],\"order_desc\":true,\"row_limit\":10000,\"truncate_metric\":true,\"show_empty_columns\":true,\"comparison_type\":\"values\",\"annotation_layers\":[],\"forecastPeriods\":10,\"forecastInterval\":0.8,\"orientation\":\"vertical\",\"x_axis_title_margin\":15,\"y_axis_title_margin\":15,\"y_axis_title_position\":\"Left\",\"sort_series_type\":\"sum\",\"color_scheme\":\"supersetColors\",\"only_total\":true,\"show_legend\":true,\"legendType\":\"scroll\",\"legendOrientation\":\"top\",\"x_axis_time_format\":\"smart_date\",\"y_axis_format\":\"SMART_NUMBER\",\"y_axis_bounds\":[null,null],\"rich_tooltip\":true,\"tooltipTimeFormat\":\"smart_date\",\"extra_form_data\":{},\"dashboards\":[]}", + "query_context": "{\"datasource\":{\"id\":" + str(datasets[idx]["id"]) + ",\"type\":\"table\"},\"force\":false,\"queries\":[{\"filters\":[],\"extras\":{\"having\":\"\",\"where\":\"\"},\"applied_time_extras\":{},\"columns\":[{\"timeGrain\":\"P1D\",\"columnType\":\"BASE_AXIS\",\"sqlExpression\":\"" + row['xAxisConcept'] + "\",\"label\":\"" + row['xAxisConcept'] + "\",\"expressionType\":\"SQL\"}],\"metrics\":[{\"expressionType\":\"SQL\",\"sqlExpression\":\"OBS_VALUE\",\"column\":null,\"aggregate\":null,\"datasourceWarning\":false,\"hasCustomLabel\":false,\"label\":\"OBS_VALUE\",\"optionName\":\"metric_qll9scn97y_rj89e0zvnw\"}],\"orderby\":[[{\"expressionType\":\"SQL\",\"sqlExpression\":\"OBS_VALUE\",\"column\":null,\"aggregate\":null,\"datasourceWarning\":false,\"hasCustomLabel\":false,\"label\":\"OBS_VALUE\",\"optionName\":\"metric_qll9scn97y_rj89e0zvnw\"},false]],\"annotation_layers\":[],\"row_limit\":10000,\"series_columns\":[],\"series_limit\":0,\"order_desc\":true,\"url_params\":{},\"custom_params\":{},\"custom_form_data\":{},\"time_offsets\":[],\"post_processing\":[{\"operation\":\"pivot\",\"options\":{\"index\":[\"" + row['xAxisConcept'] + "\"],\"columns\":[],\"aggregates\":{\"OBS_VALUE\":{\"operator\":\"mean\"}},\"drop_missing_columns\":false}},{\"operation\":\"sort\",\"options\":{\"by\":\"OBS_VALUE\",\"ascending\":false}},{\"operation\":\"flatten\"}]}],\"form_data\":{\"datasource\":\"" + str(datasets[idx]["id"]) + "__table\",\"viz_type\":\"echarts_timeseries_bar\",\"x_axis\":\"" + row['xAxisConcept'] + "\",\"time_grain_sqla\":\"P1D\",\"x_axis_sort\":\"OBS_VALUE\",\"x_axis_sort_asc\":false,\"x_axis_sort_series\":\"name\",\"x_axis_sort_series_ascending\":true,\"metrics\":[{\"expressionType\":\"SQL\",\"sqlExpression\":\"OBS_VALUE\",\"column\":null,\"aggregate\":null,\"datasourceWarning\":false,\"hasCustomLabel\":false,\"label\":\"OBS_VALUE\",\"optionName\":\"metric_qll9scn97y_rj89e0zvnw\"}],\"groupby\":[],\"adhoc_filters\":[],\"order_desc\":true,\"row_limit\":10000,\"truncate_metric\":true,\"show_empty_columns\":true,\"comparison_type\":\"values\",\"annotation_layers\":[],\"forecastPeriods\":10,\"forecastInterval\":0.8,\"orientation\":\"vertical\",\"x_axis_title_margin\":15,\"y_axis_title_margin\":15,\"y_axis_title_position\":\"Left\",\"sort_series_type\":\"sum\",\"color_scheme\":\"supersetColors\",\"only_total\":true,\"show_legend\":true,\"legendType\":\"scroll\",\"legendOrientation\":\"top\",\"x_axis_time_format\":\"smart_date\",\"y_axis_format\":\"SMART_NUMBER\",\"y_axis_bounds\":[null,null],\"rich_tooltip\":true,\"tooltipTimeFormat\":\"smart_date\",\"extra_form_data\":{},\"dashboards\":[" + str(dashboard['id']) + "],\"force\":false,\"result_format\":\"json\",\"result_type\":\"full\"},\"result_format\":\"json\",\"result_type\":\"full\"}", + "query_context": "", + "slice_name": row["Title"], + "datasource_type": "table", + "viz_type": "echarts_timeseries_bar", + }, + headers={"Authorization": f"Bearer {tokens['access_token']}"}, + ) class Api(BaseSupersetView): query_context_factory = None + @event_logger.log_this + @api + @handle_api_exception + @expose("/v1/sdmx/dashboard", methods=("POST",)) + def sdmx_upload_dashboard(self) -> FlaskResponse: + try: + uploaded_file = request.files['file'] + file_content = uploaded_file.read() + spec = yaml.safe_load(file_content) + base_url = request.url_root + + tokens = requests.post( + base_url + "api/v1/security/login", + json={ + "password": "admin", + "provider": "db", + "refresh": False, + "username": "admin", + }, + ).json() + + datasets = load_databases(spec, base_url, tokens) + + dashboard = create_dashboard(spec, base_url, tokens) + + create_charts(spec, datasets, dashboard, base_url, tokens) + + return self.json_response({"status": "OK"}) + except Exception as e: + raise e + return self.json_response( + json.dumps({'error': str(e)}), + status=500, + ) + + @event_logger.log_this + @api + @handle_api_exception + @expose("/v1/sdmx/", methods=("POST",)) + def sdmx_upload(self) -> FlaskResponse: + try: + data = request.json + sdmx_url = data["sdmxUrl"] + base_url = request.url_root + + message = read_sdmx( + sdmx_url + ) + + dataset_uuid = uuid.uuid4() + + engine = create_engine(f"sqlite:///dbs/{dataset_uuid}", echo=False) + + for dataset in message.payload.keys(): + df = message.payload[dataset].data + df.to_sql(str(dataset) + " " + str(datetime.datetime.now()), con=engine) + + tokens = requests.post( + base_url + "api/v1/security/login", + json={ + "password": "admin", + "provider": "db", + "refresh": False, + "username": "admin", + }, + ).json() + + response = requests.post( + base_url + "api/v1/database/", + json={ + "sqlalchemy_uri": f"sqlite:///dbs/{dataset_uuid}", + "database_name": f"{dataset_uuid}", + }, + headers={"Authorization": f"Bearer {tokens['access_token']}"}, + ) + + database_id = response.json()["id"] + + tables = requests.get( + base_url + f"/api/v1/database/{database_id}/tables/?q=(schema_name:main)", + headers={"Authorization": f"Bearer {tokens['access_token']}"}, + ).json()["result"] + + datasets = [] + for table in tables: + response = requests.post( + base_url + "/api/v1/dataset/", + json={ + "table_name": f"{table['value']}", + "database": f"{database_id}", + "schema": "main", + "is_sdmx": True, + "sdmx_url": sdmx_url, + }, + headers={"Authorization": f"Bearer {tokens['access_token']}"}, + ) + datasets.append( + { + "dataset_id": response.json()["id"], + "table_name": response.json()["result"]["table_name"], + } + ) + if not sdmx_url: + return self.json_response( + {'error': 'sdmxUrl is required'}, + status=400, + ) + return self.json_response({"status": "OK"}) + except Exception as e: + return self.json_response( + json.dumps({'error': str(e)}), + status=500, + ) + + @event_logger.log_this @api @handle_api_exception From c3c7ba141957fe5811ac60ab993dfc255b78affd Mon Sep 17 00:00:00 2001 From: "andres.sole" Date: Wed, 6 Sep 2023 15:44:43 +0200 Subject: [PATCH 02/26] feat sdmx: added logo.png --- superset-frontend/src/assets/logo.png | Bin 0 -> 21073 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 superset-frontend/src/assets/logo.png diff --git a/superset-frontend/src/assets/logo.png b/superset-frontend/src/assets/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..cd5d475bf484414f217d3e848e012b2831cc954e GIT binary patch literal 21073 zcmeEtWmH{F(k>*p1PksG++7at?ry=|-7UClf(Hu@L4&)KKyY`5AP0Bu<{kNF*8Tp> zTKE1NvLJ`OyQ{matGnuXs)$rlkVHnnLx6yQK$eygQ-Oeh>;^uY!@&SQ>t<+y7bue6 zYFe%;Mjj-N&JN~Qwq_)*UXErYW}a5&5D=b=Md>!~1g*%QUT$b>p*n;DM>pPrsQ8A? zFY-8?tNgNw6)0@XmyItV-;1EEg5jX zo^UUl@DP8qW)M*OcpiOk@^n6zn&~Hynp^$(0rQ6H3H|A|`DWg4{3nh1(WjPxcUOLW zHyB%3vVD`+-y3pLh>Z~ZI$_sf%D9xct&00^#>lZJ-CRbg#(nELPx%}g=c5G0@03Ou zui7ouN2yySo^^cpbW5IqxHSHhr@(P8Mf~ii>m8En$o)EkzkYgZ*hyBMiXf5p+C^hiuu;*d}G zHuR<%@29r_1}a$^7H2XYE{RRE%n6SbO$J*KJ39MyKa0*UQl!oI0lkxh#Cn-2YH_`%rjaV#9ATHYur-ddLN zE?c8AU5ksO&d4Q2nN%LvZ~r^XKvo4$x|j2fX;)TxMayE6#Z}Shtd`5$+?B>Z9d1iR zJ~_^ZL4pc`-hX1)`1zF1cQ;pNeua-CPTTw0w>m$}*mx}8k=x|fKOaA!)TRk)(>Z)I z{`x$Gr>WjqJS0qo&O>grW0f9bQwSmSc}Tl!7dnUd#@us_=eeK!i7kRZm)9Lue&Srq6J$59zosVx*CkOgZN{ZRJnMD~m7Qq4}9fAlL8qqy#FWcV(I(PyLQVk^;?qt7gt z2Bh-XCXxq#bmElqu+}aS8I|SkciD(XCfhj#xt#c=Tzz~h^H7$Ut7KEr<@dF;CFdHB zZ+fkNNk4*Q z|6rH4_t-Ok05A; zcG@UP4ey}LiX`lbd7#QRKI5BZyhEwUO^Zf0^LKfJ?xlmzyfA~wD!=K?LeNW@D7Y7# z>-;0CPTevuVX=427``L?QH#1eU}Oua(|w26h?N%my;>EP`?th|azHIT9|XS-LZg$Q zd#qM26cP#I2cNey)Ue;DCBi%ev*`(q^%f;E`A70A4*apu~aP zqYyd#iYRRcfqLg`h-UWzDjirb>3dE9dR@Ry>8(n$b)ZxMQC1 zqEj?1dx8PZh<;+_8A`~HGEtLc(y}gD=D*%&()5`n2f4MMXs^EMkXmnNP_g-xfY*W& zCtYi>k3#o8gnQQ7ZyBcT=4L{Okm5n(r4i)gOZMjMqjBV}l3nV}9J%<{N9PvAI0fS3 zmIg1owtg4)ESY5)!;A05N!SuZv}36Z@c{35Ue>CZ_Zavydf(Sf}G8iNgCpjHk>UwYWoOJ*znL@5aoZEu3gl zig7^m$L#F@T(86_5{UiZla67 z=gYNinb62*Vr|T-)@+hM5pfA&Q$9Rh6%F%mRHK{<;w<<*Ysh<$1>KlVR!S+7Kt~dP*fK z19V>@0u)zh-*eaAxOji%@C`Gi-b%S7k>3(;&{ls~gm%FSPC}T8!cU(}ciDJzPUBf3 z)`nV|5fv|YSKCP5m*UT*qvVKGUCLGla+lSg`c1icY=>hp3I4(6$4Qp!`IF4zi4#^JIC1Y; ze<*+Xu;GcpX{foys*|+|X5GAya`x7OxG3eb*U(!E6Gw(n% zekkBBv@nJXV`V`V8kOmGfQt%4S)7*qqBDH*eTBO})3;9q|Wj!c-<2H3J z#|bWsq#L>a5I(Ng_RZ2d8RsuoHOCohN2@FdJ$M0*+G$39GEfYG&5}{AiCjyq4lKh`4 z5FI&pErmUzO}|FqtW$A0+p@o%CV3K_hc1_9N&=lZouQ7s`E6c->W&_8if0M>ndU9M z{WjqEUI@%42Zb02`7VjIQk0hwe^Lj84O#~>>$8~Fh9mF&vBxF?hY|Hzu z9Pt8uKN5FOm^lBlPQ(u36u$7QA1*!J6ZA%8AU$*L#>l&#Elf88C}iOoeItcD0ro$F z+mkU9etl*9T&)KCSyi!IC^#aYXA_Ro6Pf?(($H?nO0M1O%MAFIIOaVG=2Dn=3# zj6rWxGH>15B5|0z#wqmF_42`8lIqsOk>?~xcdyP>$Td06(!EV2@Q;zG?sjQ#7+NBx z$9zLd;?b$XA)4bRk!xBhV(Y0#Z&5TAIJ%F>O1eu6ErHl4tlGp6o1mqq)!u6ytZR5J z5Dmhrq-yM#a-Y_96UGWhf36ldxZ{+0>8DDxIHLB1KmDxC>t5f>Ru?Ut++V2%k#ddH zSA!g)pGT2^i~OMoLxGo;2p1l+TSyl}^kA4b&DT)l$864h2=%L zw(CxR6^X1j2CCn(wx93={Wyfz9!WntduSXo(uCO(UF*s3e#lIwrE?Z!PV8U8{*^A> zp9$c;+#gV1eq+!VrkteKe|{Jm7$yT1rVG&?>Whedyj2=Y+P`9Ez;kBYfSX_W2E*cF zvCEvSy_)Ky z{6r}Bc2h8HVl>N8?6;t_4zvr3hG;Osej0TA>FS11i z>gMd!+(24ev)LjMUUnPNh@=WJ$mYOfD7XCR^}N)N-o+AGnCRdtdcqg;sQx4rHt`|- zBbNDSofCohuZdZKhnR4I?`ZlR5Hvb(a?C6MgrdtMw*EM;>mKx1tkQk3w zxo-sxcGkegI|VJ#0JSBfdWV*%%ng3GTMftG|s{rr_PUK z^NdXmjjWjB4w^-)2MyNW^6F&q=3zb(n>CU~FkcE|q<)Krd9RAu=m}Zoxz3JqTQAxQ zZYqYOp43Xzh!e9&&(pyPm%&%QRrzWJM}YVyi3%oXQmp$B(S1<1`$3VK^wW+U^2U}O zYiYMBYr<@c$TJUb7?Q7349;$}>RaD^ANC?ps6+eC9o1<_DVshb%5|FdeV#}Orj8qP^8sQ8I|U76T(C`d z_n`4icfYnKtfnxkZ(vgkH%*IF6Y-q)v^RZZauBSdQj!$GnhmV8A(bXw$4ku%)~9c&Q0Kaw?@I;(dL)db?H zNyQOAqWF)`(ksZfNGDSKzA#=QE)cTBw&bvVc$jb0r4m}lZJOkL*1?eSO^kz~I5#gd*)Chrkfgj}>AI zM?|M5$2m)#{Kt1s5IhRb2{#t=B>?w@s`HnXs@=O;UD?Ia_yGUX@JdgA8-6k^c_k822WK-9c1CtaW(ILj zD|c2h0R$3|v#B|+ikQUTMF79?lUcgDI`T3xd3bm*dayA%I9o8W@bK_3F|#tUvN8ZI z7+k#UU5z{$>|MxTMf^>Mn3;=-vz4Q(m4iLWt4t$f2RB!KGBV&j$v@_2=O{1#Px1CH zf7b$_4<=6|M;wtVA2>JVj{?{HZY5-Nlq+;gc;O1;%Chl%#?@Io! zB1}#G+27I4+4isHn3^z|*_zn_O^NKl`xV=tKT8y9Ub$nh^2NNq(-oHMY znX#I4n6R-i@NjdoGO%+SvoP?mn6NRh8kw1Km^1TmvKgEHizsP(7gr;D6SG%QfN(}D zKn{<&84Ifkml1<0D~l-uyP2s8gAtDj2LlHy54#B$3$r;Z7stPdP;|BetkTH#UuX3y z$`laA&dzRTWX#IOz|6tJ#=vgE!OFnH!@ZQ%tL11Q$$)lLBce~AUw!Yk@* zX5{MNtmfcg%TM+?C6d>kf0;K4=x>`MW#t02@OriQzuUa3nbY5X``Zz)wfgH73CUk( z%WGuvw?SNt+|B;BBS62uy)v;hvbQh;&iCI9>L1Uo{+G?-He%&rW9DRLFfwE3VPH36 z<6+=tVK-+m=3zB9S5$;CTsz$6j%*lK!2@F*0>dOg+uSTTs{|I!KQuLl2e4FLWA z_6)dQfZK}cpV!sjb@r;p|BqjP-;4i`BLJfRJIH^e-~X!Xzv}vrH1Hoa{;zfYS6%;+ z2L7YQ|Fy3FXX--uAHZW~4@5y80Gw&JeR~B!AsAy>Nim3**MDEzi<5yT@Qza2E)Woi z7_Z-u5E)rGz(ZJ9X?bzjJ$NXDxA1sDc*qbCBoNYK!fKw2$9^831cQmE4%n@5(OE7e z!ccEWoQ=OpC8CFzPHKQ;(=Z86eY>yuvyX2~*Z8y>Pfb@27EZ^MlgzYh6&pXmM5c(b zKvLkKN<|Ificvc2K(uzP68BYMN5k{8@w2tA&)(b>#AF{Ltgz|N%%~UDXwhRebtBt5 zhTtzd!Um@u#A8K6&$cI;#+!QQpo~Y6hsB|SSX{qsJ4*1wrApF(G`AJ2uBbGK*>B&= zD$@9ku$(%Nkt2I$LmHMnOI=cj64zXig$YLYzlmMKa0P^}!*Dh63nQepX(0;zyij4F zYTil2S7hKFE9so!?i>rI`XNx{S*3ETF56juwzDaE>{M6y%SFN3|$6a#W+&W`e6f zJ@2ebR)N1b*88@8_%;8t>-N3D^BIU%r)9lzZUd1os)o-c+xW+E-%!!?F{sGcs617W z=AztOZbE&(lp|n_2r-qmTF+T_LsY>l%MA43QcG95s zKpnBBWbMNuc|_!aTD|K=EFuRp13ei7FG0M+h^XmT;?>UC2#+#ryj$~~j5v@cRHW*oC0Xzy3AdkgWtI<^->7h%2hLa9I4FqVbZBSsM~K;|eUExh8Buk+8`x^p^lV9%MUfI^77}{$*YujF8o=|Hy|0G zlX9oZ+9(@+V%_z*R$&`&Fnw?e9_Stvxxmw=p%IF|@)lyqK5(*DYdb-*6elB7(EFl1 zO+W*mUni$rrG~%MRr9R?Lwxiu_ZTQA&Rwqfcoxopsakk^pS$gf7t}$6(mxCiHkK6G zpNNWt2TLupuzTMpyZFychx5fNw)hvG$*L+^ugmV_Ubd_9#w%5UZ!h~lRL7ai6_*vj zR)5wsF<<>O1R)5j`9MsgY$V24KM%sxVPW9`#qiODQKktt?`bmlHC@~m9!Vl})?XE} zcc?u}ukjUK!0q70D;nyZA6jZZ6R%hf1QA zE;*G434Q=uu0`oGTuy%xtT~Fo(&quu6rB}~j28zq+$MU9h^ZP7B9FeHe6HFuwL=VC z;u6YT)_}{_pc_6{9(k+7;1yQYm0P#RF?p7fm7RlkLDl~@Pkc@t{BIJn-^Uw`wFPR9 ztLL}WVM92MhrI_W)dx)(JMQA-q7rc2wyK)=s_<*z-8)Knt6b5{zR~0zPlV|xpMxvP z6GRuV1e0q%6URCc4$eh3fF1^m&qkVMT#;hmKoHUV>aL10HHh9&hz74%Jq{CI=p8rk z>T|>^BFTRM>$gi#BR?gc2}ct*pTAhf+Pq|H$mLALoW_-w+e$I2 zbE{z>#9(eWN{>Ue@QKJW`%7HqB;N5_Zx7QXN`5o99mc)NgW+%36F&Ho)H+ZMV&Q3~ z?fJbc8;4VRFyZLD=ej4y5{lxoJ_%Xk_iK z!2XteO2XL2uoNWP%&%Vhm^fhjZEtsig*~cjgC#}|m&HD-zs(D>yF&mDd?Eb38x{@H zUrK*u>wWpnw`bDdfgjhF9)HgcUM_=LBnP|>lHJ+!(D1V|G9^(+74c$W^m|R1$1**0 z72yzr;|-XGoXsbT{S@Un&nAx&-eb|<`z4hJe5G4=sWd4v`BcCCw;8LAC)~Q1xV>He zoE#0(6Gu7aA!b-s;fjHPYJXQFRLB-&lBUp2FYl8sYRp+-CDElu$hB{PvyZ+{XM2%<3NTT9(BU zR5ne(@6`RNb-)+xtcS~1JMUo1KxY4jWHlzUMfw9=EEU7_cU#!l1XBIzc?Ws${B>#3 zf+}qV99P^bs!UpO={y)??DZRRqWENQpU$MD-_>rC4GUa;52Qp={Pne>=B-&QAw-~C zVjj0zLi11E)O<J<12sqmBFt~6QxP1OlEF6Ud;v`6tfsJ9grnXDX?=CJ2%F`$?X{>&XJUM1`*K= zhvojj#*~$%uSY`KT3Zzi&u?(@tY?RPUNxxI{={E^QF5{`?yrKRj<}GhNE#N!zcR?R zlV|o3XG`KF`7;;e^hbR$IFU!B;M**=%*ZzRkYb|TH5H+@s=NJMz#*5mXe?)BNqp_b zZ-$@nY@%5DaB?H}CiuARor;0gmeLPG~dUw<1&rZGT_=13XT z#oa^n91A;-ydzRZyif=ixMes0WwzhOb4vu-dyw*@j#k(2ZFYs(#&y`iv97izYOPL{ zJ;^9ZDwr1?P2wCuXM|sGMZd6RvheVQ+vxkOh+5j-|4>AR>Ov=oAQHM zHi6NCsq`K{9V7bE)ZE!7`xVMY-xZ~?f3^EnUR!C%P4i|0m|71*cE|6ju$8#@PzN`_ z2A`n&VGhsiMPVp%oNEj?{em7^qZ(gwG6(7QXfaewm$O94)c)lN?(Vp80L47-Lo<@zM$G z{yuT;wx7S`y^ilUYQ$CVm8dmq9~h;DRZ&+Y_oX9~@2v2kZbfW!V+VJYIQ@d8?$L$2Jt4w9ZVW#`}qJHzjB zzcaF?%xp2pD}42SNJ$%%T_R68q(QhcNFSK1%`TN$n2^r#n8hi?X-OyH8Ay1+V5MebL^qjbB#dXVyvr8s4^ zvdwKIP7+8649AIv4u8(X2?@CSNi?7aE%_y#YojvCX4)b0O_y?SW|UY`6s90t=i3(^FL-|Us5o0Rt=yOv%~3sbYGyGoGo|j%jTFjbJD$)j&Y5Ut+rtQ8J`B)*wDHs)ntaZ+0S z*~>1|u_i_wyh6ty2!xHUJ-bfD&+;^3tGhkEv-UlxLI)rrK~XcNm-SQ4?ERnHL&r z(MG`ljvGnh^1QTeU%zWC_Gu^|pLIz7)^DrmF&?J2l;tFB=7R#DtKYob#&p5s&@A1M zfHS7iu2wlK+HEh5RcuixJj;U-H0C#CsU30Rn*j&tbky16S*%f5)T_8VlRnafQl-6v z_m8L!x=HVyam(MnU(AWUl{3O3qFwzu;>$DXV=OCh$^pJ#6$ZS%R0bC%PHwu{FEjFq zz1H{bLRDAim)9bm2d-^R4tgb|We9H}3>TH{vLgBbebOA|v?Yvn! zRw^Y*Py%s03B;N z@9})qMzhgXRwhDLv}vYIh)(rwx@PMsW@Ep^uBJBHU&bgX1z4re$^ym}pJKknwKJ2| zbg>JaS3p_nTy!LTt%V2R2-JiEZfK*WC6KLlwUO zTXe>l!SoGszK`jf5De9`n`;rkv!g+FNW~(FYUsTOq9CXX!Ori2izkqJ$A`T$V+J_# z=AMTj)b*`ck`OK+e*4bC18MZ!(RMMQE;24z-Fs5;-hf}{C7Uht)qNasJvh0TB4G^h z#3?2`ffm!t?WNcXN>*{+?cWh^66Twu>bFR9u8Zr`R&whilq)^~{ z$qR+&i+X4wJz`s0?aq_?@8G4#;N zV2nAEN|k_(%svwhJr3Zrnue$1_O8yYr>~Gc0{1SuNQl_QU>MbO28OJIg#_y@c7Lcb|>M_S#)}#h`9Cenpj+*cxhE`Z~M61Hys*Ml6OVSHkp~ zL?9BwPYil)Yg7C+wl&`XlVxLR=#*!<>f5@#1YDFlFkw!GEks8qWM@LkX~*p*laQZ5 z!Z5_M92SCcRz{Fi0#-e+x$GvZZRhz@L-l9?PSDK@^W(AFC&Jy+qXGhFxJxkl<@05n z!aBAEBO9K-L^AFGSo$(vF~89QLelyWjc_|YT(&k57k{`3p6Ek`nwyPt#+7!GJZjg- zmQqFogH^8lRE+RQcW%;sYxs`n4qWOn|Ly1A;Cpo6}AQF z-PS8;g;5rRdnVXo@JaBeuaVXzPqVkK<{t*Bz>kTlw@r&6*vboE`rV?3>7d89+{e2e=CA1wNP=y=cka1UAHp}tSQZeB%o<Erfd zf9{u$*7iA5`1O+%Rx+)ERQhU0rju89QdaLAFu$4^lok^y(t9NUfwzk9V$N5!rr(pz z!7sDjXh8I!f*WAugw~T~z8fxC9D7@>ifuLnJ>?;#(6qc7>s6=CseLbm--*XPlw>wlKRafbV92?Kd#^WwkzYh1A3Cprc{B zsbYgo#lV#PF49OC2%VhAe1MzyrO5*yY^TQP6`<*asam%TJ#G^6*3n~^mm(AzV1=-= zdrhbv4u!;JKYvs-g!^=Rzs@GEJtQ9yR6N#eZrZ8sgj^Z&M_+hX&Yb-cnUqgJ#yvhw znvrns9j(O;bg8GzIn}j$LDKzFcJsRrHe29sDtYd?c~WmS0a_9Sg@p z(F!xl2IugEaTpqVJt-1|BxB{7ZvPtNSrj^ay-LFR* ztl|Jrw#kAEz-`84`b4&nZyvH>G_rd zrQmC`nvdQfqM0RgrVSal*W;TOl!B9i?rGfzlN#scedIcyo*08o)COzV>46CRSr+MG zNnT3iFi$cr;B$1qH@#3?BK@90#`aUs z*CM~7mLl{%vZsT~>r!j`Y8TEzb15AKI|V4;kov)U)D;v-WUYN5vSOc8#`DgWa#u=Y`3bpc8K8bY6R*M>` z(C@W<6A!)EU!}hHp)s{%jjhJ8h8~(QR74{DaHp!uSo63LKu`g0PxrzALUpB;g=jjv z6|t?Hrqxf&pY}2JDcSLm>+u*M4P7tS<_FgzNux*)v`fsq3=Mc4P9_H%D(90^jcp!v zR;Po|a_BBMo;&6@mDI`jZRaCc7jmkzfc2LF@2rY5#;R_}0%hQWYg_AW-CE}0b%2?c;!16tJr%yD0?+JA88 z#vxtywhSrddo^?MrOH@f&{2fJfXBFUPXJ4qSd*nWMKL*6I>5?2f4b_^`6=@o!^cPA zT{BXS_xq~IUTlx|TMJK1iI|e^LXKoc^q%iq!p1dxY%nXuevH6ySpJW5SqW z;c64i>U+iYmvaO-$-X|7cv+pCm{ct6?GFQ}oeub$`P}i0&_2Pugw<^Vpi6Ukb8V1t z5(fXOyi%#Z-WbsqhODCPPci%E*Bw5+wjC%&WQ*h^`6l~Dg|0t2Zh`|euoMe`us(Nk zdD;X5U4PF4+^qra6b|<@N)<`iJCm&zvDXjGD;wwn592+_zF&b6^J4Dx z*)Kjrp*Gg=;xx**nrEmLd$QKwhF{kA_l$Jf*XJ!AcC-7N2@Y77QNlI}PW` z@EWAk!;p;Snk*Ovh(WO#qj%fiMg?A|Y;*xYC0qLk2$XNPQt^`%m@Nlgw{H*9gh# zAgY#_m8k{rf3VS^T@Xw}9!Kl+`Uv-8_t}Lqtn5I@@~G&YhDzf z_+~zD3A}MQ12BX`bIJVr(r3HvjDYp)nP)-0mN7m917jr4D%o|v6E%H0ywAoG&h46g zR!p3}wX7!W4>#H`4;x9gk23>$w_b?Vql>)UwH3B_JjA0rVDL6$9hU_wVg~-=@IgIa z@92zTLgYK;fif*crHUQVUgNZ>$@ey-Q?rtyYYdNruzTQ%*j+P&KqbGv(6b-rwP_T| zz4)eEE&wxGzE|csxRBT}>nKVYZfvjs-z5S8 zu~@O?#>O_VEvlq8sRAwh;^h#_R~G10 zgFF`fpg9s=w6ZNRQk-=s)Qx>r7>J6loGQyv1O*Jt%ZXlI6g9q@eblNInD2@v{mxKm z9P`^N6QIkY2S?LVMNC547fNf}2w>qrri>9N@L~WXI)hVB{+wI2EW{*y^*ruC6PWvh z%J)8x%^0vvuIp z@%I=3XaiwgvaW}E>&;QC;u|1PSMDF!c@(IHtmGElqogh_{-%+Wg(}dW#ZQ+$H&hru za-D+-?P?)!{*ElJq=fx5yG&x4`DG%$zu+#o#{uz$>(mI?0w|CeL>(66kW@Nxe zPV?F=i6(U8^RXp~ck00+vhwU19D3Uv5*z>2cKswz6WIt-j_wb>+;=2v1~9YI)B+vP zH+ia;1SYB(k>e+G{4FVYe)G?aC4JDa>mAEixAy?WFg18O_~P?NVD;J@z&!Bl7I&v{ z2Pkn$OKgFF780{84wp>%v`sR7YPQaBaa_#V_U_5@%2TIQRawI%%3pGobYzaUjF03W zA1^=5TDJj5CRY}h$--b%q zqInFw8kUxwTiX>6yszZhM@725hJ@p?6R<9yY4>yC7Be1zb{A?oQ`} zj}LftAEc*0&UXT2G8{-fbxxC0I2@om)Z%n8rh@7ExPuksDus?Zjjb(5w0%_NVijv< z<0c!hfMg{tpEtKpuJ#6S4gv2|!}9^H-)0MyX#Gd90pK+h=6;}2X;@H2(AoRgIOD|< z{J@H(WRob4-PfT#m$fB5biEy^LI>!{q}>pdH-2qSbN3BmTxy^FX};k@Kwu2L)&lqh z;bvVh_0vsQvU}ET(mDhSK!6$=AOKns;c1^AboTR_41f^qRL`%86{hL|O^?*5+3NY5 zq+zzie6E_;T$`RA?Y9+H=9mEgJrj^hMl8TS(QO|P5n2!qnhcY?AC=w6&~inNuX#2A zoM6h3(kw_5pldX&pQ&s>ACIz74oV&Y$Am?I5Rs3adHXunF-NpwAc80 zGOdM%`M^TnlNq>>FDM@j0YYM@(LRM@P}W2%@tp?7BZW#U-zeZLko~4{2SGT?4ooNXq4V zVtk%8iCQ>wPP(-F9^K+;-LEUza=83lX@c zUm|R+wqKiBq@tj=e}hrji3OapVJq8Z0^Gq<+A8#Ti%#?Yq4JxvuDoEm$1sa$#hHdp zUsWoAbh0J-_`qNkR{bRg0TNTk5bNp3;&jI8wntaMhqf(TdGj_61BTN4aDK)o|6tcSV}Q+bdf`Wh zySNd5r?_Q~?$uaGiMHcCmj27~kXS_ZC1D*o!LE`TQm$TV8}r@|ON}M$m$iGC&KGvD zGdrm{q4EZy2YRDs|`80u{p1A|c?6 z0B1+kjGn3eU4YAtgmoK#2T%Om@BHKJQfWHc-^92zX!(O%t^8%F&Z_by7F6M(3R4#; z#?2*(T%G}|-oAD>l-5rp6{sht(GDjoQ58*63IeIz?~pHU`d_2E8hNj*AM8ba%u;is z0DCspR`hdTYPI~2aUKA_1KON{gb;9i2!4*E(w_qF^Rf6G44Z?xi}1aS=c=1L*EgJKKO8AU}ArK3o!DpfwpnT+mhi|6X3eG@At>WW1kXe{wjW?oCpJ7L$!bMLiMXI zp#TM+rXATS(Q~8Q=dAhHT^oFZFDmAK{2=GfMkwrQw+>&ttTlrK%Xd;ngx8g~oplkt zuZ&Gr&y{PEyM)4CwQCASI#F7{?1@FeyiL7B~hk+ zJ}smHn91>zwK#xbeq~h~syZ+~?*=0Y49UhwY-Dg3N`!9k1w1As*M8xYWw%d0zbcZ8 ze>ZqTJRfrY?Om3`iK!nIs|k6HAXluf!}n#FrFh#P?Hy+$xsVWR%dMDJ6C2a#+n=;> z0R-XFs#c*V=UIQyfE@SH)Ej1NfahoW&$FvZ@~%OA(hdPBBt&C8wdn#NDPiq)}xb?K;fCIC2dC(wBM}d63NpT$j)c?r|E4l6hj@5Iij9(kk z;^pc{k{fL3|3^SqCRM3@YGmp|Vu1PBr;`b=Sz8|zEL;_Ng-PAz&f;C_&le0)#j?Ev z+XksQiBBIHM4)_xs>q0cU5U74)5vQABwbjK$ElFgJ+M{D{n-h zL~&|k_r8^^_J(-8-JayD#6U&ALpJcv=jJ>u{q%%e`nyhFGXh*X@iLzNfXo~1D>HDL zr7tMgWIbBm$4fDhsdAsp8vqnxWNSwEla(a08irD{V7Gp8Rb=B-VC{JC%}bM(P&arh zbtZ?1h?C99SY9AicOEm3lbeNA!8TOv8Z(}L)q9ntb52|HPWtj&%5&iuv7MLHo|~;X z7E?C$AdnaPqt%(_Qc;tx4eZmluY?@!o6mCfv|~eVNgpquigak%=_Pp%fQmKOe1h?( zJBk4dbF-p49q{&8v}@y4K6|Ikc{?6BHAgUmh&5LaNY{Hl577Yh?WPB&rWcdq)1PT^ z`ZKk|;n?^vwS)&w9y>)P*-;hcmKS2=Hh{WK^1->D~^ z&O}sSK7wOKWvh0Bj2`#r)6Lw(40#R0#`02kPJz{W9*FU5#cKEfCMc|yAc#30>uxAK z3GV+r!J>_S-KSwx)4y<6Fja9~ydSNWc^h=PoP7OBG^$C4ON+)H&ZDw&aTJ~vrnr?C>X z*=m{6?BZE!E;sJj6{-ta&|lYx>6)EG8~w8+~&qH z4fE902<^bA?q$KtXOG*YpAIs!!m!f*pKYiB;U=Kuub69Z`gL@ z@a4Qq_0HW+2!{~_P%tVc_0!NK5pe#%2R-x(&)ZEq(l&aeM`4jUzz{R3HX)2U^tZi5 zN4HN!N8I{rFk5X_{TI9a;`K~dvoZ#YP$)pj%4=Dcao&0Lr}} zzEbcI3CIrTUUS=pkZbAHlhFQN)5t&S&PP7GxE34yP2KS@CJpfJQngYuBlxa6AORi* zE6tz11DESzNCXTS`$y$z7PEYG%pLa#`MJHRZv zPnisQD~GL&7onwVfRtb4kdWZSDI&wTP8C?%t!%>b>((N_ekrG=xGc7Tpds=_4;hd_ z^0l&n2lh(as)1fV%IS|Hw&mGC-9zKnuV_uD#ympF>e+&t7p#ujIw=2D0|i6AsZ?t> z+V(L523d|f^{A@pySyN#)PRgFschKAEvXp?ci;B_nFu5wV@B;(EKi7a5}9xSzux!# zHNi&C#;M|HC(r1WT7fBpNN`n2&i@v><%m|DZ^9UIwapU~fKCw0_OO>fd^nxl#hUzR z%!e^dOZbattk34ZfdIO66G&9O)`R4(@rPEQ&|>Q08vODPk@*c=Th==-rN62tS>$WZ zC%cNR7Wh1zn zd(q~q$_1zN4MYYJ!2xLM?7f@u_$MpfMG-J>_`q3mGB&dS8zPXT7lp>nLRLKPK7{@z z5f0WN0I2WV6Cfn`b$tUA6M22aQM9KE{H(AywzZWms5!jl5{i>My_jabHFeonwFXR07X`h8T@ourD8fDj*vgsmO|0VNTw!4Ws7`mT14`D@Zmb~ z@Qgc#a;j_;KZP~GdgiupxVmmm#BIyP2Wl)$vPoxfYbuxEF0|GwxG*OhRZ*!7wl~}Z zKqa1*4yS8^d~WbS1yUbY&)GE6fVp|z&6$vh6bVZ%@X+)5j3x%(Vk2*6^K}O&j?_o- zT>&=cVt;z1K$Xq`D8N*EwpyP`3W@wb0hI-6`WLw>-N?~TP>T2F%AO&uJVULVwL!&t zFnoA93DNl0fgK1mcw=gA9xom*!-bYEA`8Ll1tv#?z{5jJMDKiaCtf^WhN-!E1RA{Y ztphtyoE$IMA`x3LL9E9jr9y+!cxvc}h9{tgyd)O5-1t)Yr-4sP4kljYs&qqJJ}Lak zQ5i-_&yYk85_(z^qGCKjwzFulmXsq{FRZMM2*Ki_1+SG=;jN3;c>85F_b=rKE+xrx zSq_U93*NeT4X>3|VR6xdwGkos?!jG13=4Ai<5P&9yMN&7#+Rx;ntaeW8IX%FR~e7i z35}ATH_;}jbRHO~C1$;_V(qAWs;(6;ohXOVWMaj!eHd42(~%cz)JoM(@}JG#bsa#8Stz<-v(5{NKYLqp5!gIu9+r zcvmqVEzMccb5Lpw%Nb&Ia?PJy`f)~nnQ+y_7kkp6CjV0PPs#@^ex!^;E9OYME5mB3 z6)KdTPV5;qsb@$#Z;%atxU#1y`NFB5cnz_T2&i#03Yy zTd!lJRo^>^|N8hMj3zUJeSPq?-J1{}5=g`&seH~D(YT+xLHz#-h zJ}F~JInj`79I^7rF`f{4U7=Q}PGUMM*l;h-$<`qYw zOigqx77NZbw&UMVl*4E;BRx7C&mG*wx&hMgJe1a_Gvxo>2{Ar3@)n%|pT_$jRsVeJ zPAz_D>WAhDfD-ysV|*@pj{4HNT8xMf{JcH!jlEm3D{~zz0N$>=Vf)RNAIPu?YBU;L zY3qgs!0xOR{KLIlY%P{ZWOId>n=|D9-3c*1HS!jn0iVYEAXWc->rO3xXzGV#Jh@_p zu2?J7aePO8>4er|vsf%>=pD2TAa~6~CcJdJBTgpJYBYGbcmq-+LlI=So!mZeHrtx$ zrbdM`;ytsy2xBuwB!mVnqc5flgBp5q*4mY>SZh4^-HCsEq8w_%H}`HO-ud&BVr~DwV`vPEix!0V8EhM) zBLo_}5$N!n*xQ?~)Q3^1H6;|^6>DwERnt^%ebL6JJ#R|K2>JA(20vVQ&KT2^D+Vb< zA9CGZ1+dZ+WP`Nma5w{qiBV`Yn!m&YBSJ-4jzZ~4wN!=DQ@VentM!{y!=n-_?`p@*~H1TyoOv?jUF1KPCGM>PUB3QV-18z$#ZOLnnD5mLm0)o#-|p) zG&`EB)rZg7@YU*P=&bz6F&>wkw7WaV70XjHW~g{hxbd6=Pl_d`Mn66pF*>z8LRaiRC^!GkP?J-tXDO$r z>WuyW|7sN@bEWVu-_-yB002ovPDHLk FV1h%rf0+OP literal 0 HcmV?d00001 From 4b6c5d4841b0c939fd8e178110795db1ffc1f6f3 Mon Sep 17 00:00:00 2001 From: "andres.sole" Date: Wed, 6 Sep 2023 15:49:49 +0200 Subject: [PATCH 03/26] feat sdmx: added logo.png --- superset-frontend/src/assets/{ => images}/logo.png | Bin 1 file changed, 0 insertions(+), 0 deletions(-) rename superset-frontend/src/assets/{ => images}/logo.png (100%) diff --git a/superset-frontend/src/assets/logo.png b/superset-frontend/src/assets/images/logo.png similarity index 100% rename from superset-frontend/src/assets/logo.png rename to superset-frontend/src/assets/images/logo.png From ed3850b37c51cdb61806d7918a49ad752b6c885d Mon Sep 17 00:00:00 2001 From: "andres.sole" Date: Wed, 6 Sep 2023 16:07:24 +0200 Subject: [PATCH 04/26] feat sdmx: disable example load --- docker/.env-non-dev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/.env-non-dev b/docker/.env-non-dev index a86ddbd193..249fb3c9d1 100644 --- a/docker/.env-non-dev +++ b/docker/.env-non-dev @@ -46,7 +46,7 @@ REDIS_HOST=redis REDIS_PORT=6379 SUPERSET_ENV=production -SUPERSET_LOAD_EXAMPLES=yes +SUPERSET_LOAD_EXAMPLES=no SUPERSET_SECRET_KEY=TEST_NON_DEV_SECRET CYPRESS_CONFIG=false SUPERSET_PORT=8088 From 6e5b6d91d40e90a8e8825d636cd29b28a9a97152 Mon Sep 17 00:00:00 2001 From: "andres.sole" Date: Mon, 11 Sep 2023 21:41:12 +0200 Subject: [PATCH 05/26] improvement ui: better modal ui/ux --- .../ImportDashboardSDMXModal/Modal.tsx | 35 +++++++++++++++---- .../src/components/ImportSDMXModal/Modal.tsx | 31 +++++++++++++--- .../src/pages/DatasetList/index.tsx | 2 +- 3 files changed, 57 insertions(+), 11 deletions(-) diff --git a/superset-frontend/src/components/ImportDashboardSDMXModal/Modal.tsx b/superset-frontend/src/components/ImportDashboardSDMXModal/Modal.tsx index 77ca500681..5ab560cedd 100644 --- a/superset-frontend/src/components/ImportDashboardSDMXModal/Modal.tsx +++ b/superset-frontend/src/components/ImportDashboardSDMXModal/Modal.tsx @@ -4,6 +4,7 @@ import './Modal.css'; // Assuming you place your CSS in this file import { SupersetClient } from '@superset-ui/core'; import { Upload, Space } from 'antd'; import { set } from 'lodash'; +import { useToasts } from 'src/components/MessageToasts/withToasts'; import { Input } from '../Input'; import Button from '../Button'; @@ -13,24 +14,45 @@ const Modal = ({ isOpen, onClose }) => { } const [sdmxFile, setSdmxFile] = useState([]); + const [isLoading, setIsLoading] = useState(false); + const { addDangerToast } = useToasts(); const onUpload = () => { const formData = new FormData(); - + setIsLoading(true); formData.append('file', sdmxFile[0]); SupersetClient.post({ endpoint: 'api/v1/sdmx/dashboard', postPayload: formData, - }).then(res => { - window.location.reload(); - }); + }) + .then(res => { + setIsLoading(false); + window.location.reload(); + }) + .catch(err => { + setIsLoading(false); + addDangerToast('There was an error loading the SDMX file'); + setSdmxFile([]); + }); }; return (
e.stopPropagation()}>

Import SDMX Dashboard YAML definition

+

+ A sample YAML file can be downloaded{' '} + + here + + . This is a proof of concept of what a declaration of graphs through + configuration files may look like.{' '} + + Learn more about this project + +

+
{
diff --git a/superset-frontend/src/components/ImportSDMXModal/Modal.tsx b/superset-frontend/src/components/ImportSDMXModal/Modal.tsx index 9c78bd3864..7cc25fc919 100644 --- a/superset-frontend/src/components/ImportSDMXModal/Modal.tsx +++ b/superset-frontend/src/components/ImportSDMXModal/Modal.tsx @@ -5,29 +5,49 @@ import { SupersetClient } from '@superset-ui/core'; import { Space } from 'antd'; import Button from '../Button'; import { Input } from '../Input'; +import { useToasts } from 'src/components/MessageToasts/withToasts'; + + const Modal = ({ isOpen, onClose, children }) => { + if (!isOpen) { return null; } - + const [sdmxUrl, setSdmxUrl] = useState(''); + const [isLoading, setIsLoading] = useState(false); + const { addDangerToast } = useToasts(); const onUpload = () => { + setIsLoading(true) + + SupersetClient.post({ endpoint: 'api/v1/sdmx/', jsonPayload: { sdmxUrl, }, }).then(res => { + setIsLoading(false); window.location.reload(); - }); + }).catch(err => { + setIsLoading(false); + console.log(err) + + addDangerToast("There was an error loading the SDMX Url"); + setSdmxUrl(''); + } }; return (
e.stopPropagation()}> -

Import SDMX

+

Import SDMX Dataset

+

+ SDMX (Statistical Data and Metadata eXchange) is an international standard for exchanging statistical data and metadata. Supported by major international organizations like the IMF, World Bank, and OECD, it is widely used in various domains like agriculture, finance, and social statistics. + Learn more +

{ onClick={onUpload} buttonStyle="primary" disabled={sdmxUrl.length === 0} + loading={isLoading} > - Load + { !isLoading ? "Load" : "Loading..." } + SDMXHub +
diff --git a/superset-frontend/src/pages/DatasetList/index.tsx b/superset-frontend/src/pages/DatasetList/index.tsx index cbfbfdb2c1..5e938688c5 100644 --- a/superset-frontend/src/pages/DatasetList/index.tsx +++ b/superset-frontend/src/pages/DatasetList/index.tsx @@ -624,7 +624,7 @@ const DatasetList: FunctionComponent = ({ }; buttonArr.push({ - name: 'Import SDMX url', + name: 'Import SDMX dataset', onClick: toggleImportFileModal, buttonStyle: 'tertiary', }); From 78a01b3a31730723fe92ee3dc9ed7c52e26de65a Mon Sep 17 00:00:00 2001 From: "andres.sole" Date: Mon, 18 Sep 2023 10:32:46 +0200 Subject: [PATCH 06/26] feat backend: added i18n and major refactor --- superset/charts/data/api.py | 18 +- superset/connectors/sqla/models.py | 14 + .../2023-09-16_14-57_13aa403b69e9_.py | 42 +++ superset/sdmx.py | 338 ++++++++++++++++++ superset/views/api.py | 276 ++------------ 5 files changed, 421 insertions(+), 267 deletions(-) create mode 100644 superset/migrations/versions/2023-09-16_14-57_13aa403b69e9_.py create mode 100644 superset/sdmx.py diff --git a/superset/charts/data/api.py b/superset/charts/data/api.py index 5d575fedbb..81aaa5f1fc 100644 --- a/superset/charts/data/api.py +++ b/superset/charts/data/api.py @@ -53,10 +53,7 @@ from superset.utils.core import create_zip, get_user_id, json_int_dttm_ser from superset.views.base import CsvResponse, generate_download_headers, XlsxResponse from superset.views.base_api import statsd_metrics -from sdmxthon import read_sdmx -from sqlalchemy import create_engine -import datetime -import os +from superset.sdmx import load_database if TYPE_CHECKING: from superset.common.query_context import QueryContext @@ -239,19 +236,8 @@ def data(self) -> Response: slice = db.session.query(Slice).filter_by(id=json_body['form_data']['slice_id']).first() datasource_id = slice.datasource_id datasource = db.session.query(SqlaTable).filter_by(id=datasource_id).first() - database = db.session.query(Database).filter_by(id=datasource.database_id).first() if datasource and datasource.is_sdmx and json_body['form_data']['force']: - message = read_sdmx(datasource.sdmx_url) - datasource.sdmx_uuid = database.database_name.split(" ")[0] - engine = create_engine(f"sqlite:///dbs/{datasource.sdmx_uuid}", echo=False) - - for dataset in message.payload.keys(): - df = message.payload[dataset].data - table_name = str(dataset) + " " + str(datetime.datetime.now()) - df.to_sql(table_name, con=engine) - datasource.table_name = table_name - - db.session.commit() + load_database(datasource.sdmx_url, datasource) try: diff --git a/superset/connectors/sqla/models.py b/superset/connectors/sqla/models.py index 52e4e80706..434b273ada 100644 --- a/superset/connectors/sqla/models.py +++ b/superset/connectors/sqla/models.py @@ -112,6 +112,7 @@ ) from superset.utils import core as utils from superset.utils.core import GenericDataType, MediumText +import os config = app.config metadata = Model.metadata # pylint: disable=no-member @@ -537,6 +538,7 @@ class SqlaTable( is_sdmx = Column(Boolean, default=False) sdmx_url = Column(String(1024), nullable=True) sdmx_uuid = Column(String(256), nullable=True) + concepts = Column(Text, nullable=True) database: Database = relationship( "Database", backref=backref("tables", cascade="all, delete-orphan"), @@ -1476,6 +1478,17 @@ def after_insert( target.load_database() security_manager.dataset_after_insert(mapper, connection, target) + @staticmethod + def before_delete( + mapper: Mapper, + connection: Connection, + sqla_table: SqlaTable, + ) -> None: + if sqla_table.is_sdmx: + session = Session.object_session(sqla_table) + os.remove(f"dbs/{sqla_table.sdmx_uuid}") + session.delete(sqla_table.database) + @staticmethod def after_delete( mapper: Mapper, @@ -1498,6 +1511,7 @@ def load_database(self: SqlaTable) -> None: sa.event.listen(SqlaTable, "before_update", SqlaTable.before_update) sa.event.listen(SqlaTable, "after_insert", SqlaTable.after_insert) +sa.event.listen(SqlaTable, "before_delete", SqlaTable.before_delete) sa.event.listen(SqlaTable, "after_delete", SqlaTable.after_delete) sa.event.listen(SqlMetric, "after_update", SqlaTable.update_column) sa.event.listen(TableColumn, "after_update", SqlaTable.update_column) diff --git a/superset/migrations/versions/2023-09-16_14-57_13aa403b69e9_.py b/superset/migrations/versions/2023-09-16_14-57_13aa403b69e9_.py new file mode 100644 index 0000000000..e4b077059d --- /dev/null +++ b/superset/migrations/versions/2023-09-16_14-57_13aa403b69e9_.py @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""empty message + +Revision ID: 13aa403b69e9 +Revises: 449e43460c4e +Create Date: 2023-09-16 14:57:19.194377 + +""" + +# revision identifiers, used by Alembic. +revision = '13aa403b69e9' +down_revision = '449e43460c4e' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('tables', sa.Column('concepts', sa.Text(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('tables', 'concepts') + # ### end Alembic commands ### diff --git a/superset/sdmx.py b/superset/sdmx.py new file mode 100644 index 0000000000..39412d878d --- /dev/null +++ b/superset/sdmx.py @@ -0,0 +1,338 @@ +from sqlalchemy import create_engine +from sdmxthon import read_sdmx +import uuid +import datetime +import yaml +import pandas as pd +from sdmxthon.webservices import webservices +from superset.connectors.sqla.models import SqlaTable +from superset.extensions import security_manager +from superset.models.core import Database +from superset.models.dashboard import Dashboard +from superset.models.slice import Slice +from superset.dashboards.commands.create import CreateDashboardCommand +from superset.databases.commands.create import CreateDatabaseCommand +from superset import db +import json + + +def load_database(sdmx_url, dataset_instance=None): + message = read_sdmx(sdmx_url) + + dataflow_id = get_dataflow_id(sdmx_url) + agency_id = get_agency_id(sdmx_url) + ws = None + + if agency_id == "BIS": + ws = webservices.BisWs() + elif agency_id == "ECB": + ws = webservices.EcbWs() + elif agency_id == "ESTAT": + ws = webservices.EstatWs() + elif agency_id == "ILO": + ws = webservices.IloWs() + try: + metadata = ws.get_data_flow(dataflow_id, references="descendants") + except Exception: + raise Exception(dataflow_id) + + data = list(message.payload.values())[0].data + df, concepts_name = generate_final_df_and_concepts_name(data, metadata.payload) + if dataset_instance is None: + dataset_uuid = uuid.uuid4() + database = CreateDatabaseCommand( + { + "sqlalchemy_uri": f"sqlite:///dbs/{dataset_uuid}", + "database_name": f"{dataset_uuid}", + } + ).run() + else: + dataset_uuid = dataset_instance.sdmx_uuid + database = dataset_instance.database + + engine = create_engine(f"sqlite:///dbs/{dataset_uuid}", echo=False) + + for dataset in message.payload.keys(): + df.to_sql(str(dataset) + " " + str(datetime.datetime.now()), con=engine) + + schemas = database.get_all_schema_names(cache=False) + tables = database.get_all_table_names_in_schema(schemas[0], force=True) + + for schema in schemas: + security_manager.add_permission_view_menu( + "schema_access", security_manager.get_schema_perm(database, schema) + ) + + for table in tables: + if dataset_instance is None: + table_instance = SqlaTable( + table_name=f"{table[0]}", + database=database, + schema="main", + is_sdmx=True, + sdmx_url=sdmx_url, + sdmx_uuid=dataset_uuid, + concepts=json.dumps(concepts_name), + ) + else: + table_instance = dataset_instance + table_instance.table_name = f"{table[0]}" + db.session.add(table_instance) + db.session.commit() + + table_instance.fetch_metadata() + + return table_instance + + +def create_dashboard(spec): + dashboard = CreateDashboardCommand({"dashboard_title": spec["DashID"]}).run() + + return dashboard + + +def get_locale_column_if_exists(data, column, locale): + if locale not in ["en", "es", "fr"]: + raise Exception("Locale not supported") + + for column_data in data["columns"]: + if column_data["column_name"] == column + f"-{locale}": + return column + f"-{locale}" + + return column + + +def create_charts(spec, datasets, dashboard, locale="en"): + for idx, row in enumerate(spec["Rows"]): + if not row["DATA"]: + continue + + if row["chartType"] == "VALUE": + chart = Slice( + dashboards=[dashboard], + datasource_id=datasets[idx].id, + is_managed_externally=True, + slice_name=row["Title"], + datasource_type="table", + params='{"datasource":"' + + str(datasets[idx].id) + + '__table","viz_type":"handlebars","query_mode":"aggregate","groupby":[],"metrics":[{"aggregate":null,"column":null,"datasourceWarning":false,"expressionType":"SQL","hasCustomLabel":false,"label":"OBS_VALUE","optionName":"","sqlExpression":"OBS_VALUE"}],"all_columns":[],"percent_metrics":[],"order_by_cols":[],"order_desc":true,"row_limit":10000,"server_page_length":10,"adhoc_filters":[],"handlebarsTemplate":"

\\n {{#each data}}\\n {{this.OBS_VALUE}}\\n {{/each}}\\n

","styleTemplate":"\\n.data-chart {\\n text-align: center;\\n font-size: 6em;\\n}\\n","extra_form_data":{},"dashboards":[' + + str(dashboard.id) + + "]}", + query_context='{"datasource":{"id": ' + + str(datasets[idx].id) + + ',"type":"table"},"force":false,"queries":[{"filters":[],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":[],"metrics":[{"aggregate":null,"column":null,"datasourceWarning":false,"expressionType":"SQL","hasCustomLabel":false,"label":"OBS_VALUE","optionName":"","sqlExpression":"OBS_VALUE"}],"orderby":[[{"aggregate":null,"column":null,"datasourceWarning":false,"expressionType":"SQL","hasCustomLabel":false,"label":"OBS_VALUE","optionName":"","sqlExpression":"OBS_VALUE"},false]],"annotation_layers":[],"row_limit":10000,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"' + + str(datasets[idx].id) + + '__table","viz_type":"handlebars","query_mode":"aggregate","groupby":[],"metrics":[{"aggregate":null,"column":null,"datasourceWarning":false,"expressionType":"SQL","hasCustomLabel":false,"label":"OBS_VALUE","optionName":"","sqlExpression":"OBS_VALUE"}],"all_columns":[],"percent_metrics":[],"order_by_cols":[],"order_desc":true,"row_limit":10000,"server_page_length":10,"adhoc_filters":[],"handlebarsTemplate":"

\\n {{#each data}}\\n {{this.OBS_VALUE}}\\n {{/each}}\\n

","styleTemplate":"\\n.data-chart {\\n text-align: center;\\n font-size: 6em;\\n}\\n","extra_form_data":{},"dashboards":[40],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}', + viz_type="handlebars", + ) + dashboard.slices.append(chart) + db.session.add(dashboard) + db.session.add(chart) + + elif row["chartType"] == "PIE": + chart = Slice( + dashboards=[dashboard], + datasource_id=datasets[idx].id, + is_managed_externally=True, + slice_name=row["Title"], + datasource_type="table", + params='{"viz_type":"pie","groupby":["' + + get_locale_column_if_exists( + datasets[idx].data, row["legendConcept"], locale + ) + + '"],"metric":{"aggregate":null,"column":null,"datasourceWarning":false,"expressionType":"SQL","hasCustomLabel":false,"label":"OBS_VALUE","optionName":"metric_2o7qsp2myck_m08ex1fqyyp","sqlExpression":"OBS_VALUE"},"adhoc_filters":[],"row_limit":10000,"sort_by_metric":true,"color_scheme":"supersetColors","show_labels_threshold":5,"show_legend":true,"legendType":"scroll","legendOrientation":"right","label_type":"key_value_percent","number_format":"SMART_NUMBER","date_format":"smart_date","show_labels":true,"labels_outside":true,"label_line":true,"outerRadius":70,"innerRadius":30,"extra_form_data":{},"dashboards":[' + + str(dashboard.id) + + "]}", + query_context="", + viz_type="pie", + ) + dashboard.slices.append(chart) + db.session.add(dashboard) + db.session.add(chart) + + elif "LINES" in row["chartType"]: + datasets[idx].fetch_metadata() + chart = Slice( + dashboards=[dashboard], + datasource_id=datasets[idx].id, + is_managed_externally=True, + slice_name=row["Title"], + datasource_type="table", + params='{"datasource":"' + + str(datasets[idx].id) + + '__table","viz_type":"echarts_timeseries_line","x_axis":"' + + row["xAxisConcept"] + + '","time_grain_sqla":"P1D","x_axis_sort_asc":true,"x_axis_sort_series":"name","x_axis_sort_series_ascending":true,"metrics":[{"expressionType":"SQL","sqlExpression":"OBS_VALUE","column":null,"aggregate":null,"datasourceWarning":false,"hasCustomLabel":false,"label":"OBS_VALUE","optionName":"metric_2pzqjh4ddnr_9fsj62hsc9c"}],"groupby":["' + + get_locale_column_if_exists( + datasets[idx].data, row["legendConcept"], locale + ) + + '"],"adhoc_filters":[],"order_desc":true,"row_limit":10000,"truncate_metric":true,"show_empty_columns":true,"comparison_type":"values","annotation_layers":[],"forecastPeriods":10,"forecastInterval":0.8,"x_axis_title_margin":15,"y_axis_title_margin":15,"y_axis_title_position":"Left","sort_series_type":"sum","color_scheme":"supersetColors","seriesType":"line","show_value":false,"only_total":true,"opacity":0.2,"markerSize":6,"zoomable":false,"show_legend":true,"legendType":"scroll","legendOrientation":"top","x_axis_time_format":"smart_date","rich_tooltip":true,"tooltipTimeFormat":"smart_date","y_axis_format":"SMART_NUMBER","y_axis_bounds":[null,null],"extra_form_data":{},"dashboards":[' + + str(dashboard.id) + + "]}", + query_context='{"datasource":{"id":' + + str(datasets[idx].id) + + ',"type":"table"},"force":false,"queries":[{"filters":[],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":[{"timeGrain":"P1D","columnType":"BASE_AXIS","sqlExpression":"' + + get_locale_column_if_exists( + datasets[idx].data, row["xAxisConcept"], locale + ) + + '","label":"' + + get_locale_column_if_exists( + datasets[idx].data, row["xAxisConcept"], locale + ) + + '","expressionType":"SQL"},"' + + get_locale_column_if_exists( + datasets[idx].data, row["legendConcept"], locale + ) + + '"],"metrics":[{"expressionType":"SQL","sqlExpression":"OBS_VALUE","column":null,"aggregate":null,"datasourceWarning":false,"hasCustomLabel":false,"label":"OBS_VALUE","optionName":"metric_2pzqjh4ddnr_9fsj62hsc9c"}],"orderby":[[{"expressionType":"SQL","sqlExpression":"OBS_VALUE","column":null,"aggregate":null,"datasourceWarning":false,"hasCustomLabel":false,"label":"OBS_VALUE","optionName":"metric_2pzqjh4ddnr_9fsj62hsc9c"},false]],"annotation_layers":[],"row_limit":10000,"series_columns":["' + + get_locale_column_if_exists( + datasets[idx].data, row["legendConcept"], locale + ) + + '"],"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{},"time_offsets":[],"post_processing":[{"operation":"pivot","options":{"index":["' + + get_locale_column_if_exists( + datasets[idx].data, row["xAxisConcept"], locale + ) + + '"],"columns":["' + + get_locale_column_if_exists( + datasets[idx].data, row["legendConcept"], locale + ) + + '"],"aggregates":{"OBS_VALUE":{"operator":"mean"}},"drop_missing_columns":false}},{"operation":"rename","options":{"columns":{"OBS_VALUE":null},"level":0,"inplace":true}},{"operation":"flatten"}]}],"form_data":{"datasource":"9__table","viz_type":"echarts_timeseries_line","x_axis":"' + + get_locale_column_if_exists( + datasets[idx].data, row["xAxisConcept"], locale + ) + + '","time_grain_sqla":"P1D","x_axis_sort_asc":true,"x_axis_sort_series":"name","x_axis_sort_series_ascending":true,"metrics":[{"expressionType":"SQL","sqlExpression":"OBS_VALUE","column":null,"aggregate":null,"datasourceWarning":false,"hasCustomLabel":false,"label":"OBS_VALUE","optionName":"metric_2pzqjh4ddnr_9fsj62hsc9c"}],"groupby":["' + + get_locale_column_if_exists( + datasets[idx].data, row["legendConcept"], locale + ) + + '"],"adhoc_filters":[],"order_desc":true,"row_limit":10000,"truncate_metric":true,"show_empty_columns":true,"comparison_type":"values","annotation_layers":[],"forecastPeriods":10,"forecastInterval":0.8,"x_axis_title_margin":15,"y_axis_title_margin":15,"y_axis_title_position":"Left","sort_series_type":"sum","color_scheme":"supersetColors","seriesType":"line","show_value":false,"only_total":true,"opacity":0.2,"markerSize":6,"zoomable":false,"show_legend":true,"legendType":"scroll","legendOrientation":"top","x_axis_time_format":"smart_date","rich_tooltip":true,"tooltipTimeFormat":"smart_date","y_axis_format":"SMART_NUMBER","y_axis_bounds":[null,null],"extra_form_data":{},"dashboards":[ ' + + str(dashboard.id) + + ' ],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}', + viz_type="echarts_timeseries_line", + ) + dashboard.slices.append(chart) + db.session.add(dashboard) + db.session.add(chart) + + elif "BARS" in row["chartType"]: + chart = Slice( + dashboards=[dashboard], + datasource_id=datasets[idx].id, + is_managed_externally=True, + slice_name=row["Title"], + datasource_type="table", + params='{"datasource":"' + + str(datasets[idx].id) + + '__table","viz_type":"echarts_timeseries_bar","x_axis":"' + + get_locale_column_if_exists( + datasets[idx].data, row["xAxisConcept"], locale + ) + + '","time_grain_sqla":"P1D","x_axis_sort":"OBS_VALUE","x_axis_sort_asc":false,"x_axis_sort_series":"name","x_axis_sort_series_ascending":true,"metrics":[{"expressionType":"SQL","sqlExpression":"OBS_VALUE","column":null,"aggregate":null,"datasourceWarning":false,"hasCustomLabel":false,"label":"OBS_VALUE","optionName":"metric_qll9scn97y_rj89e0zvnw"}],"groupby":[],"adhoc_filters":[],"order_desc":true,"row_limit":10000,"truncate_metric":true,"show_empty_columns":true,"comparison_type":"values","annotation_layers":[],"forecastPeriods":10,"forecastInterval":0.8,"orientation":"vertical","x_axis_title_margin":15,"y_axis_title_margin":15,"y_axis_title_position":"Left","sort_series_type":"sum","color_scheme":"supersetColors","only_total":true,"show_legend":true,"legendType":"scroll","legendOrientation":"top","x_axis_time_format":"smart_date","y_axis_format":"SMART_NUMBER","y_axis_bounds":[null,null],"rich_tooltip":true,"tooltipTimeFormat":"smart_date","extra_form_data":{},"dashboards":[]}', + query_context='{"datasource":{"id":' + + str(datasets[idx].id) + + ',"type":"table"},"force":false,"queries":[{"filters":[],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":[{"timeGrain":"P1D","columnType":"BASE_AXIS","sqlExpression":"' + + get_locale_column_if_exists( + datasets[idx].data, row["xAxisConcept"], locale + ) + + '","label":"' + + get_locale_column_if_exists( + datasets[idx].data, row["xAxisConcept"], locale + ) + + '","expressionType":"SQL"}],"metrics":[{"expressionType":"SQL","sqlExpression":"OBS_VALUE","column":null,"aggregate":null,"datasourceWarning":false,"hasCustomLabel":false,"label":"OBS_VALUE","optionName":"metric_qll9scn97y_rj89e0zvnw"}],"orderby":[[{"expressionType":"SQL","sqlExpression":"OBS_VALUE","column":null,"aggregate":null,"datasourceWarning":false,"hasCustomLabel":false,"label":"OBS_VALUE","optionName":"metric_qll9scn97y_rj89e0zvnw"},false]],"annotation_layers":[],"row_limit":10000,"series_columns":[],"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{},"time_offsets":[],"post_processing":[{"operation":"pivot","options":{"index":["' + + get_locale_column_if_exists( + datasets[idx].data, row["xAxisConcept"], locale + ) + + '"],"columns":[],"aggregates":{"OBS_VALUE":{"operator":"mean"}},"drop_missing_columns":false}},{"operation":"sort","options":{"by":"OBS_VALUE","ascending":false}},{"operation":"flatten"}]}],"form_data":{"datasource":"' + + str(datasets[idx].id) + + '__table","viz_type":"echarts_timeseries_bar","x_axis":"' + + get_locale_column_if_exists( + datasets[idx].data, row["xAxisConcept"], locale + ) + + '","time_grain_sqla":"P1D","x_axis_sort":"OBS_VALUE","x_axis_sort_asc":false,"x_axis_sort_series":"name","x_axis_sort_series_ascending":true,"metrics":[{"expressionType":"SQL","sqlExpression":"OBS_VALUE","column":null,"aggregate":null,"datasourceWarning":false,"hasCustomLabel":false,"label":"OBS_VALUE","optionName":"metric_qll9scn97y_rj89e0zvnw"}],"groupby":[],"adhoc_filters":[],"order_desc":true,"row_limit":10000,"truncate_metric":true,"show_empty_columns":true,"comparison_type":"values","annotation_layers":[],"forecastPeriods":10,"forecastInterval":0.8,"orientation":"vertical","x_axis_title_margin":15,"y_axis_title_margin":15,"y_axis_title_position":"Left","sort_series_type":"sum","color_scheme":"supersetColors","only_total":true,"show_legend":true,"legendType":"scroll","legendOrientation":"top","x_axis_time_format":"smart_date","y_axis_format":"SMART_NUMBER","y_axis_bounds":[null,null],"rich_tooltip":true,"tooltipTimeFormat":"smart_date","extra_form_data":{},"dashboards":[' + + str(dashboard.id) + + '],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}', + viz_type="echarts_timeseries_bar", + ) + + dashboard.slices.append(chart) + db.session.add(dashboard) + db.session.add(chart) + + db.session.commit() + + +def create_codelist_dataframe(codelist, concept_name): + codelist_list = [] + for id, code in codelist.items.items(): + item = {"id": id} + for lang_code, strings in code.name.items(): + item[f"{concept_name}-{lang_code}"] = strings["content"] + + codelist_list.append(item) + + return pd.DataFrame(codelist_list) + + +def generate_insight_dict(metadata_payload): + result = {} + codelists = metadata_payload["Codelists"] + structure = metadata_payload["DataStructures"] + if len(structure) != 1: + raise Exception("One structure expected") + structure = list(structure.values())[0] + for id, component in structure.dimension_descriptor.components.items(): + codelist = component.representation.codelist + if codelist: + codelist = create_codelist_dataframe(codelist, component.id) + result[id] = {"name": component.concept_identity.name, "codelist": codelist} + return result + + +def generate_final_df_and_concepts_name(data, metadata_payload): + insight_dict = generate_insight_dict(metadata_payload) + concepts_names = {} + for code, component in insight_dict.items(): + concepts_names[code] = component["name"] + if component["codelist"] is not None: + data = data.merge( + component["codelist"], + left_on=code, + right_on="id", + how="inner", + suffixes=("", f"_{code}"), + ) + to_drop = [column for column in data.columns if column.startswith(f"id_")] + data.drop(columns=to_drop, inplace=True, errors="ignore") + + return data, concepts_names + + +def get_dataflow_id(sdmx_url): + if "stats.bis.org" in sdmx_url: + full_id = sdmx_url.split("/data/")[1].split("/")[0] + if "," in full_id: + return full_id.split(",")[1] + return full_id + elif "sdw-wsrest.ecb.europa.eu" in sdmx_url: + full_id = sdmx_url.split("/data/")[1].split("/")[0] + if "," in full_id: + return full_id.split(",")[1] + return full_id + elif "ec.europa.eu" in sdmx_url: + return sdmx_url.split("/data/")[1][:-1] + elif "www.ilo.org" in sdmx_url: + full_id = sdmx_url.split("/data/")[1].split("/")[0] + if "," in full_id: + return full_id.split(",")[1] + return full_id + raise NotImplementedError("This SDMX provider is not supported yet") + + +def get_agency_id(sdmx_url): + if "stats.bis.org" in sdmx_url: + return "BIS" + elif "sdw-wsrest.ecb.europa.eu" in sdmx_url: + return "ECB" + elif "ec.europa.eu/eurostat" in sdmx_url: + return "ESTAT" + elif "www.ilo.org" in sdmx_url: + return "ILO" diff --git a/superset/views/api.py b/superset/views/api.py index 0ed948b76f..d3f73318fc 100644 --- a/superset/views/api.py +++ b/superset/views/api.py @@ -32,15 +32,12 @@ ) from superset.legacy import update_time_range from superset.models.slice import Slice +from superset.models.core import Database from superset.superset_typing import FlaskResponse from superset.utils import core as utils from superset.utils.date_parser import get_since_until from superset.views.base import api, BaseSupersetView, handle_api_exception -from sqlalchemy import create_engine -from sdmxthon import read_sdmx -import uuid -import datetime -import requests +from superset.sdmx import get_dataflow_id, get_agency_id, generate_final_df_and_concepts_name, load_database, create_dashboard, create_charts import yaml if TYPE_CHECKING: @@ -48,287 +45,64 @@ get_time_range_schema = {"type": "string"} -def load_databases(spec, base_url, tokens): - result = [] - for row in spec["Rows"]: - if not row["DATA"]: - result.append(None) - continue - - sdmx_url = row["DATA"].split(" ")[0] - - message = read_sdmx(sdmx_url) - dataset_uuid = uuid.uuid4() - - engine = create_engine(f"sqlite:///dbs/{dataset_uuid}", echo=False) - - for dataset in message.payload.keys(): - df = message.payload[dataset].data - df.to_sql(str(dataset) + " " + str(datetime.datetime.now()), con=engine) - - response = requests.post( - base_url + "api/v1/database", - json={ - "sqlalchemy_uri": f"sqlite:///dbs/{dataset_uuid}", - "database_name": f"{dataset_uuid}", - }, - headers={"Authorization": f"Bearer {tokens['access_token']}"}, - ) - - database_id = response.json()["id"] - - tables = requests.get( - base_url + f"api/v1/database/{database_id}/tables", - params={"q": f"(schema_name:main)"}, - headers={"Authorization": f"Bearer {tokens['access_token']}"}, - ).json()["result"] - - for table in tables: - response = requests.post( - base_url + "api/v1/dataset", - json={ - "table_name": f"{table['value']}", - "database": f"{database_id}", - "schema": "main", - "is_sdmx": True, - "sdmx_url": sdmx_url, - }, - headers={"Authorization": f"Bearer {tokens['access_token']}"}, - ) - result.append(response.json()["data"]) - return result - - -def create_dashboard(spec, base_url, tokens): - response = requests.post( - base_url + "api/v1/dashboard/", - json={ - "certification_details": None, - "certified_by": None, - "css": "", - "dashboard_title": spec["DashID"], - "external_url": None, - "is_managed_externally": True, - "json_metadata": "", - "owners": [1], - "position_json": "", - "published": False, - "roles": [1], - "slug": None, - }, - headers={"Authorization": f"Bearer {tokens['access_token']}"}, - ) - - return response.json()["data"] if "data" in response.json() else response.json() - - -def create_charts(spec, datasets, dashboard, base_url, tokens): - - for idx, row in enumerate(spec["Rows"]): - if not row["DATA"]: - continue - - if row["chartType"] == "VALUE": - response = requests.post( - base_url + "api/v1/chart/", - json={ - "cache_timeout": 0, - "certification_details": None, - "certified_by": None, - "dashboards": [ dashboard["id"] ], - "description": None, - "is_managed_externally": True, - "owners": [1], - "datasource_id": datasets[idx]["id"], - "params": "{\"datasource\":\"" + str(datasets[idx]["id"]) + "__table\",\"viz_type\":\"handlebars\",\"query_mode\":\"aggregate\",\"groupby\":[],\"metrics\":[{\"aggregate\":null,\"column\":null,\"datasourceWarning\":false,\"expressionType\":\"SQL\",\"hasCustomLabel\":false,\"label\":\"OBS_VALUE\",\"optionName\":\"\",\"sqlExpression\":\"OBS_VALUE\"}],\"all_columns\":[],\"percent_metrics\":[],\"order_by_cols\":[],\"order_desc\":true,\"row_limit\":10000,\"server_page_length\":10,\"adhoc_filters\":[],\"handlebarsTemplate\":\"

\\n {{#each data}}\\n {{this.OBS_VALUE}}\\n {{/each}}\\n

\",\"styleTemplate\":\"\\n.data-chart {\\n text-align: center;\\n font-size: 6em;\\n}\\n\",\"extra_form_data\":{},\"dashboards\":[" + str(dashboard['id']) +"]}", - "query_context": "{\"datasource\":{\"id\": " + str(datasets[idx]["id"]) + ",\"type\":\"table\"},\"force\":false,\"queries\":[{\"filters\":[],\"extras\":{\"having\":\"\",\"where\":\"\"},\"applied_time_extras\":{},\"columns\":[],\"metrics\":[{\"aggregate\":null,\"column\":null,\"datasourceWarning\":false,\"expressionType\":\"SQL\",\"hasCustomLabel\":false,\"label\":\"OBS_VALUE\",\"optionName\":\"\",\"sqlExpression\":\"OBS_VALUE\"}],\"orderby\":[[{\"aggregate\":null,\"column\":null,\"datasourceWarning\":false,\"expressionType\":\"SQL\",\"hasCustomLabel\":false,\"label\":\"OBS_VALUE\",\"optionName\":\"\",\"sqlExpression\":\"OBS_VALUE\"},false]],\"annotation_layers\":[],\"row_limit\":10000,\"series_limit\":0,\"order_desc\":true,\"url_params\":{},\"custom_params\":{},\"custom_form_data\":{}}],\"form_data\":{\"datasource\":\"" + str(datasets[idx]["id"]) + "__table\",\"viz_type\":\"handlebars\",\"query_mode\":\"aggregate\",\"groupby\":[],\"metrics\":[{\"aggregate\":null,\"column\":null,\"datasourceWarning\":false,\"expressionType\":\"SQL\",\"hasCustomLabel\":false,\"label\":\"OBS_VALUE\",\"optionName\":\"\",\"sqlExpression\":\"OBS_VALUE\"}],\"all_columns\":[],\"percent_metrics\":[],\"order_by_cols\":[],\"order_desc\":true,\"row_limit\":10000,\"server_page_length\":10,\"adhoc_filters\":[],\"handlebarsTemplate\":\"

\\n {{#each data}}\\n {{this.OBS_VALUE}}\\n {{/each}}\\n

\",\"styleTemplate\":\"\\n.data-chart {\\n text-align: center;\\n font-size: 6em;\\n}\\n\",\"extra_form_data\":{},\"dashboards\":[40],\"force\":false,\"result_format\":\"json\",\"result_type\":\"full\"},\"result_format\":\"json\",\"result_type\":\"full\"}", - "slice_name": row["Title"], - "datasource_type": "table", - "viz_type": "handlebars", - }, - headers={"Authorization": f"Bearer {tokens['access_token']}"}, - ) - elif row["chartType"] == "PIE": - response = requests.post( - base_url + "api/v1/chart/", - json={ - "cache_timeout": 0, - "certification_details": None, - "certified_by": None, - "dashboards": [dashboard["id"]], - "description": None, - "is_managed_externally": True, - "owners": [1], - "datasource_id": datasets[idx]["id"], - "params": "{\"viz_type\":\"pie\",\"groupby\":[\"" + row['legendConcept'] + "\"],\"metric\":{\"aggregate\":null,\"column\":null,\"datasourceWarning\":false,\"expressionType\":\"SQL\",\"hasCustomLabel\":false,\"label\":\"OBS_VALUE\",\"optionName\":\"metric_2o7qsp2myck_m08ex1fqyyp\",\"sqlExpression\":\"OBS_VALUE\"},\"adhoc_filters\":[],\"row_limit\":10000,\"sort_by_metric\":true,\"color_scheme\":\"supersetColors\",\"show_labels_threshold\":5,\"show_legend\":true,\"legendType\":\"scroll\",\"legendOrientation\":\"right\",\"label_type\":\"key_value_percent\",\"number_format\":\"SMART_NUMBER\",\"date_format\":\"smart_date\",\"show_labels\":true,\"labels_outside\":true,\"label_line\":true,\"outerRadius\":70,\"innerRadius\":30,\"extra_form_data\":{},\"dashboards\":[" + str(dashboard["id"]) + "]}", - "query_context": "", - "slice_name": row["Title"], - "datasource_type": "table", - "viz_type": "pie", - }, - headers={"Authorization": f"Bearer {tokens['access_token']}"}, - ) - elif "LINES" in row["chartType"]: - response = requests.post( - base_url + "api/v1/chart/", - json={ - "cache_timeout": 0, - "certification_details": None, - "certified_by": None, - "dashboards": [dashboard["id"]], - "description": None, - "is_managed_externally": True, - "owners": [1], - "datasource_id": datasets[idx]["id"], - "params": "{\"datasource\":\"" + str(datasets[idx]['id']) + "__table\",\"viz_type\":\"echarts_timeseries_line\",\"x_axis\":\"" + row["xAxisConcept"] + "\",\"time_grain_sqla\":\"P1D\",\"x_axis_sort_asc\":true,\"x_axis_sort_series\":\"name\",\"x_axis_sort_series_ascending\":true,\"metrics\":[{\"expressionType\":\"SQL\",\"sqlExpression\":\"OBS_VALUE\",\"column\":null,\"aggregate\":null,\"datasourceWarning\":false,\"hasCustomLabel\":false,\"label\":\"OBS_VALUE\",\"optionName\":\"metric_2pzqjh4ddnr_9fsj62hsc9c\"}],\"groupby\":[\"SEX\"],\"adhoc_filters\":[],\"order_desc\":true,\"row_limit\":10000,\"truncate_metric\":true,\"show_empty_columns\":true,\"comparison_type\":\"values\",\"annotation_layers\":[],\"forecastPeriods\":10,\"forecastInterval\":0.8,\"x_axis_title_margin\":15,\"y_axis_title_margin\":15,\"y_axis_title_position\":\"Left\",\"sort_series_type\":\"sum\",\"color_scheme\":\"supersetColors\",\"seriesType\":\"line\",\"show_value\":false,\"only_total\":true,\"opacity\":0.2,\"markerSize\":6,\"zoomable\":false,\"show_legend\":true,\"legendType\":\"scroll\",\"legendOrientation\":\"top\",\"x_axis_time_format\":\"smart_date\",\"rich_tooltip\":true,\"tooltipTimeFormat\":\"smart_date\",\"y_axis_format\":\"SMART_NUMBER\",\"y_axis_bounds\":[null,null],\"extra_form_data\":{},\"dashboards\":[1]}", - "query_context": "{\"datasource\":{\"id\":" + str(datasets[idx]['id']) + ",\"type\":\"table\"},\"force\":false,\"queries\":[{\"filters\":[],\"extras\":{\"having\":\"\",\"where\":\"\"},\"applied_time_extras\":{},\"columns\":[{\"timeGrain\":\"P1D\",\"columnType\":\"BASE_AXIS\",\"sqlExpression\":\"" + row["xAxisConcept"] + "\",\"label\":\"" + row["xAxisConcept"] + "\",\"expressionType\":\"SQL\"},\"SEX\"],\"metrics\":[{\"expressionType\":\"SQL\",\"sqlExpression\":\"OBS_VALUE\",\"column\":null,\"aggregate\":null,\"datasourceWarning\":false,\"hasCustomLabel\":false,\"label\":\"OBS_VALUE\",\"optionName\":\"metric_2pzqjh4ddnr_9fsj62hsc9c\"}],\"orderby\":[[{\"expressionType\":\"SQL\",\"sqlExpression\":\"OBS_VALUE\",\"column\":null,\"aggregate\":null,\"datasourceWarning\":false,\"hasCustomLabel\":false,\"label\":\"OBS_VALUE\",\"optionName\":\"metric_2pzqjh4ddnr_9fsj62hsc9c\"},false]],\"annotation_layers\":[],\"row_limit\":10000,\"series_columns\":[\"SEX\"],\"series_limit\":0,\"order_desc\":true,\"url_params\":{},\"custom_params\":{},\"custom_form_data\":{},\"time_offsets\":[],\"post_processing\":[{\"operation\":\"pivot\",\"options\":{\"index\":[\"" + row["xAxisConcept"] + "\"],\"columns\":[\"SEX\"],\"aggregates\":{\"OBS_VALUE\":{\"operator\":\"mean\"}},\"drop_missing_columns\":false}},{\"operation\":\"rename\",\"options\":{\"columns\":{\"OBS_VALUE\":null},\"level\":0,\"inplace\":true}},{\"operation\":\"flatten\"}]}],\"form_data\":{\"datasource\":\"9__table\",\"viz_type\":\"echarts_timeseries_line\",\"x_axis\":\"" + row["xAxisConcept"] + "\",\"time_grain_sqla\":\"P1D\",\"x_axis_sort_asc\":true,\"x_axis_sort_series\":\"name\",\"x_axis_sort_series_ascending\":true,\"metrics\":[{\"expressionType\":\"SQL\",\"sqlExpression\":\"OBS_VALUE\",\"column\":null,\"aggregate\":null,\"datasourceWarning\":false,\"hasCustomLabel\":false,\"label\":\"OBS_VALUE\",\"optionName\":\"metric_2pzqjh4ddnr_9fsj62hsc9c\"}],\"groupby\":[\"SEX\"],\"adhoc_filters\":[],\"order_desc\":true,\"row_limit\":10000,\"truncate_metric\":true,\"show_empty_columns\":true,\"comparison_type\":\"values\",\"annotation_layers\":[],\"forecastPeriods\":10,\"forecastInterval\":0.8,\"x_axis_title_margin\":15,\"y_axis_title_margin\":15,\"y_axis_title_position\":\"Left\",\"sort_series_type\":\"sum\",\"color_scheme\":\"supersetColors\",\"seriesType\":\"line\",\"show_value\":false,\"only_total\":true,\"opacity\":0.2,\"markerSize\":6,\"zoomable\":false,\"show_legend\":true,\"legendType\":\"scroll\",\"legendOrientation\":\"top\",\"x_axis_time_format\":\"smart_date\",\"rich_tooltip\":true,\"tooltipTimeFormat\":\"smart_date\",\"y_axis_format\":\"SMART_NUMBER\",\"y_axis_bounds\":[null,null],\"extra_form_data\":{},\"dashboards\":[ " + str(dashboard['id']) + " ],\"force\":false,\"result_format\":\"json\",\"result_type\":\"full\"},\"result_format\":\"json\",\"result_type\":\"full\"}", - "query_context": "", - "slice_name": row["Title"], - "datasource_type": "table", - "viz_type": "echarts_timeseries_line", - }, - headers={"Authorization": f"Bearer {tokens['access_token']}"}, - ) - elif "BARS" in row["chartType"]: - response = requests.post( - base_url + "api/v1/chart/", - json={ - "cache_timeout": 0, - "certification_details": None, - "certified_by": None, - "dashboards": [dashboard["id"]], - "description": None, - "is_managed_externally": True, - "owners": [1], - "datasource_id": datasets[idx]["id"], - "params": "{\"datasource\":\"" + str(datasets[idx]["id"]) + "__table\",\"viz_type\":\"echarts_timeseries_bar\",\"x_axis\":\"" + row['xAxisConcept'] + "\",\"time_grain_sqla\":\"P1D\",\"x_axis_sort\":\"OBS_VALUE\",\"x_axis_sort_asc\":false,\"x_axis_sort_series\":\"name\",\"x_axis_sort_series_ascending\":true,\"metrics\":[{\"expressionType\":\"SQL\",\"sqlExpression\":\"OBS_VALUE\",\"column\":null,\"aggregate\":null,\"datasourceWarning\":false,\"hasCustomLabel\":false,\"label\":\"OBS_VALUE\",\"optionName\":\"metric_qll9scn97y_rj89e0zvnw\"}],\"groupby\":[],\"adhoc_filters\":[],\"order_desc\":true,\"row_limit\":10000,\"truncate_metric\":true,\"show_empty_columns\":true,\"comparison_type\":\"values\",\"annotation_layers\":[],\"forecastPeriods\":10,\"forecastInterval\":0.8,\"orientation\":\"vertical\",\"x_axis_title_margin\":15,\"y_axis_title_margin\":15,\"y_axis_title_position\":\"Left\",\"sort_series_type\":\"sum\",\"color_scheme\":\"supersetColors\",\"only_total\":true,\"show_legend\":true,\"legendType\":\"scroll\",\"legendOrientation\":\"top\",\"x_axis_time_format\":\"smart_date\",\"y_axis_format\":\"SMART_NUMBER\",\"y_axis_bounds\":[null,null],\"rich_tooltip\":true,\"tooltipTimeFormat\":\"smart_date\",\"extra_form_data\":{},\"dashboards\":[]}", - "query_context": "{\"datasource\":{\"id\":" + str(datasets[idx]["id"]) + ",\"type\":\"table\"},\"force\":false,\"queries\":[{\"filters\":[],\"extras\":{\"having\":\"\",\"where\":\"\"},\"applied_time_extras\":{},\"columns\":[{\"timeGrain\":\"P1D\",\"columnType\":\"BASE_AXIS\",\"sqlExpression\":\"" + row['xAxisConcept'] + "\",\"label\":\"" + row['xAxisConcept'] + "\",\"expressionType\":\"SQL\"}],\"metrics\":[{\"expressionType\":\"SQL\",\"sqlExpression\":\"OBS_VALUE\",\"column\":null,\"aggregate\":null,\"datasourceWarning\":false,\"hasCustomLabel\":false,\"label\":\"OBS_VALUE\",\"optionName\":\"metric_qll9scn97y_rj89e0zvnw\"}],\"orderby\":[[{\"expressionType\":\"SQL\",\"sqlExpression\":\"OBS_VALUE\",\"column\":null,\"aggregate\":null,\"datasourceWarning\":false,\"hasCustomLabel\":false,\"label\":\"OBS_VALUE\",\"optionName\":\"metric_qll9scn97y_rj89e0zvnw\"},false]],\"annotation_layers\":[],\"row_limit\":10000,\"series_columns\":[],\"series_limit\":0,\"order_desc\":true,\"url_params\":{},\"custom_params\":{},\"custom_form_data\":{},\"time_offsets\":[],\"post_processing\":[{\"operation\":\"pivot\",\"options\":{\"index\":[\"" + row['xAxisConcept'] + "\"],\"columns\":[],\"aggregates\":{\"OBS_VALUE\":{\"operator\":\"mean\"}},\"drop_missing_columns\":false}},{\"operation\":\"sort\",\"options\":{\"by\":\"OBS_VALUE\",\"ascending\":false}},{\"operation\":\"flatten\"}]}],\"form_data\":{\"datasource\":\"" + str(datasets[idx]["id"]) + "__table\",\"viz_type\":\"echarts_timeseries_bar\",\"x_axis\":\"" + row['xAxisConcept'] + "\",\"time_grain_sqla\":\"P1D\",\"x_axis_sort\":\"OBS_VALUE\",\"x_axis_sort_asc\":false,\"x_axis_sort_series\":\"name\",\"x_axis_sort_series_ascending\":true,\"metrics\":[{\"expressionType\":\"SQL\",\"sqlExpression\":\"OBS_VALUE\",\"column\":null,\"aggregate\":null,\"datasourceWarning\":false,\"hasCustomLabel\":false,\"label\":\"OBS_VALUE\",\"optionName\":\"metric_qll9scn97y_rj89e0zvnw\"}],\"groupby\":[],\"adhoc_filters\":[],\"order_desc\":true,\"row_limit\":10000,\"truncate_metric\":true,\"show_empty_columns\":true,\"comparison_type\":\"values\",\"annotation_layers\":[],\"forecastPeriods\":10,\"forecastInterval\":0.8,\"orientation\":\"vertical\",\"x_axis_title_margin\":15,\"y_axis_title_margin\":15,\"y_axis_title_position\":\"Left\",\"sort_series_type\":\"sum\",\"color_scheme\":\"supersetColors\",\"only_total\":true,\"show_legend\":true,\"legendType\":\"scroll\",\"legendOrientation\":\"top\",\"x_axis_time_format\":\"smart_date\",\"y_axis_format\":\"SMART_NUMBER\",\"y_axis_bounds\":[null,null],\"rich_tooltip\":true,\"tooltipTimeFormat\":\"smart_date\",\"extra_form_data\":{},\"dashboards\":[" + str(dashboard['id']) + "],\"force\":false,\"result_format\":\"json\",\"result_type\":\"full\"},\"result_format\":\"json\",\"result_type\":\"full\"}", - "query_context": "", - "slice_name": row["Title"], - "datasource_type": "table", - "viz_type": "echarts_timeseries_bar", - }, - headers={"Authorization": f"Bearer {tokens['access_token']}"}, - ) - class Api(BaseSupersetView): query_context_factory = None - + # @has_access_api @event_logger.log_this @api @handle_api_exception @expose("/v1/sdmx/dashboard", methods=("POST",)) def sdmx_upload_dashboard(self) -> FlaskResponse: try: - uploaded_file = request.files['file'] + uploaded_file = request.files["file"] file_content = uploaded_file.read() spec = yaml.safe_load(file_content) - base_url = request.url_root - - tokens = requests.post( - base_url + "api/v1/security/login", - json={ - "password": "admin", - "provider": "db", - "refresh": False, - "username": "admin", - }, - ).json() - - datasets = load_databases(spec, base_url, tokens) + locale = request.form.get("locale", "en") + datasets = [] + + for row in spec["Rows"]: + if not row["DATA"]: + datasets.append(None) + continue + sdmx_url = row["DATA"].split(" ")[0] + datasets.append(load_database(sdmx_url)) - dashboard = create_dashboard(spec, base_url, tokens) + dashboard = create_dashboard(spec) - create_charts(spec, datasets, dashboard, base_url, tokens) + create_charts(spec, datasets, dashboard, locale) return self.json_response({"status": "OK"}) except Exception as e: - raise e return self.json_response( - json.dumps({'error': str(e)}), + json.dumps({"error": str(e)}), status=500, ) + # @has_access_api @event_logger.log_this @api @handle_api_exception @expose("/v1/sdmx/", methods=("POST",)) def sdmx_upload(self) -> FlaskResponse: try: - data = request.json - sdmx_url = data["sdmxUrl"] - base_url = request.url_root - - message = read_sdmx( - sdmx_url - ) - - dataset_uuid = uuid.uuid4() - - engine = create_engine(f"sqlite:///dbs/{dataset_uuid}", echo=False) - - for dataset in message.payload.keys(): - df = message.payload[dataset].data - df.to_sql(str(dataset) + " " + str(datetime.datetime.now()), con=engine) - - tokens = requests.post( - base_url + "api/v1/security/login", - json={ - "password": "admin", - "provider": "db", - "refresh": False, - "username": "admin", - }, - ).json() - - response = requests.post( - base_url + "api/v1/database/", - json={ - "sqlalchemy_uri": f"sqlite:///dbs/{dataset_uuid}", - "database_name": f"{dataset_uuid}", - }, - headers={"Authorization": f"Bearer {tokens['access_token']}"}, - ) - - database_id = response.json()["id"] - - tables = requests.get( - base_url + f"/api/v1/database/{database_id}/tables/?q=(schema_name:main)", - headers={"Authorization": f"Bearer {tokens['access_token']}"}, - ).json()["result"] - - datasets = [] - for table in tables: - response = requests.post( - base_url + "/api/v1/dataset/", - json={ - "table_name": f"{table['value']}", - "database": f"{database_id}", - "schema": "main", - "is_sdmx": True, - "sdmx_url": sdmx_url, - }, - headers={"Authorization": f"Bearer {tokens['access_token']}"}, - ) - datasets.append( - { - "dataset_id": response.json()["id"], - "table_name": response.json()["result"]["table_name"], - } - ) + json_data = request.json + sdmx_url = json_data["sdmxUrl"] + if not sdmx_url: return self.json_response( - {'error': 'sdmxUrl is required'}, + {"error": "sdmxUrl is required"}, status=400, ) + + load_database(sdmx_url) + return self.json_response({"status": "OK"}) except Exception as e: return self.json_response( - json.dumps({'error': str(e)}), + json.dumps({"error": str(e)}), status=500, ) - @event_logger.log_this @api @handle_api_exception From 9fb794099312bb0f23852e175c8059d3fa46fc08 Mon Sep 17 00:00:00 2001 From: "andres.sole" Date: Mon, 18 Sep 2023 10:33:06 +0200 Subject: [PATCH 07/26] fix front-end: major revamp of the modals --- .../src/assets/images/favicon.png | Bin 10863 -> 53484 bytes .../ImportDashboardSDMXModal/Modal.css | 24 ---- .../ImportDashboardSDMXModal/Modal.tsx | 124 +++++++++++------- .../src/components/ImportSDMXModal/Modal.css | 24 ---- .../src/components/ImportSDMXModal/Modal.tsx | 50 ++++--- 5 files changed, 98 insertions(+), 124 deletions(-) delete mode 100644 superset-frontend/src/components/ImportDashboardSDMXModal/Modal.css delete mode 100644 superset-frontend/src/components/ImportSDMXModal/Modal.css diff --git a/superset-frontend/src/assets/images/favicon.png b/superset-frontend/src/assets/images/favicon.png index 2bf2c4bde23c00cef5c3255bf26ec0fbd18dd896..9ddf9adc61cb736e3b21d204e3407b905b42bb8c 100644 GIT binary patch literal 53484 zcmeFZ`9IX}_Xj+tAxmn;9tjhYY$YOzGL{yij3u&ER0yT4S*B#Ikg{*3!q|$0Y*Q-A z)IFd_u2>PB|9%vk41iByH^1yM2JohAz(!jMWgNsrkX4fN!u8HlK%ze%4qyI(_9&Ly~ zkG^(%G3k}x@Gc}*?i63}0rbw#C#j4V8u*3FS9-}WsA)%Q?5Y<}Pij-=YBRSS(K&gg ziAx!S)kU%W^PdPiihgEn2&ZHS3jNQ2B-f!dF#r4k@_T7X3QCeB%a8j%OTda8sI-6o z2*ZbUK*j3DM@IbLg+owlP_6&|5t@rr1!a`=jzIXoyVDI}L!JHCk5D8GMg%4O;S3)C zuRT*xA*%oSQHU-Yy$;29ZYK`+FTov9l#T!Tk**{gNW!%%IpW`ftDpwf{aZt^AV~-& zO@u)L3wigTg^rwi7wh$7XN>g44VZtule`e{-@@D==y(Z{)b8b&zk><9RCB2 ze;~*IK;wU)!3pvCePl9=SK+y|52|X)BE>)1d?359Q_ME#Ka@-I`UDb$uWX=%`M|1fjAe*->0cPoy8SHQKcL9{LGrWFYx98E7!uvLJ9mN%uI=e zCfm4@tm!sOmXVEK9F2h{YU?~()78|}=+kUp1}#{H;WZ%#6!!9h+~$=EndHmfJpRTe zc((5Ft!;&(9KN)+V;$2jwuz5eyE|76Rps0A2sEo+H9DFl7e&>g?$h}Gy`zD>RtbML zvQd*fucOx&^Mmz>NIrbx=~SuaD!%oYXDKvYuVM6WzTWKDp|$2j;TLP`1i{l^GZj}q zK!y+8pDNK<{lMQXDq30rH18^=F)nZ4g3j|quOfd7igkysE}=G?oZ4WqPHfAHhnU90 zCsQP?IaZ5(4q=tjQvL5{*Ne^jxmzo5xL{40pWVyr-dx9wl{(2F@{>MttDr;*cUMWV ze2h}a+1%2i9pvaZcoP-!}mm>g+n{f;Ba$Q1_GtC z_@(@L|CcV3Dm!*Ob?zV&m&;~tD*0Gy{r77WISWo!c_)f{f4x6e(=X$hMROPDNYm2P z)D-g-dT)qu3K2)Z5YF(*tY_&#iW`nHb$L`AwWCJeq8>%iAyDW>-l}Gow`^4B4`y7<_D<4_BNLRa#FEJpt9ahk~xN<^or70+fBfs6ES1mM{ zPow2&K% zU%Kt(-qK-dU*p^#$fyzB$};B!dDvXurJnHB&8^`{>H_!aAqgLHV59cjtFfbvgDLge z6>4 zzP7SBw+K&~l_>>w0f)A>#!^Y8` zcTcFr$H&*1BwhcBeC95E=7HPx`>TlZT-Qusc{D9Rzcl~Iix;lxUep=?n$P9k?p1BY zwv|)g9;$K1fJ_8DAln@SzBMb_BgJBx#|`mQA+atd6H{tU_7~m1I5Q(-AC4P&FKL8F1$8bcCKt<6#W~FTlB+q4rAHqz z_y`SZkRpCzN(r@o4M_9hD%e2_4JXLrBC)qZk_-9cYO=Wqv11rtVboeSL?}IYu}I*9 zL#sjoYL2d5c0p0L?<9?$MD;BMkw6;&@F=!Aa&LI?p1I^H-VN6|FDb)oRnCA+dK}JvtnFg4+oFcv zwN@&h!j(~Yi1h1bTctO_W7rA^1{5))IhHy@B^{h;f^nzya_@;I%2IZ-DYV?GLSAl$ z?DCO%04o5*=7uB-;vYo5Jt{MkZB9=!e|3i+Q4!#mZp6uKXVyE>d|#h9)-$P8YVao0 zcXwZ4NH=cW&?>U4nP(iXxM#C+`UVI}5}+{;}r!6|;* z8JR7}PD@~?u~mi>tes+Dr)Q%Zv(a+*_!8v?YL?GrP}@!nP4t8zE2-c?`YqR-#aKHz zCZJMUr9oBt(^qV?BUnB;-K*h@^~{p_6l>c=*miDuw)^U~v2smQ=ld4gI85bb)=PeH zDYH0gYYB}p{~ADj?tp1@BmHN~&K7~ni_^zjG7q@Zn8j0j`?YTroIyD71Hxe1 z{%i9jSQf|uAj^U4%6}NfntQtYndS)YH6_qmiQ z3F4NDJ~j(=9J|L@a`MZl%TIV?g+X2nVQ10M@8@eb7ToIk!>Mihk+wkafA{%Nk9+x6 z_{v|aJ#_#<6;6aSM;Sj@W!Mk!tCYq>HF_P$PxW8m`}&Z~_%*g(X$4LHl>=SmPU?bu z&M@tlQ5kU{kUW2C*NPyoaG(Q|;@p)8h$2a1Ui41t`PcNNNJc$s+>hB@NbVOo!#0Vi z3oB9xvVHU~6+`raLVx5D(EsC4?QzZCkm9XdOfG!JWrUz+=vF|%>I}0TtjBH%sgwo_ z$E{U+VH721KHx?SY|e`I)LGH#If#HazohoEbQ*C2QC62r*^=sLY$!jC`P45X6U~l1 zscQuQ{%TaXg9R$!uO$kZV>LQuAN7y*P|%Rl_5s{?T;J@s;!_xK1j#ICGvq{Ir^doy zgva|9&Z^g#jEWg$<&(Q~w|lCtY{3dS)*I=@tY{K()M4UVS(8P9)x}7!$;m&DwtVEp z^yQJ24Oh-eaxZwoApK_4YV3IkK`z{vR@c!uBu6$OT_@7Dt|&F0Sb1qSXjVn^toMFI zb-I%NY^quZ33eL;KRjDwmrvgQNmf!DHh?Vxoq7fs{#f}8=dMGLsZQ507Iu^6qVk9o zO~yTirVR732MR1t+=+N%w(}O2a}5b5rKhL6m3R00^4}@o``xFb(>A>!d&QTqk6~ez z1%r5HoPO`|S(_zEq4=Zbpy_%u5QA2 z2{E&E`s)r4$P;_dTSb?ylo^zB$adc*pIqIp1=Iv(t(6PN8D*WANWN})clspVzx1@?wU%(Dftq0N3~HvxyaCJoRLtRVy&p~&R~3xdE;cTH z*yP{~*Ww!Q&cK;2s*lkM%7iL{jg2pvA*TpJo)%OoRgP}tZ#aQT-C#4Yl{_Ef=<#ZP z<#+V}T`E^kd$XErl7~oF#=cBcU!9xEjBcFBUT2i6!&7ZXJQ})^XGkLPbC40c+G@P1 z4;GR15wTZnUx^*CJ!92mqP_0OXdj(^x?ttaUJiW-z z66`!rb8Bl~aj$!^v5nNyU+|*ueoI}m@pq6^zzB)sqJgq9tY^yrWt#NAPn0RWKXr@0 zVkE`YyU(XTaQQ>UD>+W&I2;awA1AziYs|9Bhe%GI+_%v1xDXpSFF_t%lYR7{E<7hG z1xnyjOU-3H*Dm$s;_zE7kFJs*$7sx%D+LXrw*)oz6OX#MT@hIEj1T3o>DqS{NOFM) z3?-!sPE-5B4e3v`Z1UJ0qeDi>vN%{aa#u>y>asEp&daiRWwNXGm-7!P9_DoSTLm{C ztf)3Y0aW|$au2IltD6FFQ;z3i;hg%Q97b_&5%Y43w z#C9;o^IT9-_FjPWG!g&l(00E|=VL58%C{z_P~g}9>X_x~QJQaK{BmLei4x zdijRHN-6V18nkuz)UYff8yn^mKcHxRUqHq>37;`#Vu$w7ZKFMiQcuETzh*MNvxNF9 zV=C|_cn@`Ed`wnpz3~8(?0@8({&6IY3ZV>vAu6;+=C3S>H9A3QUa}z6X6`zG9<(X^ z)BT(mwK@y%W%w~DE<@LRrKNok5+8c-NfT?=*6o?UFJsTzyF>xGdR zxZ>nO5Puvln%J(^mY<+pK8e$mJ0Ch*AAFoBu`&1vAL0kUkTuI0(yU6#!3;h2rLg<- z+|-Xmh0#+=+|~m!T18*^$uZ)x>ik^{lA`bd zI>vfVMcsEhh-J_!=oeu#7OCHzNc6z?Gq7@Jt-osD#^7{)|Lb)8Ub96$BopG z_pEghMA&i2b~OcjfTVyQH5sFj`|t&nlHC&zSzSV;-v;s+RW6)am0k?(?41#@7n_^Q z-fG^_V3;23xpJDFP^Sz?{zq&@?z>Kvp8}RueIE$>#*u?0SLMe3V{(9GBlOd|%8=n- zvtf|GhN4ZR3xCX!mRzyRe>5L@N>=N`+eh2ckI8=RTwE(Q=UIc2U>i{R&l_7X>h3)i zy(2BD%7Ssd1Cb{NE|BM=a{2jI3fho=j6%4dq%?2dhVQ4yxr0`&d}wcPXAGGfL<|%~ z(&72N9@UCwIR|IY5rEWkU)nNPN-q17CDG^R9;v|0lE10f5iYsE=2D?Vss*Tng)<}l94u99WLS;YHC-4dbS;8 z>H9c1ZyFcN!?6{EsDbp>%ESYFWfU5betWg9385&0vkW-==Tzw6Qpa(X%F=(heQ7$W z)H&55W22obnzrEwTj*j1IusCB&2Anz>$Qe|(T3#FoKX>QhQrYGaZGSpXh@Acv!dB@8HyH4Lt^E7PD#@mW}UxW zkFx%mwHgnEeACIbQlp`vD=1k(1YtL=Ullg?fn3IT+5%qSBF|s{nuViBj~3t3lKI6l z1Y_iUS#>`fhU|l?wj|uXfp;PGlSfZJFfr0t5w=?Zgk@u;H7L?3TZlLilwuu$IyQ2)8i>r9plV=IAuVED8$HKvkY^ZSv>8S62KiRiw7$39JHzgux+Y=sF} z>2M9h^_G>p59=%>{+nTEcgBF8+sDW+HjMa~o!*!~~T52WG}T z2}x!iAy~*Bi9PUlL!P`|hQJu7Vy zc>X(#tQ_(#N+Du}o-8Yj&DG8Mta-n~rDog5*==w7Oo+R41d<9?)+vYB9TLAh6!$BDZ%@rBlp-e<=UCHZVfQo%jdh2AN#h5Uc!BP1V#?lX{(Nyz3 zgsxOR|Hp9n1RxF?(U@ELZ)TW(hbla;Bj2)3Mlj{c%FF*TzVus|O?4qRV(;2`U-oK- zFx9X2Y&F+GR=!e{?*M+u zi#!_C+dv}StvG&X1&6cj+4|T%6pM7O8aQ&+=@D0@2RQH6aw0s1wq>zh;2*rw555(@ z2?dFy+ua_p(oqNJOuo3Yc%V|!$R_&#i9Cn-98gcB)!kQ8QeDZtq%zk|rS71GiC`(b z4_C3j{G{GMI(dF-UlH}#%Ay2h(NE`2;jvK=S{4))6l@cybH(XiJonp;(>X&WK}@N> z1&)C5_5@=5)At)4*m#pLJDpZoK@E_^{=J`di;l1+C4 zOi2#s7v?qju5V$Y++9T(-{B#z?A03R-XQL?_;s?17Gyk;3KbO>S5jB1{&ymmzR8Z! z<-E~}W9$0vHm94MC3m#C>2;xR!5E6ycA&l=lTRFXc?{rWWEjGxyQfBW z8~PUi&&zwZK0i^oHQJ~)Vn|(sy6=TsS4m7EaFjmL6&Z)Ix&yKh;nQeo-4aQ;?6qX_ z(VKf5_3E$eNe_S4J#p5b79$Y%BU)}c6&RQizokfh+a?(}T3tmj02jwII1KiuVOICV zB&{a+I%8=dXfnTu-e)h1_h_76)MNHF(K#O~sO;Wd?&9d^sN{5AprrgnAr%OmK#dy( z#_uYeTT%q8CSSPHJzH`V9?8;}dEh~1)(K4KZ49ps;IkiMATJGiRw2ZO;Sd71nH^p0 zOJhdu7n<&?(`uVeTwlV$z{lPRNv<&^^kp6!5h!hKYoqN`Rh`ZkD2>@tn%rwfAWJMs zXb&uPcutm)=Wnoe4kA?F0D^6vx_}?jXy?cHOLrpn))w(Xi>G;dG44uX!yeAp3UJ`m zYR1u9nx{v*(!5Ho6Ivcls*o4QI*aQXpa7o3mjv7jsBLJr^PU|HpT_oxnptzRVV3X! z9-kp<>j5db){V*g&Q9;^_%UMdY0j|{Y4k@<^P^JsPfxumPV@4KkQLbPesDmMJb&n1 zm@;5jPC!lHc&zuHVRC-a9+x`iOw}c$u~G#n6abPW2S@nDNoJe^`WH><^>x56==FF+ zla)qDhxHUlk8tOTz;I(``Z7D9UMP?yt<4+tQ!9M4vmF2AN-i^7Y7NnT*!LmXZr}Gn zD`1BCAk@ZM)kj8Ch0OxsR_%yriA;IB0WQMa} z%9$}!yQfbQTMRZQ@Z!!@Km-HUoZ;%5e0O&eShkr>Xxd`V*9!=(5**!#haFd znZ7oYBS$tthe@*_M(pO!^}u-Qt^q&YmBn>8MAr?vK8!Btj+`ED{86*)m_gmJsiAK? zQkS3I`Rv)8?g@3By}O=js(op9Da+7SQ+vaB*Llvr{!w#*sI$G+!yn~k=VigHnw8q}pSshtv* z9n@V9h2HBrxB)yk;rxcaayty)=kcX=j`#&F-}kutlLOfO)_}UEieW8k(ceCaN940N ztAu6@Q6x{W;?Qm_dBZF-(DQ?c>LN4!$m-)n`@SbBa%)L(TyEc<{hA409F|b#kE;{y zdhE7qJ!yBY%jYGg*81>o427su@m`aCWy;mB*#YTn!cjSj7rET^P^8zevUCtCB1Tsn zZ5;Ho<^do0u>g&^gLeZRPh{%WyFT2DaRFcPiEcdX zVxPpGD(!iLJo)9}wNtron#pb7bPC>Cr+GCvS$kiKcYUai!Jf+iP#B?yMqxP(AjgRb zpk)dgy*x_n_`D|X-cDRD!gyHIn3ol!pBf!!-&aUBR4=Q*j_oW}j%tiwlgGpmwYSBd z*akJXs3ZlI#b%s^!5rYXDs^s&I_c9(KaQWe8|&iJD*lT9uHL9f*?K$x!M?tQWsx(y zlNV}37)u-dGJ>Da-3v2wgdUYYEmO>(tkcr(4mksB)p8diORI9Qh z%ykxZ_Pky$RaXZ0nO#Z=%u}Jzi}(p_YEr z^6I7K(MGu&Zfip+=b9le2$ugDvHZd{Tj1}hHYL31iNHV4P%PY-UcWV7M2eE zAmc*FVndgfmc~O&iO2K3@i8v%o=BJ7y0zE^r6?n5T7^smcz2aJHF>0Y@s}yLUb}_! zY^~YTw6pfd8OrUXuBZ5^KDNa3?(ic|6l0nGAn=&(BG7#7OT7)Tv6adBGu zX$&gazc^wVr?{U@wObM&AS0svs8NF0@gc*aKWQncYmNT*>bC;BLJvS*=1Yr7%gE^4 z75c*(`etZ^ggiGaCq{DQxx6QRYuSlH=>yV3EMqL)y5FIF(?8wI z<9*4vTD?`O8cyUf^mO$IY%{adu3cDd}>OiJ))Zx0W9Mq|HLpA+d?)v9>9Yjl* zb_gr}YQ=c$TjdLyF+pd&H$4Jq`U5Qw&G*`Ry-m%nd~t^1Czc~d_W{u<`q2()>+xbC84Zy& zMWeHY5Wst#PP6n-HcKYpC5JakNHKSU!qvA3Fxu&08Qx7paFwh}je zy5^*ro6yu|=NUho=HBn?wiGa>iQ0~#3r_{noHvny36VctqNIzQZaozjb6Qe&2OlDA z*165p>-B;J(1kwQ;dTGZ(cV7@qi~YC$(~pEVm{eR~XITE@5Fr+jb{HyjCC z0DEF@Uljm%*#c<#p&-bV7p*4ocBt$k#J^aX8#3e=aDF*Vcd2H=;_dY z)hzo_aB_h-^i}78ZX@8cA2TJM2g+FXm{VAYw*}O25$rOes_voQ0JK%AVi16)t!?_m zUZRjXFj0P(G2A)`sSm=?)mx_%Q}F|7-P@Z=ZW-t0oWBBFH=y?9r&?G#(9<514KWs~ zKky;C5`c2^l%ZqL6--J^wdozB@B2|>FYXe?(NSYfEO#~4zI!d?uG8E_8nZMj;)KI> zvWY>uXNafs%q5)P0_w2mm_4S|v_lW*0Fnm8-q&50$D}-Kpxw}0*EtqwLRb<__=Zuk z?5O(kT$D#jC4>^WHRJ1^@h9~`GWfK2Y@60p(6!0)cTl67{36p$sFj5l(X?m9mPAH|9nLE!xxwH$SE$qmv$Qh z8zztMKX0OMzH=QLjkyJ84@OkJtoLhSu^y)yH>RAZVl1I)%;PyZIl*1+qIrw#$tMCCdz3M6ky=r#no+#bhRtm+Hz}V;FE)5;W)uKJ zD7xKVdSuOKCu$g_5Pg!t5Ys=n}gbk(9H^5nr> zS=9wn#HJ+E?*iEwdE~2C#(4MWKx$IwU3D^@DY$KO1nz7fq<^>5Qc_+xyj~~~2L#y< zc8WE%<|ADMK60*SH2>;bYrSg$!AFR5t-mm0I+Kkyb-mYfUd2@5^!h;g;y_+7m!v%S zo2PUZy4ogdAnOHQPwAzAc+mYsE$Z#&lAAuZq5X&Xq?I5`|FcGSu>$4><|l_owPO8l z*$s%3=VP+_6U2`rvlr*6J>kdHjijo)++!4)l#Hdup6^WQ64(C@fD&`zWK(A6_JOpj zPXWPq$y0w``JcDuCB*@6A+l|Es+7uJU~eBqr0uE}xAMcGx8}-gTVVte+hOhk!q(aL zQqlZfW)#iNcO^HAJq2;JzOPF*Yz&Y(twd?ARm882sa*HGjuhfGS)tH0`&B9jO`Zs0Y5$UXz(aMtV^;xY_IE$+Q=@lLPe$>y3Nw-=CZNX`?5m zXU*A@0$fuo2bu(NF^BLynUs)usB0m2^aYGbcs&D*TYRlqPCMu0RF31hdXSj8b4XjP zOnkY})m+QHFe%=g9x5sS2zozQVE)h;G{03{5YyiH$Ppx`+wtVnaYY6)Kw@4WAE1GD zPKb=)J$uFCLX<9~%YBe88>Sp<%N+yM>aS)fi_&Yy%cV*2M0$lx?>lH=sT>2XZTaD> zis9h@T5)kWeYZ!SE%Wzs#7m2zNgU63E_qpNNV`kv6d+s7pT11Ki-ZS}5nhw-P$A3e z-z-t+AFZJdhO?>lv!HFQ`OZi9*oP3+q`OUiV%eb=zZuG-&-hCY5%e6pQt}k7yM4p{ zTXQW1{*7lodGHp;mF690niFZA?L%SKmgU`n7hvN|6^6saeJ%vt)?K(F1nD7(eU|4J zB~&4V@|*p6ZEyKJPcZ$ty`3ZXZS%g7&Lw;r!7INZ>wk z!o-Ddn%AW3mYzF>ZzUl&a>$_SY^wiqy1S%%VjqHso^I?dmyYJgz^Cce4HLM&~_8fy!oK+IEZZPi?g-&a-92nXES8#t@~r5zF?HfZoyd~ z%2W8Fk(6Oy{IQX*rT#Q!7(lcDCV8gdyPhP{#*(@ceIg#ITt4=tbKRZhsw5YY?H6B}W8uw%S7P{<;8Lxpg zvQIMw_UgkqSooIvhBuF(j~5=<_-kd(hOy-3m!X49trQ@WsvD_2`t-uaL&gG-n;IG) zJ!;uM%g1bL-u!|dzT9mnq23#kn&uUL)_UgiWiF%Jx?V_;kRxn<4QO%a88_UV$fiHQ zx0$>v8UG4R*C$FaUU}QAwFZb9tDIzXp@nNAGR*7le1%SQa^CQDzJ9rP%;w%bQF+hG zh)ws&{TI#!-o2r_6T;h_^J@F`ume>!dOwfVsO_QlB{_Auj-9;MaSD3Z2)BN!#jt1$ zqh#K}JY5dV({aG@?R^Wxr)OpQGcjpimOv*R>ll6cwi;-cBe39GOxtr#sD(fO+SKn- z))#feAE@s{;F%`>-ifb}fY3brrFhw7YI!(x5Xw(p@E)LXv@7pi3Val^JvN6)^2L6aWEPeKe-_pBU_o1PO zh;_t+btJ-3I>1q`ncBO2u;62x&hp}fPoE#v!nCW(;jwT`9&iYyQPUfUS{m&Hx)jY1M$3i0X>lWmehA=+?@n5``*5 z)%^S%a*0CSWKHl9?fj>Q8sQhiB??-a-`BO@){OvN92gf>Gu#4&+qTxq(=b36OG4Ve zY!C*8*HvSA2wNSTz3{K7kFv+c?a`XleQ{3Kbjh!B_Rce!I6s%WgPdjNBz|-H!$*D1 zfxw&90SGL$RIDMLKqlBe1wRY^n9siLqyD#vneXrwMi}N_5JIN=W=>jJMa*0n+{N@K z<&a(O+OxSjcXZce9xldJu#*7)FlXSjcEJt}hMSc*ZIzKu`h+?FRbbt;;}c1ka_zJW zm?&BpFKwe?4`d-e;!D%sz;$a;11Q9m;xS6`aWZj(X5c}$%EsgQly_u*+rsT7B52N@ z2S_7;1HDC{;_s)!G|vvAWne3GsB!L179{R7)?d6^>1f5w5?2@*L#CqDl-?a{;h(J~ z8!#O9ss_YI{O;}@NiTm({9dD@);66jW+}ohg-r(88$}PLjc^y(*DY=4=jZR>ua@}G zBcE5`i0&khMul|{G`9$Wg;qiR(_~n!*bPDY9(g|DqlZ?tc*}UE{&)MZ@-^EHKL!*l*b zSy2M&d@`tGrz6ifH$*iCoPMWwJB~Z;J~{mMWkYqn|4t|j!?659RAV{eOyVDNN)e7T zi9CY)F8PWNrbXA_r>@ELyjL9LT(l+91WhW-Pv(IpD+)JXZsH+5;!evahbz#XCxNfA zfy|5YOV=R+yj~YKml5$Syn^>|j+AExd7s>^+=tcofgBVLpk>bu#!mLx!er1OOa@6@ zH_rL&cPzhyyl?U?4~c|*;#b5$_-N5;-J}JtB?y>-<^5fQZh4xo8c?WIq~2X&Z@M#rl{3f zfKVxT*Uf!qPP(9*Be*$pU76!9zG%QAuUFesqZRANkm=4J2d5$T4v3S(bkISC^-nZ6k^17q>;d^X6i~Yb{ z?$5UJP(BvI83(0dN%GVeiB<*j1L8P$b|h{ayKGPOj7J?Jqx|CIvsC^5BbjPU(+tg2~%FW^>ose+bT z_4_;#(jO?JM1nWuN_wsir}?gXzrj@Uaq)Vd*x;Wssa;O^OXc@HUUQu0hf5rHoVuQ# z-kue9EBe9gBd^KLY6@@ofMgv=(^6_GiC=gT8I9A`1yY_9D+1jt^78M+dMJywqpx!= z#*q8Y$Ff(HCnJpSOGs$~S_*EHNBJO1%Qlsij-BvmA4}?T#J{q6YRKCbv|9jXd-CmH z|2}{s=|iS#B`uIB<$0V30_ciYEwwCd|I6nsmr{nIy=zh}UMpn#;VlZ1!!1tWpNemsd%DwhGt z^tNuE<8Zdno~xVZK-uAV6l|Wmwbjgg8ykwx4|8r7J7)E9Ff|2Z%Xq3YqgI8&RQsFVBtX z=n*&EQWt-T6)_EAqlAR?Y_40Kq3fVt^pFTX!54;H0D5-2*Sp5`OF{D23+25*!*Ic* z=*+{)`;bhXv`(l|xGV@E-A83~zKz(Sx6(jQkfWd+B0YN&!jN{IA5Gb4i{dst0V^x3 zuU-S*Z7;ZOZG8KU# zeZXs^g89#~LUq6)9U)v><~gyLerS(W-C#rk(|N`j=SMvAG{WmNX**FZ%I9ti%t^UA zI+EC=w^ly|AGfCVkjV3Ow-oQZ!iqcv#o?}(o)kgM_9B?AD*_aGF!T`9I=$F#KHZR1 zB^M>3MV*v5t-kZ5GPZ*n-&C^rgt6rJSD3`X!n8)fv|>@)EsvyBR;M?!O%dElfi&}e z`{DF=+_o=Z6p*;xvZz|!n$&m=S|~vsfC#q8KKY?`=YizLR6(WMMk&F>Aai(XZf;J` z+1WYjXFMO4bOE`z8G+oss11r9#>P?xp-C1p`l$T^Fd#gH?`w45C_aJpdo7D>H+Fd7 zn@X-GGL|YTvH$DtkG0LTWjHM|0AV~rhkSCDdQ;&QitfMw@n{^`$WwKuVMj9p8j!m`XaI< zjG7;Qdvfi7NsA$3&*-yT&=i@2D z=Jem@D$T$@O24OyKm1f5dMV=}d8_o@#N0@PZe2t@6zy-WmBq&#R@14h$0Y$r77QHO zD^K~ypMn7Sn`;>9LA2rd0Zc|VSz<%cC>ytW>1gVqjGYv|o+wH#fD*Cx-6`{Lh-qqw>IjHSu?E)o%!Ydmm(*jx9!!z%Y9+O@N&hB?#iVIRos_r9fccZdUqo$GeA zO?R$;W;C!H5Y-Syc)ZXt`{&McA}QKq+`YTo0CbI>jzv1DmmVPm;qhA1Lg^JGE|^$K zq|SX_vpx0FTA=W^Ml@Qkh1&>gu%4|GnNHSIZBvjY1wdbim+je2KAZ-OjF8w!A_%4T zl?YCbw8LqyULBy<>nLMH6`{C2oIx#iyBsHASp^0_o9de$$ouA$TAJlG*=e2dXR@?V zvGV(!!xpxI^o~0IxNfza!%*F|AQK_q{en>HoUal5e17{5)gPhr)RCT!;d+x=` z-4bKvJM*J3j`LtFEwv!n$-*s`a@L%l%uCL}J%Vw(zJR(HxHqO4bUA*YCX?7vlkT-y z8gmK<(0+L9yNg?ZaJn0j#>)dk@L-WU!6H+)`O3&5NpPU7tSp&!h$>i4!IAw>^xzT` zTARr+b>&+c{0uG&>S1kjf3H7Q=-O}dvV`2;SXYV|N`8O&hg+((gWK)xzbH%VMSf66QO zR~S06t*uQ$1-h?X3Y?(`_x5-FvE^F`Iwp8c4nL9na0a>x5(#j(ulxPI?Zp=^rMIiw zLauU{uvxq3q*Y)unwmFy{jIIuaQW_1;poNyQF}RdSw$o#3s`m_*2tabe6@DHo0s@wckOj~15X_Z zVpLXSbOh|jKEX-nCjU2H@2Yc?q*r$k@os zwNkp{Xqsim=~EjH*1mx@#>FVF+tW`picE`Nm=hVCWwRGMYTSEYq$eudFNxu-cNB?! z6^2w{uOV^n>o5G%iULhoFy%9ky!ADJV2`=rk9_VwgfIrOkcQ@ffO|@KyK?`aou3~h zO!m$m*yf2P#gIq6)>=#cfZR+VF7GaHW8YPnX-lw9NbgxZl!qnNA(3VmW;;;8aG@t> ztysvb<2YX!U(0}*Qg^s8uw+0`>Y2G`i>shGlIP!0RBx6ovTEjLt7r!=%yUK02jtqI z?!=4i=oX}radV%kekLf77w2~-v;3kaG&N?$h|E8^yPfKs5nMjnw@|*T>JVAXG|Q(& zt#iDBR=Z%QJ+S9pzx&F~s`zkJ)<=liK9HGzG7vrpjSFNp&W!UAr8yzGk)i)^o>vsS zbQ_$2b5nw10K{V3kB#VUE(OqTFN)afR-pf6&JTa;}s9?M1BVggYg`NIF^f zbp$Mrom;~C*Lxf=xr} z3b!b_piD?)bR4quD>jm(z?MYN@`t}5j0vf=QUamAB~>;-wlfW#ir=0VXFHN02O;4v z*&Ru4ycCT-YT8lksCpHIuKkr3F#Fr*)LKjeSaW8}^i}8cK3>A;dcZuDO_mnorchn& zt?qCM^rARbh?DMfn+N}aIl^we-q1JM$X!;o*2>)k{s_t4P}VU2l-DsQZX;Kk_wLOG zWMf1(@)(wiRY^XPCK==Jz&%8n-WFrdb{OEvayCs6vklqGT<_k1fq@@eUc2!AU=5(`)n-)mN9tMrCJgX`ZLkDu%?$goHsqb9FaC-X&$-Y&j ztrAYAfP>nnPyKq-+j5fgCSjnn)7GnH^8yT$CB%)tl{H(5j5KCjHm6>M4ZKu5(W&bl z_Ld!`c|CvJXTv&GE&LM=dKaMvlarOX3i^_R`1Ghbd)68Rq9tFYrN<>LYrEii z0~ymmC+qTVW+7v#HTzq9;9LCf&BVZWR^HB^#kkUfm*%?rmuAXg%6G##?71f^SuaPY z;d!Lh67lTm{nzqQL-nnd)Emc?ugGrmtR~lLzxy6qr0?MFKA8m*iL@^zBGJ372cR)+ z63=s_P+fK5cEsLYWj)_{4Oc0{u;V-Ky@df2$uK33l>2f~%6q68o6gPMO%Gdvw_OkJ zT(Uw!3KZfgj)iS6Zec495q(?;G6G1{%@3;ZS5n~IWffw-YgdKlj)b=;$+txuvF}!{ z4YI>+NLl$0?&UNcP0dR6kF&3f%z!MG(4=lQ9loQ`v6obwH_D%4DI`b)Ma$#-R7GK_ zffn-oC?X{eTs$H-S&Hzdul#*ULB`eh@$Gu&tWwmLI#;QLrR^=V4~vddnkTp5#l~#o z%TnN;kCrS9dc1k;S6Es50NQ|)X;aHgNF6T~YE$7a1S9EgiXY-T8}-1HksOZ94@7xKI^X zXXcewZN-_iNNSZ4k(OGqU**r)vuC$SR~ck&q;DbuqjO5)lM0JUaN+#Q<5wB!3Z2IwM1cr}dg)g(`ZvmLIBWx1PusRF-60%a(M z^cY$i8Q{!@DJLYZ+)#wOKzRJ}CU0(Xe;6e=e^g2C1`=x#cjDg7cCSH!1$MF{H_wRG z049G--TE{;S=*dY8&9x05GL>WgnY?BZ^90BCADi0u1hQA*tUz2aJ0(6n%|Y_YrVh= z=!VegyxYQeVk`0}vc$q#voK+F83>$vBTnC44g!Wm_ZYpEz)lL|wylJ+G0>vO_MO)@ z(QjEI??<5}P-u~VQ;KJ!`^gX=+#9bwz(c(cSdR4Ni<90-YJvzxQ-#^h;J6?7RuO1q z++@=pQH10_4k)Vl9cQI&Dj@60G&VMNfA%hSRiULQmvf{I$YF_cFT9 zX4)$UvQN6*LA6bsJGMLb$7A@|c4)4pP1G>@67OICZ=EWJ%+Lg%D?72Z9SP$$dOTcNq;-0p-`P<+i?>23Lz#=9=Z8&OT#fX@bE0`dwc3 z{bh{1Ve3sEUFVbEB`v;*``{hDIG2cHatBRrM3_dtKfYev*VJ*BbN^V?WZrq{&AD2YWY?tnQzbm0e_@|(c-uU6uKq+PeXhEgKJIM4P}?b56*Ou&#LbCyT!Zp? z`3xZmKR4gbm;q7_&BSTg5N?mQeGm~u#|TioOg~j@H!v{B&^p_imig$TdQAe4kV4}~ zTjug{)$ScNGv_v4L#fgx(yDfb7Bv1$M5&_|Ps9v-8lY>wRJ1-uDpd$LoBdH^OWr3r zt;-OT_q8@RtkdR1|2;ksgD*L+vz78Sh%71x z8lAWoxf_#EsuF`~3r8HD;(Q{|U*W!gP~-7Lr@7Or^=mo~*uC~G?QdvqyV%+*;Gw^3 znh3C3*}Uj#KP_eHfk~Qkp!TrWY~;;`d2oR@gUyST*b@BdJ7^kz-0^s~+ntUU_bis<==Wi0`ldTUxtL3k?reE4?yIw2#Ig)o%X@ZX5QUBzlHGrF6X-9c5{*7 z2I%^OyYvrwu$$AX2+zvP*>DBIXdS^gD*cq?O^0f3O)P66h5dJoK#!EG zplL@A&Wzclvz~KS?zbP#FFxjF>^4O(c!sL#Uf-QlPP;ysWQzWqf6XZ?Fln~cW=JKdmJT(s&C)mFzH;{)mT!nPEsB(3-roXs$bBlYaM1? zdTd0dG_Izl{#-=67O6k7@pHiLO$OX2b%}Kc3DbHs(&8Vkc6;Kx>8m-jDfd5KW`Dw_ za`I&a@`p&eiNrFMp{QJ;=M*u*_h?5;-T5SpIo$&PZRcZ?pHrqFR*Bc7X@%~%!|&ly!2ULMNRDv(;iyCqy@Q-LhBp0d zc5{APe!RE3bM5I%m!|xNhllG9+Mp)_`As`~a32i=n{MpY7JL#H7spF&PWJLSYWT+C zBV>DX@~R64yKb0D%aXM8&%`Hj-#^R%JOA0I^d3ee>4d*G)=9_YU2C}?Lscl+#=CbKJp>*BSt{%;%oma2zd~uJr4y&qMx4g8F zSv&E(dr5Io{hW$8n;Ytgp>l~Uk)BTY`?S&Jz6l2Mj+(t~jHoPbd4inhSYf&Q)nz7Q z*(bEa^w<3re<7hsWn2YUL#A#K{w9^BM=y(`uM0Sgcryz`shj#H)r@@GMqY7*oaNbB zpy6bl;pR+}__CNJjCRFp*c0s5N^#E0e{fT`zvr}Z-^|z;nl6`^d?B7XH?)~I+z~Hq z+R?!OQH}7-_&d=|S(j~bFOTO(f02!1DDBtluH=>|=r}oh@T%!tYS`}Job$sk$hMmI zqXGvQ(k7hvWYo&Q*{v0q*Lqq)ltVMux(5R`N!&EJ|%5-I`rrXqC zqGsPqB7K-}`3Ul22U4K^_mYt5E&jsDQK1zR4x7^t;a@Sxr>#Fk_?r{yQ%|2h^)NK& zBH{%hUTIknuLM4%o4P(isn+#qhG!nE*zy9oA)!sr`SxNnO87w5QFQZF7#7(cD7v}c zUoM0fwL#bk8)r`Ez<+bZdO5D=yev;?4puMKCY&+L=^WV0x$d|wvEPs&vDulUxz~rZ zaJD(=>0p6%=_GC8NRp%r!1Q{@k?Cxu$V>ORt4|X4ZFKSdo{@s!@rdA?F7h}015LBL z`y-Pu&wAZ;Jg#0{EJ`|b|8NhQd?Gk_YVKnBk=kB{{+TxTQve_M*@%U-NBRCcML0;h z9_%K3aT{j5jc$%)!)jM;@9lFsQQw9k9TGB6WTRHq%6CbSY2GV|>+)>xapV#)c zMbB|;HznQ^6BEnV(F|XM*5ro^uV>k>Yd?%#pM_;2~GkMq;>2zw>Bj~ial zK>T&_vCQEFw{Py{%O8dvcWlBuRt!lWInRG8M2b6iG%@1m2Pp$eJ5yLrA5)h_@7H{{ zu~}hz)07tB?HJRa&kdK({OF!mXH zgfU2EkH|h4OJ%7vA!`weEFn8H5raxvWXhHnH43F7yB2htNE=GF3P~vY{+y5Q`+2^v z=l8sR|J9xQvJm(EKz%&k+$AB5PQ-ykII^*&r6q$?Q+!Aw4_4Rh^ zQfw5XRJvGLsA>tfW&aw~D$iMBHit3%2Vvcj$@%UrF{x#&s#3s zAb%UvW)|MEM-rbw$#NRLx98itX9{_@Rz-%5=6a@*fQ&Rczr$j3<^daSS!lJI%9ODp z7g@rd9K=iOP7E-*7R~q!p8wmveJu(|MMEx1-E?>)WUy79UVkca9tdcgU?GtaW5WYuV2$)Kx_9=P!yNQyx`4qr4 z;@hU!`st1hEHUr`+9GiSLo*n!^jZNE)lV1lBVTM(rsfu!Gx3D~zO(XM)A^}bF$ao? z6*Ty_+QZtUg@eqsd3Qq2B#}!6w`e_V zQr|=QP~OzkB-YfYU6ONHP>F($j@^o*3nS8)B3|Wbp`~CZO@eOrJXVU-_rlx>-aP6yi1yz|4 z?G3R?R;xWoThftyfIpltDF~m^#M`GiQT`aMKCQ)auP*s10T5|yH6xDLGKrPGPs%z{ z8lWvX<46g{5CPAS#6|}+(+iziSf4p0WgmPMsrLr9MNYTrQ}_8<2!_^@m4Gnf+K2AP+W`x6kF7$5!5) zRb@YOuH`q+a%}C)$C!>Rm%xR&NaaH3QA9@dlgSLU`yxkM4HC;-?AG5w=)VQmtk#&? zPEJZ_j(KYKDmnTJJ=U;$mQA!-@EldFl?77A#m7<^f}3>;o76#WH71;3t8D|Q1=$U@ zU*u+2DHNmQ0Rdxk59Y}AWgPoI;5)w0C+_L`Dd7^WwX`R1Sn0(_?$+Do#Yr*JC|~?Y}wG$R7=6t}>&!@wLS8l$NEnedIjpD69ABWQZFk*bjn$XQZg8}v$?a4xk)8>Wu~(|cFaq5NnGn1 zL&JRf17ksZxF@7T+@kRgAQ^x1^wm4NuG@^D#px}L*RRmGo9LhnJ-Yv*wrRsngNmCx zeXO7rWOw-SUfn}%lrQP@638v7qp&!JVR6VR?G;nJfoO>Mv|_CG-`cY&jHXQfb_A}a ziCoM5gHQQW2jIe|b`T2{0a}0fr*nBATs`%E+-Nvu<_3AXehbTlC*NW$~@(=trxcmpJx6 zCJ=Xa6K2PKX3qkNO0QK~dfz0*&-&Hk2pbdxf*ROMy$6E@wY zT)LrA|MFgoWGgq>{5u6ADC#mwkGU0q!XMIBCWf_lvagqmuDis&gX#g7^8C=;h{o~HPpYk0-(n0sryA=2UC)d1TF8*2^IA(s1 z_Csz`9okj4kjc2XF$B_0rZ2u@Zy6FVXf7>gl=^2`kRGN7<3)hb1Hm?GA#Ll z$~CV8lodR_){~Cc*p87=nY?njbfH>I1xIaI6l^m<(d0 z#A)HI&@>mb(9mE^ek$KwBik|x`cDO-0&Du$Wj|aYdCO{ghH<&kWbp@B&W@wm|w-`3|U#(QrT%bJNgwCBt{@* z6Vm#u&3LS<9knE`d*vq`!LgYLTDqb0ar84+{egj0bmS`FVM=ds257Gv|am z&al~~-X)YBWb-iw?;jr>TI^Ola2sct%tft$L9u_c5Aw{OGiWFvM%f!!_GBZb>e$Qr`a+FYJ35gJjk#`LkHtanB3m!Yk;?I==*?fB2 z^iqZmU(~YvI%z2PonP35>^9tNh{ZQ1BrYkO-&Rz&eep(uw#JlK2C2@!Zb zwcP4&gjXUFAOPL3Pno}+#J%rMoeg<1IYs1vq_lz1o(~SGBbDduns(ww>QOpui{59D zYpsSlQEJoZvtv`*5;Evq?0dn5;)Lu$igK>sK+a7lz)upNRS(G5@n$a=4)+{_n6n*g z;|vk##?Ml$-GZ4binfaN6fO`f0ff&!ni#)ea+==zxp|9*Hb!6cPTvvtHQY*gkvKr{ zn_CN*?VAeoi$0lL?_El7QRJSnpkTukU=y>k&NzCVym|WKqRM1Z_(x~o=KB@Bi@I@n zN_Phyu)Qhzu9kLKO{_i$NIp#}LYo{Jg6W$kC!^Wg5=s9cj9Duyk|(4`w=oUA zXmfk6!Cb=ElGKy95j#&nMAiE#lzO>-?k{Jl#e_FnG!S0i$vc}&!%Cszs<>e!jgXe? zr=kH(6}V{x^-N5A=WwxMad$^E=p!6|x7+#Qh4jJWbRS%H9hIq1WcJ|*?_~MqNd+;m zK?6a|{WBipy5I?Dbb26u&xb{WwV8;u5QKDK#hwctZ@JFV@B)Y311y{^YwR5AhWsB9 z;9i`Ddww`nVv5WNJ873o$@g{H+_M!>yOxMl6Tg{Y#WJ=x=Bxw(ipa|itmSjYd}rn4 z-egi4x!J=DucUL&T!i(Qz};V6dbEd+WjK_h{MIi$3vLjwi}=-Uws*4JaDN6gz^vcr z=Xd0BypirRFL}mE62!?GCow-7+?wwSmkrST_6cS~O+v_WTVm{BN8_Acwoj-Fb^Ig< zKPK%ak7nq6rJ^vm5q5~dik%iwYrKmaxz$idJmIX)WfQIfPkiY6Uzjp;$d8Ykww=4D z{?E>}D~a4__y8ozghHWGc*OP+9&*1H@dTORj2DK|{LW|yIgIIe9k+=aHiI0DX;a+$ zq+Jm{IF~17MHz2YB53YqjlE}A5bqnca<$%riv4@J{M}@eALGYdMb9@F6HlFeOBLhW zy8yt`(Tth!u-3AATQBTgth-pa8B3G- zH`*~hf>xf%2&7*pXi#Xo$(T~7eCDTPZOxpd41%n&&0Z0Y!$amv+J@`9bDq|HSEOyX zGnF!<9vQQHx}w3Ecq$^>s-CsEOPP%lPTs%vN4-{3_$#SE>c7o1bkKFm8hQ6WN;d!h z?)F`<+sB@nMpyZJ87Lb?y@``$EG7#oZ_GuKfw}vCn>cddCKIt@t{>yHfV! z932)wNi8$aj6Ct{tB8aoI+sU9v2cO-=ck=|Lln;10wUT)#=^?oY1>D`Zp1(Hc294I=%$Z0^LIT!nmanA~|K(>Qch(}qL$W(o zY|rvj+3xYLhUlTZZ8{!+gZS#OUf#*x2NHH?oIQWMz2#Ek^kDE~wn3p3AQ8}p+doIc zUah4v^LCM&8?lhV`md}Rt73FeiLuIcbw)|t&4 z;i3gXyiX@jR|S?fhvP;bpyC%MG)EFso^z69LmVG)2b*&N1L5 zSlirz99CNP%RersHZFnqS2 z6RoN)Hg-uVa1U?69>;}@*+hppoyA8QHhf z**uF_pgE$}u5T^5c z{CSbbE`>68J@kKLvTLaM-5bW!6)^iB?<7YF9eJm^V2h6mubS!Dh!zC8Sy#Bjw;#=x zfteM;*Azxmc>$2I9WFh#m9nCP|7Gw^B3lN%hPXmtzEb#k62XVIUYqdcN2W$){*WzQ zGf-)tnj)5_M&m}TNfdixA;)^=&_E%1;obcFJU@FjkAtB##5C^5E^4% zpiK}~!O2s9?M zr>o1!pY-mKWB_VDLJ)L1oLcqI>%FZnF(Kd#fi(KV>WDPb(zv2EwfpOOPou6^Z!YJN z=o{m3HZ7^6j}OjVT=mGS;#gdgZ(v>1A6^}?FMM>NUl=8GKjQye2fR=y`%~MQAYa&# z9eur39%F~k{5pW9?dQl+aejET`b!<1UWU&awe5Ef@(BzoYvhy)c+%}N7cHiGcLy+? z0Z%D2)21!3ua~j)51c9IQz-w|Oir*&gvryhPKyQ2 z@{nokVT3QA&fj%zyQJ(ecCM|LXW5RNH-5b8#i>K=4evODbgspK|H;k%tK5OB=$VMe zLtoX89aW3Bxlfr9BVHACPzsAmYTzKA&~PQ3zY3(yF0#k2`?`yZ$_*^xwx>^5H`%Cf zK#I$$|EstZU1ucvcHPq-wIsqiZ7x@F*|<~sy%Wr&mIf=hBML*i1ZuM#Sqzg4`+UP@ z-20V?e_|&bHi_Y+PoaWGg<_nD;mr}o*;Qb|0K2C zw|zB~t7uRJOGf%a8+?*0c3M2R+HtAUjO;FEdySsv$xfiY-2XvI81H@=-*E7jNG8Mn zWLs`F{6>Bcplhti>OA$_lK1SWjD&AFpE=ARM#FywI(jFRnw)af(nC=-Ht_W#Lu5AE zQe@|Ku5-0j3hEWrYii$ z+&a`Z10KA&`bb*9o@280W)W{W)5)jli1SC;qxTZ}zf!5{=qu?h#&t=i0tgsUX?>aWQLi_9FxjLh2!lX z?VT<`wc^L0e$PtCsNVGHve|=ua(6T+6+0m( z5U=RyL1om#C@ilRqI*?C0PncJW_j4>u+10=uX90l{mv>coX}c-I^x0C>N59j7Oo+IBOng=_R;=y1q6xVRVtbB=jf9Pv)GT@Zu78W zPlb+~YlfR$35IkAgEb-qD%dVW5GKv>-8T}af_X5f5I}h2L#g0a?Pr$%v3`@Us!)!S zK!2%yViLtILL{~7l`iPt!KVVGhBixP=6ln#B!uCp`3+@U0vC{O2I7(V^qfx%KXwSC z<9T~Rnx%jCg?GC35eto0?9#%xd00Z3i`T#HK#MeRlBnrVs#=)Q8}S3OCiweD6)0+W ze5|eFg_nE}l9_Evg+a;L(u^X7vitPq=v|Ke-q^^k$T1Z!ftbEb*8cpCPQP3LY=sqc zp$%hZ#}*&Mb)gkUD&Vb1wx#OgJACDfPwNlxE|bZ5vhn=d*u5y(eEyIRw{xOW>Dsqr zMb)U(ly$E6+?g}Wbh=NZgkYwSsOQjkWD1Kg_;%(GbV;jQ z|I*N-jt>T?4Gj>kGRV{4Xtwm!WVuOad{nE(rluT#{7_abUG1)j(c9E?iX?~`7s<4Z z)(}XBsvS~?ikS%!#JAOql>%8WFB@oX?3s`O&La+}jnVpmek!Stfb$G~w&qs%a zrnNx?ePh^O-?JbJpX3X4_@ZhU1IIiC>YpU}%xH-b$4Q7>igVNu02`*HACw6g+= zXm)9@I8`TQiGMDb+NUiL-Y5g&Zr3cAkKQiRL?dNjN<=f=VzkAQf3K%cRm=K9NIKr{ z;B1SO__G-ya&PX#(qarRXnt|e_(>?+M7tJLx3C2UbFXpS8sX?wB+6#gDe_o?6%vkL z>60tGV(BPw8V-XQ0+&=N7P<%Pbx3N@#&j{&=&>^!v_|JVzotG+y%$B=t}c?v&i+sO z9U4~q+Urg*0-F*-@Z~XV_{@h(ts#mSPW9k-wz|iwfJpz?e(XYH`?(ErWtKD;86@Qi zMT-t<(CxF8w#)x$OKkzZOftD)JMp25#NNzpBG5YAia!*(v2;MRR=ODJzorTA!=W`%hq}U8#&d5M_|txtgduVZzyi^Nio)gLKDhjQU?WF;PfZ z76P`WZy!y!m;5j<^Fx;-SuOpZ@aR1Y*iV%fcT`&K8Qm_XY;1j)(fTY9#^(e*J>MLJ zOt?J*%>0yr;@cTg|7N^iHP=^9EOy$KL{V2PTv^m%<4h)9L7Ef5KZnFpN_N+uR|gc| zQr!7`>~Gh3V&J?454u(e={k27H)_}|Lrb$y-Q!IC(Lelfc+mY<%BY6jaCfq#Xny?h zuSZBpyI?Z$6Qud@7R!AF>b_^`Ym=kQMLXi6gf*f68g%N^+d~aYpH-axt)0Pi2g*1` zQt&}E9HUVG&s40tVDd8@ZsaMn-2=RxPs-zsZVxAVk+3%ARK{oWBQVd+9_35i%`Ohn zuAjH-=R9*Unz(3I+WaySH$rPDOjoaql#!YL!2XJ!f}Gc4N&Yqg5)Zn6p=3uh`P&yN zl^TR_j>7=irc}lMkf@9+eFqwmZr_NT+(l(f zWas3>_5uhFs^Ll2=!ITu2|4VsE)9269nGpq2^~*~+psawtY9~+Gmj%aeItKPxN_3s zd3v@GuT6)d&q$wxee}$^n6~LSbg&&Mr=`beOs|Im9HWl$zto7GCnJsRc3Z>B$$&{x z+!VV9qxs}-JGY2g`6fWM1v-h=z9j3cvW9K=?yUo>cUCo^1n$prx)0XyPbEy|X~HRL z5jqHmHynBW?&XF9_a3S!u>t~|ro4J-SXTvNsbq^yHU_%$(2vLaG&Ol~9oTWC4DF+n zIlR!**p}8E3on#9p{)HaR$12DK|qi5c1pumh@jsL+wFlLJ%&_0;S`FlIZRa5O>Y@4 z3`#v^3U5I-)h`}*2)}I$h$euV$-S`k?8AS1qL1wNfd&SjCw6uAcz@_pdwR>YSOI!K zBJ@!gjpxTaeV(g47jlAra{iMK2V7R-;bz^j@6o*CYOV>;WLnbH`g&ZXQZZI!zK%Zj z!%M?kvJT#1ay_T10;m9_C=^N%y8qQD(|b+_%lbW-lB$p9;Ob zpy0Lv^@G3FLU6BSI9Qwx}Q~M?2E73)1ElbH#Y~@AB$V7n| zt6~QoQbZ{SA~%}VmwU$V23yqJzoXBKCEP>rwDFfF$bcp-V^mIHe!Mrn0ix@nZlK zw5wgLJi!gv$j!mBS|-hU5f@Xsad+*3HpqF!vYI;@$@h~+Wvs}xhsL5`CqA~;G}|kZ zqNOEc%orR@=~##(o;U{a`$QMOPl;I`jA1>~-I6?Ccf7%0B6J#hHqdfbJNs0=YQ+Ug z?V`0H{);dvb1RX{;tTG34Fc`FwPNb84B)m#L6n2zQW)RlP;VZ2=k3hdf{#ec4r8?E&~^amI>bc<>Y!LQl$sYS zx(=tWodcGAfmr6*cq(9ZdBJ3RxpdsCq>Z@9vUOG7^YWlz7aM1|tiz_~9L0eCkf#0{ z8Z`q@E4Yl8ma_R7x77=nj(B`;gkjvSAS=%y@F||0)ghd+P=v3%i9%65Jfvhn;X6`@ z2Zlh;%gd|N$^mP^xfOb`v=hxn&gI>PTfv-*G|a#oO(swF|3eiIrgQ|={BCLvfYLG{ zd`3=!EdR4D$_|S0974oJTZ?4a(CiG zm8D(wTrWLwBbg|zzPEnbt}C){WcX7LB}08MlW6^u>%t)23Ep+Cv;aSSOoTz`d#KI50DWKwC@Py z9szCb8l8tA`{N96_KodRC)BVy6S*JG`;mzF(YnDLyPoStLj zJx|xJ*&WgTg-9P46|c=#G8fH=RhvRM-_N;t>f#%XM&~5&Z_j1+T9VtHex~k_^Ipdq zW@^;2c+j8hBqx<-&)3>)DRS9Fbh*Vx9j`^OZIF@Xr$o&D@EBqhelu>80$Fn;XX#Yw z!g>GK==qYsZ4oX07B}NXr34_QqDr;K^|o-1U8k;dlocuFhc{#UH=ruH7L@@>YwB#< zwymSjSwkOFB@Xni-mu*Wu=08;(AW~k$A$fkn>K(-&Vt-&Lb4?wTVV@;9D8E4`@UOy zbzqYGdbB&meWsuh3xq?X3Z>E%hDkk<*s!5-?*4PPXCfC8C}e?M*Zpjy(+exJ7!j#J z5P+8R7`|It4WY$B6Y=%HYRKT-+Ku8~rUfbJ`R-108@`!6NMt`JL5B<&E2O1xXQiJ1x{Qfq=MWhxfbU{?h`uT^qTe3IH?_x*#uEHBn*T_iPqp6U{Wt+NL=NE1?wD z^#z1M+?#b|O7d(|sV5uF9qFrygBp z;X{>#o#gh7BQHP?mdm>w6&ChLGiEkmIs9p8IW7M#a~P5WpGojSqiqXI*I;;vFVK=D zQVyVsOJnM^W(2vS3=&Tcc)9tiIo_raU{m1^Y4 zDlqvvVhrQfsz2<nFx{8#AoB9*Om&yT&$D%=-2Eo3JXP`4tHa%=rj^=i z0l?em7(|S(&H6^3xq1rJbmnAro9oK@$y*v&DS9BZc(h(gLAz7qn9U=7Xwc@;d-)N4 zUju7+$M9sWp}m+j1!l>$|5gw~6;^uRNS{1G1G7n6CCHB}QYFD8KHLm;d9 zq2`;#koDM8M~ZQ$sLkkzo|uW+_H0n8v&;CN(K+ax<)NunBd_;KbkpIX+o6GmAX5Fv z1y{|dtBpQ?BO!z~f?n}=vUa;(#iMnxTi9P&Z?N+ka*RkPtL{tv290f)+e&*T`i1wh zui^9aymTFlNAn^12bR3t-iXanC0cP@6f;dxgV+|s8hdn|%M^vZ=ic82TjFWxaNCOT zd4fg&Y(~Bq4vn>~99Lft$v9`l$r|D&ol$)QQe&flr8fGM$fD@-f$I{+O65W~AoCxu zj(V<$ngMR4cy7S!;hhKkN#Azuh6g`-KVr}E-x(_fS<2mIJ|`pdhKD!tYBUTo7uksV zAykLM%tqejmEM?%IAYylt;l(tFG!-6W-J5f-I;p)a9T&pRBWK+@eU}s0#Hd0_oWW) z|8Sh{x3_R+ufax|1Nqy0@L)3I(dIOdZBsQG@8rXk0OLpx_r&Msq=&v0)zDIF{o_Nt zdbtFBbxFlAG`h46kG&J{NM>8DOj)-$%WDskdtIz6o zKSKZ01}OAPhd1EW55Z$A)XZO}83>2qaxa z-F_zWT)1*4`H58TbauBdTa8@H@CVZ;T-^QdsFl(+2-=<7okSIccGd>+bUyi!kvE~g z7?SEz1G}Ihd+E}TFR)X-kf%p_&(bpuMdmB%HLcHG>Du1Fkexd;lp(kdktHEg)qqTf zT^RE(f=-Gp7TNQmnC@o}i8Pe<%i^1Op6|4=JK*%EOyi1Nc|qb zW$EChxrMOv!0P40=x}LcU$PE%8IgTgil0KJKALG65W3LVytIa5^0&hM|IFSfTJPk{ zcmiltqy=NzLPaOt0;Ja(=Jw1TpK;$omiz4dGbukQLQE|Dl?1xo?e8%`eHLx_sWY( z=I;n-2`~F^j5DXKXy9wrq46hC)!K(%V`ex^Nk(4@uqoYcyba`o0V> z=21ELXygGPig|hb{b4U-?^6wl78+T|HqGY&vYs-j7o>P0u}O<|y4dk>_j~9C0%h-}ZDu!CB^3 zB2pMm?xv&H5O}KUb09$x#f6C#^Gmf}$#C5$jcCM=Ydv4OWKhO4e@V*1yBQH43u0h* z?}lkrgd!;3#ly4=>ya{;Ip zoxJV-A6FdewR4rl)LbjuSPZQ-S^S66wHNriQ0^+UPt92Zs@2Es`A~uHP%_vdTD0Eo zujygA6NvRP2eCGMG#M-iNxA3}&(r~fDB|759Jk)2HOOI-gp5|8p8R&k!d97d)vhN-OJ#`PHY1M}9sZd@c z)Rna6z)c(nxZmA1wrx8#;di<+^2d?*a(cgMgrG|V#m$Ud{;pd3!m;J;_^9Li&hDL+ zMMu^u1t#I(I1@ASlS4kda_Za63WQ-NBkl21qD~LJU)8|xY4zkKL~cXR%S@YJSsc0+ zl&7Vq*S%|q)B4dJhBTZ;5S9AT{^9I@71)#s>`YJ$>;F$Ah8x*~+@aaPqhO8$bqDVB zVw_BR#H{nc#vLaQLr(UMR6RMzj8?$s);^hyMjh^VdbiluXrusmmpbC>lRR-X`tdewIS*Tn(EWIP<*^mM zj74Xe9YkPJ;egXx!rSOOGa3j7Q^;i2aY~?+qgY~O|pqr8v$-FpNLO=N!0fxJ?j^rnE zV$e85zr;sSBAONWBd^ppg%{?)08_UJ0`hzLm8(6tDf!S7=xg+r+`ErKNgbrM6nLyw zcx)*azp(RE?M3E2W|445HBtndf=w$5j5lK5kx9u7B8dE&LnTKt13-%%S^X>$2)1f~ zmyaxL(Tefkl`A~9=z{MyZ0y}hIq)Gpb>#BS%a+}HINF3VZuj)&QN>U!!dW}!K{7n9n}tdU;2>2fJc! z_qMYHUQI14-!*vg$%C@Ci=2`EST@>1F}Q}FBSewEL~A$*k4SYpGN^v7kcOh<%B&ZwV%+Fym zH0cq&7NB=dTEklHiLo6-eNhQGcha*CB>Kf?|3F$u_3H!!%6k)z5`5;D27Er1sYWb3 zKIorpiRY<-^+q3l)Bu!1{N@kh z=u+DJQ7pNF8`g5X&&i{btyjWYkwmqVmwcd3rB&-SpZ%VU05`AV=d4z+tNqT=3yl|Z z(VeC>>_=9GsA|r&B!KIPXc!T#nu}G&fuo?@_x08Agce1aoZ8K4p^SyI52j5sC_Ftx zV)-FqSVbQ_y6@IkC9*^A>49Sno-V;U;>|ZFN@1W+$4@og>~*dm1EWqrQCDSH*^kY+ z1T7(C=Rde~;DZUS8~HP0b9CVMUYfvmnM$WLH%-AMMofZyDDf(wd2#6z88qM=M79sniJcOno0Tc3)L{Vlgy1 zV1zwec&R^D#G&f)qEJ?n=sMFTszSFnLF0o98Xsawy)KlWYqBwcBSz;F&Y~StThb4S z!0S9{c>E1GNBxZD#Jj5s&T%2-h$kM`m;fE=r(ms{F+Z0#kZ@4JW+x$;pL%Qe72}YX zX`%_cj(*uUcFd<4l*rL(U%Y;SywAx|XWsovD%O4HhUIOAqls5U7xYeu z@5Ac6#3oxP+0yCfc9k0)j3G=q;=5ZCH@aB?#mUd96mEo4<1_U{NX50A>#Zi-Gosfp zv_(wtZtvIoQbyuC+{u$X-FfAQ8?s^B*q9y3V}tp9kU=qN-^BQ2B1MTv(suwjG-mK8YcOkVV_OYz{5w|EZh z|J#rL`g+V9cdCJact-aEwrL~`k9~KoU!5tw_A;H*KTs% zup4s==e`f$$&VUE)Sv%2#GNBYoZwU5_V%RwU&f29+c_u+Z#7s6|4W%T6Y3QV%Hwx9 zV2iaUgSTJ~D6NOkWJmV_u^&&tmca!aCvS}LB$aFbhhcn2fXIwmy_|*&Xt5$FPsvj4v5x}6QMq9R`DFR8XD3m_?GX1S{ z_WfSOKjW+XCY71LXU{Y`3x?b_fuWNxSRZg&o-oVm4imSxmb3&|@qb;OFSec@ol^soC+K@^3KbJR+QN8Y$QTAJ!NJ_&9a zHY^O>nTZ(;QBXjfhLBV%gqUZTaUffe`UoyCH#op3JR$#fLmAE*A!&E28SE0=N1{|H z7A{hb+e>__$yVM2)tGY04b4xsk4A`V0|=k_4P zSZ&JJLz`jGBciWhWs)Zf(T3$2>FY;x;9pBNgk=+npu#lI8@4}tM@ZBnL_#aU7@VpP zcOkYP<$$-yl=SZ2EoiMMHA`F!3R~&D(Wbq0h;KpLqOCENDoG+v1-1 zP7X!ef_xz!ryZg|HC4iVE!l{W9v~ zgdeR_%avgqZ4Sk-yw(r_NkRbW;pc4^A!tq!$Y>HrFyIfz}2gMST+;L68+8k1PkPx%$(;eD}{ry zj~(S2SfBNV1JgvUD7%TrUVIX)U6rFwOb%K)A))~B>~8-?b{ZG=Ms;El3DB!V4t!}& z-?I!t8RAVqs4p2Tm`uL_>w04)GS)e(oHZtPFRTYJ7Q&V^eG&Rq1=Jp3oxI6jRZnuK z=2S=9({ds+X)6wE2FuV?McAxK4O;p5j#FJRg?*A7L+LFqHKhg~duo=Bn^zgRk>jt} zxQOH;hiIB;Xw`g;`{%@O*L50*se*nFki_Y2|g>PB7A+aJCG ze>~G>Ks!B8omjfNT>qfG6+rqgugCTgqo>1~8XvDBR&xqv36ueAP4!{c7@9WHo*k;X z5l;B8iAyY;L8qR{_7|_TB%gFyI{F)C*qMTRpprY9w}s%Z=lA~9h`LxyNdaAsRRZ+r5`UPBS!ck*7cX4pe@xD~ZdlpN^agzr|{ zp8uEiDf(#epbk707Ua>11<)cn!cIO&tSTr@56KKd1HqU#HQXMwB*qs-I9~-HeP2EM z8x!ohl(}BMZnYldKb}LMb3%8pbsfBz?P%{;!mX;^QIQ8 z$FIad>LiZt1PdFluEu<+2r?@qlscx4=!I=5Hl0Z#JN(`Wpu0U zuj*+_xMk-m#Cn6&5wDeNZ1Qt^5#elE##b*th_P3ZGHMe+Ed}sEmYUn*3y3rCWiw(s z2*WR3-wER~y%RoW9eG;H_)xm=420(tO8W*WscK`OVs$)WuV&JXHIQ_s4K5~x0lU_O zs}zA9O(i{C+bm60y-g)~V4&x^e5a$MBU;$AzRref8nbn>ve{^S@3p|HA4W#f+69dO zUJSLq$|kYhcNX7!C9UP7n`_E$D5Ibzv)3x0qi(^{%E{kG2p5P77$Jn)8Pr9MrXUDz z6KSPEVPhT-h4@E=MCgg&W%3C#Kpox z;)X~qS&e>cPC(52H_JczNnVHYL~s{VHaqFL7G&!tsk4^v%AMHH-e7*lbri0vd(@yi z>kqAvPA+53O*V+gW9s$Td2gN+%5)@NPw`%hH7}fi`|jc=CK4b%6(nayv{X-#&aCvn2X^LiMPo;59PWAIcC%T z?$6Aba<`n&;V3CwBltf)fI}Ol%Y`9A|9wsfZ=9^#GZR}&k3Cw6U(R-;$l8+=e#`Uk zn}+><4ieR)by?D6Fk66%z+B#c;Y>i6+XwP#71EHaI01Ha8L)NlB`0Nk01B8%Ij{tyW>_0XeVO7tFO&p zauIaX6v1$+LQpeJ_cF##Bh|g``-#C{;5b?}g^3E+L5YC+r=DZ-O^L!dC$y?kZ*Tr_ zx#8`t-JNc5rXU7CFmo}{g<-&0APo0>NW3PbWeT%f0IHEp_G@m!BqJ_BFc4)7^Lt8X zFKmRfn%J|q-r2H$#-2R=0&KctmoUK-_Nlx|b58vqU4p*ML&aa6%E*8F^}1${JVzEB zPP7LGt+4IDQAzXgVXz7O9Vj$H_Flo`jc<4MEqz6{SG_}$ZuDOrTGEw`sYRQiv*fz% zrGKzKCK@>cMn$VQ;iKUb-^9ZODS(wqDiATRcX0z7^80bw`7qy`Pj)GwA+ zR`o}GGT7|eIVPv4KAdMlZ}d^Y@SwySC0ey(-Ri{Jf(VcETj6yM4LEkG>?rNp*_Gn~ zK41?o1~3rc*T;Y!CqV;oOz`cK3N`h_B8J^&prlO6_EOKLAJ{E~;=AQOo_P1`7E@QK zWA>diS7(eW;>i6PZkDv$l0=D6B*yC2Rw&2Me2M5=Ns~U=btoz*<)t6^1Iu8&YmA9D zEW^G>d=*Vlx;L|*yp58-mq_Srq~t-9MW(u zg^96S!lX&ftz_>$Hdf*mSg3Lcw8D;L3!$F_V|t%Hf+jxmD5-Wu`K-zg_cXb==e+=2 zCFAYmv9mDXnPM^Xs)3Hq0h80xT$mbI+nx9h@mg*XADrRmO{{S9W7sgH@H=-G%q!2t zp?VkAD%NEi&DcGdeYN2x{hrq2-ER$G6mcn>oB7T^Z~s@1^p+<(w_A35A`HVP*5r>% zY6tGcpALp$MhFI?fqN1mR!Qln@8CvGBkNDL9L}$c7A}b2jXEd5fzR!CWmUC_EG?{^kw67ToeH}I_;r(UvKoC6j zSN$xJiW?Npk7Qfj_j$u0{B`N({U@K8@uf!Rwj)ZOjXtYe&KHVzS&`Kcbs(oItgj6| zv*!wS^QkRwA)Z+7zx~_>K9dIWnQY=%{J=Nd8I2-OC(AWYMlYVM{vshO>oFK^ZU5}< zXiDlxT;2=QGppF;Bj10n9hioWB@e9^(M{$t2I@onHyOE;?SACyTLPH;-^~u7#;MHs z3-sQP!$(hTtosr;p>1p&Z+?Ct1$U4lj_=Sh&z5u$fKe26mxmXWxwNHk(;6NjJQV+U z@ol3$N$eVb0o=XhU2vn404k}U!C6M27+t2HV}DuCksTET^Z7J>*uX0b%sSFNp>H7N zdm7q`R{Fzj+%UwXXbU{GsL$*L?a!rE+qGnea*9njzy^x5wt=*ns=8y;%o8~#yRMIA zm;A;A%0FQ(?o7{VW{EA{R&{X#eOs>vi)Ye?{53$ui*W)sOCBkS3 zTx2$txq6D8cX=z{zsT*bWPvWYbCaML;ef|QM&Q_zr@P6{h?a5Fr?`WUDKqZGgFy!q z`wy2wYiSyq;4}IP&rCs+&Sa9m2_;osWC)&UbIbLN*q@BT$UwwE^$5VFXgVq*-fv%u5Z6 z*#v;b9ZZ)i8a3H#nPCuhguC8Gis}wzypMh?u)_D8bK6>kwUMyP{gBq{J5 z1nS$iJ^TqwamkFf%wf|4Ul59#j695=JWg+h>O73zn7IMq;XTxp^8G)YALflZFYhLK7F&PKuNL1#+7%tg&~r z#|8+BQ_&iPsA-PK_s;igs$ve~{%Xz?lGJ|QD{spJ6U9uB8Jvh>NvXX|f5HcnVmmkR zJHQGKTOatA1W>XW-*K0w4LqD3H_~B1p^;u(L!t!cgw~nbvfxn1BTj;oAOMh|M$e*x1bvJ@=|yD0Kr%iwZW3Wf+|_u^ zObH+_TG@=RZEb!}p{z4A=Cb9dbMC6N-grI6I&VN&I>-?a78X|CUGBSf=)UhD220)y zNY|0AchGCQk1ASCyC5rI zZu36{FAxQ1m0`OFRtBaZz?IC|ex|TThiHU;W(kO8^44OLaDZll5i}F{3MYYHMdmfh z_!z&wm;|Fd+M_NgG9}jan6P`-q$pc1MI`|nIg(?rK^tcc!#aL;vX2`DaQV2`{kgK% znw-lqWsX7*AUUk(vH1yXI-0%#MiCHjG$^5nPUQWFXcyR`Hvn6-&$zN;t3t=S4zY6o z$$_zC=NN+aw}J|dVBoxS>Pf5P>-2^a-7f$O5o=2eNIQh#vqghDHRrlhaT7M1h@tQ zQd5vA+|`vu?i-i=1TJ~omd=I%Y)F+)D;OWHp)#bXZ!5^(F0*GV{n+xi;+{|OTaIX? zMYcFCr=Lmw0%chr-S7E$$@uct{~gCZE>OU!0Kxn!z$dAuhRLj5_gi*PSUp>#|Oztw9S@|aQ zU!H418Sp=YuErh+x_!1}S?%(f+gc$MimU{Le2rYq;9u`S4VXZz@;l}@FoxndiObCA z%9hHM(*Qfd{RFj-CbC7LDsqLw7T*Sc@SXCPLqR69g?-~M{A^8t4Mc(kFtpAOqIpB0 z8{cXSQGX{n`OnoxwLg}jsmzZJZDi$l3l3xM?^-A}!6tqcaOM|=GIlK0lENZt;UyXTm?Lt9j5 zwLM3gtAcs!tpZg69)~?08`M(!&rtUg@T^D+vV|IF@P%hA)I~Nv-OImE@g%1wnKzdA z&Ey?c7FU;lvw0WKt;fS|7v1c*9~p_yxli=^m(CnDGkd#YTtB=7XJ${w%yt_XPKpf0 zZB3f|>6;fd^s`-WIHx`TS?AJ)f6!A4%jKF)G5clwUCq(AE;+vTDPMd0X4e6^GGC02 zj*eQormN_Gh9}(o1d?YpIz>>{-NVJDy71DU zE{U(~X4q>c^CP^PP3o8S>8!0$w%l5K@MU;@`6CB%%{-e;vb~F#{2qC<F zb(GJC)YF5F5AM<{zVxTm>x}PdoX*-@*1mr}+nKCPn^g6m;x+CJz^3B~V2(i- ze16AcNk!43(djP-i*(mMS=Od*+P*pX`H!Of_|hpjzARs|=k+cO+$xDm;$Y(-aggQV zFK?|B(^e&UcNku3PV)&hLuuq?k3hfnO?o>x28KX*}RoV z$Pd=?Zbn7dm&L;G=(^nyYrNK{*)-8NBq1B*(%#BN4$>OIOv?l%qI)hO;+T*V*f{B9{&AjlGEo;Z8__|AXdffc+1X`_7jO{8Wt zGHtuwD0+IN@8rh4k4mx2PP=~SdLkbHo^A!+!(#n|B;>jm;JSC^l8T>9ACmpF8(n^B zU+-?-&zIL%4Sqm|isN6=O(M2Uhg{&d_Hjy^VEhEZ) zYto8V*~X-zl8AeyvR?bOFWEES^O||P^?2ORKk$7#+~3~!@tX5G=XuU~p6BbOc~9f@ zNKOYgPyJnxh#_vD4`m`oBM;ic>}Ovp(uR9=|L{ZE0?a))`EbLWC=*S0z@DjWw8n_v zmXXbhw4T%H7TWOtVOiTfSf#RzwQ7fhey05zr+KHX&HSMOq&}v9CfN{MX2roBDMmpK z49Vo%DA|!oLCV*2x%Cr$>7SCU?Sh{4Q_BHA6o|cu1upOxmhqL4_8`1$cAs??bu0qD zKRq>-YN3vqoJ9U{p`An18IBLNbEspAO3nO4Iod>ObB^%u>}aR&MshN$ScVH!a2W?l zK_XY_+>WtgW;yk4b7qQ?$N)iR?^&$YgmEn_%i46X7U35=IwgKuW#kRDPWto@ zRQkC>c_PKmAlvj!zx##<;oA@XS|mE%PpvC0HdByb`{td4FkX%xLOnS0XN=pG4eRm@ zV@D(47uA@5dKtZCB@9RD;8nnttVQ8?-(ytjPd?}R9}B?b*IV^{IX__Wc1Eypqph2)aP#4cUFs6w`JkLMQmd5* zwX%<4wQ_+A;uS&0(eli(;@v z2$gxyX?p9~dt{RN@Q*6~*RvJMP^{%zq}Hu9!Echo5bl96y%567#Zqc| zYA&fQjJ2&Ott^{q%wx_Kw?w^TZ;6+%2jB)#ceFuiPDK->G_{RCjXK3fsHU4qH=q8s z9ZN+bp90Q|)X5KzwRw!WmMMC~>8aw@@BH64QU_8P<^anDRz?Xjs?VSw2j)!8*+%7~ z2`%c{##3NaFvBb-q9M$J184~4YBd-wkpYm!ZDe}pzYxkTmv0o8oxM!0^NY?>#~_v$ zh4wLqeO2?JYc*t}LssrpHJY%_p@#po3bM+o=%x#d_Lt{|%bq>+$D2)YTHm71Z%Y`e zQIcT+qg@y|jMFZ>oCn*(H`ibioTX7Sht<<83V1_DW7c|V_fP~w+gjyhSq;Vo)K;<# z`F8-QAi`n%+)IHq5$7A#Dq63ZGcmMr4qp#Mny?iDEe9d`10YG)g+ndXBf_BQu(&9>!-lv>mTXoaTTJ>$cDVUobC^ZtS zr4TwZG0wP1l*wI`# z19zHng#(e;s)P5eg8YhNS&d@Z>EPtlapmo?G5M%hlA0%CN5^2E<{st1h8|ZujNTC+ zggAln+HtgsM(q(03ivZ6Q*W}L=_w^*=*iAv+cfG#9@L4e-fZ_;uz3Di=9Ed?%q?nN z;%)&IEi4j8h(s$IMS|~48dx8EZntNv z6MXE3ESz|MOP!F6Y8EV3F~T!F8NDxe#;ei8fl1*yoBMxg6EDC@1utt`to7_^mnK)n zRdVp4rvC5^b~YI}aDgLUX&nb=2VZAerifAC%yUks^`JX-y_gH%GbQ96P_d;}(7uP0 z#0~~2%mT2oV&Fwhc!G-sw2*;j#q@{e){n`%C^t+J?0iPrUQAg@xC*nO zt-S-Q>t9|(8V1VgF8Q6t9+(gpsC6?xhV|`@S-9J~T@j{VgkaokiuN`;eE9RNM+`Sh zlKDMiJ*Hu@EJ3~IoD;c(qdWxWP>fcUK7922gaMTt{F1Qe?0Sqfwg78rW^*71%~7=% zCcf?5Fz9jO#zjtMXw{a2w1AJh#J=x)eQh@;WF( z9$z%jBBAE!ATQV2yym+n66i?ktG*W&-N!BtLocYK7UfNX9Gu4dN}`vB5sDYzD5lj3dh+qBoDfo2bK9tM_@;hUHU!EGf2cSm+a_aj;?uA`k zG@by)rSdkd2IG986Yvk+D&UT9aNIaXdh>)YOb#8ZW3E_SYhdY`izk3{vQUi*!dz`+ zY_hV+W*k#5oMup#Ngr&Ni^W(}2Sle|u+2lDGBl573FV2Eruql|j-%GS$QI%^WhoB) zfz=kc12?vw2jVG4DV>Af9~=_RpgU5mZ&EW#|8^F~I@Aw@PVYp%<0=`|s~}DTEfl+= zoM=)igFc1x^l#t36~ramI{v5f8;?_FLKl41Dl%KTwh_ah!uw;#tF;IcFAsZ!8X=R!%@j&lrIbDx7QQGnPY6>_Ac-{t@xV^!3B|P}=L4V#TuH2{9Jn{KR za7pD*8so7gDX+1xGDwJBFtw?Wh=NP+j3XJ|FiDsaMuE-VFL?M%Y?W0G+1ncsZQM^` z`(FfTFQ%Y6chKVU`QWuucD}MX@R{>atf69;+~d#h?MEAb@ib?vf@TWKc&{0ue^T59fRQtY%g?E4b_1Fs)J!u#o^{S=aB9XysyObQp|G7 ziha!|p!Cu3$ufa*Ot4P6fTJAc0tI`?RIx^S<4DR5gMf}Wt&EV-*J2T_@2AOd)I+Iz z3i|Zv`f6~9p$sgj6`{~Ht4#bN7hu*_tw@s}YU&_fU}M$`r_S2`<6Z0E1%i~gxVXpi ziEr&sOCMl3-=f+@u}(0sfBq4mHGl4~3{(m0p%N|=&M;M*HZaqJ2R_~~5tJYXj$BU< z&gfyyNjRJV2D7CPKrZ_!gU%1Zw;T9L`Su2?RS3d4SQ3wf&0*A z-Gp*_df45Z!F+}aXvUN0GW$@cAUy`Bf=DyL&7vHfVW`hclE@2L!_>Oh;wq^i8-Hb; zDqy9BF<3PIxB^+sN5lM^c!#h5^h2qFC^&GIl`hQomkrjLs+1n*k?!A**{%fJ@d=yP zAkV?J8%c4h57fSFwAM{(_<7c$3nk$yr0nX=?I>lk2&{1c>$L8KcEaZk6!DIF=-uCn zCjt|&S)?NjJr;Q2GHAZT&|^-r(bf~`-$@GZ!vtr?A{1984#b0meLHVzMM zLfsU*m=C7mqTmt3r?A8?EbO~paY;-iRxRnE`*d_~Yjfczw!pxWvL*!`ZdI%_vnI`S z!X)-`=kCKA;Wv;CR@o;vLh`KELvh|7*P5=bcJ^(+vZleESyWXuqYycJNy!J%8}L5 z%L5zkM7*=sR2s>p)^!OCJ8#1*E(41^Cc+dPMU5LB@Y{j2;|OP3*Opbq@uwMNVWw>4a*AK0+%7`pL$tya;KInX~p=g-~H&>)o`!CWvN^mNg{%=3VGL2D|Q zZ-<1L$E)d=QpfV!)h4AFMA{SjRKqhhHW%c70@ex{bex3#VekUaCP4TPsbf10q#bHX zHs$SA)N`wGciWF;q6S)Fa-dHMV5D^LySuxm>0R5K8>^GZguwot#Npvh+l4iN^eZ%7SnCgswB z^QB*??MI*%?+}V$Sg$d&`@Kd@$%Nax$T^EIzfl5iQl$#+tqs&>N1+-Em6u}4PXf1e zL0C$!aVmLAP@vhCyEwx=Obwh2Ra!si1OwL~@==BR8izlts!&0F?RxgBUP*99i7S9I|wEcQ(>F|rzo(ak{VrZeQrzoe`(Ywmt zr%8Paf90&fE83pvW^(1hs{Z8T-)n8cl1n;pOA-%)837Vl~UT?Z2|rh*-{wF zD?mAl&(4=a?>Wo?%GJYsft}qo0tz&u%iQX@OoCGho$6yp>Wf*ZCt(ggfy}sc2~8rR z=Yh}g8{u2OZ7Vx5a(|`c1NiZAGRZpEG5;Eyu7AGy$rqxy<&{n<+b@hItAH#_AX?#@ zoHq!5i86MIHPA^nvoGbbE>OoR>>Oynkm*1xC}oD75WMpHd7y{KV8ifA$j_r>|Gd(E zpTe}@87RmVcN`=sPiUOiyNJ^{OufB^$t5bHoNrG@I zAA;@mb7EuA$nLgZJv{`BG9uowOW=W3SQ7bN!h5&@w+qXsEXI|Hj%Dl6Q!SNoSH7#+;f zqGv!rjqw>ZFt#2{dmG?z1iL*ucN~Z`)Ihf?`(>l6joQ>?qvy6hZnPE+vL?AczC8j8 znzfw+;(_Ab!!OWN{wB)HbArR@uL0kRm&Jd-{^B>ie-3nqhpJyAc7v$R)JamA+{Shj zf(IhE+Tf;)ENzwb?yb!(&Nx-^OX@C$N2{cFO}ednp{-pJaAih%=q+dL+`=`q7p@da zk>`|bpdG;`CD`aRW$uCYd_9?zcVtu}7UMpA8mKn-4tnS+w8Mk~Og8#yFnQ`eVLF#k z?waHizA<{_4{F`!&D9z)>{0a7pEh6vJFf2WnHyenLy?5?pu_LK6*)0SBbr4GhMg8; zNJ2BV{Gm+Gs?55Q)s|Xa0&Yi{j!xE6C z2b3M6^^g@LHqXJk+sYuX%}&Z-tJstZheO!7WuH4Xc@~%7g~Q<8#DDn*D~m61pm-9n zdhH_ekYMERKfpBq++2=noJAlep$rC_U+u=87oA&j`3E3NC`0)y6~PF&60; zUD_@2GFwl;fLSVxrNUU!7?=oKQgS~;!;<)1* TsqMNr_;1fHvz@s{4rl%k%%};s literal 10863 zcmch7WmH_jvM%oKmM}oD;DdVx86^0iK@wmH&fxA&a0%}2u0b;p+}#5N50>B%;E{9B zIq&_r=l#2P*4k_BuBod2s=B&YSM7>WQ<1~PqQXKzK)_Xym(f5#Kt%cbV4yuy#xWHC z{z2(1tLqGbeROs+b~Hl(L189l00moPb2AMyW2lG2keL_)0y2Z8rmnNDvXY1?%$C#m zFAt}?t^Kn!0)m*NyS=fgwV5-(#LV2%PMrRE_y>!#wK)BMfznl0 z14zRh%>V+N+#IId+(3Y!Fei@yAFlv6I{?TH1qZAJHl)M|FBua^1s0%#mD~)K^P>(CnNaZWBD&x=zqi_E&Cs_a6NUchmkBu!cKYhQh;-bU6KJmI*dTO{@x2`e6Mhkl0)=@7~p0^_i z0pSe+0s_G^0R{1YatOdcK)^skKnO%Y@CP6u{Lc>mgIPX39>c69ILQ;~oa_+Y^jb~h z2)M=#6-FcxS=r+_Tcx^gRjs128C)@h6qAjR>}Ww7Er`TkHi2V5nvao{ zJgq_#LqFuAYi;Z4zHGbIDWXqD4uZ@-P@*I zUVG8rSnp6`<587pSNma{yiFlhF7^GTsD4PjiLn=Y2tfCY*Fo+{W>GaHT3b*m^b2Ju zX5{`i>$^Ezbr$ybK5OhpZ;ii~eMfd8V75u9?G29BddT6hDZDdO{Z-zzhYpvxbSbHY zh1e2lzvRs}@y$I!Jb4l)2!3=xo*=)% zc0ra&ZLJb}FK$SAUR^5~B91y{lAz*YF$C>qh*>+!KyIF1x{<^xZ7)fa;f@5GHAc`Wd+Oim%!XOVD~AixEV(QT)} zA~{?_Q#AyNy}F1A=Ok-ZSuBD37R|qTbAQ%&h6^OKtzj?)>6+hkWv?9au6^jgUsek2 zS@G}IQ4RFN%COnYc(6B~z*;G?i^7O{I^AUb`i_52WGp`?zBBuNuG5#9a%GbsSo_R` zre^-}bCl24SE=SNJnowgj{tI{Y^*cI2R3*N<*S@DH)%Wd>ql|830gbAK&C%=?U*c$ z?1F5awJpU)z?E?#va}kYWB8dc3EJ?datpwS)N$r|0XX)Ue|tRByDPHM08r^!ZR1LFZ5hDGnRm5Kf9d>#-e5n%)P zWS~}^#XacJ3bL%4JCgc+v$u(0un_|*O-G2R6wMA{< zylPLZa8lIqaL8shI2b}81cwpt4j+dHtbtBG(U%WNgu9M)h33D08)B}eipo6RZgpOB zx~k28Rr{WLU8?hBU8*^qid{ohdQvzAV}yYLVMHUahyIL*lF%?WnnFhV^AIw)ef z5+1WKb@|Zx5gl1TXr3_Z%6Ruq*&n@>ZjHMqHITzbaETELPDyZff121K-r}?QV z5K!CRV4zIg6I+bw>Nuw}a_WY~W1%!gvi?Y-Y-PguW z!cLLyj!n~conAX&u^|;z)pHtLbpV?vmpo-L65mBFSlS}ed#LRic5TW7k>5hI|mys~U8)w@eZ{fTz7 zL_vWAH8+o`Ya~-38HU$qv};#KzqmrNkhd;`D0mZwp#FH{$rDYG66$L#%crk1r563D zBzlG_0^KbmAdb$EVz6V%4rK18>_>}%=XHwKk7A>3PBs!qI8(#3$@yc5V8OhSds2yo zU+4|kNL()^BDxN3)IWhW z!%KMr)0S0);iH?wMdDQAFUU@c32G)nGzcEUYTXx$C~5^KX;NfFC$d!o1}z-tw66|_ z#%oB`J99E|n>4x46Mnj7iu--Ou@qC4MjM2Dn25u-HOnvQ#gYuB#hZ%^c}bE+&=R zWP(!1k1=>t_v$4SIQRJ8#sG_l8kRHGNBnnRSzQQ{l_DoUy20D_pqc9JF_L>Tq`pOPXO23s#Uce-zN=d z?>HkkiEh|_IaPG4=`cBEQtihhJFA&6wy^pj+xOBFo0Es*m+ZIcb<;ieU;T_`Lg`ee z=k4^;4=UawiKfR6++*HnC&=b#eq5O^k>mi?Uz<)9acKJT`d#!)Ef>-j;tiFkGozax z7|^N~UW>U*v$0AZ6{3zgu=;&&r}!Gz_KJ8goF{-HHN~y;fqm+M2_tiiiEyFQiRVMg z8u<%3(ebW`Kk7Vi(M8&1TM!v4p6zZbs$b##b%6Y}G@YlViqk~dF@X)#&c;fF{Q3Nl zUW{+uH54lfVeWwh88*}u^ydjrLolqP{KuFoB$pnHtYTC<=gBi}QnN~4ckb;e;;il) zb5O|kmreVUqI9u7hFh=QzHz=uFBFWEJSQsuGo*eP1xJu2{8SdGoGSVH?vp>8qr=F! z^qJgDvI_Z2VK>wqZI`^DroeNR%(?f0J+Un<4$)9;BT#8B9>VNpstgPkL0KJ|$KfR_A0p%it~ zd{(ujqLKlvKSru|(nckKY42pb|BI`^Ib!w{%EPwPfDvmk0k#cdP9@)QEMGFJO#tm| zW!CE>;)_bgje55RX;zH44$7kB$Pxy9qL`xNZE4bh6&Q%xU zLE@3CbfSpz*s$+G(AVQAHdEyiFjXMub}86RS5QrFaUDN$|9JUZ2{97nazf|0%-vIh z*FI%OeQUB+GO!2Zq`q91lyFNS4<87!XjmIJn`sMlc_p%-Vb24I;WV5h#}7b`JLzpY za##B`Pw$0J6Qb)e$+KO}GE3i-Hn#UU8~zv+>(fV_;tc+ied>PKGbAVwhlv&H@4MMu z7Mb8~xYlN%h30W4;@qvUE5IV|P4=0_iAh!~nE*A=voWhhu5CBy#&zVITNO)=n5ipN zi*iqbCNQRn@A^uc;D+^=s6ltkJnj+D#dEH;tZ}cAjU~TYF_h_NGVT%?A5bX5l?z|v zjN+nlM>sZ0XA8Smq0vpenz}J z!c&go#%NkK$S`;P0dNh`$IMO+(i?-a`C2N~D(u{DU48N4y$)pU?3W`hh>M4NUP7!) zk8iudjpe+zXj_4*oJ@)iBPje~>>fbI(7?Z-&=NNz* zZIC4>%6=3BCx{$}>5CP<+K_rfnfskEiN}W=2k;vP(1tBo)iG4e=G#v+cBJ*xzGlKt z+J#9`R1x}GPvjVE%p-+Df=nfajGnH|@uyc1Z9NsM5reG z)%Y9f4rSp(B_)4^Vo4?ibyN2?ym)pr(i*Iisbq38c1fVbfdePTkoi&si1PUL#By1I zU&iB8xVfo|PZl5xPY$_p5oR3~1CC$lmRaz?KxA%>q1+=#w6#%D_SP!DE*I3o185)u z?FP~WK^2_7*Md)j?=Fv$qGI3Y5JMUt*j`$6rK0fTdLy4(CHrw?jXJIBAq=fC!9^aB^iOZQOc&{r^h zBZ0k_aNL1Qht+tVXlybj@kiWxOm5O=454K(^CwO(bjzkyq-a2<)g)>@Fw*IAMIo2( zmY((^dS!WI?E7LHbYqJU1!M5*OuJS747obZ}n7yz9j_Q)`bSnVnK3xdnV4 zGO(BQKl~8B@2R1_*@=!^PlWcp?Xd^+8 z-)fFJB#;v}e2VO70+bB?Vk`q-3~|@v7>cc45!#$Vw6l9>)RNkd0)7A`-8boAHVMC= zcncp$Uizzs6Jx8NdIAqq+ zH}HQ#H*)d@di+SRxbd1WkhHoD(m~1M^sq1ej$Rbc8$VM$6YfT?ekf=S4X;;z;txV{ z@#}*AcAbx_>nWq|2<0wn0?_0wq2C}=Mu;h4wjcco-kp1Bj6Ri=xM~yqD4b5UlNR7h zikhL!Nnh-D$AqhY4)jRiZ~$*Q87Bt&lqh($V7ANj{`8i&2Bqez_j;UIJX+2CX>1lQ zbr{4@T)drI`H|iJBRu@>b*x%IEe@Vs=sbhc&RytOJiG^W!xGRNQV*MenRRAOFpvBQ*b zqe}p8BwG+@{=(GT)1vTYw%RSs z`2YeG7Qe6l75U_y>gA`9RT3K(Y2zqImwb~uS4xTxvH9hS^?4Ep@Rr~Ei)&vMRnun( z)5it?&P?Ig4&C#%=9W#0E>ul|L8zb<_V(BVZ*zNb*=&(BNR}+1Jm3Ai)f^@|t)-)^ ztZ*Psj=HW+@N!^M)u_bSI)yN&0FW_nPO?tw%A^1QORdWp?3#-PFZ+&r(~9c!<*C<;4@wGSWE1dDM?TK|mE?sR!S(YQYOO#L$a6u7D` zKLl{I>=G;JF=yR=nsJj0m_X1Keh}Rwztv7`9_B}R;Rx$4IQLZ<`qFU%x+Lkr^1W8f&W&lc|$k zDUDFoJM1jzF;N4}?=dJZQ`STRlJ@s03AkQqkXz2p5?V==XI?%gWQ6(tzO!a0kUV2~ z=Ss4bV7WaUVkHNz*6+k#o_()jK8umEVlwmoLuKaO7?`dRW~l?L#!X3#zi_R!f0O76t_AF? z#TAVjAa@ZTxz^u%_Y0Wfqj9KWc1-=6)*tlQ`D!_Jf5rEED{pOqy)&S2EC=g!Lm{ne z?E83-z>$dR(z zlxHodRC3*fln9`!!>reKxEl|AvZ<0TX4UcX#=+jxNV>X`_`-D0D&Q~No zwz_{JRfp>ue)GrP{JlI_y|lnMYZmP@tM`8GUh0jNa!8AV?bl@M9bS~YHU+ZUraHhm zedRt^*GCn)Lg_B|1%hBi=Wk^X`SKro2cr4BNJzOn9btI;Y3{;i8>4a#Lr-m+pMO%> zz^Ydlu7B`{q~2VAyD+twJegI+3FdZmpzb0Qo%j-<>a}c&-E6>$#x?MAb^51-+De-w z5ekpi4=8uJYbZ|^v8}zpd|Dm_jO3us{j|XvfZ>f^IJFUEtwb|XUE?Z>Wx%+y@`~-< zaheP2FA;f7S@n^gZQ_cQYXjp(l?$Sh2*Ay8p`1YaD2NHop!{CYQ(%qB@P2T(Q&a4i z=Jms^r4-I{9H>8$Tt%LEyM@MU4C zcE!?TG$vc6;;=zD=C8NbWpVh7@BnKR)L@oey|bL}s?w-u=ANj+zR77LU* zBOC1NS{Sb0>=_ln*cT`dxGfCs9+L=nU#Cp0W+pg3!;6ch7Q zMQ*=%zVGNRdyIQ~gtO#-5lfVIIs{>CHvV!wq8Y0u(aBEF)5{QSA`J#cQ%#U6 zmsriNGY5;8K18LukzA2O4mKM~w-*~&>m9#e%O%`j+S*>3ZH?scwJDkgb4*p53%~DW zu0jp)sc%a63C9mHOY4ZHNgi4Glwwtb+7Yg+uJl|+n$FfsNo=To#}tOSqM(#<>}t%8 z^Km^nYQdg{CzI*%jWq}`&8+2&&0|%YrbVKQ(nai=%Gj6d$i~IWb|9@I?V=hK zE49J8xErV3+6-FZVKUJ9z_SrPl1JtG>zeMo%IiW4R}$mBx6z9%Xug2MNhtX&-`BEk zip3!O!4)#3#WdDsxd1SahFFJ|d!3L@U($T~`iXYmCgZ(C&8?xBj)(Bm z+f7y{zvNnes136U6B}IzvWl0s@1oLck;jzhbU%q@GFnk{Q!6C7*@!6I8HV3VPg{=~ zR1j4;{;EBa z%I^7F1_+!jvv&K{IlvM&6}h%9$8l!d)?2N0^|^Kqsu}(8Nhn34EPjw5&DhCvK* z*96)4vY8OU$)qHem5DBekI6Bmto#FCK;-#7?X-wt(qh}0EevSjL+2RqX#1H!`Pd_7+X(g;7n-fPsLAjo_w#P{ z?Mw$(xu-rSCe_v7<`SDL(2=es_lPb_CH7u1Wgh071Gh`jd*rcW1ITU~P7a2pH`qrz zYy~Fr$paqNZOn4#(gO;H7*O((T{C?c=C)PnpKoZsCOw~Prb8hHk8XU2b74DszuW2C ziji|@g>@+bp50XxjV+;*@U>Q>AVpj#1`wT=R8=x;7*A9I%y^hQtTx}TL_=RJM9Sy6 z^ic1PO+H_TinPB)>m=tTc_dHZ&gH{Vy|e*ffs~M|-{uXde~lnlgKD+0>sBPvpXFZn zzZyQ3>-MIV{Y)OuEG*srNyj+QO@97Dh9H&NehsbnOF+=-{o7L4KAB~g{-*71q~`2U zwO1mTKf*8)B&U_T_T7g)9taf|9~qx35dgFoyxj%wo#%ggBRTR{XEjBXiURsRh}nuh zk!?{W&=k|0@M8IBBFDTdEZ)vby4*ZZdR$WcUHCEIR5Y~nNzju4W7MV9m+Zax*}YX? z;^oBp4YUBB;L3@oUT}ss-DM!FcgmGl+;<_N^SxOpE9RgYCBTV;vzNb-QyH_YnC1OkmCUC@d(kV5zkU!2;7<7=@-7)- z8?Z5uz3r^{;5l~Ape^B2B`BbttRZ-xn;?p1lHP-A4DNQWVMjiVR))t&?Kx8_ZcvDO zI_voLex<&ItH3WCNu0WJ-Yy?)ldCQSzn~UG$t1iwtb-IgjLQhODp6FkyOH#5m`_o) zY^!|Y9AQ=+=4PjN9sgcR9Y)^HfPph0`lO-zy1Ig~H7+VK@3m^a4j$8bOKc_Q`?btC zqCJHnk!iBCv>Fo8Fy=_h070ezaJQo*<4#pcKOIA3+Aa=(!Q^f0$sc3Y<>o!W7 zNgmJ{j!2HqbK*hBf$-KYUW!2Ehl6p&oissed+>9SQr^IP>|5?P*^7Iq zJD1Lf-*VuQZ^GFU6ua%f{Bk(+?$2&qtUL;N^$i!~@q^UXW2a%?AB1`|p+WJr;ft)I zWnuj4Uc>=#IseYq`F9S8E@~U0*01fapb|U#Hl=jWUqqKWmHW$ml!)+qDj;2k?LB)j z3}hhz?o=bk-?sC9Z=0O}$9rIj#=`_boM}rlPm|Q3jhVPbhe`BxW%p_~hAAfEs8=@k ze|8HyE?aI*h~ST!yVoZ8p|Wh(JNGR4SN;alJuy-#_I0T*HVIHZHq~0&k169bU+J#j z3wx<~q9W2EP2IabG19?$U1*p_>##N{;yRo*oCb){m7?GMBKSsX^YzN)dLw_3>ta{( ziqRsAmrXp}8m1_;iLKLompf7OY9>**l$9=;sKxKre!Rl({mNmZ!r8YgdXeo_B#RaQC)7 z)=gkmU_@HR(f_I!KsrO}`;eE@0qULal8T1&0zCGy0w*zPxcFLB*JuL4naYNJazV9U z>kcysOS!jeDzituD}|aZR>GcZ2THU( zOzTfJ!yu#Hi3vPI%fCKO7R3V531sde#D{0W!wB%13oI&FQ35uz!UU%Tk&)1&vdF6i ze*|#FQbKA!Ueu$`fV&>H3%nKUh3Se^^Z!v%*tue+)!nM05sr}UQ_)iegZJYy4e?!= zaH>BvqWF*qxEE%Ez&*OUo1o!#T(J)g65gAFra9gBug4o_g8yPN0=;jaAq5acIn4X!5gO~kl4fGRXKaF7=ZvgmxFKZ#-bL9>eO0=xVks_YU=L)hhT=KV93{-W$St;E%++;t<<@p>2j}_x-lx7$bq`E+p zdOdM{mdu4X9%YPG)0zStvFOnJOqZYQN)Kaj%*aPR&huXKr*GIXM;whD@a6gf0{v9E zf@H~8#k7Qd=1jE(lbp0=br)8CT?#_eKc8z}XP2TEmN^D8>!$0*cgf{%iuz}bN$iEr z8@~JgNveB$(joDsa&jgv`{Cj0RV2vCgi82lx#dOu{otl$(mEm;;YvtFnv?ow{-bY! z{q9OvSQLf`iT{BF&5DD)-i#0>x$2Zz5s#8R3sp&%zO26eg z+V@v6BkjZ)d0@T$yR9-*ZNC*1SMSo!n1FO>l7)H^J+JtWnJTcPuwyYt7_F1C@cFT4 zoAjWc@%w_ZI#I`c_Bq*ZbWVPUF{v8dgjFQ8c)tToNI!d;pJsp;tV|3af0Mj#jK3*3 z%xDyI@1LAdXW`6CsaIVUvh2GJNz62*tNw6qidbT;G`ZO!{fA~fpGSQA0CC_K363Vi zs9Hm$p^3^CPgYbxMi8z9hi7B0(-4M0JzqyqPlSeWwCCiB1(`_?c5&uiMz=N&{LxQ; z(bCGZ)h?IXw3y!4AUZkL?j^kq9i{nQ6&9`3X3|HejA7gEGZ2 zy98y+0#{P5?v7^lx?*%)w}grO($ow`CwH^zt%RQNQsME=hf2Trc%jg}@O$pAd%oPq z@+k6L^HTI)XX5YO8FSM9C0|d~F@BT0ArRS>HuF=jUdV&851x5vl9wc)1gx zR$R|%=2DpoZ+jO+Io087;RjO=2hlLIJ0L9gJ#<7yMaiBf#DB31t0rG8yTz@STf(|L zwcvje(~RhqH|w3loz*$i+B^5!O{`<94T#Dj!tb}%T`K9QX_@Ob9mLq)MDJbH_K{T7 zwZT}giHe%s^fvRRwbA^E)TG^3suH>O3Kz;Q)oc{*j5$KkpC$^%c&jcb-@)B-*9q}4 z6N1K&*Hg{?oGUHqF>+xzVq)vZ+*kF3-U0QMp`%~>1aqT1?2s2sEH%Nj zx5gjFBQraYHAcmD`rm2o`ezT>e=zH==Lwq| ZK~$tZ9<}6*^=}KPf~<;6wbX~e{{#0!mxcfU diff --git a/superset-frontend/src/components/ImportDashboardSDMXModal/Modal.css b/superset-frontend/src/components/ImportDashboardSDMXModal/Modal.css deleted file mode 100644 index 1e166544e2..0000000000 --- a/superset-frontend/src/components/ImportDashboardSDMXModal/Modal.css +++ /dev/null @@ -1,24 +0,0 @@ -.modal-overlay { - position: fixed; - top: 0; - left: 0; - right: 0; - bottom: 0; - /* background-color: rgba(0, 0, 0, 0.7); */ - display: flex; - justify-content: center; - align-items: center; - } - - .file-uploader { - padding: 20px; - } - - .modal-content { - background-color: #fff; - padding: 20px; - border-radius: 8px; - max-width: 500px; - width: 90%; - position: relative; - } \ No newline at end of file diff --git a/superset-frontend/src/components/ImportDashboardSDMXModal/Modal.tsx b/superset-frontend/src/components/ImportDashboardSDMXModal/Modal.tsx index 5ab560cedd..1eea89d532 100644 --- a/superset-frontend/src/components/ImportDashboardSDMXModal/Modal.tsx +++ b/superset-frontend/src/components/ImportDashboardSDMXModal/Modal.tsx @@ -1,26 +1,27 @@ import React, { useState } from 'react'; import PropTypes from 'prop-types'; -import './Modal.css'; // Assuming you place your CSS in this file import { SupersetClient } from '@superset-ui/core'; -import { Upload, Space } from 'antd'; -import { set } from 'lodash'; +import { Upload, Space, Select } from 'antd'; +import Modal from 'src/components/Modal'; import { useToasts } from 'src/components/MessageToasts/withToasts'; -import { Input } from '../Input'; +import { FormLabel } from 'src/components/Form'; import Button from '../Button'; -const Modal = ({ isOpen, onClose }) => { - if (!isOpen) { - return null; - } - +const SdmxDashboardModal = ({ isOpen, onClose }) => { const [sdmxFile, setSdmxFile] = useState([]); const [isLoading, setIsLoading] = useState(false); + const [locale, setLocale] = useState('en'); const { addDangerToast } = useToasts(); + if (!isOpen) { + return null; + } + const onUpload = () => { const formData = new FormData(); setIsLoading(true); formData.append('file', sdmxFile[0]); + formData.append('locale', locale); SupersetClient.post({ endpoint: 'api/v1/sdmx/dashboard', @@ -31,6 +32,7 @@ const Modal = ({ isOpen, onClose }) => { window.location.reload(); }) .catch(err => { + console.log(err); setIsLoading(false); addDangerToast('There was an error loading the SDMX file'); setSdmxFile([]); @@ -38,54 +40,76 @@ const Modal = ({ isOpen, onClose }) => { }; return ( -
-
e.stopPropagation()}> -

Import SDMX Dashboard YAML definition

-

- A sample YAML file can be downloaded{' '} - - here - - . This is a proof of concept of what a declaration of graphs through - configuration files may look like.{' '} - - Learn more about this project - -

+ + + + } + > +

+ A sample YAML file can be downloaded{' '} + + here + + . This is a proof of concept of what a declaration of graphs through + configuration files may look like.{' '} + + Learn more about this project + +

+
+ + Load YAML Dashboard file + { + setSdmxFile([value]); + return false; + }} + > + + + -
- - { - setSdmxFile([value]); - return false; - }} - > - - - - -
- + Select Locale + setSdmxUrl(evt.target.value)} - /> - {children} + - SDMXHub - - -
- -
-
+ + }> +

+ SDMX (Statistical Data and Metadata eXchange) is an international standard for exchanging statistical data and metadata. Supported by major international organizations like the IMF, World Bank, and OECD, it is widely used in various domains like agriculture, finance, and social statistics. + Learn more +

+ + SDMX Url + setSdmxUrl(evt.target.value)} + /> + SDMXHub + + ); }; -Modal.propTypes = { +SdmxImportModal.propTypes = { isOpen: PropTypes.bool.isRequired, onClose: PropTypes.func, children: PropTypes.node, }; -export default Modal; +export default SdmxImportModal; From a9bfc577b6220bdb260b1abc8863669adc3412bf Mon Sep 17 00:00:00 2001 From: "andres.sole" Date: Mon, 25 Sep 2023 18:57:25 +0200 Subject: [PATCH 08/26] feat sdmx: added explore tab in add dataset --- .../ImportDashboardSDMXModal/Modal.tsx | 2 +- .../src/components/ImportSDMXModal/Modal.tsx | 173 ++++++-- .../src/components/ImportSDMXModal/style.css | 3 + superset/connectors/sqla/models.py | 5 +- superset/sdmx.py | 383 ++++++++++++------ superset/views/api.py | 63 ++- 6 files changed, 450 insertions(+), 179 deletions(-) create mode 100644 superset-frontend/src/components/ImportSDMXModal/style.css diff --git a/superset-frontend/src/components/ImportDashboardSDMXModal/Modal.tsx b/superset-frontend/src/components/ImportDashboardSDMXModal/Modal.tsx index 1eea89d532..bbf5d81a93 100644 --- a/superset-frontend/src/components/ImportDashboardSDMXModal/Modal.tsx +++ b/superset-frontend/src/components/ImportDashboardSDMXModal/Modal.tsx @@ -29,7 +29,7 @@ const SdmxDashboardModal = ({ isOpen, onClose }) => { }) .then(res => { setIsLoading(false); - window.location.reload(); + window.location.pathname = `/superset/dashboard/${res.json.dashboard_id}`; }) .catch(err => { console.log(err); diff --git a/superset-frontend/src/components/ImportSDMXModal/Modal.tsx b/superset-frontend/src/components/ImportSDMXModal/Modal.tsx index 899dadfb4e..388b2103fd 100644 --- a/superset-frontend/src/components/ImportSDMXModal/Modal.tsx +++ b/superset-frontend/src/components/ImportSDMXModal/Modal.tsx @@ -1,74 +1,173 @@ -import React, { useState } from 'react'; +import React, { useState, useEffect } from 'react'; import PropTypes from 'prop-types'; import { SupersetClient } from '@superset-ui/core'; -import { Space } from 'antd'; +import { Select, Space } from 'antd'; import Button from '../Button'; import { Input } from '../Input'; import { useToasts } from 'src/components/MessageToasts/withToasts'; import Modal from 'src/components/Modal'; import { FormLabel } from '../Form'; +import Tabs, { TabsProps } from 'src/components/Tabs'; +import { isPlainObject } from 'lodash'; +import "./style.css" +const SdmxImportModal = ({ isOpen, onClose }: { isOpen: boolean, onClose: () => {} }) => { - -const SdmxImportModal = ({ isOpen, onClose }) => { + const { TabPane } = Tabs; if (!isOpen) { return null; } - - const [sdmxUrl, setSdmxUrl] = useState(''); - const [isLoading, setIsLoading] = useState(false); const { addDangerToast } = useToasts(); + const [sdmxUrl, setSdmxUrl] = useState(''); + const [isLoadingDataset, setIsLoading] = useState(false); + const [currentTab, setCurrentTab] = useState('1'); + const [agencies, setAgencies] = useState([] as string[]); + const [selectedAgency, setSelectedAgency] = useState(null as null | string); + const [dataflows, setDataflows] = useState({}); + const [isLoadingDataflows, setIsLoadingDataflows] = useState(false); + const [selectedDataflow, setSelectedDataflow] = useState(null as null | string); + const [numberOfObservations, setNumberOfObservations] = useState(5); + + const loadDataflows = (agencyId: string) => { + if (!dataflows[agencyId]) { + setIsLoadingDataflows(true) + SupersetClient.get({ + endpoint: `api/v1/sdmx/agency/${agencyId}` + }).then(res => { + setDataflows({ ...dataflows, [agencyId]: res.json }) + }).catch(err => { + addDangerToast("There was an error loading the available dataflows"); + }).finally(() => { + setIsLoadingDataflows(false) + }) + } + } + const onUpload = () => { setIsLoading(true) + if (currentTab === '1') { + SupersetClient.post({ + endpoint: 'api/v1/sdmx/', + jsonPayload: { + sdmxUrl, + }, + }).then(res => { + setIsLoading(false); + window.location.reload(); + }).catch(err => { + setIsLoading(false); + addDangerToast("There was an error loading the SDMX Url"); + setSdmxUrl(''); + }) + } else if (currentTab === '2') { + SupersetClient.post({ + endpoint: 'api/v1/sdmx/', + jsonPayload: { + agencyId: selectedAgency, + dataflowId: selectedDataflow, + numberOfObservations + }, + }).then(res => { + setIsLoading(false); + window.location.reload(); + }).catch(err => { + setIsLoading(false); + addDangerToast("There was an error loading the SDMX dataflow"); + }) + } + }; - - SupersetClient.post({ - endpoint: 'api/v1/sdmx/', - jsonPayload: { - sdmxUrl, - }, + useEffect(() => { + SupersetClient.get({ + endpoint: 'api/v1/sdmx/agency' }).then(res => { - setIsLoading(false); - window.location.reload(); + setAgencies(res.json as string[]) }).catch(err => { - setIsLoading(false); - console.log(err) - addDangerToast("There was an error loading the SDMX Url"); setSdmxUrl(''); + }) + }, []) + + const handleAgencyChange = (agencyId: string) => { + if (!dataflows[agencyId]) { + setDataflows({ ...dataflows, [agencyId]: [] }) } - }; + setSelectedAgency(agencyId) + + loadDataflows(agencyId) + setSelectedDataflow(null) + setNumberOfObservations(5) + } return ( - - + }> -

- SDMX (Statistical Data and Metadata eXchange) is an international standard for exchanging statistical data and metadata. Supported by major international organizations like the IMF, World Bank, and OECD, it is widely used in various domains like agriculture, finance, and social statistics. - Learn more -

- - SDMX Url +

+ SDMX (Statistical Data and Metadata eXchange) is an international standard for exchanging statistical data and metadata. Supported by major international organizations like the IMF, World Bank, and OECD, it is widely used in various domains like agriculture, finance, and social statistics. + Learn more +

+ setCurrentTab(tabId)}> + + + SDMX Url setSdmxUrl(evt.target.value)} /> - SDMXHub - - + + + + + Select Agency + setSelectedDataflow(dataflowId)} + style={{ width: '100%' }} + value={selectedDataflow as string} + disabled={isLoadingDataflows} + loading={isLoadingDataflows} + options={dataflows[selectedAgency].map((option: any) => { + if (isPlainObject(option.name)) + option.name = option.name.en.content + return { + label: option.name, + value: option.unique_id, + } + })} /> + Last time observations + setNumberOfObservations(Number(evt.target.value))} /> + } + + + + + ); }; diff --git a/superset-frontend/src/components/ImportSDMXModal/style.css b/superset-frontend/src/components/ImportSDMXModal/style.css new file mode 100644 index 0000000000..2d7d24b1ed --- /dev/null +++ b/superset-frontend/src/components/ImportSDMXModal/style.css @@ -0,0 +1,3 @@ +.ant-tabs-ink-bar { + background: #d9d9d9 !important; +} \ No newline at end of file diff --git a/superset/connectors/sqla/models.py b/superset/connectors/sqla/models.py index 434b273ada..97ba82a3c9 100644 --- a/superset/connectors/sqla/models.py +++ b/superset/connectors/sqla/models.py @@ -1484,11 +1484,12 @@ def before_delete( connection: Connection, sqla_table: SqlaTable, ) -> None: - if sqla_table.is_sdmx: + try: session = Session.object_session(sqla_table) os.remove(f"dbs/{sqla_table.sdmx_uuid}") session.delete(sqla_table.database) - + except Exception as ex: # pylint: disable=broad-except + pass @staticmethod def after_delete( mapper: Mapper, diff --git a/superset/sdmx.py b/superset/sdmx.py index 39412d878d..ad9a3784a6 100644 --- a/superset/sdmx.py +++ b/superset/sdmx.py @@ -14,29 +14,32 @@ from superset.databases.commands.create import CreateDatabaseCommand from superset import db import json +import re +import sqlite3 def load_database(sdmx_url, dataset_instance=None): message = read_sdmx(sdmx_url) - - dataflow_id = get_dataflow_id(sdmx_url) agency_id = get_agency_id(sdmx_url) + dataflow_id = get_dataflow_id(sdmx_url) ws = None + data = list(message.payload.values())[0].data + if agency_id == "BIS": ws = webservices.BisWs() elif agency_id == "ECB": ws = webservices.EcbWs() elif agency_id == "ESTAT": - ws = webservices.EstatWs() + ws = webservices.EuroStatWs() elif agency_id == "ILO": ws = webservices.IloWs() try: + metadata = ws.get_data_flow(dataflow_id, references="descendants") - except Exception: - raise Exception(dataflow_id) + except Exception as e: + raise Exception(e, dataflow_id) - data = list(message.payload.values())[0].data df, concepts_name = generate_final_df_and_concepts_name(data, metadata.payload) if dataset_instance is None: dataset_uuid = uuid.uuid4() @@ -94,19 +97,70 @@ def create_dashboard(spec): def get_locale_column_if_exists(data, column, locale): if locale not in ["en", "es", "fr"]: raise Exception("Locale not supported") + try: + for column_data in data["columns"]: + if column_data["column_name"] == column + f"-{locale}": + return column + f"-{locale}" + except Exception: + for column_name in data.columns: + if column_name == column + f"-{locale}": + return column + f"-{locale}" - for column_data in data["columns"]: - if column_data["column_name"] == column + f"-{locale}": - return column + f"-{locale}" - return column +def get_locale_value(df, column, locale): + if locale not in ["en", "es", "fr"]: + raise Exception("Locale not supported") + return df[get_locale_column_if_exists(df, column, locale)][0] + + +def substitute_string(string, df, locale="en"): + pattern = r"\{\$([^\}]+)\}" + for column in re.findall(pattern, str(string)): + string = string.replace( + f"{{${column}}}", + get_locale_value( + df, + column, + locale, + ), + ) + return string + + +def modify_key(original, old_key, new_key): + return {new_key if k == old_key else k: v for k, v in original.items()} + + def create_charts(spec, datasets, dashboard, locale="en"): for idx, row in enumerate(spec["Rows"]): if not row["DATA"]: continue + conn = sqlite3.connect(f"dbs/{datasets[idx].sdmx_uuid}") + query = "SELECT name FROM sqlite_master WHERE type='table';" + table_names = conn.execute(query).fetchall() + + # Convert list of tuples to list of strings + table_names = [name[0] for name in table_names] + + # Query the database and load the result into a pandas DataFrame + query = f'SELECT * FROM "{table_names[0]}"' + df = pd.read_sql(query, conn) + + # Close the connection + conn.close() + + mentioned_columns = ["OBS_VALUE"] # OBS_VALUE is always available + pattern = r"\{\$([^\}]+)\}" + + for key in list(row.keys()): # iterating on a copy since we modify the dict + matches = re.findall(pattern, str(row[key])) + mentioned_columns += matches + + row[key] = substitute_string(row[key], df, locale) + if row["chartType"] == "VALUE": chart = Slice( dashboards=[dashboard], @@ -114,17 +168,38 @@ def create_charts(spec, datasets, dashboard, locale="en"): is_managed_externally=True, slice_name=row["Title"], datasource_type="table", - params='{"datasource":"' - + str(datasets[idx].id) - + '__table","viz_type":"handlebars","query_mode":"aggregate","groupby":[],"metrics":[{"aggregate":null,"column":null,"datasourceWarning":false,"expressionType":"SQL","hasCustomLabel":false,"label":"OBS_VALUE","optionName":"","sqlExpression":"OBS_VALUE"}],"all_columns":[],"percent_metrics":[],"order_by_cols":[],"order_desc":true,"row_limit":10000,"server_page_length":10,"adhoc_filters":[],"handlebarsTemplate":"

\\n {{#each data}}\\n {{this.OBS_VALUE}}\\n {{/each}}\\n

","styleTemplate":"\\n.data-chart {\\n text-align: center;\\n font-size: 6em;\\n}\\n","extra_form_data":{},"dashboards":[' - + str(dashboard.id) - + "]}", - query_context='{"datasource":{"id": ' - + str(datasets[idx].id) - + ',"type":"table"},"force":false,"queries":[{"filters":[],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":[],"metrics":[{"aggregate":null,"column":null,"datasourceWarning":false,"expressionType":"SQL","hasCustomLabel":false,"label":"OBS_VALUE","optionName":"","sqlExpression":"OBS_VALUE"}],"orderby":[[{"aggregate":null,"column":null,"datasourceWarning":false,"expressionType":"SQL","hasCustomLabel":false,"label":"OBS_VALUE","optionName":"","sqlExpression":"OBS_VALUE"},false]],"annotation_layers":[],"row_limit":10000,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"' - + str(datasets[idx].id) - + '__table","viz_type":"handlebars","query_mode":"aggregate","groupby":[],"metrics":[{"aggregate":null,"column":null,"datasourceWarning":false,"expressionType":"SQL","hasCustomLabel":false,"label":"OBS_VALUE","optionName":"","sqlExpression":"OBS_VALUE"}],"all_columns":[],"percent_metrics":[],"order_by_cols":[],"order_desc":true,"row_limit":10000,"server_page_length":10,"adhoc_filters":[],"handlebarsTemplate":"

\\n {{#each data}}\\n {{this.OBS_VALUE}}\\n {{/each}}\\n

","styleTemplate":"\\n.data-chart {\\n text-align: center;\\n font-size: 6em;\\n}\\n","extra_form_data":{},"dashboards":[40],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}', viz_type="handlebars", + params=json.dumps( + { + "datasource": f"{datasets[idx].id}__table", + "viz_type": "handlebars", + "query_mode": "raw", + "groupby": [], + "metrics": [ + { + "aggregate": None, + "column": None, + "datasourceWarning": False, + "expressionType": "SQL", + "hasCustomLabel": False, + "label": ", ".join(mentioned_columns), + "sqlExpression": ", ".join(mentioned_columns), + } + ], + "all_columns": mentioned_columns, + "percent_metrics": [], + "order_by_cols": [], + "order_desc": True, + "row_limit": 10000, + "server_page_length": 10, + "adhoc_filters": [], + "handlebarsTemplate": f'{{{{#each data}}}}\n

\n {{{{this.OBS_VALUE}}}}\n

{row["Subtitle"]}{{{{/each}}}}\n', + "styleTemplate": "\n.data-chart {\n text-align: center;\n font-size: 6em;\n}\n.data-label {\n text-align: center;\n font-size: 2em;\n width: 100%;\n display: block;\n}", + "extra_form_data": {}, + "dashboards": [dashboard.id], + } + ), + query_context="", ) dashboard.slices.append(chart) db.session.add(dashboard) @@ -137,13 +212,43 @@ def create_charts(spec, datasets, dashboard, locale="en"): is_managed_externally=True, slice_name=row["Title"], datasource_type="table", - params='{"viz_type":"pie","groupby":["' - + get_locale_column_if_exists( - datasets[idx].data, row["legendConcept"], locale - ) - + '"],"metric":{"aggregate":null,"column":null,"datasourceWarning":false,"expressionType":"SQL","hasCustomLabel":false,"label":"OBS_VALUE","optionName":"metric_2o7qsp2myck_m08ex1fqyyp","sqlExpression":"OBS_VALUE"},"adhoc_filters":[],"row_limit":10000,"sort_by_metric":true,"color_scheme":"supersetColors","show_labels_threshold":5,"show_legend":true,"legendType":"scroll","legendOrientation":"right","label_type":"key_value_percent","number_format":"SMART_NUMBER","date_format":"smart_date","show_labels":true,"labels_outside":true,"label_line":true,"outerRadius":70,"innerRadius":30,"extra_form_data":{},"dashboards":[' - + str(dashboard.id) - + "]}", + params=json.dumps( + { + "viz_type": "pie", + "groupby": [ + get_locale_column_if_exists( + datasets[idx].data, row["legendConcept"], locale + ) + ], + "metric": { + "aggregate": None, + "column": None, + "datasourceWarning": False, + "expressionType": "SQL", + "hasCustomLabel": False, + "label": "OBS_VALUE", + "sqlExpression": "OBS_VALUE", + }, + "adhoc_filters": [], + "row_limit": 10000, + "sort_by_metric": True, + "color_scheme": "supersetColors", + "show_labels_threshold": 5, + "show_legend": True, + "legendType": "scroll", + "legendOrientation": "right", + "label_type": "key_value_percent", + "number_format": "SMART_NUMBER", + "date_format": "smart_date", + "show_labels": True, + "labels_outside": True, + "label_line": True, + "outerRadius": 70, + "innerRadius": 30, + "extra_form_data": {}, + "dashboards": [str(dashboard.id)], + } + ), query_context="", viz_type="pie", ) @@ -159,54 +264,64 @@ def create_charts(spec, datasets, dashboard, locale="en"): is_managed_externally=True, slice_name=row["Title"], datasource_type="table", - params='{"datasource":"' - + str(datasets[idx].id) - + '__table","viz_type":"echarts_timeseries_line","x_axis":"' - + row["xAxisConcept"] - + '","time_grain_sqla":"P1D","x_axis_sort_asc":true,"x_axis_sort_series":"name","x_axis_sort_series_ascending":true,"metrics":[{"expressionType":"SQL","sqlExpression":"OBS_VALUE","column":null,"aggregate":null,"datasourceWarning":false,"hasCustomLabel":false,"label":"OBS_VALUE","optionName":"metric_2pzqjh4ddnr_9fsj62hsc9c"}],"groupby":["' - + get_locale_column_if_exists( - datasets[idx].data, row["legendConcept"], locale - ) - + '"],"adhoc_filters":[],"order_desc":true,"row_limit":10000,"truncate_metric":true,"show_empty_columns":true,"comparison_type":"values","annotation_layers":[],"forecastPeriods":10,"forecastInterval":0.8,"x_axis_title_margin":15,"y_axis_title_margin":15,"y_axis_title_position":"Left","sort_series_type":"sum","color_scheme":"supersetColors","seriesType":"line","show_value":false,"only_total":true,"opacity":0.2,"markerSize":6,"zoomable":false,"show_legend":true,"legendType":"scroll","legendOrientation":"top","x_axis_time_format":"smart_date","rich_tooltip":true,"tooltipTimeFormat":"smart_date","y_axis_format":"SMART_NUMBER","y_axis_bounds":[null,null],"extra_form_data":{},"dashboards":[' - + str(dashboard.id) - + "]}", - query_context='{"datasource":{"id":' - + str(datasets[idx].id) - + ',"type":"table"},"force":false,"queries":[{"filters":[],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":[{"timeGrain":"P1D","columnType":"BASE_AXIS","sqlExpression":"' - + get_locale_column_if_exists( - datasets[idx].data, row["xAxisConcept"], locale - ) - + '","label":"' - + get_locale_column_if_exists( - datasets[idx].data, row["xAxisConcept"], locale - ) - + '","expressionType":"SQL"},"' - + get_locale_column_if_exists( - datasets[idx].data, row["legendConcept"], locale - ) - + '"],"metrics":[{"expressionType":"SQL","sqlExpression":"OBS_VALUE","column":null,"aggregate":null,"datasourceWarning":false,"hasCustomLabel":false,"label":"OBS_VALUE","optionName":"metric_2pzqjh4ddnr_9fsj62hsc9c"}],"orderby":[[{"expressionType":"SQL","sqlExpression":"OBS_VALUE","column":null,"aggregate":null,"datasourceWarning":false,"hasCustomLabel":false,"label":"OBS_VALUE","optionName":"metric_2pzqjh4ddnr_9fsj62hsc9c"},false]],"annotation_layers":[],"row_limit":10000,"series_columns":["' - + get_locale_column_if_exists( - datasets[idx].data, row["legendConcept"], locale - ) - + '"],"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{},"time_offsets":[],"post_processing":[{"operation":"pivot","options":{"index":["' - + get_locale_column_if_exists( - datasets[idx].data, row["xAxisConcept"], locale - ) - + '"],"columns":["' - + get_locale_column_if_exists( - datasets[idx].data, row["legendConcept"], locale - ) - + '"],"aggregates":{"OBS_VALUE":{"operator":"mean"}},"drop_missing_columns":false}},{"operation":"rename","options":{"columns":{"OBS_VALUE":null},"level":0,"inplace":true}},{"operation":"flatten"}]}],"form_data":{"datasource":"9__table","viz_type":"echarts_timeseries_line","x_axis":"' - + get_locale_column_if_exists( - datasets[idx].data, row["xAxisConcept"], locale - ) - + '","time_grain_sqla":"P1D","x_axis_sort_asc":true,"x_axis_sort_series":"name","x_axis_sort_series_ascending":true,"metrics":[{"expressionType":"SQL","sqlExpression":"OBS_VALUE","column":null,"aggregate":null,"datasourceWarning":false,"hasCustomLabel":false,"label":"OBS_VALUE","optionName":"metric_2pzqjh4ddnr_9fsj62hsc9c"}],"groupby":["' - + get_locale_column_if_exists( - datasets[idx].data, row["legendConcept"], locale - ) - + '"],"adhoc_filters":[],"order_desc":true,"row_limit":10000,"truncate_metric":true,"show_empty_columns":true,"comparison_type":"values","annotation_layers":[],"forecastPeriods":10,"forecastInterval":0.8,"x_axis_title_margin":15,"y_axis_title_margin":15,"y_axis_title_position":"Left","sort_series_type":"sum","color_scheme":"supersetColors","seriesType":"line","show_value":false,"only_total":true,"opacity":0.2,"markerSize":6,"zoomable":false,"show_legend":true,"legendType":"scroll","legendOrientation":"top","x_axis_time_format":"smart_date","rich_tooltip":true,"tooltipTimeFormat":"smart_date","y_axis_format":"SMART_NUMBER","y_axis_bounds":[null,null],"extra_form_data":{},"dashboards":[ ' - + str(dashboard.id) - + ' ],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}', + params=json.dumps( + { + "datasource": f"{datasets[idx].id}__table", + "viz_type": "echarts_timeseries_line", + "x_axis": row["xAxisConcept"], + "time_grain_sqla": "P1D", + "x_axis_sort_asc": True, + "x_axis_sort_series": "name", + "x_axis_sort_series_ascending": True, + "metrics": [ + { + "expressionType": "SQL", + "sqlExpression": "OBS_VALUE", + "column": None, + "aggregate": None, + "datasourceWarning": False, + "hasCustomLabel": False, + "label": "OBS_VALUE", + } + ], + "groupby": [ + get_locale_column_if_exists( + datasets[idx].data, row["legendConcept"], locale + ) + ], + "adhoc_filters": [], + "order_desc": True, + "row_limit": 10000, + "truncate_metric": True, + "show_empty_columns": True, + "comparison_type": "values", + "annotation_layers": [], + "forecastPeriods": 10, + "forecastInterval": 0.8, + "x_axis_title_margin": 15, + "y_axis_title_margin": 15, + "y_axis_title_position": "Left", + "sort_series_type": "sum", + "color_scheme": "supersetColors", + "seriesType": "line", + "show_value": False, + "only_total": True, + "opacity": 0.2, + "markerSize": 6, + "zoomable": False, + "show_legend": True, + "legendType": "scroll", + "legendOrientation": "top", + "x_axis_time_format": "smart_date", + "rich_tooltip": True, + "tooltipTimeFormat": "smart_date", + "y_axis_format": "SMART_NUMBER", + "y_axis_bounds": [None, None], + "extra_form_data": {}, + "dashboards": [str(dashboard.id)], + } + ), + query_context="", viz_type="echarts_timeseries_line", ) dashboard.slices.append(chart) @@ -220,36 +335,59 @@ def create_charts(spec, datasets, dashboard, locale="en"): is_managed_externally=True, slice_name=row["Title"], datasource_type="table", - params='{"datasource":"' - + str(datasets[idx].id) - + '__table","viz_type":"echarts_timeseries_bar","x_axis":"' - + get_locale_column_if_exists( - datasets[idx].data, row["xAxisConcept"], locale - ) - + '","time_grain_sqla":"P1D","x_axis_sort":"OBS_VALUE","x_axis_sort_asc":false,"x_axis_sort_series":"name","x_axis_sort_series_ascending":true,"metrics":[{"expressionType":"SQL","sqlExpression":"OBS_VALUE","column":null,"aggregate":null,"datasourceWarning":false,"hasCustomLabel":false,"label":"OBS_VALUE","optionName":"metric_qll9scn97y_rj89e0zvnw"}],"groupby":[],"adhoc_filters":[],"order_desc":true,"row_limit":10000,"truncate_metric":true,"show_empty_columns":true,"comparison_type":"values","annotation_layers":[],"forecastPeriods":10,"forecastInterval":0.8,"orientation":"vertical","x_axis_title_margin":15,"y_axis_title_margin":15,"y_axis_title_position":"Left","sort_series_type":"sum","color_scheme":"supersetColors","only_total":true,"show_legend":true,"legendType":"scroll","legendOrientation":"top","x_axis_time_format":"smart_date","y_axis_format":"SMART_NUMBER","y_axis_bounds":[null,null],"rich_tooltip":true,"tooltipTimeFormat":"smart_date","extra_form_data":{},"dashboards":[]}', - query_context='{"datasource":{"id":' - + str(datasets[idx].id) - + ',"type":"table"},"force":false,"queries":[{"filters":[],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":[{"timeGrain":"P1D","columnType":"BASE_AXIS","sqlExpression":"' - + get_locale_column_if_exists( - datasets[idx].data, row["xAxisConcept"], locale - ) - + '","label":"' - + get_locale_column_if_exists( - datasets[idx].data, row["xAxisConcept"], locale - ) - + '","expressionType":"SQL"}],"metrics":[{"expressionType":"SQL","sqlExpression":"OBS_VALUE","column":null,"aggregate":null,"datasourceWarning":false,"hasCustomLabel":false,"label":"OBS_VALUE","optionName":"metric_qll9scn97y_rj89e0zvnw"}],"orderby":[[{"expressionType":"SQL","sqlExpression":"OBS_VALUE","column":null,"aggregate":null,"datasourceWarning":false,"hasCustomLabel":false,"label":"OBS_VALUE","optionName":"metric_qll9scn97y_rj89e0zvnw"},false]],"annotation_layers":[],"row_limit":10000,"series_columns":[],"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{},"time_offsets":[],"post_processing":[{"operation":"pivot","options":{"index":["' - + get_locale_column_if_exists( - datasets[idx].data, row["xAxisConcept"], locale - ) - + '"],"columns":[],"aggregates":{"OBS_VALUE":{"operator":"mean"}},"drop_missing_columns":false}},{"operation":"sort","options":{"by":"OBS_VALUE","ascending":false}},{"operation":"flatten"}]}],"form_data":{"datasource":"' - + str(datasets[idx].id) - + '__table","viz_type":"echarts_timeseries_bar","x_axis":"' - + get_locale_column_if_exists( - datasets[idx].data, row["xAxisConcept"], locale - ) - + '","time_grain_sqla":"P1D","x_axis_sort":"OBS_VALUE","x_axis_sort_asc":false,"x_axis_sort_series":"name","x_axis_sort_series_ascending":true,"metrics":[{"expressionType":"SQL","sqlExpression":"OBS_VALUE","column":null,"aggregate":null,"datasourceWarning":false,"hasCustomLabel":false,"label":"OBS_VALUE","optionName":"metric_qll9scn97y_rj89e0zvnw"}],"groupby":[],"adhoc_filters":[],"order_desc":true,"row_limit":10000,"truncate_metric":true,"show_empty_columns":true,"comparison_type":"values","annotation_layers":[],"forecastPeriods":10,"forecastInterval":0.8,"orientation":"vertical","x_axis_title_margin":15,"y_axis_title_margin":15,"y_axis_title_position":"Left","sort_series_type":"sum","color_scheme":"supersetColors","only_total":true,"show_legend":true,"legendType":"scroll","legendOrientation":"top","x_axis_time_format":"smart_date","y_axis_format":"SMART_NUMBER","y_axis_bounds":[null,null],"rich_tooltip":true,"tooltipTimeFormat":"smart_date","extra_form_data":{},"dashboards":[' - + str(dashboard.id) - + '],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}', + params=json.dumps( + { + "datasource": f"{datasets[idx].id}__table", + "viz_type": "echarts_timeseries_bar", + "x_axis": get_locale_column_if_exists( + datasets[idx].data, row["xAxisConcept"], locale + ), + "time_grain_sqla": "P1D", + "x_axis_sort": "OBS_VALUE", + "x_axis_sort_asc": False, + "x_axis_sort_series": "name", + "x_axis_sort_series_ascending": True, + "metrics": [ + { + "expressionType": "SQL", + "sqlExpression": "OBS_VALUE", + "column": None, + "aggregate": None, + "datasourceWarning": False, + "hasCustomLabel": False, + "label": "OBS_VALUE", + } + ], + "groupby": [], + "adhoc_filters": [], + "order_desc": True, + "row_limit": 10000, + "truncate_metric": True, + "show_empty_columns": True, + "comparison_type": "values", + "annotation_layers": [], + "forecastPeriods": 10, + "forecastInterval": 0.8, + "orientation": "vertical", + "x_axis_title_margin": 15, + "y_axis_title_margin": 15, + "y_axis_title_position": "Left", + "sort_series_type": "sum", + "color_scheme": "supersetColors", + "only_total": True, + "show_legend": True, + "legendType": "scroll", + "legendOrientation": "top", + "x_axis_time_format": "smart_date", + "y_axis_format": "SMART_NUMBER", + "y_axis_bounds": [None, None], + "rich_tooltip": True, + "tooltipTimeFormat": "smart_date", + "extra_form_data": {}, + "dashboards": [str(dashboard.id)], + } + ), + query_context="", viz_type="echarts_timeseries_bar", ) @@ -264,8 +402,9 @@ def create_codelist_dataframe(codelist, concept_name): codelist_list = [] for id, code in codelist.items.items(): item = {"id": id} - for lang_code, strings in code.name.items(): - item[f"{concept_name}-{lang_code}"] = strings["content"] + if isinstance(code.name, dict) : + for lang_code, strings in code.name.items(): + item[f"{concept_name}-{lang_code}"] = strings["content"] codelist_list.append(item) @@ -307,32 +446,20 @@ def generate_final_df_and_concepts_name(data, metadata_payload): def get_dataflow_id(sdmx_url): - if "stats.bis.org" in sdmx_url: - full_id = sdmx_url.split("/data/")[1].split("/")[0] - if "," in full_id: - return full_id.split(",")[1] - return full_id - elif "sdw-wsrest.ecb.europa.eu" in sdmx_url: - full_id = sdmx_url.split("/data/")[1].split("/")[0] - if "," in full_id: - return full_id.split(",")[1] - return full_id - elif "ec.europa.eu" in sdmx_url: - return sdmx_url.split("/data/")[1][:-1] - elif "www.ilo.org" in sdmx_url: - full_id = sdmx_url.split("/data/")[1].split("/")[0] - if "," in full_id: - return full_id.split(",")[1] - return full_id - raise NotImplementedError("This SDMX provider is not supported yet") + full_dataflow_id = sdmx_url.split("data/")[1].split("/")[0] + if "," in full_dataflow_id: + dataflow_id = full_dataflow_id.split(",")[1] + else: + dataflow_id = full_dataflow_id + return dataflow_id def get_agency_id(sdmx_url): if "stats.bis.org" in sdmx_url: return "BIS" - elif "sdw-wsrest.ecb.europa.eu" in sdmx_url: + elif "ecb.europa.eu" in sdmx_url: return "ECB" - elif "ec.europa.eu/eurostat" in sdmx_url: + elif "ec.europa.eu" in sdmx_url: return "ESTAT" - elif "www.ilo.org" in sdmx_url: + elif "ilo.org" in sdmx_url: return "ILO" diff --git a/superset/views/api.py b/superset/views/api.py index d3f73318fc..7ebd9f1053 100644 --- a/superset/views/api.py +++ b/superset/views/api.py @@ -37,16 +37,20 @@ from superset.utils import core as utils from superset.utils.date_parser import get_since_until from superset.views.base import api, BaseSupersetView, handle_api_exception -from superset.sdmx import get_dataflow_id, get_agency_id, generate_final_df_and_concepts_name, load_database, create_dashboard, create_charts +from superset.sdmx import load_database, create_dashboard, create_charts import yaml +from sdmxthon.api.api import get_supported_agencies +from sdmxthon.webservices import webservices if TYPE_CHECKING: from superset.common.query_context_factory import QueryContextFactory get_time_range_schema = {"type": "string"} + class Api(BaseSupersetView): query_context_factory = None + # @has_access_api @event_logger.log_this @api @@ -59,7 +63,7 @@ def sdmx_upload_dashboard(self) -> FlaskResponse: spec = yaml.safe_load(file_content) locale = request.form.get("locale", "en") datasets = [] - + for row in spec["Rows"]: if not row["DATA"]: datasets.append(None) @@ -71,7 +75,7 @@ def sdmx_upload_dashboard(self) -> FlaskResponse: create_charts(spec, datasets, dashboard, locale) - return self.json_response({"status": "OK"}) + return self.json_response({"status": "OK", "dashboard_id": dashboard.id}) except Exception as e: return self.json_response( json.dumps({"error": str(e)}), @@ -86,16 +90,23 @@ def sdmx_upload_dashboard(self) -> FlaskResponse: def sdmx_upload(self) -> FlaskResponse: try: json_data = request.json - sdmx_url = json_data["sdmxUrl"] - - if not sdmx_url: - return self.json_response( - {"error": "sdmxUrl is required"}, - status=400, - ) - load_database(sdmx_url) + if "sdmxUrl" in json_data: + sdmx_url = json_data["sdmxUrl"] + elif "agencyId" in json_data: + agency_id = json_data["agencyId"] + supported_agencies = get_supported_agencies() + if json_data["agencyId"] not in supported_agencies.keys(): + return self.json_response( + { + "error": f"Agency {agency_id} not supported. Supported agencies: {list(supported_agencies.keys())}" + }, + status=400, + ) + dataflow_id = json_data["dataflowId"].split(":")[1].split("(")[0] + sdmx_url = supported_agencies[agency_id]().get_data_url(dataflow_id, last_n_observations=json_data["numberOfObservations"]) + load_database(sdmx_url) return self.json_response({"status": "OK"}) except Exception as e: return self.json_response( @@ -103,6 +114,36 @@ def sdmx_upload(self) -> FlaskResponse: status=500, ) + @event_logger.log_this + @api + @handle_api_exception + @expose("/v1/sdmx/agency", methods=("GET",)) + def sdmx_get_agencies(self) -> FlaskResponse: + try: + agencies = list(get_supported_agencies().keys()) + return self.json_response(agencies) + except Exception as e: + return self.json_response( + json.dumps({"error": str(e)}), + status=500, + ) + + @event_logger.log_this + @api + @handle_api_exception + @expose("/v1/sdmx/agency/", methods=("GET",)) + def sdmx_get_dataflows(self, agency_id) -> FlaskResponse: + try: + return self.json_response( + get_supported_agencies()[agency_id]().get_all_dataflows() + ) + + except Exception as e: + return self.json_response( + json.dumps({"error": str(e)}), + status=500, + ) + @event_logger.log_this @api @handle_api_exception From 20ea0069d9a94e45d4467f5669665c9147863ced Mon Sep 17 00:00:00 2001 From: "andres.sole" Date: Mon, 25 Sep 2023 20:46:17 +0200 Subject: [PATCH 09/26] lint sdmx: fixed lint --- .../ImportDashboardSDMXModal/Modal.tsx | 28 ++- .../src/components/ImportSDMXModal/Modal.tsx | 209 +++++++++++------- .../src/pages/DatasetList/index.tsx | 4 - 3 files changed, 151 insertions(+), 90 deletions(-) diff --git a/superset-frontend/src/components/ImportDashboardSDMXModal/Modal.tsx b/superset-frontend/src/components/ImportDashboardSDMXModal/Modal.tsx index bbf5d81a93..0e0f1b1821 100644 --- a/superset-frontend/src/components/ImportDashboardSDMXModal/Modal.tsx +++ b/superset-frontend/src/components/ImportDashboardSDMXModal/Modal.tsx @@ -5,10 +5,17 @@ import { Upload, Space, Select } from 'antd'; import Modal from 'src/components/Modal'; import { useToasts } from 'src/components/MessageToasts/withToasts'; import { FormLabel } from 'src/components/Form'; +import { RcFile } from 'antd/lib/upload'; import Button from '../Button'; -const SdmxDashboardModal = ({ isOpen, onClose }) => { - const [sdmxFile, setSdmxFile] = useState([]); +const SdmxDashboardModal = ({ + isOpen, + onClose, +}: { + isOpen: boolean; + onClose: () => void; +}) => { + const [sdmxFile, setSdmxFile] = useState([] as RcFile[]); const [isLoading, setIsLoading] = useState(false); const [locale, setLocale] = useState('en'); const { addDangerToast } = useToasts(); @@ -32,7 +39,6 @@ const SdmxDashboardModal = ({ isOpen, onClose }) => { window.location.pathname = `/superset/dashboard/${res.json.dashboard_id}`; }) .catch(err => { - console.log(err); setIsLoading(false); addDangerToast('There was an error loading the SDMX file'); setSdmxFile([]); @@ -57,20 +63,27 @@ const SdmxDashboardModal = ({ isOpen, onClose }) => { } > -

+

A sample YAML file can be downloaded{' '} - + here . This is a proof of concept of what a declaration of graphs through configuration files may look like.{' '} - + Learn more about this project

- Load YAML Dashboard file + Load YAML Dashboard file { - Select Locale - + Select Agency - handleAgencyChange(agencyId)} + style={{ width: '100%' }} value={selectedAgency as string} - options={agencies.map((option, index) => { - return { - label: option, - value: option, - } - })} + options={agencies.map((option, index) => ({ + label: option, + value: option, + }))} loading={isLoadingDataflows} disabled={isLoadingDataflows} /> - {selectedAgency && + {selectedAgency && ( <> Select Dataflow - + setSelectedDataflow(dataflowId) + } style={{ width: '100%' }} value={selectedDataflow as string} disabled={isLoadingDataflows} loading={isLoadingDataflows} options={dataflows[selectedAgency].map((option: any) => { + let optionName = option.name; + if (isPlainObject(option.name)) - option.name = option.name.en.content + optionName = optionName.en.content; + return { - label: option.name, + label: optionName, value: option.unique_id, - } - })} /> + }; + })} + /> Last time observations - setNumberOfObservations(Number(evt.target.value))} /> - } + + setNumberOfObservations(Number(evt.target.value)) + } + /> + + )} - ); diff --git a/superset-frontend/src/pages/DatasetList/index.tsx b/superset-frontend/src/pages/DatasetList/index.tsx index 5e938688c5..c2b8a4f2a2 100644 --- a/superset-frontend/src/pages/DatasetList/index.tsx +++ b/superset-frontend/src/pages/DatasetList/index.tsx @@ -739,10 +739,6 @@ const DatasetList: FunctionComponent = ({ ); }; - const openModal = () => { - setIsModalOpen(true); - }; - const closeModal = () => { setIsModalOpen(false); }; From c5fd23e2ace386ba6bbac8df24ad537bfc701b9c Mon Sep 17 00:00:00 2001 From: "andres.sole" Date: Mon, 25 Sep 2023 20:46:37 +0200 Subject: [PATCH 10/26] fix chore: poinbt to the hackathon image --- docker-compose-non-dev.yml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose-non-dev.yml b/docker-compose-non-dev.yml index d3a006cdd8..8ca4924af2 100644 --- a/docker-compose-non-dev.yml +++ b/docker-compose-non-dev.yml @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -x-superset-image: &superset-image apachesuperset.docker.scarf.sh/apache/superset:${TAG:-latest-dev} +x-superset-image: &superset-image andressole/sdmx-insight x-superset-depends-on: &superset-depends-on - db - redis diff --git a/docker-compose.yml b/docker-compose.yml index 4c74e5d968..c72b5aae3b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -x-superset-image: &superset-image apachesuperset.docker.scarf.sh/apache/superset:${TAG:-latest-dev} +x-superset-image: &superset-image andressole/sdmx-insight x-superset-user: &superset-user root x-superset-depends-on: &superset-depends-on - db From 5c41114e6afbad8e23308c2bb26e1121146814b0 Mon Sep 17 00:00:00 2001 From: "andres.sole" Date: Mon, 25 Sep 2023 21:01:35 +0200 Subject: [PATCH 11/26] fix font-end: avoid getting TabsPane on every render --- superset-frontend/src/components/ImportSDMXModal/Modal.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/superset-frontend/src/components/ImportSDMXModal/Modal.tsx b/superset-frontend/src/components/ImportSDMXModal/Modal.tsx index 1118d8c684..9045102e62 100644 --- a/superset-frontend/src/components/ImportSDMXModal/Modal.tsx +++ b/superset-frontend/src/components/ImportSDMXModal/Modal.tsx @@ -11,6 +11,8 @@ import { Input } from '../Input'; import Button from '../Button'; import './style.css'; +const { TabPane } = Tabs; + const SdmxImportModal = ({ isOpen, onClose, @@ -18,8 +20,6 @@ const SdmxImportModal = ({ isOpen: boolean; onClose: () => void; }) => { - const { TabPane } = Tabs; - const { addDangerToast } = useToasts(); const [sdmxUrl, setSdmxUrl] = useState(''); From cb0a1fe301f73917885d5ce03dcfb71d4a1fbc89 Mon Sep 17 00:00:00 2001 From: "andres.sole" Date: Mon, 25 Sep 2023 21:06:28 +0200 Subject: [PATCH 12/26] fix chore: added latest to docker image ref --- docker-compose-non-dev.yml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose-non-dev.yml b/docker-compose-non-dev.yml index 8ca4924af2..1d5db3c4b8 100644 --- a/docker-compose-non-dev.yml +++ b/docker-compose-non-dev.yml @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -x-superset-image: &superset-image andressole/sdmx-insight +x-superset-image: &superset-image andressole/sdmx-insight:latest x-superset-depends-on: &superset-depends-on - db - redis diff --git a/docker-compose.yml b/docker-compose.yml index c72b5aae3b..f855f10770 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -x-superset-image: &superset-image andressole/sdmx-insight +x-superset-image: &superset-image andressole/sdmx-insight:latest x-superset-user: &superset-user root x-superset-depends-on: &superset-depends-on - db From 8df72e1038cd542f42996f4715cf6741a657d153 Mon Sep 17 00:00:00 2001 From: "andres.sole" Date: Mon, 25 Sep 2023 22:52:53 +0200 Subject: [PATCH 13/26] chore: working on image creation --- docker-compose-non-dev.yml | 6 ++---- requirements/base.txt | 1 + 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docker-compose-non-dev.yml b/docker-compose-non-dev.yml index 1d5db3c4b8..5d73d47dcc 100644 --- a/docker-compose-non-dev.yml +++ b/docker-compose-non-dev.yml @@ -23,10 +23,8 @@ x-superset-volumes: - ./docker:/app/docker - superset_home:/app/superset_home - ./dbs:/app/dbs - - ./superset:/app/superset - - ./superset-frontend:/app/superset-frontend - - + # - ./superset:/app/superset + # - ./superset-frontend:/app/superset-frontend version: "3.7" services: diff --git a/requirements/base.txt b/requirements/base.txt index 4928876a13..a3472cbbcc 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -365,6 +365,7 @@ zipp==3.15.0 # via # importlib-metadata # importlib-resources +sdmxthon==2.3.1 # The following packages are considered to be unsafe in a requirements file: # setuptools From ee667758a539696d7446618eee6148f1c62f41a4 Mon Sep 17 00:00:00 2001 From: "andres.sole" Date: Wed, 27 Sep 2023 13:31:00 +0200 Subject: [PATCH 14/26] deps: sdmxthon 2.3.2 --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index a3472cbbcc..b6bc1e05c0 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -365,7 +365,7 @@ zipp==3.15.0 # via # importlib-metadata # importlib-resources -sdmxthon==2.3.1 +sdmxthon==2.3.2 # The following packages are considered to be unsafe in a requirements file: # setuptools From 96e25e160a62dacc1829cbc573d804316d6b677b Mon Sep 17 00:00:00 2001 From: "andres.sole" Date: Wed, 27 Sep 2023 13:31:20 +0200 Subject: [PATCH 15/26] feat sdmx: allow searching data flows --- .../src/components/ImportSDMXModal/Modal.tsx | 31 ++++++++++++++----- .../src/components/ImportSDMXModal/style.css | 4 +-- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/superset-frontend/src/components/ImportSDMXModal/Modal.tsx b/superset-frontend/src/components/ImportSDMXModal/Modal.tsx index 9045102e62..98094bd451 100644 --- a/superset-frontend/src/components/ImportSDMXModal/Modal.tsx +++ b/superset-frontend/src/components/ImportSDMXModal/Modal.tsx @@ -1,9 +1,10 @@ import React, { useState, useEffect } from 'react'; import PropTypes from 'prop-types'; import { SupersetClient } from '@superset-ui/core'; -import { Select, Space } from 'antd'; +import { Space } from 'antd'; import { useToasts } from 'src/components/MessageToasts/withToasts'; import Modal from 'src/components/Modal'; +import Select from 'src/components/Select/Select'; import Tabs from 'src/components/Tabs'; import { isPlainObject } from 'lodash'; import { FormLabel } from '../Form'; @@ -107,6 +108,11 @@ const SdmxImportModal = ({ } }; + const filterDataflow = ( + input: string, + option?: { label: string; value: string }, + ) => (option?.label ?? '').toLowerCase().includes(input.toLowerCase()); + const handleAgencyChange = (agencyId: string) => { if (!dataflows[agencyId]) { setDataflows({ ...dataflows, [agencyId]: [] }); @@ -172,13 +178,19 @@ const SdmxImportModal = ({ Select Agency