From 9c55f717fcce6848dc93821fbf65ac691dad7a98 Mon Sep 17 00:00:00 2001 From: Carolina Gomes Date: Thu, 19 Oct 2023 16:37:08 -0300 Subject: [PATCH] =?UTF-8?q?altera=C3=A7=C3=B5es=20nos=20par=C3=A2metros?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../rj_smtr/br_rj_riodejaneiro_gtfs/flows.py | 18 ++---------------- pipelines/rj_smtr/constants.py | 6 +----- 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/pipelines/rj_smtr/br_rj_riodejaneiro_gtfs/flows.py b/pipelines/rj_smtr/br_rj_riodejaneiro_gtfs/flows.py index e88711e00..9d1cf7743 100644 --- a/pipelines/rj_smtr/br_rj_riodejaneiro_gtfs/flows.py +++ b/pipelines/rj_smtr/br_rj_riodejaneiro_gtfs/flows.py @@ -41,20 +41,6 @@ labels=[constants_emd.RJ_SMTR_DEV_AGENT_LABEL.value], ) -gtfs_captura_parameters = { - "dataset_id": "Parameter('dataset_id', default=None)", - "source_type": "Parameter('source_type', default=None)", - "table_id": "Parameter('table_id', default=None)", # agency - "partition_date_only": "Parameter('partition_date_only', default=None)", # True - "extract_params": "Parameter('extract_params', default=None)", # - "primary_key": "Parameter('primary_key', default=None)", -} - -gtfs_captura = set_default_parameters( - flow=gtfs_captura, - default_parameters=gtfs_captura_parameters, -) - gtfs_materializacao = deepcopy(default_materialization_flow) gtfs_materializacao.name = "SMTR - MaterializaĆ§Ć£o dos dados do GTFS" gtfs_materializacao.storage = GCS(constants_emd.GCS_FLOWS_BUCKET.value) @@ -66,7 +52,6 @@ gtfs_materializacao_parameters = { "dataset_id": constants.GTFS_DATASET_ID.value, "dbt_vars": constants.GTFS_MATERIALIZACAO_PARAMS.value, - "table_id": constants.GTFS_TABLE_CAPTURE_PARAMS.value, } gtfs_materializacao = set_default_parameters( @@ -92,7 +77,7 @@ run_captura = create_flow_run.map( flow_name=unmapped(gtfs_captura.name), project_name=unmapped(constants_emd.PREFECT_DEFAULT_PROJECT.value), - parameters=gtfs_captura_parameters, + parameters=constants.GTFS_TABLE_CAPTURE_PARAMS.value, labels=unmapped(LABELS), ) @@ -106,6 +91,7 @@ run_materializacao = create_flow_run( flow_name=gtfs_materializacao.name, project_name=constants_emd.PREFECT_DEFAULT_PROJECT.value, + parameters=constants.GTFS_MATERIALIZACAO_PARAMS.value, labels=LABELS, upstream_tasks=[wait_captura], ) diff --git a/pipelines/rj_smtr/constants.py b/pipelines/rj_smtr/constants.py index 350be7148..f90f3c21c 100644 --- a/pipelines/rj_smtr/constants.py +++ b/pipelines/rj_smtr/constants.py @@ -394,8 +394,4 @@ class constants(Enum): # pylint: disable=c0103 }, ] - GTFS_MATERIALIZACAO_PARAMS = ( - { - "dbt_vars": {"date_versao_gtfs": "YYYY-MM-DD"}, - }, - ) + GTFS_MATERIALIZACAO_PARAMS = {"date_versao_gtfs": "YYYY-MM-DD"}