From e7fed7b013fced26a435914fc66cc25360123e44 Mon Sep 17 00:00:00 2001 From: Rafael Carvalho Pinheiro <74972217+pixuimpou@users.noreply.github.com> Date: Mon, 13 Jan 2025 09:29:59 -0300 Subject: [PATCH 1/4] =?UTF-8?q?corrige=20deduplica=C3=A7=C3=A3o=20(#385)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipelines/migration/br_rj_riodejaneiro_bilhetagem/flows.py | 2 +- queries/models/br_rj_riodejaneiro_bilhetagem/integracao.sql | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pipelines/migration/br_rj_riodejaneiro_bilhetagem/flows.py b/pipelines/migration/br_rj_riodejaneiro_bilhetagem/flows.py index a43f302b..fc4e7ee0 100644 --- a/pipelines/migration/br_rj_riodejaneiro_bilhetagem/flows.py +++ b/pipelines/migration/br_rj_riodejaneiro_bilhetagem/flows.py @@ -2,7 +2,7 @@ """ Flows for br_rj_riodejaneiro_bilhetagem -DBT: 2025-01-06 +DBT: 2025-01-13 """ from copy import deepcopy diff --git a/queries/models/br_rj_riodejaneiro_bilhetagem/integracao.sql b/queries/models/br_rj_riodejaneiro_bilhetagem/integracao.sql index 0e8feebc..a772109e 100644 --- a/queries/models/br_rj_riodejaneiro_bilhetagem/integracao.sql +++ b/queries/models/br_rj_riodejaneiro_bilhetagem/integracao.sql @@ -191,7 +191,7 @@ from complete_partitions where id_integracao not in (select id_integracao from integracoes_teste_invalidas) qualify row_number() over ( - partition by id_integracao, id_transacao, priority - order by datetime_processamento_integracao desc + partition by id_integracao, id_transacao + order by datetime_processamento_integracao desc, priority ) = 1 \ No newline at end of file From 481d92855074e060db6c2c1ffe22449831e44012 Mon Sep 17 00:00:00 2001 From: Victor Miguel Rocha Date: Mon, 13 Jan 2025 15:40:27 -0300 Subject: [PATCH 2/4] =?UTF-8?q?Remove=20teste=20de=20Ordem=20de=20Servi?= =?UTF-8?q?=C3=A7o=20regular=20do=20gtfs=20(#386)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * remove verificação de tab * atualiza changelog --- .../br_rj_riodejaneiro_gtfs/CHANGELOG.md | 5 +++++ .../br_rj_riodejaneiro_gtfs/utils.py | 19 ++++++++----------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/pipelines/migration/br_rj_riodejaneiro_gtfs/CHANGELOG.md b/pipelines/migration/br_rj_riodejaneiro_gtfs/CHANGELOG.md index fc4bd135..f7616363 100644 --- a/pipelines/migration/br_rj_riodejaneiro_gtfs/CHANGELOG.md +++ b/pipelines/migration/br_rj_riodejaneiro_gtfs/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog - gtfs +## [1.2.1] - 2025-01-13 + +### Corrigido +- Removido teste de Ordem de Serviço regular (https://github.com/prefeitura-rio/pipelines_rj_smtr/pull/386) + ## [1.2.0] - 2025-01-03 ### Adicionado diff --git a/pipelines/migration/br_rj_riodejaneiro_gtfs/utils.py b/pipelines/migration/br_rj_riodejaneiro_gtfs/utils.py index 0f106d98..9d7a9a13 100644 --- a/pipelines/migration/br_rj_riodejaneiro_gtfs/utils.py +++ b/pipelines/migration/br_rj_riodejaneiro_gtfs/utils.py @@ -204,18 +204,15 @@ def processa_ordem_servico( None """ - if ( - len([sheet for sheet in sheetnames if "ANEXO I:" in sheet]) != 1 - and regular_sheet_index is None - ): - raise Exception( - "More than 1 regular sheet in the file. Please specify the regular sheet index." - ) + # if ( + # len([sheet for sheet in sheetnames if "ANEXO I " in sheet]) != 1 + # and regular_sheet_index is None + # ): + # raise Exception( + # "More than 1 regular sheet in the file. Please specify the regular sheet index." + # ) - if regular_sheet_index is None: - regular_sheet_index = next( - (i for i, name in enumerate(sheetnames) if "ANEXO I" in name), None - ) + regular_sheet_index = next((i for i, name in enumerate(sheetnames) if "ANEXO I" in name), None) quadro_geral = pd.DataFrame() From 9029a719c68c0ac9509d74da335698c176fef91e Mon Sep 17 00:00:00 2001 From: Victor Miguel Rocha Date: Mon, 13 Jan 2025 18:06:29 -0300 Subject: [PATCH 3/4] Registra flow do gtfs (#387) --- pipelines/migration/br_rj_riodejaneiro_gtfs/flows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/migration/br_rj_riodejaneiro_gtfs/flows.py b/pipelines/migration/br_rj_riodejaneiro_gtfs/flows.py index 43a11702..b5e72a59 100644 --- a/pipelines/migration/br_rj_riodejaneiro_gtfs/flows.py +++ b/pipelines/migration/br_rj_riodejaneiro_gtfs/flows.py @@ -2,7 +2,7 @@ """ Flows for gtfs -DBT 2024-12-20 +DBT 2025-01-13 """ from prefect import Parameter, case, task From f5cd9d40b2db2247d35348a74ca699da4f172c25 Mon Sep 17 00:00:00 2001 From: Rafael Carvalho Pinheiro <74972217+pixuimpou@users.noreply.github.com> Date: Wed, 15 Jan 2025 22:50:31 -0300 Subject: [PATCH 4/4] aumenta tamanho da pagina do gps validador (#388) --- pipelines/constants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipelines/constants.py b/pipelines/constants.py index d61a8a17..63ee1a31 100644 --- a/pipelines/constants.py +++ b/pipelines/constants.py @@ -203,8 +203,8 @@ class constants(Enum): # pylint: disable=c0103 WHERE id > {last_id} AND id <= {max_id} """, - "page_size": 1000, - "max_pages": 70, + "page_size": 35000, + "max_pages": 2, }, "primary_key": ["id"], "interval_minutes": 5,