From c950dbd124256afa0ebcd928944645adedefb968 Mon Sep 17 00:00:00 2001 From: PedroSiqueira1 Date: Thu, 9 Jan 2025 14:52:42 -0300 Subject: [PATCH 1/5] feat: Add registro_cancelamento_produto and licenca_transferida and some updates --- pipelines/sislic/dump_db_sislic/schedules.py | 53 +++++++++++++++++++ .../alvara_obra_privada.sql | 5 ++ .../certidao_aceitacao.sql | 2 + .../certidao_habite_se.sql | 1 + .../licenca_transferida.sql | 7 +++ .../registro_cancelamento_produto.sql | 6 +++ 6 files changed, 74 insertions(+) create mode 100644 queries/models/adm_licenca_urbanismo/licenca_transferida.sql create mode 100644 queries/models/adm_licenca_urbanismo/registro_cancelamento_produto.sql diff --git a/pipelines/sislic/dump_db_sislic/schedules.py b/pipelines/sislic/dump_db_sislic/schedules.py index e0e7462..d164935 100644 --- a/pipelines/sislic/dump_db_sislic/schedules.py +++ b/pipelines/sislic/dump_db_sislic/schedules.py @@ -498,6 +498,7 @@ "dump_mode": "overwrite", "execute_query": """ SELECT + num_lic, licenca, num_proc, cod_dlf, @@ -559,9 +560,12 @@ ConcedeNumeracao, NumConcedida, OBS, + mat_func, cancelado, + licgratis, OrgaoCentral, CODORGAO_ESPECIAL, + OrdemEmissao, Licenca_Loteamento, LicencaAvulsa, LoteamentoNovoModelo, @@ -971,6 +975,7 @@ Num_Hab ,Num_Proc ,Num_Lic + ,Habite_se ,Cod_DLF ,Dt_Emissao ,Dt_Certidao @@ -1049,12 +1054,14 @@ SELECT Num_Aceitacao ,Num_Proc ,Num_Lic + ,Aceitacao ,Cod_DLF ,Dt_Emissao ,Dt_Certidao ,Matricula_RGI ,Nr_Oficio_RGI ,PAL + ,DescricaoLote ,Mat_Tec_Resp ,Cancelado ,Obs @@ -1082,6 +1089,52 @@ """, "biglake_table": True, }, + + "licenca_transferida": { + "materialize_after_dump": True, + "materialization_mode": "prod", + "dump_mode": "overwrite", + "execute_query": """ + SELECT + num_lic_Origem, + num_lic_Destino, + DtTransferencia, + Mat_Func, + Nome_Func, + Cargo_Func, + MAQUINA_UTILIZADA, + CODORGAOSIGMA_ORIGEM, + CODORGAOSIGMA_DESTINO + FROM SMU_PRD.dbo.tbLIC_LicencasTransferidasCLU_CRU + + """, + "biglake_table": True, + }, + + "registro_cancelamento_produto": { + "materialize_after_dump": True, + "materialization_mode": "prod", + "dump_mode": "overwrite", + "execute_query": """ + SELECT + cod_dlf, + tipo, + Nr_documento, + dt_cancelamento, + mat_func, + nome_func, + cargo_func, + motivo, + complmotivo, + responsavel, + outroresp, + dt_autorizacao, + MAQUINA_UTILIZADA + + FROM SMU_PRD.dbo.tbLIC_DocCancelados + """, + "biglake_table": True, + }, } sislic_clocks = generate_dump_db_schedules( diff --git a/queries/models/adm_licenca_urbanismo/alvara_obra_privada.sql b/queries/models/adm_licenca_urbanismo/alvara_obra_privada.sql index 8af2fdf..bc024b3 100644 --- a/queries/models/adm_licenca_urbanismo/alvara_obra_privada.sql +++ b/queries/models/adm_licenca_urbanismo/alvara_obra_privada.sql @@ -1,4 +1,5 @@ SELECT +CAST(CAST(num_lic AS FLOAT64) AS int64) AS id_licenciamento, licenca AS numero_alvara_licenca, num_proc AS numero_processo, cod_dlf AS id_orgao_sislic, @@ -6,6 +7,7 @@ CAST(dt_emissao AS DATETIME) AS data_emissao, CAST(dt_inicio_validade AS DATETIME) AS data_inicio_validade, CAST(dt_venc AS DATETIME) AS data_vencimento, CAST(CAST(prazo AS FLOAT64) AS int64) AS prazo, +CAST(CAST(Dt_alvara AS DATETIME) AS int64) AS data_passe_alvara, num_darm_pg AS numero_darm_pago, CAST(vl_lic AS float64) AS valor_darm, requerente AS requerente, @@ -59,9 +61,12 @@ edif_pav AS descricao_edificacao_concatenada, CAST(concedenumeracao AS bool) AS concede_numeracao, numconcedida AS numeracao_concedida, obs AS observacao, +CAST(CAST(mat_func AS FLOAT64) AS INT64) AS matricula_tecnico, CAST(cancelado AS BOOL) AS cancelado, +CAST(lic_gratis AS BOOL) AS licenca_gratis, CAST(orgaocentral AS BOOL) AS licenca_orgao_central, codorgao_especial AS cod_orgao_especial, +OrdemEmissao AS ordem_emissao, CAST(licenca_loteamento AS BOOL) AS licenca_loteamento, CAST(licencaavulsa AS BOOL) AS licenca_avulsa, CAST(loteamentonovomodelo AS BOOL) AS loteamento_novo_modelo, diff --git a/queries/models/adm_licenca_urbanismo/certidao_aceitacao.sql b/queries/models/adm_licenca_urbanismo/certidao_aceitacao.sql index 8750c2c..8b26ad2 100644 --- a/queries/models/adm_licenca_urbanismo/certidao_aceitacao.sql +++ b/queries/models/adm_licenca_urbanismo/certidao_aceitacao.sql @@ -2,12 +2,14 @@ SELECT Num_Aceitacao AS id_aceitacao, Num_Proc AS numero_processo, Num_Lic AS id_licenciamento, + Aceitacao AS numero_certidao, Cod_DLF AS id_orgao_sislic, CAST(Dt_Emissao AS DATETIME) AS data_emissao, CAST(Dt_Certidao AS DATETIME) AS data_certidao, Matricula_RGI AS matricula_rgi, Nr_Oficio_RGI AS numero_oficio_rgi, PAL AS numero_pal, + DescricaoLote AS descricao_lote, Mat_Tec_Resp AS matricula_tecnico_responsavel, CAST(Cancelado AS BOOL) AS cancelado, Obs AS observacao diff --git a/queries/models/adm_licenca_urbanismo/certidao_habite_se.sql b/queries/models/adm_licenca_urbanismo/certidao_habite_se.sql index 194cd1b..b3fcb53 100644 --- a/queries/models/adm_licenca_urbanismo/certidao_habite_se.sql +++ b/queries/models/adm_licenca_urbanismo/certidao_habite_se.sql @@ -2,6 +2,7 @@ SELECT Num_Hab AS id_habite_se, Num_Proc AS numero_processo, Num_Lic AS id_licenciamento, + Habite_se AS numero_certidao, Cod_DLF AS id_orgao_sislic, CAST(Dt_Emissao AS DATETIME) AS data_emissao, CAST(Dt_Certidao AS DATETIME) AS data_certidao, diff --git a/queries/models/adm_licenca_urbanismo/licenca_transferida.sql b/queries/models/adm_licenca_urbanismo/licenca_transferida.sql new file mode 100644 index 0000000..b0ac543 --- /dev/null +++ b/queries/models/adm_licenca_urbanismo/licenca_transferida.sql @@ -0,0 +1,7 @@ +SELECT + CAST(CAST(num_lic_Origem AS FLOAT64) AS int64) AS id_licenciamento_origem, + CAST(CAST(num_lic_Destino AS FLOAT64) AS int64) AS id_licenciamento_destino, + CAST(DtTransferencia AS DATETIME) AS data_transferencia, + CODORGAOSIGMA_ORIGEM AS id_orgao_SISLIC_origem, + CODORGAOSIGMA_DESTINO AS id_orgao_SISLIC_destino +FROM `rj-smdue.adm_licenca_urbanismo_staging.licenca_transferida` \ No newline at end of file diff --git a/queries/models/adm_licenca_urbanismo/registro_cancelamento_produto.sql b/queries/models/adm_licenca_urbanismo/registro_cancelamento_produto.sql new file mode 100644 index 0000000..a4940e1 --- /dev/null +++ b/queries/models/adm_licenca_urbanismo/registro_cancelamento_produto.sql @@ -0,0 +1,6 @@ +SELECT + cod_dlf AS id_orgao_SISLIC, + tipo AS tipo, + Nr_documento AS numero_documento, + CAST(dt_cancelamento AS DATETIME) AS data_cancelamento +FROM `rj-smdue.adm_licenca_urbanismo_staging.registro_cancelamento_produto` \ No newline at end of file From 55c0963f0b25108a55fe4028eaff05f3eaac78ce Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 17:54:45 +0000 Subject: [PATCH 2/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../ISSUE_TEMPLATE/notifica--o-de-problema.md | 2 +- .../requerimento-de-funcionalidade.md | 2 +- pipelines/sislic/dump_db_sislic/schedules.py | 24 +++++++++---------- .../area_protecao_cultural.sql | 4 ++-- .../darm_licenca_alvara.sql | 2 +- .../licenca_loteamento.sql | 2 +- .../licenca_transferida.sql | 2 +- 7 files changed, 18 insertions(+), 20 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/notifica--o-de-problema.md b/.github/ISSUE_TEMPLATE/notifica--o-de-problema.md index acc81b6..9294f8f 100644 --- a/.github/ISSUE_TEMPLATE/notifica--o-de-problema.md +++ b/.github/ISSUE_TEMPLATE/notifica--o-de-problema.md @@ -3,7 +3,7 @@ name: Notificação de problema about: Notifique um problema que está ocorrendo para que possamos corrigí-lo. title: "[BUG]" labels: bug -assignees: +assignees: --- **Descreva o problema** diff --git a/.github/ISSUE_TEMPLATE/requerimento-de-funcionalidade.md b/.github/ISSUE_TEMPLATE/requerimento-de-funcionalidade.md index 347d97f..51d9919 100644 --- a/.github/ISSUE_TEMPLATE/requerimento-de-funcionalidade.md +++ b/.github/ISSUE_TEMPLATE/requerimento-de-funcionalidade.md @@ -3,7 +3,7 @@ name: Requerimento de funcionalidade about: Sugira novas funcionalidades que podem ser úteis a todos os desenvolvedores. title: "[FEATURE]" labels: enhancement -assignees: +assignees: --- **Essa funcionalidade está relacionada a um problema?** diff --git a/pipelines/sislic/dump_db_sislic/schedules.py b/pipelines/sislic/dump_db_sislic/schedules.py index d164935..3e108b1 100644 --- a/pipelines/sislic/dump_db_sislic/schedules.py +++ b/pipelines/sislic/dump_db_sislic/schedules.py @@ -1089,34 +1089,32 @@ """, "biglake_table": True, }, - "licenca_transferida": { "materialize_after_dump": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": """ - SELECT - num_lic_Origem, - num_lic_Destino, - DtTransferencia, - Mat_Func, - Nome_Func, - Cargo_Func, - MAQUINA_UTILIZADA, - CODORGAOSIGMA_ORIGEM, + SELECT + num_lic_Origem, + num_lic_Destino, + DtTransferencia, + Mat_Func, + Nome_Func, + Cargo_Func, + MAQUINA_UTILIZADA, + CODORGAOSIGMA_ORIGEM, CODORGAOSIGMA_DESTINO FROM SMU_PRD.dbo.tbLIC_LicencasTransferidasCLU_CRU """, "biglake_table": True, }, - - "registro_cancelamento_produto": { + "registro_cancelamento_produto": { "materialize_after_dump": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": """ - SELECT + SELECT cod_dlf, tipo, Nr_documento, diff --git a/queries/models/adm_licenca_urbanismo/area_protecao_cultural.sql b/queries/models/adm_licenca_urbanismo/area_protecao_cultural.sql index 8a24acc..36f90db 100644 --- a/queries/models/adm_licenca_urbanismo/area_protecao_cultural.sql +++ b/queries/models/adm_licenca_urbanismo/area_protecao_cultural.sql @@ -1,8 +1,8 @@ SELECT ID_AREA_PROTECAO_CULTURAL AS id_area_protecao_cultural, REGEXP_REPLACE( - NORMALIZE(UPPER(DS_AREA_PROTECAO_CULTURAL), NFD), - r'\p{M}', + NORMALIZE(UPPER(DS_AREA_PROTECAO_CULTURAL), NFD), + r'\p{M}', '' ) AS nome_area_protecao_ambiental FROM `rj-smdue.adm_licenca_urbanismo_staging.area_protecao_cultural` \ No newline at end of file diff --git a/queries/models/adm_licenca_urbanismo/darm_licenca_alvara.sql b/queries/models/adm_licenca_urbanismo/darm_licenca_alvara.sql index 9a657a7..13ac46b 100644 --- a/queries/models/adm_licenca_urbanismo/darm_licenca_alvara.sql +++ b/queries/models/adm_licenca_urbanismo/darm_licenca_alvara.sql @@ -6,5 +6,5 @@ cod_Lic AS cod_Lic, cod_Compl_Lic AS id_complemento_tipo_licenca , compl_Livre AS comentario_licenca, CAST(Valor AS FLOAT64) AS valor , -Formula AS formula +Formula AS formula FROM `rj-smdue.adm_licenca_urbanismo_staging.darm_licenca_alvara` \ No newline at end of file diff --git a/queries/models/adm_licenca_urbanismo/licenca_loteamento.sql b/queries/models/adm_licenca_urbanismo/licenca_loteamento.sql index 102202a..9a822b4 100644 --- a/queries/models/adm_licenca_urbanismo/licenca_loteamento.sql +++ b/queries/models/adm_licenca_urbanismo/licenca_loteamento.sql @@ -3,5 +3,5 @@ CAST(Id_Lote AS string) AS id_lote , CAST(num_lic AS string) AS id_licenciamento, CAST(CAST(Quantidade AS FLOAT64) AS int64) AS quantidade_lotes, CAST(CAST(Categoria AS FLOAT64) AS int64) AS categoria_lote, -CAST(CAST(TpLote AS FLOAT64) AS int64) AS lote_aprovado +CAST(CAST(TpLote AS FLOAT64) AS int64) AS lote_aprovado FROM `rj-smdue.adm_licenca_urbanismo_staging.licenca_loteamento` \ No newline at end of file diff --git a/queries/models/adm_licenca_urbanismo/licenca_transferida.sql b/queries/models/adm_licenca_urbanismo/licenca_transferida.sql index b0ac543..1b7e052 100644 --- a/queries/models/adm_licenca_urbanismo/licenca_transferida.sql +++ b/queries/models/adm_licenca_urbanismo/licenca_transferida.sql @@ -1,7 +1,7 @@ SELECT CAST(CAST(num_lic_Origem AS FLOAT64) AS int64) AS id_licenciamento_origem, CAST(CAST(num_lic_Destino AS FLOAT64) AS int64) AS id_licenciamento_destino, - CAST(DtTransferencia AS DATETIME) AS data_transferencia, + CAST(DtTransferencia AS DATETIME) AS data_transferencia, CODORGAOSIGMA_ORIGEM AS id_orgao_SISLIC_origem, CODORGAOSIGMA_DESTINO AS id_orgao_SISLIC_destino FROM `rj-smdue.adm_licenca_urbanismo_staging.licenca_transferida` \ No newline at end of file From 078a71534feeca9e11621c64d964208f3b6e3f97 Mon Sep 17 00:00:00 2001 From: PedroSiqueira1 Date: Thu, 9 Jan 2025 15:40:11 -0300 Subject: [PATCH 3/5] fix: remove columns from staging --- pipelines/sislic/dump_db_sislic/schedules.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/pipelines/sislic/dump_db_sislic/schedules.py b/pipelines/sislic/dump_db_sislic/schedules.py index 3e108b1..8f57a2d 100644 --- a/pipelines/sislic/dump_db_sislic/schedules.py +++ b/pipelines/sislic/dump_db_sislic/schedules.py @@ -1098,10 +1098,6 @@ num_lic_Origem, num_lic_Destino, DtTransferencia, - Mat_Func, - Nome_Func, - Cargo_Func, - MAQUINA_UTILIZADA, CODORGAOSIGMA_ORIGEM, CODORGAOSIGMA_DESTINO FROM SMU_PRD.dbo.tbLIC_LicencasTransferidasCLU_CRU @@ -1118,16 +1114,6 @@ cod_dlf, tipo, Nr_documento, - dt_cancelamento, - mat_func, - nome_func, - cargo_func, - motivo, - complmotivo, - responsavel, - outroresp, - dt_autorizacao, - MAQUINA_UTILIZADA FROM SMU_PRD.dbo.tbLIC_DocCancelados """, From 4fbda198de0231583d0f3b7faab32c38a8c71d41 Mon Sep 17 00:00:00 2001 From: PedroSiqueira1 Date: Thu, 9 Jan 2025 15:48:53 -0300 Subject: [PATCH 4/5] run dbt model --- pipelines/templates/run_dbt_model/flows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/templates/run_dbt_model/flows.py b/pipelines/templates/run_dbt_model/flows.py index 7fc5469..0f84216 100644 --- a/pipelines/templates/run_dbt_model/flows.py +++ b/pipelines/templates/run_dbt_model/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -MATERIALIZA MODELOS DO DBT. +MATERIALIZA MODELOS DO DBT. """ from copy import deepcopy From deac861acc00776a8f52bdf5c916df1ccbfdd736 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 18:49:39 +0000 Subject: [PATCH 5/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pipelines/templates/run_dbt_model/flows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/templates/run_dbt_model/flows.py b/pipelines/templates/run_dbt_model/flows.py index 0f84216..7fc5469 100644 --- a/pipelines/templates/run_dbt_model/flows.py +++ b/pipelines/templates/run_dbt_model/flows.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -MATERIALIZA MODELOS DO DBT. +MATERIALIZA MODELOS DO DBT. """ from copy import deepcopy