From 4ad0a3f9665c5d645a18cf0973b6ddc15942c841 Mon Sep 17 00:00:00 2001 From: Diego Oliveira Date: Wed, 24 Jul 2024 15:50:07 -0300 Subject: [PATCH] V2.0.0 (#1727) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [infra] Version 1.7.0 python-package * [infra] fix update_columns test * [infra] remove unused import * [infra] add to_partition utility function * [infra] add test for to_partitions * [infra] pump package version 1.6.9-b2 * [infra] add break_file feature * Revert "[infra] pump package version 1.6.9-b2" This reverts commit 0cba449c23e348468531d74f1edb35c91325d547. * feat: add `connection_id` to external data configuration * fix(Datatype): add connection id for external configuration * feat: add automatic management of BQ connection * chore: fix linting issues * feat: add test folder to gitignore * feat: release beta version * feat(Connection): add `service_account` property * feat(Base): add IAM stuff * chore: fix linting issues * feat: automatic granting roles to BigLake service account * feat: better error handling, set biglake permissions is now optional * feat: release beta version * chore: modify log message * chore: make all partitions string * chore: merge master * add __version__ atribute (#1488) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * add option to change copied table name (#1489) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * fix: pylint * changing python_path fixture * adding shapely as dependency to downgrade if already installed, like Colab * pylinting files * bump version 1.6.10-beta.1 * bump version 1.6.10 * updating version * return update_columns to Table class * authentication methods in base class * method to return dataset id from slug using graphql * method to return table id from slug of dataset and table using graphql * using variables in graphql query * change default & log downloaded path * authentication with graphql * change version in pyproject.toml * chore: refactor connection imports and make working dir default for storage download * chore: make staging the default mode for storage download * chore: make staging the default mode for storage download * fix: pylint * chore: release new beta version * small corrections in 1.6.11 * methods to retrieve metadata from graphql api * adding data to api_data_dict * adding exists_in_api method * changing is_updated method * method to get a request in graphql * logging errors with loguru, instead of print * writing yaml files before updating the database * helper to convert case from snake_case to camelCase and vice versa * start refactoring the query to use alias * refactoring: clean edges and nodes from graphql response * moving graphql queries for separated files and others * improving unit tests and graphql for api_metadata * changing api_response for compatibility with current yamls * Hotfix storage init args (#1576) * [dados]br_ibge_estadic.indicadores (#1535) * dados * dados * title * observation_level * escolaridade --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * [dados] br_ipea_avs (#1530) * up br_ipea_avs * Ajeitando os comentários da equipe de dados. * Alterando a temporal_coverage * Ajustando o PR.3 * Create code * Delete code * Create br_avs_ipea * Add files via upload * Delete br_avs_ipea * Subindo novamente toda a base, devido as alterações. * Ajustando o PR * Delete br_ipea_avs.ipynb * update * update * update * update * update --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * Update table_config.yaml (#1553) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * [dados] br_me_clima_organizacional (#1548) * dados * partner_organization --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * [dados] br_me_exportadoras_importadoras (#1521) * Sobe br_me_exportadoras_importadoras * Faz correções no script * Corrige erros no table_config apontados na correção do PR * Corrige erros no table_config apontados na correção do PR v2 * Corrige erros no dicionário * Corrige tipo do CEP * Corrige o tipo do dado do CEP no publish.sql * Delete dataset_config yamls --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Laura Amaral <100051996+laura-l-amaral@users.noreply.github.com> * [dados] world_fao_production (#1536) * Abrir PR world_fao_production * Corrige erros apontados na correção do PR * Corrige erros apontados na correção do PR v2 * Altera nível de observação do table_cofing da tabela item * Corrige erros apontados na correção do PR * Corrige a partição e altera a nome da variável ano para year * remove palavra repetida da descrição --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Laura Amaral <100051996+laura-l-amaral@users.noreply.github.com> * [dados] world_wb_wwbi.country_finance (#1538) * dados * update * data * add observacoes --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * [dados-atualizacao] update pib municipio (#1559) * update pib municipio * updates * updates * Update table_description.txt --------- Co-authored-by: Gabrielle Carvalho <77730866+gabrielle-carv@users.noreply.github.com> * [dados] br_ibge_estadic (#1560) * update * update metadata_modified * update metadata_modified * update metadata_modified --------- Co-authored-by: Laura Amaral <100051996+laura-l-amaral@users.noreply.github.com> * [dados] br_ibge_munic (#1534) * dados * update table_config * update * code * code novo * sigla_uf --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * [dados] br_me_siconfi.uf (#1546) * upload br_me_siconfi_uf * update review gabs * Update table_config.yaml * Update table_config.yaml * Update table_config.yaml --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Laura Amaral <100051996+laura-l-amaral@users.noreply.github.com> * [dados] br_ipea_avs (#1564) * up br_ipea_avs * up br_ibge_estadic * Ajustando os comentários da equipe de dados. * Ajustando cobertura temporal. * Delete README.md * Delete dataset_config.yaml * Delete publish.sql * Delete schema-prod.json * Delete schema-staging.json * Delete table_config.yaml * Delete table_description.txt * update * update * update * update * update * update * update * Delete br_ibge_estadic_educação.ipynb apagando código estadic * Delete publish.sql apagando publish.sql * Delete schema-prod.json apagando schema-prod * Delete schema-staging.json apagando schema-staging * Delete table_config.yaml apagando table_config * Delete table_description.txt apagando table_description --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * [dados] br_bcb_estban (#1561) * Abre PR do conjunto br_bcb_estban * Corrige erros apontados na correção do PR * Delete dataset_config.yaml * Corrige erros apontados na correção do PR v2.0 --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Laura Amaral <100051996+laura-l-amaral@users.noreply.github.com> * update dicionario br_me_rais (#1567) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * [dados] br_ibge_estadic (#1531) * up br_ipea_avs * up br_ibge_estadic * Ajustando os comentários da equipe de dados. * Ajustando cobertura temporal. * Delete README.md * Delete dataset_config.yaml * Delete publish.sql * Delete schema-prod.json * Delete schema-staging.json * Delete table_config.yaml * Delete table_description.txt * update * update * update * update * update * update * update * update * update dicionario br_ibge_estadic * update br_ibge_estadic * update dicionário br_ibge_estadic --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * [dados-atualizacao] update `br_sgp_informacao.despesas_cartao_corporativo` (#1570) * update cartao corporativo * Update table_config.yaml * [dados] world_spi (#1555) * update world_spi * update * update * update_2 * update world_spi * fix: dataset_id --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Crislane Alves <58278652+crislanealves@users.noreply.github.com> * add required args * [dados-atualizacao] br_inep_indicadores_educacionais (#1566) * [dados-atualizacao] atualiza os dados para 2022; atualiza table_config, cria script em python * Update table_config.yaml * Update table_config.yaml * Update table_config.yaml * Update table_config.yaml --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --------- Co-authored-by: Gabrielle Carvalho <77730866+gabrielle-carv@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Patrick Teixeira <105399231+tricktx@users.noreply.github.com> Co-authored-by: Gabriel Pisa <61624649+folhesgabriel@users.noreply.github.com> Co-authored-by: Laura Amaral <100051996+laura-l-amaral@users.noreply.github.com> Co-authored-by: Arthur Gusmão Co-authored-by: Crislane Alves <58278652+crislanealves@users.noreply.github.com> Co-authored-by: Fernanda Scovino Co-authored-by: Lucas Moreira <65978482+lucasnascm@users.noreply.github.com> * removing coverage from query * fixes in dataset and table config files creation * owner_org and exists_in_api methods * removing references to REST API and treat errors in login * adjustments in data_dict, tests for publish * initializing RemoteAPI for mutations * mutation to create a dataset * adjustments in data_dict and others * corrections and new table for publish tests * removing IDE settings from project * prevent exclusion of tmp_bases when it already exists * change filename to avoid conflict in tests * commiting notebook file as it does not rollback, despite the fact that contents are identical * stable exists method * metadata_modified as datetime * chore: remove yaml dependecy from metadata.py * chore: remove yaml dependecy from metadata.py * feat: remove more code * feat: remove more code * initial structure * chore: refactor graphql requests * chore: refactor graphql requests * chore: refactor graphql requests * chore: make publish_sql * feat: add backend class for handling interaction with graphql * chore: clean some code and comment parts where table_config are needed * chore: clean some code and comment parts where table_config are needed * chore: add dataset config query * chore: add table config query * feat: create dataset and use API metadata * chore(deps): remove unnecessary deps * chore: minor cleanup * chore: delete file * feat: fix occurences of `table_config` * feat: add API url to config init * feat: add structure for `Metadata.create` * chore: more table modifications * feat: table create using data_columns and partitioned data * feat: some refactor and finish table.create * chore: better casing * chore: better casing * chore: better logging * chore: update table.create docstring * chore: clean config files * feat: refactor table.publish and table.update * chore: make publish.sql from staging schema * feat: get partition dict from storage * chore: rename some methods * chore: update and publish only acts in prod and uses the staging table schema to generate the prod publish query and update schema * chore: load schema using SchemaField, remove code that depends on template * chore: refactor init process * chore: remove upload function from cli * chore: remove upload function from cli * chore: clean unused imports, redo poetry packages and release 2.0.0-b1 * chore: add a new dependencie requests-toolbelt * chore: add tomlkit and better error if columns does not have name * chore: error handling and make publish and update get info from api if existis * fix: typo in _get_columns_from_data and better infos * chore: add tomlkit * chore: error handling in case that API is off * chore: error handling in case that API is off * hotfix: chang metadata base_url * chore: get backend metadata from cloud tables * feat: bump beta version * chore: change mode in table.delete * chore: pump version * chore: no more version number on files * feat: implement external warnings and messages * feat: add csv_delimiter and allow csv_allow_jagged_rows * fix: pump bd version and add new parameters csv_delimiter and csv_allow_jagged_rows * chore: cleanup * chore: remove compressed r package * chore: refactor dependency management * chore: fix linting issues * chore: remove pylint action * fix: change install instructions * feat: pump bd version * fix: change install instructions * feat: pump bd version * feat: add new parameter csv_skip_leading_rows and setup.py * feat: pump bd version * chore: fix conflicts * chore: add `all` extra * chore: lint * feat: create branch v2.0.0 * fix: add csv delimiter to schema * feat: expand credential scope to drive and bq * chore: start cleaning tests * chore: add timeout to pypi warning * fix: log in wrong position * fix: log in wrong position * fix: log in wrong position * fix: fix conflicts * Fix #1718 (#1719) * [Infra] Version 2.0.0b16 (#1678) * [infra] Version 1.7.0 python-package * [infra] fix update_columns test * [infra] remove unused import * [infra] add to_partition utility function * [infra] add test for to_partitions * [infra] pump package version 1.6.9-b2 * [infra] add break_file feature * Revert "[infra] pump package version 1.6.9-b2" This reverts commit 0cba449c23e348468531d74f1edb35c91325d547. * feat: add `connection_id` to external data configuration * fix(Datatype): add connection id for external configuration * feat: add automatic management of BQ connection * chore: fix linting issues * feat: add test folder to gitignore * feat: release beta version * feat(Connection): add `service_account` property * feat(Base): add IAM stuff * chore: fix linting issues * feat: automatic granting roles to BigLake service account * feat: better error handling, set biglake permissions is now optional * feat: release beta version * chore: modify log message * chore: make all partitions string * chore: merge master * add __version__ atribute (#1488) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * add option to change copied table name (#1489) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * fix: pylint * changing python_path fixture * adding shapely as dependency to downgrade if already installed, like Colab * pylinting files * bump version 1.6.10-beta.1 * bump version 1.6.10 * updating version * return update_columns to Table class * authentication methods in base class * method to return dataset id from slug using graphql * method to return table id from slug of dataset and table using graphql * using variables in graphql query * change default & log downloaded path * authentication with graphql * change version in pyproject.toml * chore: refactor connection imports and make working dir default for storage download * chore: make staging the default mode for storage download * chore: make staging the default mode for storage download * fix: pylint * chore: release new beta version * small corrections in 1.6.11 * methods to retrieve metadata from graphql api * adding data to api_data_dict * adding exists_in_api method * changing is_updated method * method to get a request in graphql * logging errors with loguru, instead of print * writing yaml files before updating the database * helper to convert case from snake_case to camelCase and vice versa * start refactoring the query to use alias * refactoring: clean edges and nodes from graphql response * moving graphql queries for separated files and others * improving unit tests and graphql for api_metadata * changing api_response for compatibility with current yamls * Hotfix storage init args (#1576) * [dados]br_ibge_estadic.indicadores (#1535) * dados * dados * title * observation_level * escolaridade --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * [dados] br_ipea_avs (#1530) * up br_ipea_avs * Ajeitando os comentários da equipe de dados. * Alterando a temporal_coverage * Ajustando o PR.3 * Create code * Delete code * Create br_avs_ipea * Add files via upload * Delete br_avs_ipea * Subindo novamente toda a base, devido as alterações. * Ajustando o PR * Delete br_ipea_avs.ipynb * update * update * update * update * update --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * Update table_config.yaml (#1553) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * [dados] br_me_clima_organizacional (#1548) * dados * partner_organization --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * [dados] br_me_exportadoras_importadoras (#1521) * Sobe br_me_exportadoras_importadoras * Faz correções no script * Corrige erros no table_config apontados na correção do PR * Corrige erros no table_config apontados na correção do PR v2 * Corrige erros no dicionário * Corrige tipo do CEP * Corrige o tipo do dado do CEP no publish.sql * Delete dataset_config yamls --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Laura Amaral <100051996+laura-l-amaral@users.noreply.github.com> * [dados] world_fao_production (#1536) * Abrir PR world_fao_production * Corrige erros apontados na correção do PR * Corrige erros apontados na correção do PR v2 * Altera nível de observação do table_cofing da tabela item * Corrige erros apontados na correção do PR * Corrige a partição e altera a nome da variável ano para year * remove palavra repetida da descrição --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Laura Amaral <100051996+laura-l-amaral@users.noreply.github.com> * [dados] world_wb_wwbi.country_finance (#1538) * dados * update * data * add observacoes --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * [dados-atualizacao] update pib municipio (#1559) * update pib municipio * updates * updates * Update table_description.txt --------- Co-authored-by: Gabrielle Carvalho <77730866+gabrielle-carv@users.noreply.github.com> * [dados] br_ibge_estadic (#1560) * update * update metadata_modified * update metadata_modified * update metadata_modified --------- Co-authored-by: Laura Amaral <100051996+laura-l-amaral@users.noreply.github.com> * [dados] br_ibge_munic (#1534) * dados * update table_config * update * code * code novo * sigla_uf --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * [dados] br_me_siconfi.uf (#1546) * upload br_me_siconfi_uf * update review gabs * Update table_config.yaml * Update table_config.yaml * Update table_config.yaml --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Laura Amaral <100051996+laura-l-amaral@users.noreply.github.com> * [dados] br_ipea_avs (#1564) * up br_ipea_avs * up br_ibge_estadic * Ajustando os comentários da equipe de dados. * Ajustando cobertura temporal. * Delete README.md * Delete dataset_config.yaml * Delete publish.sql * Delete schema-prod.json * Delete schema-staging.json * Delete table_config.yaml * Delete table_description.txt * update * update * update * update * update * update * update * Delete br_ibge_estadic_educação.ipynb apagando código estadic * Delete publish.sql apagando publish.sql * Delete schema-prod.json apagando schema-prod * Delete schema-staging.json apagando schema-staging * Delete table_config.yaml apagando table_config * Delete table_description.txt apagando table_description --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * [dados] br_bcb_estban (#1561) * Abre PR do conjunto br_bcb_estban * Corrige erros apontados na correção do PR * Delete dataset_config.yaml * Corrige erros apontados na correção do PR v2.0 --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Laura Amaral <100051996+laura-l-amaral@users.noreply.github.com> * update dicionario br_me_rais (#1567) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * [dados] br_ibge_estadic (#1531) * up br_ipea_avs * up br_ibge_estadic * Ajustando os comentários da equipe de dados. * Ajustando cobertura temporal. * Delete README.md * Delete dataset_config.yaml * Delete publish.sql * Delete schema-prod.json * Delete schema-staging.json * Delete table_config.yaml * Delete table_description.txt * update * update * update * update * update * update * update * update * update dicionario br_ibge_estadic * update br_ibge_estadic * update dicionário br_ibge_estadic --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * [dados-atualizacao] update `br_sgp_informacao.despesas_cartao_corporativo` (#1570) * update cartao corporativo * Update table_config.yaml * [dados] world_spi (#1555) * update world_spi * update * update * update_2 * update world_spi * fix: dataset_id --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Crislane Alves <58278652+crislanealves@users.noreply.github.com> * add required args * [dados-atualizacao] br_inep_indicadores_educacionais (#1566) * [dados-atualizacao] atualiza os dados para 2022; atualiza table_config, cria script em python * Update table_config.yaml * Update table_config.yaml * Update table_config.yaml * Update table_config.yaml --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --------- Co-authored-by: Gabrielle Carvalho <77730866+gabrielle-carv@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Patrick Teixeira <105399231+tricktx@users.noreply.github.com> Co-authored-by: Gabriel Pisa <61624649+folhesgabriel@users.noreply.github.com> Co-authored-by: Laura Amaral <100051996+laura-l-amaral@users.noreply.github.com> Co-authored-by: Arthur Gusmão Co-authored-by: Crislane Alves <58278652+crislanealves@users.noreply.github.com> Co-authored-by: Fernanda Scovino Co-authored-by: Lucas Moreira <65978482+lucasnascm@users.noreply.github.com> * removing coverage from query * fixes in dataset and table config files creation * owner_org and exists_in_api methods * removing references to REST API and treat errors in login * adjustments in data_dict, tests for publish * initializing RemoteAPI for mutations * mutation to create a dataset * adjustments in data_dict and others * corrections and new table for publish tests * removing IDE settings from project * prevent exclusion of tmp_bases when it already exists * change filename to avoid conflict in tests * commiting notebook file as it does not rollback, despite the fact that contents are identical * stable exists method * metadata_modified as datetime * chore: remove yaml dependecy from metadata.py * chore: remove yaml dependecy from metadata.py * feat: remove more code * feat: remove more code * initial structure * chore: refactor graphql requests * chore: refactor graphql requests * chore: refactor graphql requests * chore: make publish_sql * feat: add backend class for handling interaction with graphql * chore: clean some code and comment parts where table_config are needed * chore: clean some code and comment parts where table_config are needed * chore: add dataset config query * chore: add table config query * feat: create dataset and use API metadata * chore(deps): remove unnecessary deps * chore: minor cleanup * chore: delete file * feat: fix occurences of `table_config` * feat: add API url to config init * feat: add structure for `Metadata.create` * chore: more table modifications * feat: table create using data_columns and partitioned data * feat: some refactor and finish table.create * chore: better casing * chore: better casing * chore: better logging * chore: update table.create docstring * chore: clean config files * feat: refactor table.publish and table.update * chore: make publish.sql from staging schema * feat: get partition dict from storage * chore: rename some methods * chore: update and publish only acts in prod and uses the staging table schema to generate the prod publish query and update schema * chore: load schema using SchemaField, remove code that depends on template * chore: refactor init process * chore: remove upload function from cli * chore: remove upload function from cli * chore: clean unused imports, redo poetry packages and release 2.0.0-b1 * chore: add a new dependencie requests-toolbelt * chore: add tomlkit and better error if columns does not have name * chore: error handling and make publish and update get info from api if existis * fix: typo in _get_columns_from_data and better infos * chore: add tomlkit * chore: error handling in case that API is off * chore: error handling in case that API is off * hotfix: chang metadata base_url * chore: get backend metadata from cloud tables * feat: bump beta version * chore: change mode in table.delete * chore: pump version * chore: no more version number on files * feat: implement external warnings and messages * feat: add csv_delimiter and allow csv_allow_jagged_rows * fix: pump bd version and add new parameters csv_delimiter and csv_allow_jagged_rows * chore: cleanup * chore: remove compressed r package * chore: refactor dependency management * chore: fix linting issues * chore: remove pylint action * fix: change install instructions * feat: pump bd version * fix: change install instructions * feat: pump bd version * feat: add new parameter csv_skip_leading_rows and setup.py * feat: pump bd version * chore: fix conflicts * chore: add `all` extra * chore: lint * feat: create branch v2.0.0 * fix: add csv delimiter to schema * feat: expand credential scope to drive and bq * chore: start cleaning tests * chore: add timeout to pypi warning --------- Co-authored-by: lucascr91 Co-authored-by: Mauricio Fagundes Co-authored-by: Gabriel Gazola Milan Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Fernanda Scovino Co-authored-by: Gabrielle Carvalho <77730866+gabrielle-carv@users.noreply.github.com> Co-authored-by: Patrick Teixeira <105399231+tricktx@users.noreply.github.com> Co-authored-by: Gabriel Pisa <61624649+folhesgabriel@users.noreply.github.com> Co-authored-by: Laura Amaral <100051996+laura-l-amaral@users.noreply.github.com> Co-authored-by: Arthur Gusmão Co-authored-by: Crislane Alves <58278652+crislanealves@users.noreply.github.com> Co-authored-by: Lucas Moreira <65978482+lucasnascm@users.noreply.github.com> * chore: remove cli and its dependencies (#1707) * chore: update lint and workflows (#1708) * chore: remove deprecated git hooks * chore: update github templates * chore: update github workflows * chore: update linters and pre-commit hooks * chore: update readme (#1709) * chore: remove contributors as we already removed the workflow * chore: update readme getting started links * chore: remove pylint comments (#1710) * chore: remove schemas (#1711) * feat: add containers to docs development (#1713) * feat: conform br_tse_eleicoes code to DBT tests * feat: update backend and metadata modules (#1712) * chore: move config templates to templates folder * feat: update backend and metadata modules * updates to br_tse_eleicoes, br_me_rais * updates to br_tse_eleicoes, br_me_rais * small improvements * Close #1718 * remove overwrite_cli_config --------- Co-authored-by: Diego Oliveira Co-authored-by: lucascr91 Co-authored-by: Mauricio Fagundes Co-authored-by: Gabriel Gazola Milan Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Fernanda Scovino Co-authored-by: Gabrielle Carvalho <77730866+gabrielle-carv@users.noreply.github.com> Co-authored-by: Patrick Teixeira <105399231+tricktx@users.noreply.github.com> Co-authored-by: Gabriel Pisa <61624649+folhesgabriel@users.noreply.github.com> Co-authored-by: Laura Amaral <100051996+laura-l-amaral@users.noreply.github.com> Co-authored-by: Arthur Gusmão Co-authored-by: Crislane Alves <58278652+crislanealves@users.noreply.github.com> Co-authored-by: Lucas Moreira <65978482+lucasnascm@users.noreply.github.com> Co-authored-by: Vinicius Aguiar Co-authored-by: Ricardo Dahis * feat: pump bd version * chore: remove deprecated actions * chore: add core.base in init * chore: add core.base in init * chore: add core.base in init --------- Co-authored-by: lucascr91 Co-authored-by: Mauricio Fagundes Co-authored-by: Gabriel Gazola Milan Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Fernanda Scovino Co-authored-by: Gabrielle Carvalho <77730866+gabrielle-carv@users.noreply.github.com> Co-authored-by: Patrick Teixeira <105399231+tricktx@users.noreply.github.com> Co-authored-by: Gabriel Pisa <61624649+folhesgabriel@users.noreply.github.com> Co-authored-by: Laura Amaral <100051996+laura-l-amaral@users.noreply.github.com> Co-authored-by: Arthur Gusmão Co-authored-by: Crislane Alves <58278652+crislanealves@users.noreply.github.com> Co-authored-by: Lucas Moreira <65978482+lucasnascm@users.noreply.github.com> Co-authored-by: Pedro Castro Co-authored-by: Vinicius Aguiar Co-authored-by: Ricardo Dahis --- .flake8 | 4 ++++ python-package/basedosdados/__init__.py | 3 ++- python-package/basedosdados/core/__init__.py | 0 python-package/basedosdados/upload/storage.py | 18 +++++++-------- python-package/basedosdados/upload/table.py | 7 +++++- python-package/poetry.lock | 23 +++++-------------- python-package/pyproject.toml | 2 +- python-package/setup.py | 2 +- 8 files changed, 29 insertions(+), 30 deletions(-) create mode 100644 .flake8 create mode 100644 python-package/basedosdados/core/__init__.py diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..46e07c1a1 --- /dev/null +++ b/.flake8 @@ -0,0 +1,4 @@ +[flake8] + select = C,E,F,W,B,B950 + extend-ignore = E501 + max-line-length = 88 \ No newline at end of file diff --git a/python-package/basedosdados/__init__.py b/python-package/basedosdados/__init__.py index 6e816a927..05a8a7757 100644 --- a/python-package/basedosdados/__init__.py +++ b/python-package/basedosdados/__init__.py @@ -1,6 +1,7 @@ """ Importing the module will automatically import the submodules. """ + import os import sys @@ -10,9 +11,9 @@ show_warnings() sys.path.append(f"{os.getcwd()}/python-package") - from basedosdados.backend import Backend from basedosdados.constants import config, constants +from basedosdados.core.base import Base from basedosdados.download.download import download, read_sql, read_table from basedosdados.download.metadata import ( get_columns, diff --git a/python-package/basedosdados/core/__init__.py b/python-package/basedosdados/core/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python-package/basedosdados/upload/storage.py b/python-package/basedosdados/upload/storage.py index f6291fe0e..6d901bce1 100644 --- a/python-package/basedosdados/upload/storage.py +++ b/python-package/basedosdados/upload/storage.py @@ -1,6 +1,7 @@ """ Class for managing the files in cloud storage. """ + import sys @@ -223,13 +224,13 @@ def upload( "to 'replace' to overwrite data." ) - logger.success( - " {object} {filename}_{mode} was {action}!", - filename=filepath.name, - mode=m, - object="File", - action="uploaded", - ) + logger.success( + " {object} {filename}_{mode} was {action}!", + filename=filepath.name, + mode=m, + object="File", + action="uploaded", + ) def download( self, @@ -401,8 +402,7 @@ def delete_table(self, mode="staging", bucket_name=None, not_found_ok=False): ) # Divides table_blobs list for maximum batch request size table_blobs_chunks = [ - table_blobs[i : i + 999] - for i in range(0, len(table_blobs), 999) # noqa + table_blobs[i : i + 999] for i in range(0, len(table_blobs), 999) # noqa ] for i, source_table in enumerate( diff --git a/python-package/basedosdados/upload/table.py b/python-package/basedosdados/upload/table.py index 8257d5b52..c254984f1 100644 --- a/python-package/basedosdados/upload/table.py +++ b/python-package/basedosdados/upload/table.py @@ -561,7 +561,12 @@ def create( Path, ), ): - Storage(self.dataset_id, self.table_id).upload( + Storage( + dataset_id=self.dataset_id, + table_id=self.table_id, + config_path=self.config_path, + bucket_name=self.bucket_name, + ).upload( path=path, mode="staging", if_exists=if_storage_data_exists, diff --git a/python-package/poetry.lock b/python-package/poetry.lock index 6d602f7f7..8e6bfadf0 100644 --- a/python-package/poetry.lock +++ b/python-package/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. [[package]] name = "anyio" @@ -445,12 +445,12 @@ files = [ google-auth = ">=2.14.1,<3.0.dev0" googleapis-common-protos = ">=1.56.2,<2.0.dev0" grpcio = [ + {version = ">=1.33.2,<2.0dev", optional = true, markers = "extra == \"grpc\""}, {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, - {version = ">=1.33.2,<2.0dev", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, ] grpcio-status = [ + {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "extra == \"grpc\""}, {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, - {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, ] proto-plus = ">=1.22.3,<2.0.0dev" protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" @@ -599,8 +599,8 @@ files = [ google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} google-auth = ">=2.14.1,<3.0.0dev" proto-plus = [ - {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""}, {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""}, + {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""}, ] protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" @@ -1561,9 +1561,9 @@ files = [ [package.dependencies] numpy = [ - {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, {version = ">=1.20.3", markers = "python_version < \"3.10\""}, - {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, + {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" @@ -1995,7 +1995,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -2003,16 +2002,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -2029,7 +2020,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -2037,7 +2027,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, diff --git a/python-package/pyproject.toml b/python-package/pyproject.toml index dffd6c91e..3789c5289 100644 --- a/python-package/pyproject.toml +++ b/python-package/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "basedosdados" -version = "2.0.0-beta.16" +version = "2.0.0-beta.21" description = "Organizar e facilitar o acesso a dados brasileiros através de tabelas públicas no BigQuery." authors = ["Joao Carabetta ", "Ricardo Dahis", "Diego Oliveira"] classifiers = [ diff --git a/python-package/setup.py b/python-package/setup.py index 339422e58..8083ddacf 100644 --- a/python-package/setup.py +++ b/python-package/setup.py @@ -6,7 +6,7 @@ setup( name="basedosdados", - version="2.0.0-beta.15", + version="2.0.0-beta.21", packages=find_packages(), include_package_data=True, )