From bce9db57304a7dd6d71996022dfeea5de545dd03 Mon Sep 17 00:00:00 2001 From: Rafael Date: Tue, 10 Sep 2024 10:08:26 -0300 Subject: [PATCH 01/71] testa modelo python --- queries/dbt_project.yml | 4 ++ .../models/teste_python/aux_shapes_wkt.sql | 31 +++++++++++++ queries/models/teste_python/teste_python.py | 43 +++++++++++++++++++ 3 files changed, 78 insertions(+) create mode 100644 queries/models/teste_python/aux_shapes_wkt.sql create mode 100644 queries/models/teste_python/teste_python.py diff --git a/queries/dbt_project.yml b/queries/dbt_project.yml index cb7cdb6c1..442ea5a98 100644 --- a/queries/dbt_project.yml +++ b/queries/dbt_project.yml @@ -302,3 +302,7 @@ models: staging: +materialized: view +schema: transito_staging + teste_python: + +materialized: view + +schema: teste_python + +database: rj-smtr-dev diff --git a/queries/models/teste_python/aux_shapes_wkt.sql b/queries/models/teste_python/aux_shapes_wkt.sql new file mode 100644 index 000000000..210530fbc --- /dev/null +++ b/queries/models/teste_python/aux_shapes_wkt.sql @@ -0,0 +1,31 @@ +{{ + config( + materialized="view" + ) + +}} +with shapes AS ( + select + feed_start_date, + shape_id, + shape_pt_sequence, + CONCAT(shape_pt_lon, " ", shape_pt_lat) AS lon_lat + from + rj-smtr.gtfs.shapes + where + feed_start_date = '2024-09-01' + and shape_id = "hj1m" + order by + 1, + 2, + 3 +) +SELECT + feed_start_date, + shape_id, + concat("LINESTRING(", string_agg(lon_lat, ", "), ")") AS shape_wkt +from + shapes +group by + 1, + 2 \ No newline at end of file diff --git a/queries/models/teste_python/teste_python.py b/queries/models/teste_python/teste_python.py new file mode 100644 index 000000000..a2f3cad01 --- /dev/null +++ b/queries/models/teste_python/teste_python.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +from pyspark.sql.functions import col, explode, udf +from pyspark.sql.types import ArrayType, StringType +from shapely import wkt +from shapely.geometry import LineString, Point + + +def cut(line, distance, lines): + if distance <= 0.0 or distance >= line.length: + return [LineString(line)] + coords = list(line.coords) + for i, p in enumerate(coords): + pd = line.project(Point(p)) + if pd == distance: + return [LineString(coords[: i + 1]), LineString(coords[i:])] + if pd > distance: + cp = line.interpolate(distance) + lines.append(LineString(coords[:i] + [(cp.x, cp.y)]).wkt) + line = LineString([(cp.x, cp.y)] + coords[i:]) + if line.length > distance: + cut(line, distance, lines) + else: + lines.append(LineString([(cp.x, cp.y)] + coords[i:]).wkt) + return lines + + +def cut_udf(wkt_string): + line = wkt.loads(wkt_string) + return cut(line, distance=1000, lines=list()) + + +cut_udf = udf(cut_udf, ArrayType(StringType())) + + +def model(dbt, session): + dbt.config( + materialized="table", + ) + df = dbt.ref("aux_shapes_wkt") + + df_segments = df.withColumn("segments", cut_udf(col("shape_wkt"))) + df_exploded = df_segments.select(explode(col("segments")).alias("segment")) + return df_exploded From cab4c3bb314108f94873c41ea73200ab4e341c21 Mon Sep 17 00:00:00 2001 From: Rafael Date: Tue, 10 Sep 2024 19:11:23 -0300 Subject: [PATCH 02/71] cria divisao shape --- poetry.lock | 320 +++++++++--------- pyproject.toml | 2 +- .../models/teste_python/aux_shapes_wkt.sql | 1 - queries/models/teste_python/teste_python.py | 31 +- 4 files changed, 179 insertions(+), 175 deletions(-) diff --git a/poetry.lock b/poetry.lock index 3e3c8e8f2..a53a9e987 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "agate" @@ -108,29 +108,33 @@ upload = ["gql (>=3.4,<4.0)", "requests-toolbelt (>=1,<2)"] [[package]] name = "black" -version = "23.11.0" +version = "23.12.1" description = "The uncompromising code formatter." optional = false python-versions = ">=3.8" files = [ - {file = "black-23.11.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dbea0bb8575c6b6303cc65017b46351dc5953eea5c0a59d7b7e3a2d2f433a911"}, - {file = "black-23.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:412f56bab20ac85927f3a959230331de5614aecda1ede14b373083f62ec24e6f"}, - {file = "black-23.11.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d136ef5b418c81660ad847efe0e55c58c8208b77a57a28a503a5f345ccf01394"}, - {file = "black-23.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:6c1cac07e64433f646a9a838cdc00c9768b3c362805afc3fce341af0e6a9ae9f"}, - {file = "black-23.11.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cf57719e581cfd48c4efe28543fea3d139c6b6f1238b3f0102a9c73992cbb479"}, - {file = "black-23.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:698c1e0d5c43354ec5d6f4d914d0d553a9ada56c85415700b81dc90125aac244"}, - {file = "black-23.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:760415ccc20f9e8747084169110ef75d545f3b0932ee21368f63ac0fee86b221"}, - {file = "black-23.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:58e5f4d08a205b11800332920e285bd25e1a75c54953e05502052738fe16b3b5"}, - {file = "black-23.11.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:45aa1d4675964946e53ab81aeec7a37613c1cb71647b5394779e6efb79d6d187"}, - {file = "black-23.11.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4c44b7211a3a0570cc097e81135faa5f261264f4dfaa22bd5ee2875a4e773bd6"}, - {file = "black-23.11.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a9acad1451632021ee0d146c8765782a0c3846e0e0ea46659d7c4f89d9b212b"}, - {file = "black-23.11.0-cp38-cp38-win_amd64.whl", hash = "sha256:fc7f6a44d52747e65a02558e1d807c82df1d66ffa80a601862040a43ec2e3142"}, - {file = "black-23.11.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7f622b6822f02bfaf2a5cd31fdb7cd86fcf33dab6ced5185c35f5db98260b055"}, - {file = "black-23.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:250d7e60f323fcfc8ea6c800d5eba12f7967400eb6c2d21ae85ad31c204fb1f4"}, - {file = "black-23.11.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5133f5507007ba08d8b7b263c7aa0f931af5ba88a29beacc4b2dc23fcefe9c06"}, - {file = "black-23.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:421f3e44aa67138ab1b9bfbc22ee3780b22fa5b291e4db8ab7eee95200726b07"}, - {file = "black-23.11.0-py3-none-any.whl", hash = "sha256:54caaa703227c6e0c87b76326d0862184729a69b73d3b7305b6288e1d830067e"}, - {file = "black-23.11.0.tar.gz", hash = "sha256:4c68855825ff432d197229846f971bc4d6666ce90492e5b02013bcaca4d9ab05"}, + {file = "black-23.12.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e0aaf6041986767a5e0ce663c7a2f0e9eaf21e6ff87a5f95cbf3675bfd4c41d2"}, + {file = "black-23.12.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c88b3711d12905b74206227109272673edce0cb29f27e1385f33b0163c414bba"}, + {file = "black-23.12.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a920b569dc6b3472513ba6ddea21f440d4b4c699494d2e972a1753cdc25df7b0"}, + {file = "black-23.12.1-cp310-cp310-win_amd64.whl", hash = "sha256:3fa4be75ef2a6b96ea8d92b1587dd8cb3a35c7e3d51f0738ced0781c3aa3a5a3"}, + {file = "black-23.12.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8d4df77958a622f9b5a4c96edb4b8c0034f8434032ab11077ec6c56ae9f384ba"}, + {file = "black-23.12.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:602cfb1196dc692424c70b6507593a2b29aac0547c1be9a1d1365f0d964c353b"}, + {file = "black-23.12.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c4352800f14be5b4864016882cdba10755bd50805c95f728011bcb47a4afd59"}, + {file = "black-23.12.1-cp311-cp311-win_amd64.whl", hash = "sha256:0808494f2b2df923ffc5723ed3c7b096bd76341f6213989759287611e9837d50"}, + {file = "black-23.12.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:25e57fd232a6d6ff3f4478a6fd0580838e47c93c83eaf1ccc92d4faf27112c4e"}, + {file = "black-23.12.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2d9e13db441c509a3763a7a3d9a49ccc1b4e974a47be4e08ade2a228876500ec"}, + {file = "black-23.12.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d1bd9c210f8b109b1762ec9fd36592fdd528485aadb3f5849b2740ef17e674e"}, + {file = "black-23.12.1-cp312-cp312-win_amd64.whl", hash = "sha256:ae76c22bde5cbb6bfd211ec343ded2163bba7883c7bc77f6b756a1049436fbb9"}, + {file = "black-23.12.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1fa88a0f74e50e4487477bc0bb900c6781dbddfdfa32691e780bf854c3b4a47f"}, + {file = "black-23.12.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a4d6a9668e45ad99d2f8ec70d5c8c04ef4f32f648ef39048d010b0689832ec6d"}, + {file = "black-23.12.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b18fb2ae6c4bb63eebe5be6bd869ba2f14fd0259bda7d18a46b764d8fb86298a"}, + {file = "black-23.12.1-cp38-cp38-win_amd64.whl", hash = "sha256:c04b6d9d20e9c13f43eee8ea87d44156b8505ca8a3c878773f68b4e4812a421e"}, + {file = "black-23.12.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3e1b38b3135fd4c025c28c55ddfc236b05af657828a8a6abe5deec419a0b7055"}, + {file = "black-23.12.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4f0031eaa7b921db76decd73636ef3a12c942ed367d8c3841a0739412b260a54"}, + {file = "black-23.12.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97e56155c6b737854e60a9ab1c598ff2533d57e7506d97af5481141671abf3ea"}, + {file = "black-23.12.1-cp39-cp39-win_amd64.whl", hash = "sha256:dd15245c8b68fe2b6bd0f32c1556509d11bb33aec9b5d0866dd8e2ed3dba09c2"}, + {file = "black-23.12.1-py3-none-any.whl", hash = "sha256:78baad24af0f033958cad29731e27363183e140962595def56423e626f4bee3e"}, + {file = "black-23.12.1.tar.gz", hash = "sha256:4ce3ef14ebe8d9509188014d96af1c456a910d5b5cbf434a09fef7e024b3d0d5"}, ] [package.dependencies] @@ -144,7 +148,7 @@ typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} [package.extras] colorama = ["colorama (>=0.4.3)"] -d = ["aiohttp (>=3.7.4)"] +d = ["aiohttp (>=3.7.4)", "aiohttp (>=3.7.4,!=3.9.0)"] jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] uvloop = ["uvloop (>=0.15.2)"] @@ -443,13 +447,13 @@ pyarrow = ">=3.0.0" [[package]] name = "dbt-bigquery" -version = "1.7.2" +version = "1.7.3" description = "The Bigquery adapter plugin for dbt" optional = false python-versions = ">=3.8" files = [ - {file = "dbt-bigquery-1.7.2.tar.gz", hash = "sha256:27c7f492f65ab5d1d43432a4467a436fc3637e3cb72c5b4ab07ddf7573c43596"}, - {file = "dbt_bigquery-1.7.2-py3-none-any.whl", hash = "sha256:75015755363d9e8b8cebe190d59a5e08375032b37bcfec41ec8753e7dea29f6e"}, + {file = "dbt-bigquery-1.7.3.tar.gz", hash = "sha256:8a132b614d548ea08edd0a7d23ba1c778ab77a8cbffbe177e50a14f1acfaae7f"}, + {file = "dbt_bigquery-1.7.3-py3-none-any.whl", hash = "sha256:c80665600bf2dc518b6d93f6e66e38f6642d7d794c7a19a290451f47a1dbd495"}, ] [package.dependencies] @@ -544,27 +548,28 @@ typing-extensions = ">=4.4,<5.0" [[package]] name = "dill" -version = "0.3.7" +version = "0.3.8" description = "serialize all of Python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "dill-0.3.7-py3-none-any.whl", hash = "sha256:76b122c08ef4ce2eedcd4d1abd8e641114bfc6c2867f49f3c41facf65bf19f5e"}, - {file = "dill-0.3.7.tar.gz", hash = "sha256:cc1c8b182eb3013e24bd475ff2e9295af86c1a38eb1aff128dac8962a9ce3c03"}, + {file = "dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7"}, + {file = "dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca"}, ] [package.extras] graph = ["objgraph (>=1.7.2)"] +profile = ["gprof2dot (>=2022.7.29)"] [[package]] name = "distlib" -version = "0.3.7" +version = "0.3.8" description = "Distribution utilities" optional = false python-versions = "*" files = [ - {file = "distlib-0.3.7-py2.py3-none-any.whl", hash = "sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057"}, - {file = "distlib-0.3.7.tar.gz", hash = "sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8"}, + {file = "distlib-0.3.8-py2.py3-none-any.whl", hash = "sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784"}, + {file = "distlib-0.3.8.tar.gz", hash = "sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64"}, ] [[package]] @@ -629,19 +634,19 @@ files = [ [[package]] name = "filelock" -version = "3.13.1" +version = "3.16.0" description = "A platform independent file lock." optional = false python-versions = ">=3.8" files = [ - {file = "filelock-3.13.1-py3-none-any.whl", hash = "sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c"}, - {file = "filelock-3.13.1.tar.gz", hash = "sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e"}, + {file = "filelock-3.16.0-py3-none-any.whl", hash = "sha256:f6ed4c963184f4c84dd5557ce8fece759a3724b37b80c6c4f20a2f63a4dc6609"}, + {file = "filelock-3.16.0.tar.gz", hash = "sha256:81de9eb8453c769b63369f87f11131a7ab04e367f8d97ad39dc230daa07e3bec"}, ] [package.extras] -docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.24)"] -testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)"] -typing = ["typing-extensions (>=4.8)"] +docs = ["furo (>=2024.8.6)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.6.1)", "diff-cover (>=9.1.1)", "pytest (>=8.3.2)", "pytest-asyncio (>=0.24)", "pytest-cov (>=5)", "pytest-mock (>=3.14)", "pytest-timeout (>=2.3.1)", "virtualenv (>=20.26.3)"] +typing = ["typing-extensions (>=4.12.2)"] [[package]] name = "flake8" @@ -880,13 +885,13 @@ grpc = ["grpcio (>=1.38.0,<2.0dev)", "grpcio-status (>=1.38.0,<2.0.dev0)"] [[package]] name = "google-cloud-dataplex" -version = "1.12.3" +version = "1.13.0" description = "Google Cloud Dataplex API client library" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-dataplex-1.12.3.tar.gz", hash = "sha256:c0692b6713fce6180df6d59bfb972d06f9fae264f33e7f2ba781d785f64ab8c3"}, - {file = "google_cloud_dataplex-1.12.3-py2.py3-none-any.whl", hash = "sha256:2ef7943b331ff254d2a8d2395c6a3b666837a73d32da1c4572a07453c0069dd3"}, + {file = "google-cloud-dataplex-1.13.0.tar.gz", hash = "sha256:afaa7163c06b33158be3286e9faf0c7334a1bf275dd19457f38ae7043499515f"}, + {file = "google_cloud_dataplex-1.13.0-py2.py3-none-any.whl", hash = "sha256:9c4bfadf01977aa93d904581587f57538788578b36969cc5d084efea15444e37"}, ] [package.dependencies] @@ -1273,13 +1278,13 @@ pyparsing = {version = ">=2.4.2,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.0.2 || >3.0 [[package]] name = "identify" -version = "2.5.32" +version = "2.6.0" description = "File identification library for Python" optional = false python-versions = ">=3.8" files = [ - {file = "identify-2.5.32-py2.py3-none-any.whl", hash = "sha256:0b7656ef6cba81664b783352c73f8c24b39cf82f926f78f4550eda928e5e0545"}, - {file = "identify-2.5.32.tar.gz", hash = "sha256:5d9979348ec1a21c768ae07e0a652924538e8bce67313a73cb0f681cf08ba407"}, + {file = "identify-2.6.0-py2.py3-none-any.whl", hash = "sha256:e79ae4406387a9d300332b5fd366d8994f1525e8414984e1a59e058b2eda2dd0"}, + {file = "identify-2.6.0.tar.gz", hash = "sha256:cb171c685bdc31bcc4c1734698736a7d5b6c8bf2e0c15117f4d469c8640ae5cf"}, ] [package.extras] @@ -1367,20 +1372,17 @@ six = "*" [[package]] name = "isort" -version = "5.12.0" +version = "5.13.2" description = "A Python utility / library to sort Python imports." optional = false python-versions = ">=3.8.0" files = [ - {file = "isort-5.12.0-py3-none-any.whl", hash = "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6"}, - {file = "isort-5.12.0.tar.gz", hash = "sha256:8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504"}, + {file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"}, + {file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"}, ] [package.extras] -colors = ["colorama (>=0.4.3)"] -pipfile-deprecated-finder = ["pip-shims (>=0.5.2)", "pipreqs", "requirementslib"] -plugins = ["setuptools"] -requirements-deprecated-finder = ["pip-api", "pipreqs"] +colors = ["colorama (>=0.4.6)"] [[package]] name = "jinja2" @@ -1533,16 +1535,6 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -1738,13 +1730,13 @@ files = [ [[package]] name = "mslex" -version = "0.3.0" +version = "1.2.0" description = "shlex for windows" optional = false python-versions = ">=3.5" files = [ - {file = "mslex-0.3.0-py2.py3-none-any.whl", hash = "sha256:380cb14abf8fabf40e56df5c8b21a6d533dc5cbdcfe42406bbf08dda8f42e42a"}, - {file = "mslex-0.3.0.tar.gz", hash = "sha256:4a1ac3f25025cad78ad2fe499dd16d42759f7a3801645399cce5c404415daa97"}, + {file = "mslex-1.2.0-py3-none-any.whl", hash = "sha256:c68ec637485ee3544c5847c1b4e78b02940b32708568fb1d8715491815aa2341"}, + {file = "mslex-1.2.0.tar.gz", hash = "sha256:79e2abc5a129dd71cdde58a22a2039abb7fa8afcbac498b723ba6e9b9fbacc14"}, ] [[package]] @@ -1861,18 +1853,15 @@ test = ["pytest (>=7.2)", "pytest-cov (>=4.0)"] [[package]] name = "nodeenv" -version = "1.8.0" +version = "1.9.1" description = "Node.js virtual environment builder" optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ - {file = "nodeenv-1.8.0-py2.py3-none-any.whl", hash = "sha256:df865724bb3c3adc86b3876fa209771517b0cfe596beff01a92700e0e8be4cec"}, - {file = "nodeenv-1.8.0.tar.gz", hash = "sha256:d51e0c37e64fbf47d017feac3145cdbb58836d7eee8c6f6d3b6880c5456227d2"}, + {file = "nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9"}, + {file = "nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f"}, ] -[package.dependencies] -setuptools = "*" - [[package]] name = "numpy" version = "1.26.2" @@ -1936,13 +1925,13 @@ signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] [[package]] name = "openpyxl" -version = "3.1.4" +version = "3.1.5" description = "A Python library to read/write Excel 2010 xlsx/xlsm files" optional = false python-versions = ">=3.8" files = [ - {file = "openpyxl-3.1.4-py2.py3-none-any.whl", hash = "sha256:ec17f6483f2b8f7c88c57e5e5d3b0de0e3fb9ac70edc084d28e864f5b33bbefd"}, - {file = "openpyxl-3.1.4.tar.gz", hash = "sha256:8d2c8adf5d20d6ce8f9bca381df86b534835e974ed0156dacefa76f68c1d69fb"}, + {file = "openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2"}, + {file = "openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050"}, ] [package.dependencies] @@ -2126,28 +2115,29 @@ pytzdata = ">=2020.1" [[package]] name = "platformdirs" -version = "4.0.0" -description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +version = "4.3.2" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "platformdirs-4.0.0-py3-none-any.whl", hash = "sha256:118c954d7e949b35437270383a3f2531e99dd93cf7ce4dc8340d3356d30f173b"}, - {file = "platformdirs-4.0.0.tar.gz", hash = "sha256:cb633b2bcf10c51af60beb0ab06d2f1d69064b43abf4c185ca6b28865f3f9731"}, + {file = "platformdirs-4.3.2-py3-none-any.whl", hash = "sha256:eb1c8582560b34ed4ba105009a4badf7f6f85768b30126f351328507b2beb617"}, + {file = "platformdirs-4.3.2.tar.gz", hash = "sha256:9e5e27a08aa095dd127b9f2e764d74254f482fef22b0970773bfba79d091ab8c"}, ] [package.extras] -docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] +docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.3.2)", "pytest-cov (>=5)", "pytest-mock (>=3.14)"] +type = ["mypy (>=1.11.2)"] [[package]] name = "pre-commit" -version = "3.5.0" +version = "3.8.0" description = "A framework for managing and maintaining multi-language pre-commit hooks." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "pre_commit-3.5.0-py2.py3-none-any.whl", hash = "sha256:841dc9aef25daba9a0238cd27984041fa0467b4199fc4852e27950664919f660"}, - {file = "pre_commit-3.5.0.tar.gz", hash = "sha256:5804465c675b659b0862f07907f96295d490822a450c4c40e747d0b1c6ebcb32"}, + {file = "pre_commit-3.8.0-py2.py3-none-any.whl", hash = "sha256:9a90a53bf82fdd8778d58085faf8d83df56e40dfe18f45b19446e26bf1b3a63f"}, + {file = "pre_commit-3.8.0.tar.gz", hash = "sha256:8bb6494d4a20423842e198980c9ecf9f96607a07ea29549e180eef9ae80fe7af"}, ] [package.dependencies] @@ -2363,7 +2353,6 @@ files = [ {file = "psycopg2_binary-2.9.9-cp311-cp311-win32.whl", hash = "sha256:dc4926288b2a3e9fd7b50dc6a1909a13bbdadfc67d93f3374d984e56f885579d"}, {file = "psycopg2_binary-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:b76bedd166805480ab069612119ea636f5ab8f8771e640ae103e05a4aae3e417"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996"}, @@ -2372,8 +2361,6 @@ files = [ {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-win32.whl", hash = "sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab"}, {file = "psycopg2_binary-2.9.9-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2293b001e319ab0d869d660a704942c9e2cce19745262a8aba2115ef41a0a42a"}, {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03ef7df18daf2c4c07e2695e8cfd5ee7f748a1d54d802330985a78d2a5a6dca9"}, {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a602ea5aff39bb9fac6308e9c9d82b9a35c2bf288e184a816002c9fae930b77"}, @@ -2628,13 +2615,13 @@ files = [ [[package]] name = "pymysql" -version = "1.1.0" +version = "1.1.1" description = "Pure Python MySQL Driver" optional = false python-versions = ">=3.7" files = [ - {file = "PyMySQL-1.1.0-py3-none-any.whl", hash = "sha256:8969ec6d763c856f7073c4c64662882675702efcb114b4bcbb955aea3a069fa7"}, - {file = "PyMySQL-1.1.0.tar.gz", hash = "sha256:4f13a7df8bf36a51e81dd9f3605fede45a4878fe02f9236349fd82a3f0612f96"}, + {file = "PyMySQL-1.1.1-py3-none-any.whl", hash = "sha256:4de15da4c61dc132f4fb9ab763063e693d521a80fd0e87943b9a453dd4c19d6c"}, + {file = "pymysql-1.1.1.tar.gz", hash = "sha256:e127611aaf2b417403c60bf4dc570124aeb4a57f5f37b8e95ae399a42f904cd0"}, ] [package.extras] @@ -2825,7 +2812,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -3130,19 +3116,23 @@ tornado = ["tornado (>=5)"] [[package]] name = "setuptools" -version = "69.0.2" +version = "74.1.2" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-69.0.2-py3-none-any.whl", hash = "sha256:1e8fdff6797d3865f37397be788a4e3cba233608e9b509382a2777d25ebde7f2"}, - {file = "setuptools-69.0.2.tar.gz", hash = "sha256:735896e78a4742605974de002ac60562d286fa8051a7e2299445e8e8fbb01aa6"}, + {file = "setuptools-74.1.2-py3-none-any.whl", hash = "sha256:5f4c08aa4d3ebcb57a50c33b1b07e94315d7fc7230f7115e47fc99776c8ce308"}, + {file = "setuptools-74.1.2.tar.gz", hash = "sha256:95b40ed940a1c67eb70fc099094bd6e99c6ee7c23aa2306f4d2697ba7916f9c6"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.5.2)"] +core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.text (>=3.7)", "more-itertools (>=8.8)", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib-metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.11.*)", "pytest-mypy"] [[package]] name = "six" @@ -3168,64 +3158,64 @@ files = [ [[package]] name = "sqlalchemy" -version = "2.0.25" +version = "2.0.34" description = "Database Abstraction Library" optional = false python-versions = ">=3.7" files = [ - {file = "SQLAlchemy-2.0.25-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4344d059265cc8b1b1be351bfb88749294b87a8b2bbe21dfbe066c4199541ebd"}, - {file = "SQLAlchemy-2.0.25-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6f9e2e59cbcc6ba1488404aad43de005d05ca56e069477b33ff74e91b6319735"}, - {file = "SQLAlchemy-2.0.25-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:84daa0a2055df9ca0f148a64fdde12ac635e30edbca80e87df9b3aaf419e144a"}, - {file = "SQLAlchemy-2.0.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc8b7dabe8e67c4832891a5d322cec6d44ef02f432b4588390017f5cec186a84"}, - {file = "SQLAlchemy-2.0.25-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:f5693145220517b5f42393e07a6898acdfe820e136c98663b971906120549da5"}, - {file = "SQLAlchemy-2.0.25-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:db854730a25db7c956423bb9fb4bdd1216c839a689bf9cc15fada0a7fb2f4570"}, - {file = "SQLAlchemy-2.0.25-cp310-cp310-win32.whl", hash = "sha256:14a6f68e8fc96e5e8f5647ef6cda6250c780612a573d99e4d881581432ef1669"}, - {file = "SQLAlchemy-2.0.25-cp310-cp310-win_amd64.whl", hash = "sha256:87f6e732bccd7dcf1741c00f1ecf33797383128bd1c90144ac8adc02cbb98643"}, - {file = "SQLAlchemy-2.0.25-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:342d365988ba88ada8af320d43df4e0b13a694dbd75951f537b2d5e4cb5cd002"}, - {file = "SQLAlchemy-2.0.25-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f37c0caf14b9e9b9e8f6dbc81bc56db06acb4363eba5a633167781a48ef036ed"}, - {file = "SQLAlchemy-2.0.25-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa9373708763ef46782d10e950b49d0235bfe58facebd76917d3f5cbf5971aed"}, - {file = "SQLAlchemy-2.0.25-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d24f571990c05f6b36a396218f251f3e0dda916e0c687ef6fdca5072743208f5"}, - {file = "SQLAlchemy-2.0.25-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:75432b5b14dc2fff43c50435e248b45c7cdadef73388e5610852b95280ffd0e9"}, - {file = "SQLAlchemy-2.0.25-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:884272dcd3ad97f47702965a0e902b540541890f468d24bd1d98bcfe41c3f018"}, - {file = "SQLAlchemy-2.0.25-cp311-cp311-win32.whl", hash = "sha256:e607cdd99cbf9bb80391f54446b86e16eea6ad309361942bf88318bcd452363c"}, - {file = "SQLAlchemy-2.0.25-cp311-cp311-win_amd64.whl", hash = "sha256:7d505815ac340568fd03f719446a589162d55c52f08abd77ba8964fbb7eb5b5f"}, - {file = "SQLAlchemy-2.0.25-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:0dacf67aee53b16f365c589ce72e766efaabd2b145f9de7c917777b575e3659d"}, - {file = "SQLAlchemy-2.0.25-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b801154027107461ee992ff4b5c09aa7cc6ec91ddfe50d02bca344918c3265c6"}, - {file = "SQLAlchemy-2.0.25-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59a21853f5daeb50412d459cfb13cb82c089ad4c04ec208cd14dddd99fc23b39"}, - {file = "SQLAlchemy-2.0.25-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:29049e2c299b5ace92cbed0c1610a7a236f3baf4c6b66eb9547c01179f638ec5"}, - {file = "SQLAlchemy-2.0.25-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b64b183d610b424a160b0d4d880995e935208fc043d0302dd29fee32d1ee3f95"}, - {file = "SQLAlchemy-2.0.25-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4f7a7d7fcc675d3d85fbf3b3828ecd5990b8d61bd6de3f1b260080b3beccf215"}, - {file = "SQLAlchemy-2.0.25-cp312-cp312-win32.whl", hash = "sha256:cf18ff7fc9941b8fc23437cc3e68ed4ebeff3599eec6ef5eebf305f3d2e9a7c2"}, - {file = "SQLAlchemy-2.0.25-cp312-cp312-win_amd64.whl", hash = "sha256:91f7d9d1c4dd1f4f6e092874c128c11165eafcf7c963128f79e28f8445de82d5"}, - {file = "SQLAlchemy-2.0.25-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:bb209a73b8307f8fe4fe46f6ad5979649be01607f11af1eb94aa9e8a3aaf77f0"}, - {file = "SQLAlchemy-2.0.25-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:798f717ae7c806d67145f6ae94dc7c342d3222d3b9a311a784f371a4333212c7"}, - {file = "SQLAlchemy-2.0.25-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fdd402169aa00df3142149940b3bf9ce7dde075928c1886d9a1df63d4b8de62"}, - {file = "SQLAlchemy-2.0.25-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0d3cab3076af2e4aa5693f89622bef7fa770c6fec967143e4da7508b3dceb9b9"}, - {file = "SQLAlchemy-2.0.25-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:74b080c897563f81062b74e44f5a72fa44c2b373741a9ade701d5f789a10ba23"}, - {file = "SQLAlchemy-2.0.25-cp37-cp37m-win32.whl", hash = "sha256:87d91043ea0dc65ee583026cb18e1b458d8ec5fc0a93637126b5fc0bc3ea68c4"}, - {file = "SQLAlchemy-2.0.25-cp37-cp37m-win_amd64.whl", hash = "sha256:75f99202324383d613ddd1f7455ac908dca9c2dd729ec8584c9541dd41822a2c"}, - {file = "SQLAlchemy-2.0.25-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:420362338681eec03f53467804541a854617faed7272fe71a1bfdb07336a381e"}, - {file = "SQLAlchemy-2.0.25-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7c88f0c7dcc5f99bdb34b4fd9b69b93c89f893f454f40219fe923a3a2fd11625"}, - {file = "SQLAlchemy-2.0.25-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3be4987e3ee9d9a380b66393b77a4cd6d742480c951a1c56a23c335caca4ce3"}, - {file = "SQLAlchemy-2.0.25-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2a159111a0f58fb034c93eeba211b4141137ec4b0a6e75789ab7a3ef3c7e7e3"}, - {file = "SQLAlchemy-2.0.25-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8b8cb63d3ea63b29074dcd29da4dc6a97ad1349151f2d2949495418fd6e48db9"}, - {file = "SQLAlchemy-2.0.25-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:736ea78cd06de6c21ecba7416499e7236a22374561493b456a1f7ffbe3f6cdb4"}, - {file = "SQLAlchemy-2.0.25-cp38-cp38-win32.whl", hash = "sha256:10331f129982a19df4284ceac6fe87353ca3ca6b4ca77ff7d697209ae0a5915e"}, - {file = "SQLAlchemy-2.0.25-cp38-cp38-win_amd64.whl", hash = "sha256:c55731c116806836a5d678a70c84cb13f2cedba920212ba7dcad53260997666d"}, - {file = "SQLAlchemy-2.0.25-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:605b6b059f4b57b277f75ace81cc5bc6335efcbcc4ccb9066695e515dbdb3900"}, - {file = "SQLAlchemy-2.0.25-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:665f0a3954635b5b777a55111ababf44b4fc12b1f3ba0a435b602b6387ffd7cf"}, - {file = "SQLAlchemy-2.0.25-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecf6d4cda1f9f6cb0b45803a01ea7f034e2f1aed9475e883410812d9f9e3cfcf"}, - {file = "SQLAlchemy-2.0.25-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c51db269513917394faec5e5c00d6f83829742ba62e2ac4fa5c98d58be91662f"}, - {file = "SQLAlchemy-2.0.25-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:790f533fa5c8901a62b6fef5811d48980adeb2f51f1290ade8b5e7ba990ba3de"}, - {file = "SQLAlchemy-2.0.25-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1b1180cda6df7af84fe72e4530f192231b1f29a7496951db4ff38dac1687202d"}, - {file = "SQLAlchemy-2.0.25-cp39-cp39-win32.whl", hash = "sha256:555651adbb503ac7f4cb35834c5e4ae0819aab2cd24857a123370764dc7d7e24"}, - {file = "SQLAlchemy-2.0.25-cp39-cp39-win_amd64.whl", hash = "sha256:dc55990143cbd853a5d038c05e79284baedf3e299661389654551bd02a6a68d7"}, - {file = "SQLAlchemy-2.0.25-py3-none-any.whl", hash = "sha256:a86b4240e67d4753dc3092d9511886795b3c2852abe599cffe108952f7af7ac3"}, - {file = "SQLAlchemy-2.0.25.tar.gz", hash = "sha256:a2c69a7664fb2d54b8682dd774c3b54f67f84fa123cf84dda2a5f40dcaa04e08"}, -] - -[package.dependencies] -greenlet = {version = "!=0.4.17", markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\""} + {file = "SQLAlchemy-2.0.34-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:95d0b2cf8791ab5fb9e3aa3d9a79a0d5d51f55b6357eecf532a120ba3b5524db"}, + {file = "SQLAlchemy-2.0.34-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:243f92596f4fd4c8bd30ab8e8dd5965afe226363d75cab2468f2c707f64cd83b"}, + {file = "SQLAlchemy-2.0.34-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ea54f7300553af0a2a7235e9b85f4204e1fc21848f917a3213b0e0818de9a24"}, + {file = "SQLAlchemy-2.0.34-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:173f5f122d2e1bff8fbd9f7811b7942bead1f5e9f371cdf9e670b327e6703ebd"}, + {file = "SQLAlchemy-2.0.34-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:196958cde924a00488e3e83ff917be3b73cd4ed8352bbc0f2989333176d1c54d"}, + {file = "SQLAlchemy-2.0.34-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bd90c221ed4e60ac9d476db967f436cfcecbd4ef744537c0f2d5291439848768"}, + {file = "SQLAlchemy-2.0.34-cp310-cp310-win32.whl", hash = "sha256:3166dfff2d16fe9be3241ee60ece6fcb01cf8e74dd7c5e0b64f8e19fab44911b"}, + {file = "SQLAlchemy-2.0.34-cp310-cp310-win_amd64.whl", hash = "sha256:6831a78bbd3c40f909b3e5233f87341f12d0b34a58f14115c9e94b4cdaf726d3"}, + {file = "SQLAlchemy-2.0.34-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c7db3db284a0edaebe87f8f6642c2b2c27ed85c3e70064b84d1c9e4ec06d5d84"}, + {file = "SQLAlchemy-2.0.34-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:430093fce0efc7941d911d34f75a70084f12f6ca5c15d19595c18753edb7c33b"}, + {file = "SQLAlchemy-2.0.34-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79cb400c360c7c210097b147c16a9e4c14688a6402445ac848f296ade6283bbc"}, + {file = "SQLAlchemy-2.0.34-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb1b30f31a36c7f3fee848391ff77eebdd3af5750bf95fbf9b8b5323edfdb4ec"}, + {file = "SQLAlchemy-2.0.34-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8fddde2368e777ea2a4891a3fb4341e910a056be0bb15303bf1b92f073b80c02"}, + {file = "SQLAlchemy-2.0.34-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:80bd73ea335203b125cf1d8e50fef06be709619eb6ab9e7b891ea34b5baa2287"}, + {file = "SQLAlchemy-2.0.34-cp311-cp311-win32.whl", hash = "sha256:6daeb8382d0df526372abd9cb795c992e18eed25ef2c43afe518c73f8cccb721"}, + {file = "SQLAlchemy-2.0.34-cp311-cp311-win_amd64.whl", hash = "sha256:5bc08e75ed11693ecb648b7a0a4ed80da6d10845e44be0c98c03f2f880b68ff4"}, + {file = "SQLAlchemy-2.0.34-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:53e68b091492c8ed2bd0141e00ad3089bcc6bf0e6ec4142ad6505b4afe64163e"}, + {file = "SQLAlchemy-2.0.34-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bcd18441a49499bf5528deaa9dee1f5c01ca491fc2791b13604e8f972877f812"}, + {file = "SQLAlchemy-2.0.34-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:165bbe0b376541092bf49542bd9827b048357f4623486096fc9aaa6d4e7c59a2"}, + {file = "SQLAlchemy-2.0.34-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3330415cd387d2b88600e8e26b510d0370db9b7eaf984354a43e19c40df2e2b"}, + {file = "SQLAlchemy-2.0.34-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:97b850f73f8abbffb66ccbab6e55a195a0eb655e5dc74624d15cff4bfb35bd74"}, + {file = "SQLAlchemy-2.0.34-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7cee4c6917857fd6121ed84f56d1dc78eb1d0e87f845ab5a568aba73e78adf83"}, + {file = "SQLAlchemy-2.0.34-cp312-cp312-win32.whl", hash = "sha256:fbb034f565ecbe6c530dff948239377ba859420d146d5f62f0271407ffb8c580"}, + {file = "SQLAlchemy-2.0.34-cp312-cp312-win_amd64.whl", hash = "sha256:707c8f44931a4facd4149b52b75b80544a8d824162602b8cd2fe788207307f9a"}, + {file = "SQLAlchemy-2.0.34-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:24af3dc43568f3780b7e1e57c49b41d98b2d940c1fd2e62d65d3928b6f95f021"}, + {file = "SQLAlchemy-2.0.34-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e60ed6ef0a35c6b76b7640fe452d0e47acc832ccbb8475de549a5cc5f90c2c06"}, + {file = "SQLAlchemy-2.0.34-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:413c85cd0177c23e32dee6898c67a5f49296640041d98fddb2c40888fe4daa2e"}, + {file = "SQLAlchemy-2.0.34-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:25691f4adfb9d5e796fd48bf1432272f95f4bbe5f89c475a788f31232ea6afba"}, + {file = "SQLAlchemy-2.0.34-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:526ce723265643dbc4c7efb54f56648cc30e7abe20f387d763364b3ce7506c82"}, + {file = "SQLAlchemy-2.0.34-cp37-cp37m-win32.whl", hash = "sha256:13be2cc683b76977a700948411a94c67ad8faf542fa7da2a4b167f2244781cf3"}, + {file = "SQLAlchemy-2.0.34-cp37-cp37m-win_amd64.whl", hash = "sha256:e54ef33ea80d464c3dcfe881eb00ad5921b60f8115ea1a30d781653edc2fd6a2"}, + {file = "SQLAlchemy-2.0.34-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:43f28005141165edd11fbbf1541c920bd29e167b8bbc1fb410d4fe2269c1667a"}, + {file = "SQLAlchemy-2.0.34-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b68094b165a9e930aedef90725a8fcfafe9ef95370cbb54abc0464062dbf808f"}, + {file = "SQLAlchemy-2.0.34-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a1e03db964e9d32f112bae36f0cc1dcd1988d096cfd75d6a588a3c3def9ab2b"}, + {file = "SQLAlchemy-2.0.34-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:203d46bddeaa7982f9c3cc693e5bc93db476ab5de9d4b4640d5c99ff219bee8c"}, + {file = "SQLAlchemy-2.0.34-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:ae92bebca3b1e6bd203494e5ef919a60fb6dfe4d9a47ed2453211d3bd451b9f5"}, + {file = "SQLAlchemy-2.0.34-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:9661268415f450c95f72f0ac1217cc6f10256f860eed85c2ae32e75b60278ad8"}, + {file = "SQLAlchemy-2.0.34-cp38-cp38-win32.whl", hash = "sha256:895184dfef8708e15f7516bd930bda7e50ead069280d2ce09ba11781b630a434"}, + {file = "SQLAlchemy-2.0.34-cp38-cp38-win_amd64.whl", hash = "sha256:6e7cde3a2221aa89247944cafb1b26616380e30c63e37ed19ff0bba5e968688d"}, + {file = "SQLAlchemy-2.0.34-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:dbcdf987f3aceef9763b6d7b1fd3e4ee210ddd26cac421d78b3c206d07b2700b"}, + {file = "SQLAlchemy-2.0.34-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ce119fc4ce0d64124d37f66a6f2a584fddc3c5001755f8a49f1ca0a177ef9796"}, + {file = "SQLAlchemy-2.0.34-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a17d8fac6df9835d8e2b4c5523666e7051d0897a93756518a1fe101c7f47f2f0"}, + {file = "SQLAlchemy-2.0.34-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ebc11c54c6ecdd07bb4efbfa1554538982f5432dfb8456958b6d46b9f834bb7"}, + {file = "SQLAlchemy-2.0.34-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:2e6965346fc1491a566e019a4a1d3dfc081ce7ac1a736536367ca305da6472a8"}, + {file = "SQLAlchemy-2.0.34-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:220574e78ad986aea8e81ac68821e47ea9202b7e44f251b7ed8c66d9ae3f4278"}, + {file = "SQLAlchemy-2.0.34-cp39-cp39-win32.whl", hash = "sha256:b75b00083e7fe6621ce13cfce9d4469c4774e55e8e9d38c305b37f13cf1e874c"}, + {file = "SQLAlchemy-2.0.34-cp39-cp39-win_amd64.whl", hash = "sha256:c29d03e0adf3cc1a8c3ec62d176824972ae29b67a66cbb18daff3062acc6faa8"}, + {file = "SQLAlchemy-2.0.34-py3-none-any.whl", hash = "sha256:7286c353ee6475613d8beff83167374006c6b3e3f0e6491bfe8ca610eb1dec0f"}, + {file = "sqlalchemy-2.0.34.tar.gz", hash = "sha256:10d8f36990dd929690666679b0f42235c159a7051534adb135728ee52828dd22"}, +] + +[package.dependencies] +greenlet = {version = "!=0.4.17", markers = "python_version < \"3.13\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} typing-extensions = ">=4.6.0" [package.extras] @@ -3285,18 +3275,18 @@ widechars = ["wcwidth"] [[package]] name = "taskipy" -version = "1.12.0" +version = "1.13.0" description = "tasks runner for python projects" optional = false -python-versions = ">=3.6,<4.0" +python-versions = "<4.0,>=3.6" files = [ - {file = "taskipy-1.12.0-py3-none-any.whl", hash = "sha256:38306fbc952a7ca314b8f842a74b2fc38535cdab21031fe89e714a83e6259a84"}, - {file = "taskipy-1.12.0.tar.gz", hash = "sha256:e3dd7c53f7c9c4fd17dc908b1037f545afc452907eb0953b84e91c0a9a9d809d"}, + {file = "taskipy-1.13.0-py3-none-any.whl", hash = "sha256:56f42b7e508d9aed2c7b6365f8d3dab62dbd0c768c1ab606c819da4fc38421f7"}, + {file = "taskipy-1.13.0.tar.gz", hash = "sha256:2b52f0257958fed151f1340f7de93fcf0848f7a358ad62ba05c31c2ca04f89fe"}, ] [package.dependencies] colorama = ">=0.4.4,<0.5.0" -mslex = {version = ">=0.3.0,<0.4.0", markers = "sys_platform == \"win32\""} +mslex = {version = ">=1.1.0,<2.0.0", markers = "sys_platform == \"win32\""} psutil = ">=5.7.2,<6.0.0" tomli = {version = ">=2.0.1,<3.0.0", markers = "python_version >= \"3.7\" and python_version < \"4.0\""} @@ -3408,13 +3398,13 @@ telegram = ["requests"] [[package]] name = "typer" -version = "0.9.0" +version = "0.9.4" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." optional = false python-versions = ">=3.6" files = [ - {file = "typer-0.9.0-py3-none-any.whl", hash = "sha256:5d96d986a21493606a358cae4461bd8cdf83cbf33a5aa950ae629ca3b51467ee"}, - {file = "typer-0.9.0.tar.gz", hash = "sha256:50922fd79aea2f4751a8e0408ff10d2662bd0c8bbfa84755a699f3bada2978b2"}, + {file = "typer-0.9.4-py3-none-any.whl", hash = "sha256:aa6c4a4e2329d868b80ecbaf16f807f2b54e192209d7ac9dd42691d63f7a54eb"}, + {file = "typer-0.9.4.tar.gz", hash = "sha256:f714c2d90afae3a7929fcd72a3abb08df305e1ff61719381384211c4070af57f"}, ] [package.dependencies] @@ -3425,7 +3415,7 @@ typing-extensions = ">=3.7.4.3" all = ["colorama (>=0.4.3,<0.5.0)", "rich (>=10.11.0,<14.0.0)", "shellingham (>=1.3.0,<2.0.0)"] dev = ["autoflake (>=1.3.1,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)", "pre-commit (>=2.17.0,<3.0.0)"] doc = ["cairosvg (>=2.5.2,<3.0.0)", "mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=8.1.4,<9.0.0)", "pillow (>=9.3.0,<10.0.0)"] -test = ["black (>=22.3.0,<23.0.0)", "coverage (>=6.2,<7.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.910)", "pytest (>=4.4.0,<8.0.0)", "pytest-cov (>=2.10.0,<5.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "pytest-xdist (>=1.32.0,<4.0.0)", "rich (>=10.11.0,<14.0.0)", "shellingham (>=1.3.0,<2.0.0)"] +test = ["black (>=22.3.0,<23.0.0)", "coverage (>=6.2,<7.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.971)", "pytest (>=4.4.0,<8.0.0)", "pytest-cov (>=2.10.0,<5.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "pytest-xdist (>=1.32.0,<4.0.0)", "rich (>=10.11.0,<14.0.0)", "shellingham (>=1.3.0,<2.0.0)"] [[package]] name = "typing-extensions" @@ -3478,13 +3468,13 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" -version = "20.24.7" +version = "20.26.4" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.7" files = [ - {file = "virtualenv-20.24.7-py3-none-any.whl", hash = "sha256:a18b3fd0314ca59a2e9f4b556819ed07183b3e9a3702ecfe213f593d44f7b3fd"}, - {file = "virtualenv-20.24.7.tar.gz", hash = "sha256:69050ffb42419c91f6c1284a7b24e0475d793447e35929b488bf6a0aade39353"}, + {file = "virtualenv-20.26.4-py3-none-any.whl", hash = "sha256:48f2695d9809277003f30776d155615ffc11328e6a0a8c1f0ec80188d7874a55"}, + {file = "virtualenv-20.26.4.tar.gz", hash = "sha256:c17f4e0f3e6036e9f26700446f85c76ab11df65ff6d8a9cbfad9f71aabfcf23c"}, ] [package.dependencies] @@ -3493,7 +3483,7 @@ filelock = ">=3.12.2,<4" platformdirs = ">=3.9.1,<5" [package.extras] -docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2,!=7.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8)", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10)"] [[package]] @@ -3642,4 +3632,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.11" -content-hash = "63916f4608ce386360d6407898cdde87a540c8eaeee929e872ae0342ce4db413" +content-hash = "6b299f0c7ae418b422ff8a5fd5445985d249e4e432ef04efc9407869f65b7d05" diff --git a/pyproject.toml b/pyproject.toml index 6b56c4514..c09b95984 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ authors = ["Gabriel Gazola Milan "] [tool.poetry.dependencies] python = ">=3.10,<3.11" -dbt-bigquery = "1.7.2" +dbt-bigquery = "1.7.3" google-cloud-storage = "2.13.0" prefect = "1.4.1" prefeitura-rio = "1.1.3a1" diff --git a/queries/models/teste_python/aux_shapes_wkt.sql b/queries/models/teste_python/aux_shapes_wkt.sql index 210530fbc..df6f7badc 100644 --- a/queries/models/teste_python/aux_shapes_wkt.sql +++ b/queries/models/teste_python/aux_shapes_wkt.sql @@ -14,7 +14,6 @@ with shapes AS ( rj-smtr.gtfs.shapes where feed_start_date = '2024-09-01' - and shape_id = "hj1m" order by 1, 2, diff --git a/queries/models/teste_python/teste_python.py b/queries/models/teste_python/teste_python.py index a2f3cad01..4b3c77f57 100644 --- a/queries/models/teste_python/teste_python.py +++ b/queries/models/teste_python/teste_python.py @@ -1,32 +1,44 @@ # -*- coding: utf-8 -*- +import pyproj from pyspark.sql.functions import col, explode, udf from pyspark.sql.types import ArrayType, StringType from shapely import wkt from shapely.geometry import LineString, Point +from shapely.ops import transform def cut(line, distance, lines): + wgs84 = pyproj.CRS("EPSG:4326") + utm = pyproj.CRS("EPSG:31983") + project = pyproj.Transformer.from_crs(utm, wgs84, always_xy=True).transform + line = wkt.loads(line) if distance <= 0.0 or distance >= line.length: - return [LineString(line)] + return [transform(project, LineString(line)).wkt] coords = list(line.coords) for i, p in enumerate(coords): pd = line.project(Point(p)) if pd == distance: - return [LineString(coords[: i + 1]), LineString(coords[i:])] + return [ + transform(project, LineString(coords[: i + 1])).wkt, + transform(project, LineString(coords[i:])).wkt, + ] if pd > distance: cp = line.interpolate(distance) - lines.append(LineString(coords[:i] + [(cp.x, cp.y)]).wkt) + lines.append(transform(project, LineString(coords[:i] + [(cp.x, cp.y)])).wkt) line = LineString([(cp.x, cp.y)] + coords[i:]) if line.length > distance: - cut(line, distance, lines) + cut(line.wkt, distance, lines) else: - lines.append(LineString([(cp.x, cp.y)] + coords[i:]).wkt) + lines.append(transform(project, LineString([(cp.x, cp.y)] + coords[i:])).wkt) return lines def cut_udf(wkt_string): - line = wkt.loads(wkt_string) - return cut(line, distance=1000, lines=list()) + wgs84 = pyproj.CRS("EPSG:4326") + utm = pyproj.CRS("EPSG:31983") + project = pyproj.Transformer.from_crs(wgs84, utm, always_xy=True).transform + wkt_string = transform(project, wkt.loads(wkt_string)).wkt + return cut(wkt_string, distance=1000, lines=list()) cut_udf = udf(cut_udf, ArrayType(StringType())) @@ -39,5 +51,8 @@ def model(dbt, session): df = dbt.ref("aux_shapes_wkt") df_segments = df.withColumn("segments", cut_udf(col("shape_wkt"))) - df_exploded = df_segments.select(explode(col("segments")).alias("segment")) + df_exploded = df_segments.select( + "shape_id", "feed_start_date", explode(col("segments")).alias("segment") + ) + return df_exploded From c07a87cc5109d717d077cad0071632e677abcf21 Mon Sep 17 00:00:00 2001 From: Rafael Date: Thu, 12 Sep 2024 14:53:42 -0300 Subject: [PATCH 03/71] cria tabelas segmento --- queries/dbt_project.yml | 11 ++- .../models/planejamento/segmento_shape.sql | 10 +++ .../planejamento/shapes_geom_planejamento.sql | 42 ++++++++++ .../staging/aux_segmento_shape.py | 78 +++++++++++++++++++ .../models/teste_python/aux_shapes_wkt.sql | 1 + 5 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 queries/models/planejamento/segmento_shape.sql create mode 100644 queries/models/planejamento/shapes_geom_planejamento.sql create mode 100644 queries/models/planejamento/staging/aux_segmento_shape.py diff --git a/queries/dbt_project.yml b/queries/dbt_project.yml index 442ea5a98..ca5aa45ae 100644 --- a/queries/dbt_project.yml +++ b/queries/dbt_project.yml @@ -302,7 +302,12 @@ models: staging: +materialized: view +schema: transito_staging - teste_python: - +materialized: view - +schema: teste_python + planejamento: + +materialized: incremental + +incremental_strategy: insert_overwrite + +schema: planejamento + staging: + +database: rj-smtr-dev + +materialized: view + +schema: planejamento_staging +database: rj-smtr-dev diff --git a/queries/models/planejamento/segmento_shape.sql b/queries/models/planejamento/segmento_shape.sql new file mode 100644 index 000000000..9d0826f52 --- /dev/null +++ b/queries/models/planejamento/segmento_shape.sql @@ -0,0 +1,10 @@ +{{ + config( + partition_by = { + 'field' :'feed_start_date', + 'data_type' :'date', + 'granularity': 'day' + }, + tags=['geolocalizacao'] + ) +}} \ No newline at end of file diff --git a/queries/models/planejamento/shapes_geom_planejamento.sql b/queries/models/planejamento/shapes_geom_planejamento.sql new file mode 100644 index 000000000..467cac822 --- /dev/null +++ b/queries/models/planejamento/shapes_geom_planejamento.sql @@ -0,0 +1,42 @@ +{{ + config( + partition_by = { + 'field' :'feed_start_date', + 'data_type' :'date', + 'granularity': 'day' + }, + alias = 'shapes_geom', + tags=['geolocalizacao'] + ) +}} + +WITH shapes AS ( + SELECT + feed_version, + feed_start_date, + feed_end_date, + shape_id, + shape_pt_sequence, + ST_GEOGPOINT(shape_pt_lon, shape_pt_lat) AS ponto_shape, + CONCAT(shape_pt_lon, " ", shape_pt_lat) AS lon_lat, + FROM + -- {{ ref("shapes_gtfs") }} + rj-smtr.gtfs.shapes + WHERE + feed_start_date = '2024-09-01' + AND shape_id = "hj1m" +) +SELECT + feed_version, + feed_start_date, + feed_end_date, + shape_id, + ST_MAKELINE(ARRAY_AGG(ponto_shape ORDER BY shape_pt_sequence)) AS shape, + CONCAT("LINESTRING(", STRING_AGG(lon_lat, ", " ORDER BY shape_pt_sequence), ")") AS wkt_shape, +FROM + shapes +GROUP BY + 1, + 2, + 3, + 4 diff --git a/queries/models/planejamento/staging/aux_segmento_shape.py b/queries/models/planejamento/staging/aux_segmento_shape.py new file mode 100644 index 000000000..1773975be --- /dev/null +++ b/queries/models/planejamento/staging/aux_segmento_shape.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +import numpy as np +import pyproj +from pyspark.sql.functions import col, explode, udf +from pyspark.sql.types import ArrayType, IntegerType, StringType +from shapely import wkt + +# from shapely.geometry import LineString, Point +from shapely.ops import substring, transform + + +def transform_projection(shape, from_utm=False): + bq_projection = pyproj.CRS("EPSG:4326") + shapely_projection = pyproj.CRS("EPSG:31983") + if from_utm: + project = pyproj.Transformer.from_crs( + shapely_projection, bq_projection, always_xy=True + ).transform + else: + project = pyproj.Transformer.from_crs( + bq_projection, shapely_projection, always_xy=True + ).transform + + return transform(project, shape) + + +def cut(line, distance): + line_len = line.length + + dist_mod = line_len % distance + dist_range = list(np.arange(0, line_len, distance)) + middle_index = len(dist_range) // 2 + + last_final_dist = 0 + lines = [] + + for i, _ in enumerate(dist_range, start=1): + if i == middle_index: + cut_distance = dist_mod + else: + cut_distance = distance + final_dist = last_final_dist + cut_distance + lines.append([i, substring(line, last_final_dist, final_dist).wkt]) + last_final_dist = final_dist + + return lines + + +def cut_udf(wkt_string): + line = transform_projection(wkt.loads(wkt_string)) + return cut(line, distance=1000) + + +cut_udf = udf(cut_udf, ArrayType(ArrayType(IntegerType(), StringType()))) + + +def model(dbt, session): + dbt.config( + materialized="table", + ) + df = dbt.ref("shapes_geom_planejamento") + + df_segments = df.withColumn("shape_lists", cut_udf(col("wkt_shape"))) + + df_exploded = ( + df_segments.select( + "feed_version", + "feed_start_date", + "feed_end_date", + "shape_id", + explode(col("shape_lists")).alias("shape_list"), + ) + .withColumn("id_segmento", col("shape_list").getItem(1)) + .withColumn("wkt_segmento", col("shape_list").getItem(2)) + .drop("shape_list") + ) + + return df_exploded diff --git a/queries/models/teste_python/aux_shapes_wkt.sql b/queries/models/teste_python/aux_shapes_wkt.sql index df6f7badc..210530fbc 100644 --- a/queries/models/teste_python/aux_shapes_wkt.sql +++ b/queries/models/teste_python/aux_shapes_wkt.sql @@ -14,6 +14,7 @@ with shapes AS ( rj-smtr.gtfs.shapes where feed_start_date = '2024-09-01' + and shape_id = "hj1m" order by 1, 2, From 37301fc8811d7508107c3f16f56400418358b4e5 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 13 Sep 2024 09:38:03 -0300 Subject: [PATCH 04/71] add source dados_mestres --- queries/models/sources.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/queries/models/sources.yml b/queries/models/sources.yml index 4fee3adb0..f9fec729c 100644 --- a/queries/models/sources.yml +++ b/queries/models/sources.yml @@ -162,4 +162,10 @@ sources: database: rj-smtr-staging tables: - - name: autuacoes_citran \ No newline at end of file + - name: autuacoes_citran + + - name: dados_mestres + database: datario + + tables: + - name: logradouro \ No newline at end of file From b7b57c3748be53e9e5f06b74e6b940251a1937a8 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 13 Sep 2024 09:38:23 -0300 Subject: [PATCH 05/71] cria tabela segmento_shape --- .../models/planejamento/segmento_shape.sql | 120 +++++++++++++++++- 1 file changed, 119 insertions(+), 1 deletion(-) diff --git a/queries/models/planejamento/segmento_shape.sql b/queries/models/planejamento/segmento_shape.sql index 9d0826f52..aad81455b 100644 --- a/queries/models/planejamento/segmento_shape.sql +++ b/queries/models/planejamento/segmento_shape.sql @@ -7,4 +7,122 @@ }, tags=['geolocalizacao'] ) -}} \ No newline at end of file +}} + +-- depends_on: {{ ref('feed_info_gtfs') }} +{% if execute and is_incremental() %} + {% set last_feed_version = get_last_feed_start_date(var("data_versao_gtfs")) %} +{% endif %} + +WITH aux_segmento AS ( + SELECT + feed_version, + feed_start_date, + feed_end_date, + shape_id, + id_segmento, + ST_GEOGFROMTEXT(wkt_segmento) AS segmento, + wkt_segmento, + ROUND(ST_LENGTH(ST_GEOGFROMTEXT(wkt_segmento)), 1) AS comprimento_segmento + FROM + {{ ref("aux_segmento_shape") }} +), +tunel AS ( + SELECT + ST_UNION_AGG(ST_BUFFER(geometry, 50)) AS buffer_tunel + FROM + {{ source("dados_mestres", "logradouro") }} + WHERE + tipo = "Túnel" + +), +buffer_segmento AS ( + SELECT + *, + ST_BUFFER(segmento, 20) AS buffer_completo, + FROM + aux_segmento +), +intercessao_segmento AS ( + SELECT + b1.shape_id, + b1.id_segmento, + ST_UNION(ARRAY_AGG(b2.buffer_completo) )AS buffer_segmento_posterior + FROM + buffer_segmento b1 + JOIN + buffer_segmento b2 + ON + b1.shape_id = b2.shape_id + AND b1.id_segmento < b2.id_segmento + AND ST_INTERSECTS(b1.buffer_completo, b2.buffer_completo) + GROUP BY + 1, + 2 +), +buffer_segmento_recortado AS ( + SELECT + b.*, + COALESCE( + ST_DIFFERENCE( + buffer_completo, + i.buffer_segmento_posterior + ), + buffer_completo + ) AS buffer + FROM + buffer_segmento b + LEFT JOIN + intercessao_segmento i + USING(shape_id, id_segmento) +), +indicador_validacao_shape AS ( + SELECT + s.*, + ST_INTERSECTS(s.segmento, t.buffer_tunel) AS indicador_tunel, + ST_AREA(s.buffer) / ST_AREA(s.buffer_completo) < 0.5 AS indicador_area_prejudicada, + s.comprimento_segmento < 990 AS indicador_segmento_pequeno + FROM + buffer_segmento_recortado s + CROSS JOIN + tunel t +) +SELECT + *, + ( + indicador_tunel + OR indicador_area_prejudicada + OR indicador_segmento_pequeno + ) AS indicador_segmento_desconsiderado, + '{{ var("version") }}' AS versao +FROM + indicador_validacao_shape + +{% if is_incremental() %} + + UNION ALL + + SELECT + s.feed_version + s.feed_start_date + fi.feed_end_date + s.shape_id + s.id_segmento + s.segmento + s.wkt_segmento + s.comprimento_segmento, + s.buffer_completo, + s.buffer, + s.indicador_tunel, + s.indicador_area_prejudicada, + s.indicador_segmento_pequeno, + s.indicador_segmento_desconsiderado, + s.versao + FROM + {{ this }} s + JOIN + {{ ref('feed_info_gtfs') }} fi + USING(feed_start_date) + WHERE + feed_start_date = '{{ last_feed_version }}' +{% endif %} From 9eac0d9779c904f1ab687ae39278e563b61984b4 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 13 Sep 2024 09:38:44 -0300 Subject: [PATCH 06/71] correcoes prod --- .../planejamento/shapes_geom_planejamento.sql | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/queries/models/planejamento/shapes_geom_planejamento.sql b/queries/models/planejamento/shapes_geom_planejamento.sql index 467cac822..5bc1fbfb5 100644 --- a/queries/models/planejamento/shapes_geom_planejamento.sql +++ b/queries/models/planejamento/shapes_geom_planejamento.sql @@ -10,6 +10,11 @@ ) }} +-- depends_on: {{ ref('feed_info_gtfs') }} +{% if execute and is_incremental() %} + {% set last_feed_version = get_last_feed_start_date(var("data_versao_gtfs")) %} +{% endif %} + WITH shapes AS ( SELECT feed_version, @@ -20,11 +25,12 @@ WITH shapes AS ( ST_GEOGPOINT(shape_pt_lon, shape_pt_lat) AS ponto_shape, CONCAT(shape_pt_lon, " ", shape_pt_lat) AS lon_lat, FROM - -- {{ ref("shapes_gtfs") }} - rj-smtr.gtfs.shapes - WHERE - feed_start_date = '2024-09-01' - AND shape_id = "hj1m" + {{ ref("shapes_gtfs") }} s + -- rj-smtr.gtfs.shapes s + {% if is_incremental() %} + WHERE + feed_start_date IN ('{{ last_feed_version }}', '{{ var("data_versao_gtfs") }}') + {% endif %} ) SELECT feed_version, @@ -33,6 +39,7 @@ SELECT shape_id, ST_MAKELINE(ARRAY_AGG(ponto_shape ORDER BY shape_pt_sequence)) AS shape, CONCAT("LINESTRING(", STRING_AGG(lon_lat, ", " ORDER BY shape_pt_sequence), ")") AS wkt_shape, + '{{ var("version") }}' as versao FROM shapes GROUP BY From 2582a221be51feeb844f273c26559399b6c69e3d Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 13 Sep 2024 09:39:13 -0300 Subject: [PATCH 07/71] correcao logica segmentacao --- .../planejamento/staging/aux_segmento_shape.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/queries/models/planejamento/staging/aux_segmento_shape.py b/queries/models/planejamento/staging/aux_segmento_shape.py index 1773975be..f579745a3 100644 --- a/queries/models/planejamento/staging/aux_segmento_shape.py +++ b/queries/models/planejamento/staging/aux_segmento_shape.py @@ -2,7 +2,7 @@ import numpy as np import pyproj from pyspark.sql.functions import col, explode, udf -from pyspark.sql.types import ArrayType, IntegerType, StringType +from pyspark.sql.types import ArrayType, StringType from shapely import wkt # from shapely.geometry import LineString, Point @@ -40,7 +40,9 @@ def cut(line, distance): else: cut_distance = distance final_dist = last_final_dist + cut_distance - lines.append([i, substring(line, last_final_dist, final_dist).wkt]) + lines.append( + [str(i), transform_projection(substring(line, last_final_dist, final_dist), True).wkt] + ) last_final_dist = final_dist return lines @@ -51,14 +53,14 @@ def cut_udf(wkt_string): return cut(line, distance=1000) -cut_udf = udf(cut_udf, ArrayType(ArrayType(IntegerType(), StringType()))) +cut_udf = udf(cut_udf, ArrayType(ArrayType(StringType()))) def model(dbt, session): dbt.config( materialized="table", ) - df = dbt.ref("shapes_geom_planejamento") + df = dbt.ref("aux_shapes_geom_filtrada") df_segments = df.withColumn("shape_lists", cut_udf(col("wkt_shape"))) @@ -70,8 +72,8 @@ def model(dbt, session): "shape_id", explode(col("shape_lists")).alias("shape_list"), ) - .withColumn("id_segmento", col("shape_list").getItem(1)) - .withColumn("wkt_segmento", col("shape_list").getItem(2)) + .withColumn("id_segmento", col("shape_list").getItem(0)) + .withColumn("wkt_segmento", col("shape_list").getItem(1)) .drop("shape_list") ) From 900a760beb184fcd159969c85d0c397cc1915e1b Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 13 Sep 2024 09:39:26 -0300 Subject: [PATCH 08/71] cria aux_shapes_geom_filtrada --- .../planejamento/staging/aux_shapes_geom_filtrada.sql | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 queries/models/planejamento/staging/aux_shapes_geom_filtrada.sql diff --git a/queries/models/planejamento/staging/aux_shapes_geom_filtrada.sql b/queries/models/planejamento/staging/aux_shapes_geom_filtrada.sql new file mode 100644 index 000000000..285e19979 --- /dev/null +++ b/queries/models/planejamento/staging/aux_shapes_geom_filtrada.sql @@ -0,0 +1,6 @@ +SELECT + * +FROM + {{ ref("shapes_geom_planejamento") }} +WHERE + feed_start_date = '{{ var("data_versao_gtfs") }}' \ No newline at end of file From 10f14395a3289f3ef4e6792ca43165825957f162 Mon Sep 17 00:00:00 2001 From: Rafael Date: Tue, 17 Sep 2024 09:58:51 -0300 Subject: [PATCH 09/71] corrige tratamento tunel --- queries/models/planejamento/segmento_shape.sql | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/queries/models/planejamento/segmento_shape.sql b/queries/models/planejamento/segmento_shape.sql index aad81455b..bd2c1e833 100644 --- a/queries/models/planejamento/segmento_shape.sql +++ b/queries/models/planejamento/segmento_shape.sql @@ -90,7 +90,13 @@ indicador_validacao_shape AS ( SELECT *, ( - indicador_tunel + ( + indicador_tunel + AND ( + (id_segmento > 1) + OR (shape_id < MAX(id_segmento) OVER (PARTITION BY feed_start_date, shape_id)) + ) + ) OR indicador_area_prejudicada OR indicador_segmento_pequeno ) AS indicador_segmento_desconsiderado, From f907db9bd1cb73b24f82c7116ffcbecba6953ee1 Mon Sep 17 00:00:00 2001 From: Rafael Date: Tue, 17 Sep 2024 16:35:08 -0300 Subject: [PATCH 10/71] corrige comparacao --- queries/models/planejamento/segmento_shape.sql | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/queries/models/planejamento/segmento_shape.sql b/queries/models/planejamento/segmento_shape.sql index bd2c1e833..702efcde7 100644 --- a/queries/models/planejamento/segmento_shape.sql +++ b/queries/models/planejamento/segmento_shape.sql @@ -81,20 +81,21 @@ indicador_validacao_shape AS ( s.*, ST_INTERSECTS(s.segmento, t.buffer_tunel) AS indicador_tunel, ST_AREA(s.buffer) / ST_AREA(s.buffer_completo) < 0.5 AS indicador_area_prejudicada, - s.comprimento_segmento < 990 AS indicador_segmento_pequeno + s.comprimento_segmento < 990 AS indicador_segmento_pequeno, + CAST(id_segmento AS INTEGER) AS id_segmento_int FROM buffer_segmento_recortado s CROSS JOIN tunel t ) SELECT - *, + * EXCEPT(id_segmento_int), ( ( indicador_tunel AND ( - (id_segmento > 1) - OR (shape_id < MAX(id_segmento) OVER (PARTITION BY feed_start_date, shape_id)) + (id_segmento_int > 1) + OR (id_segmento_int < MAX(id_segmento_int) OVER (PARTITION BY feed_start_date, shape_id)) ) ) OR indicador_area_prejudicada From dd3825a55febde2d2662908189d63adff2d95968 Mon Sep 17 00:00:00 2001 From: Rafael Date: Wed, 25 Sep 2024 17:15:18 -0300 Subject: [PATCH 11/71] viagens 2.0 --- queries/dbt_project.yml | 10 + .../staging/gps_segmento_viagem.sql | 113 +++++++++ .../monitoramento/staging/gps_viagem.sql | 70 ++++++ .../viagem_informada_monitoramento.sql | 32 +++ .../models/monitoramento/viagem_validacao.sql | 90 +++++++ .../models/planejamento/segmento_shape.sql | 224 ++++++++---------- .../planejamento/shapes_geom_planejamento.sql | 75 +++--- 7 files changed, 455 insertions(+), 159 deletions(-) create mode 100644 queries/models/monitoramento/staging/gps_segmento_viagem.sql create mode 100644 queries/models/monitoramento/staging/gps_viagem.sql create mode 100644 queries/models/monitoramento/viagem_informada_monitoramento.sql create mode 100644 queries/models/monitoramento/viagem_validacao.sql diff --git a/queries/dbt_project.yml b/queries/dbt_project.yml index 1226b9606..0c62678d8 100644 --- a/queries/dbt_project.yml +++ b/queries/dbt_project.yml @@ -330,3 +330,13 @@ models: +materialized: view +schema: planejamento_staging +database: rj-smtr-dev + monitoramento: + +database: rj-smtr-dev + +materialized: incremental + +incremental_strategy: insert_overwrite + +schema: monitoramento + staging: + +database: rj-smtr-dev + +materialized: view + +schema: monitoramento_staging + diff --git a/queries/models/monitoramento/staging/gps_segmento_viagem.sql b/queries/models/monitoramento/staging/gps_segmento_viagem.sql new file mode 100644 index 000000000..f9e316467 --- /dev/null +++ b/queries/models/monitoramento/staging/gps_segmento_viagem.sql @@ -0,0 +1,113 @@ +{{ + config( + materialized="incremental", + partition_by={ + "field": "data", + "data_type": "date", + "granularity": "day", + }, + incremental_strategy="insert_overwrite", + ) +}} + +{% set incremental_filter %} + data = date_sub(date('{{ var("run_date") }}'), interval 1 day) +{% endset %} +{# {% set data_versao_efetiva = ref("subsidio_data_versao_efetiva") %} #} +{% set data_versao_efetiva = ( + "rj-smtr.projeto_subsidio_sppo.subsidio_data_versao_efetiva" +) %} + +{% if execute %} + {% set data_versao_gtfs_query %} + select format_date("%Y-%m-%d", feed_start_date) from {{ data_versao_efetiva }} where {{ incremental_filter }} + {% endset %} + + {% set data_versao_gtfs = ( + run_query(data_versao_gtfs_query).columns[0].values()[0] + ) %} +{% endif %} + +with + gps_viagem as ( + select id_viagem, shape_id, geo_point_gps + from {{ ref("gps_viagem") }} + where {{ incremental_filter }} + ), + segmento as ( + select + feed_version, + feed_start_date, + feed_end_date, + shape_id, + id_segmento, + buffer, + indicador_segmento_desconsiderado + from {{ ref("segmento_shape") }} + where feed_start_date = '{{ data_versao_gtfs }}' + ), + gps_segmento as ( + select g.id_viagem, g.shape_id, s.id_segmento, count(*) as quantidade_gps + from gps_viagem g + join + segmento s + on g.shape_id = s.shape_id + and st_intersects(s.buffer, g.geo_point_gps) + group by 1, 2, 3 + ), + viagem as ( + select + data, + id_viagem, + datetime_partida, + datetime_chegada, + id_veiculo, + trip_id, + route_id, + shape_id, + servico, + sentido + from {{ ref("viagem_informada_monitoramento") }} + where {{ incremental_filter }} + ), + viagem_segmento as ( + select + v.data, + v.id_viagem, + v.datetime_partida, + v.datetime_chegada, + v.id_veiculo, + v.trip_id, + v.route_id, + shape_id, + s.id_segmento, + s.indicador_segmento_desconsiderado, + v.servico, + v.sentido, + s.feed_version, + s.feed_start_date, + s.feed_end_date + from viagem v + join segmento s using (shape_id) + ) +select + v.data, + id_viagem, + v.datetime_partida, + v.datetime_chegada, + v.id_veiculo, + v.trip_id, + v.route_id, + shape_id, + id_segmento, + v.indicador_segmento_desconsiderado, + v.servico, + v.sentido, + ifnull(g.quantidade_gps, 0) as quantidade_gps, + v.feed_version, + v.feed_start_date, + v.feed_end_date, + '{{ var("version") }}' as versao, + current_datetime("America/Sao_Paulo") as datetime_ultima_atualizacao +from viagem_segmento v +left join gps_segmento g using (id_viagem, shape_id, id_segmento) diff --git a/queries/models/monitoramento/staging/gps_viagem.sql b/queries/models/monitoramento/staging/gps_viagem.sql new file mode 100644 index 000000000..65341319b --- /dev/null +++ b/queries/models/monitoramento/staging/gps_viagem.sql @@ -0,0 +1,70 @@ +{{ + config( + materialized="incremental", + partition_by={ + "field": "data", + "data_type": "date", + "granularity": "day", + }, + incremental_strategy="insert_overwrite", + ) +}} + +with + viagem as ( + select + data, + id_viagem, + datetime_partida, + datetime_chegada, + id_veiculo, + trip_id, + route_id, + shape_id, + servico, + sentido + from {{ ref("viagem_informada_monitoramento") }} + where + + data = date_sub(date('{{ var("run_date") }}'), interval 1 day) + -- filtro de teste REMOVER + and date(datetime_partida) = date("2024-06-05") + and date(datetime_chegada) = date("2024-06-05") + ), + gps as ( + select data, timestamp_gps, servico, id_veiculo, latitude, longitude + from `rj-smtr.br_rj_riodejaneiro_veiculos.gps_sppo` + {# from {{ ref("gps_sppo") }} #} + where + data + between date_sub(date('{{ var("run_date") }}'), interval 2 day) and date( + '{{ var("run_date") }}' + ) + + ) +select + g.data, + g.timestamp_gps, + g.id_veiculo, + g.servico, + v.sentido, + g.latitude, + g.longitude, + st_geogpoint(g.longitude, g.latitude) as geo_point_gps, + v.id_viagem, + v.datetime_partida, + v.datetime_chegada, + v.trip_id, + v.route_id, + v.shape_id, + '{{ var("version") }}' as versao, + current_datetime("America/Sao_Paulo") as datetime_ultima_atualizacao +from gps g +join + viagem v + on g.timestamp_gps + between datetime_sub(v.datetime_partida, interval 10 minute) and datetime_add( + v.datetime_chegada, interval 10 minute + ) + and g.id_veiculo = v.id_veiculo + and g.servico = v.servico diff --git a/queries/models/monitoramento/viagem_informada_monitoramento.sql b/queries/models/monitoramento/viagem_informada_monitoramento.sql new file mode 100644 index 000000000..a7dacbf5b --- /dev/null +++ b/queries/models/monitoramento/viagem_informada_monitoramento.sql @@ -0,0 +1,32 @@ +{{ + config( + alias="viagem_informada", + partition_by={ + "field": "data", + "data_type": "date", + "granularity": "day", + }, + ) +}} + +select + data, + id_viagem, + datetime_partida, + datetime_chegada, + id_veiculo, + trip_id, + null as route_id, + shape_id, + servico_informado as servico, + sentido, + datetime_chegada as datetime_processamento, + datetime_chegada as datetime_captura, + '{{ var("version") }}' as versao, + current_datetime("America/Sao_Paulo") as datetime_ultima_atualizacao +from + {# {{ ref('viagem_completa') }} #} + `rj-smtr.projeto_subsidio_sppo.viagem_completa` +{% if is_incremental() %} + where data = date_sub(date('{{ var("run_date") }}'), interval 1 day) +{% endif %} diff --git a/queries/models/monitoramento/viagem_validacao.sql b/queries/models/monitoramento/viagem_validacao.sql new file mode 100644 index 000000000..78938ff7f --- /dev/null +++ b/queries/models/monitoramento/viagem_validacao.sql @@ -0,0 +1,90 @@ +{{ + config( + partition_by={ + "field": "data", + "data_type": "date", + "granularity": "day", + }, + ) +}} + +{% set parametro_validacao = 0.9 %} + +with + contagem as ( + select + data, + id_viagem, + datetime_partida, + datetime_chegada, + id_veiculo, + trip_id, + route_id, + shape_id, + servico, + sentido, + count(*) as quantidade_segmentos_verificados, + countif(quantidade_gps > 0) as quantidade_segmentos_validos, + feed_version, + feed_start_date, + feed_end_date + from {{ ref("gps_segmento_viagem") }} + where + data = date_sub(date('{{ var("run_date") }}'), interval 1 day) + and not indicador_segmento_desconsiderado + group by + data, + id_viagem, + datetime_partida, + datetime_chegada, + id_veiculo, + trip_id, + route_id, + shape_id, + servico, + sentido, + feed_version, + feed_start_date, + feed_end_date + ), + indice as ( + select + data, + id_viagem, + datetime_partida, + datetime_chegada, + id_veiculo, + trip_id, + route_id, + shape_id, + servico, + sentido, + quantidade_segmentos_verificados, + quantidade_segmentos_validos, + quantidade_segmentos_validos + / quantidade_segmentos_verificados as indice_validacao, + feed_version, + feed_start_date, + feed_end_date + from contagem + ) +select + data, + id_viagem, + datetime_partida, + datetime_chegada, + id_veiculo, + trip_id, + route_id, + shape_id, + servico, + sentido, + quantidade_segmentos_verificados, + quantidade_segmentos_validos, + indice_validacao, + indice_validacao >= {{ parametro_validacao }} as indicador_viagem_valida, + {{ parametro_validacao }} as parametro_validacao, + feed_version, + feed_start_date, + feed_end_date +from indice diff --git a/queries/models/planejamento/segmento_shape.sql b/queries/models/planejamento/segmento_shape.sql index 702efcde7..c5739f49f 100644 --- a/queries/models/planejamento/segmento_shape.sql +++ b/queries/models/planejamento/segmento_shape.sql @@ -1,135 +1,117 @@ {{ - config( - partition_by = { - 'field' :'feed_start_date', - 'data_type' :'date', - 'granularity': 'day' - }, - tags=['geolocalizacao'] - ) + config( + partition_by={ + "field": "feed_start_date", + "data_type": "date", + "granularity": "day", + }, + tags=["geolocalizacao"], + ) }} -- depends_on: {{ ref('feed_info_gtfs') }} {% if execute and is_incremental() %} - {% set last_feed_version = get_last_feed_start_date(var("data_versao_gtfs")) %} + {% set last_feed_version = get_last_feed_start_date(var("data_versao_gtfs")) %} {% endif %} -WITH aux_segmento AS ( - SELECT - feed_version, - feed_start_date, - feed_end_date, - shape_id, - id_segmento, - ST_GEOGFROMTEXT(wkt_segmento) AS segmento, - wkt_segmento, - ROUND(ST_LENGTH(ST_GEOGFROMTEXT(wkt_segmento)), 1) AS comprimento_segmento - FROM - {{ ref("aux_segmento_shape") }} -), -tunel AS ( - SELECT - ST_UNION_AGG(ST_BUFFER(geometry, 50)) AS buffer_tunel - FROM - {{ source("dados_mestres", "logradouro") }} - WHERE - tipo = "Túnel" +with + aux_segmento as ( + select + feed_version, + feed_start_date, + feed_end_date, + shape_id, + id_segmento, + st_geogfromtext(wkt_segmento) as segmento, + wkt_segmento, + round(st_length(st_geogfromtext(wkt_segmento)), 1) as comprimento_segmento + from {{ ref("aux_segmento_shape") }} + ), + tunel as ( + select st_union_agg(st_buffer(geometry, 50)) as buffer_tunel + from {{ source("dados_mestres", "logradouro") }} + where tipo = "Túnel" -), -buffer_segmento AS ( - SELECT - *, - ST_BUFFER(segmento, 20) AS buffer_completo, - FROM - aux_segmento -), -intercessao_segmento AS ( - SELECT - b1.shape_id, - b1.id_segmento, - ST_UNION(ARRAY_AGG(b2.buffer_completo) )AS buffer_segmento_posterior - FROM - buffer_segmento b1 - JOIN - buffer_segmento b2 - ON - b1.shape_id = b2.shape_id - AND b1.id_segmento < b2.id_segmento - AND ST_INTERSECTS(b1.buffer_completo, b2.buffer_completo) - GROUP BY - 1, - 2 -), -buffer_segmento_recortado AS ( - SELECT - b.*, - COALESCE( - ST_DIFFERENCE( - buffer_completo, - i.buffer_segmento_posterior - ), - buffer_completo - ) AS buffer - FROM - buffer_segmento b - LEFT JOIN - intercessao_segmento i - USING(shape_id, id_segmento) -), -indicador_validacao_shape AS ( - SELECT - s.*, - ST_INTERSECTS(s.segmento, t.buffer_tunel) AS indicador_tunel, - ST_AREA(s.buffer) / ST_AREA(s.buffer_completo) < 0.5 AS indicador_area_prejudicada, - s.comprimento_segmento < 990 AS indicador_segmento_pequeno, - CAST(id_segmento AS INTEGER) AS id_segmento_int - FROM - buffer_segmento_recortado s - CROSS JOIN - tunel t -) -SELECT - * EXCEPT(id_segmento_int), - ( - ( - indicador_tunel - AND ( - (id_segmento_int > 1) - OR (id_segmento_int < MAX(id_segmento_int) OVER (PARTITION BY feed_start_date, shape_id)) - ) + ), + buffer_segmento as ( + select *, st_buffer(segmento, 20) as buffer_completo, from aux_segmento + ), + intercessao_segmento as ( + select + b1.shape_id, + b1.id_segmento, + st_union(array_agg(b2.buffer_completo)) as buffer_segmento_posterior + from buffer_segmento b1 + join + buffer_segmento b2 + on b1.shape_id = b2.shape_id + and b1.id_segmento < b2.id_segmento + and st_intersects(b1.buffer_completo, b2.buffer_completo) + group by 1, 2 + ), + buffer_segmento_recortado as ( + select + b.*, + coalesce( + st_difference(buffer_completo, i.buffer_segmento_posterior), + buffer_completo + ) as buffer + from buffer_segmento b + left join intercessao_segmento i using (shape_id, id_segmento) + ), + indicador_validacao_shape as ( + select + s.*, + st_intersects(s.segmento, t.buffer_tunel) as indicador_tunel, + st_area(s.buffer) / st_area(s.buffer_completo) + < 0.5 as indicador_area_prejudicada, + s.comprimento_segmento < 990 as indicador_segmento_pequeno, + cast(id_segmento as integer) as id_segmento_int + from buffer_segmento_recortado s + cross join tunel t ) - OR indicador_area_prejudicada - OR indicador_segmento_pequeno - ) AS indicador_segmento_desconsiderado, - '{{ var("version") }}' AS versao -FROM - indicador_validacao_shape +select + * except (id_segmento_int), + ( + ( + indicador_tunel + and ( + (id_segmento_int > 1) + or ( + id_segmento_int + < max(id_segmento_int) over (partition by feed_start_date, shape_id) + and id_segmento_int > 1 + ) + ) + ) + or indicador_area_prejudicada + or indicador_segmento_pequeno + ) as indicador_segmento_desconsiderado, + '{{ var("version") }}' as versao +from indicador_validacao_shape {% if is_incremental() %} - UNION ALL + union all + + select + s.feed_version, + s.feed_start_date, + fi.feed_end_date, + s.shape_id, + s.id_segmento, + s.segmento, + s.wkt_segmento, + s.comprimento_segmento, + s.buffer_completo, + s.buffer, + s.indicador_tunel, + s.indicador_area_prejudicada, + s.indicador_segmento_pequeno, + s.indicador_segmento_desconsiderado, + s.versao + from {{ this }} s + join {{ ref("feed_info_gtfs") }} fi using (feed_start_date) + where feed_start_date = '{{ last_feed_version }}' - SELECT - s.feed_version - s.feed_start_date - fi.feed_end_date - s.shape_id - s.id_segmento - s.segmento - s.wkt_segmento - s.comprimento_segmento, - s.buffer_completo, - s.buffer, - s.indicador_tunel, - s.indicador_area_prejudicada, - s.indicador_segmento_pequeno, - s.indicador_segmento_desconsiderado, - s.versao - FROM - {{ this }} s - JOIN - {{ ref('feed_info_gtfs') }} fi - USING(feed_start_date) - WHERE - feed_start_date = '{{ last_feed_version }}' {% endif %} diff --git a/queries/models/planejamento/shapes_geom_planejamento.sql b/queries/models/planejamento/shapes_geom_planejamento.sql index 5bc1fbfb5..c51b3c17a 100644 --- a/queries/models/planejamento/shapes_geom_planejamento.sql +++ b/queries/models/planejamento/shapes_geom_planejamento.sql @@ -1,49 +1,48 @@ {{ - config( - partition_by = { - 'field' :'feed_start_date', - 'data_type' :'date', - 'granularity': 'day' - }, - alias = 'shapes_geom', - tags=['geolocalizacao'] - ) + config( + partition_by={ + "field": "feed_start_date", + "data_type": "date", + "granularity": "day", + }, + alias="shapes_geom", + tags=["geolocalizacao"], + ) }} -- depends_on: {{ ref('feed_info_gtfs') }} {% if execute and is_incremental() %} - {% set last_feed_version = get_last_feed_start_date(var("data_versao_gtfs")) %} + {% set last_feed_version = get_last_feed_start_date(var("data_versao_gtfs")) %} {% endif %} -WITH shapes AS ( - SELECT +with + shapes as ( + select + feed_version, + feed_start_date, + feed_end_date, + shape_id, + shape_pt_sequence, + st_geogpoint(shape_pt_lon, shape_pt_lat) as ponto_shape, + concat(shape_pt_lon, " ", shape_pt_lat) as lon_lat, + from + {# {{ ref("shapes_gtfs") }} s #} + `rj-smtr.gtfs.shapes` s + {% if is_incremental() %} + where + feed_start_date + in ('{{ last_feed_version }}', '{{ var("data_versao_gtfs") }}') + {% endif %} + ) +select feed_version, feed_start_date, feed_end_date, shape_id, - shape_pt_sequence, - ST_GEOGPOINT(shape_pt_lon, shape_pt_lat) AS ponto_shape, - CONCAT(shape_pt_lon, " ", shape_pt_lat) AS lon_lat, - FROM - {{ ref("shapes_gtfs") }} s - -- rj-smtr.gtfs.shapes s - {% if is_incremental() %} - WHERE - feed_start_date IN ('{{ last_feed_version }}', '{{ var("data_versao_gtfs") }}') - {% endif %} -) -SELECT - feed_version, - feed_start_date, - feed_end_date, - shape_id, - ST_MAKELINE(ARRAY_AGG(ponto_shape ORDER BY shape_pt_sequence)) AS shape, - CONCAT("LINESTRING(", STRING_AGG(lon_lat, ", " ORDER BY shape_pt_sequence), ")") AS wkt_shape, - '{{ var("version") }}' as versao -FROM - shapes -GROUP BY - 1, - 2, - 3, - 4 + st_makeline(array_agg(ponto_shape order by shape_pt_sequence)) as shape, + concat( + "LINESTRING(", string_agg(lon_lat, ", " order by shape_pt_sequence), ")" + ) as wkt_shape, + '{{ var("version") }}' as versao +from shapes +group by 1, 2, 3, 4 From f626975c0bf4c067cf20ae25271b1403045cbeb7 Mon Sep 17 00:00:00 2001 From: Rafael Date: Wed, 16 Oct 2024 14:59:22 -0300 Subject: [PATCH 12/71] corrige logica tunel --- queries/models/planejamento/segmento_shape.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/queries/models/planejamento/segmento_shape.sql b/queries/models/planejamento/segmento_shape.sql index c5739f49f..c13137e11 100644 --- a/queries/models/planejamento/segmento_shape.sql +++ b/queries/models/planejamento/segmento_shape.sql @@ -77,10 +77,9 @@ select indicador_tunel and ( (id_segmento_int > 1) - or ( + and ( id_segmento_int < max(id_segmento_int) over (partition by feed_start_date, shape_id) - and id_segmento_int > 1 ) ) ) From 2bb123c77fecae35ea6f9a037e4dad3d06f80fc8 Mon Sep 17 00:00:00 2001 From: Rafael Carvalho Pinheiro <74972217+pixuimpou@users.noreply.github.com> Date: Tue, 22 Oct 2024 16:24:12 -0300 Subject: [PATCH 13/71] Update viagem_informada_monitoramento.sql --- .../viagem_informada_monitoramento.sql | 33 +------------------ 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/queries/models/monitoramento/viagem_informada_monitoramento.sql b/queries/models/monitoramento/viagem_informada_monitoramento.sql index e69c97ea2..a87f3be38 100644 --- a/queries/models/monitoramento/viagem_informada_monitoramento.sql +++ b/queries/models/monitoramento/viagem_informada_monitoramento.sql @@ -1,36 +1,5 @@ {{ config( - alias="viagem_informada", - partition_by={ - "field": "data", - "data_type": "date", - "granularity": "day", - }, - ) -}} - -select - data, - id_viagem, - datetime_partida, - datetime_chegada, - id_veiculo, - trip_id, - null as route_id, - shape_id, - servico_informado as servico, - sentido, - datetime_chegada as datetime_processamento, - datetime_chegada as datetime_captura, - '{{ var("version") }}' as versao, - current_datetime("America/Sao_Paulo") as datetime_ultima_atualizacao -from - {# {{ ref('viagem_completa') }} #} - `rj-smtr.projeto_subsidio_sppo.viagem_completa` -{% if is_incremental() %} - where data = date_sub(date('{{ var("run_date") }}'), interval 1 day) -{% endif %} -======= materialized="incremental", partition_by={"field": "data", "data_type": "date", "granularity": "day"}, incremental_strategy="insert_overwrite", @@ -115,4 +84,4 @@ select *, '{{ var("version") }}' as versao, current_datetime("America/Sao_Paulo") as datetime_ultima_atualizacao -from deduplicado \ No newline at end of file +from deduplicado From 92f084d45cd4f697df22c0ef48d1d3343ac22780 Mon Sep 17 00:00:00 2001 From: Rafael Date: Wed, 23 Oct 2024 16:08:31 -0300 Subject: [PATCH 14/71] remove database dev --- queries/dbt_project.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/queries/dbt_project.yml b/queries/dbt_project.yml index 20bfcb3eb..a6503e0bb 100644 --- a/queries/dbt_project.yml +++ b/queries/dbt_project.yml @@ -326,10 +326,8 @@ models: +incremental_strategy: insert_overwrite +schema: planejamento staging: - +database: rj-smtr-dev +materialized: view +schema: planejamento_staging - +database: rj-smtr-dev monitoramento: +materialized: incremental +incremental_strategy: insert_overwrite From 7540b111727a51356ea4de36c092743c23dd6e3d Mon Sep 17 00:00:00 2001 From: Rafael Date: Wed, 23 Oct 2024 16:08:41 -0300 Subject: [PATCH 15/71] remove teste --- .../models/teste_python/aux_shapes_wkt.sql | 31 ---------- queries/models/teste_python/teste_python.py | 58 ------------------- 2 files changed, 89 deletions(-) delete mode 100644 queries/models/teste_python/aux_shapes_wkt.sql delete mode 100644 queries/models/teste_python/teste_python.py diff --git a/queries/models/teste_python/aux_shapes_wkt.sql b/queries/models/teste_python/aux_shapes_wkt.sql deleted file mode 100644 index 210530fbc..000000000 --- a/queries/models/teste_python/aux_shapes_wkt.sql +++ /dev/null @@ -1,31 +0,0 @@ -{{ - config( - materialized="view" - ) - -}} -with shapes AS ( - select - feed_start_date, - shape_id, - shape_pt_sequence, - CONCAT(shape_pt_lon, " ", shape_pt_lat) AS lon_lat - from - rj-smtr.gtfs.shapes - where - feed_start_date = '2024-09-01' - and shape_id = "hj1m" - order by - 1, - 2, - 3 -) -SELECT - feed_start_date, - shape_id, - concat("LINESTRING(", string_agg(lon_lat, ", "), ")") AS shape_wkt -from - shapes -group by - 1, - 2 \ No newline at end of file diff --git a/queries/models/teste_python/teste_python.py b/queries/models/teste_python/teste_python.py deleted file mode 100644 index 4b3c77f57..000000000 --- a/queries/models/teste_python/teste_python.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- -import pyproj -from pyspark.sql.functions import col, explode, udf -from pyspark.sql.types import ArrayType, StringType -from shapely import wkt -from shapely.geometry import LineString, Point -from shapely.ops import transform - - -def cut(line, distance, lines): - wgs84 = pyproj.CRS("EPSG:4326") - utm = pyproj.CRS("EPSG:31983") - project = pyproj.Transformer.from_crs(utm, wgs84, always_xy=True).transform - line = wkt.loads(line) - if distance <= 0.0 or distance >= line.length: - return [transform(project, LineString(line)).wkt] - coords = list(line.coords) - for i, p in enumerate(coords): - pd = line.project(Point(p)) - if pd == distance: - return [ - transform(project, LineString(coords[: i + 1])).wkt, - transform(project, LineString(coords[i:])).wkt, - ] - if pd > distance: - cp = line.interpolate(distance) - lines.append(transform(project, LineString(coords[:i] + [(cp.x, cp.y)])).wkt) - line = LineString([(cp.x, cp.y)] + coords[i:]) - if line.length > distance: - cut(line.wkt, distance, lines) - else: - lines.append(transform(project, LineString([(cp.x, cp.y)] + coords[i:])).wkt) - return lines - - -def cut_udf(wkt_string): - wgs84 = pyproj.CRS("EPSG:4326") - utm = pyproj.CRS("EPSG:31983") - project = pyproj.Transformer.from_crs(wgs84, utm, always_xy=True).transform - wkt_string = transform(project, wkt.loads(wkt_string)).wkt - return cut(wkt_string, distance=1000, lines=list()) - - -cut_udf = udf(cut_udf, ArrayType(StringType())) - - -def model(dbt, session): - dbt.config( - materialized="table", - ) - df = dbt.ref("aux_shapes_wkt") - - df_segments = df.withColumn("segments", cut_udf(col("shape_wkt"))) - df_exploded = df_segments.select( - "shape_id", "feed_start_date", explode(col("segments")).alias("segment") - ) - - return df_exploded From 2662746ae6c739a87996b7392f5ea0325cfd127e Mon Sep 17 00:00:00 2001 From: Rafael Date: Wed, 23 Oct 2024 16:08:59 -0300 Subject: [PATCH 16/71] cria arquivo viagem_planejada --- .../planejamento/viagem_planejada_planejamento.sql | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 queries/models/planejamento/viagem_planejada_planejamento.sql diff --git a/queries/models/planejamento/viagem_planejada_planejamento.sql b/queries/models/planejamento/viagem_planejada_planejamento.sql new file mode 100644 index 000000000..1c39e4688 --- /dev/null +++ b/queries/models/planejamento/viagem_planejada_planejamento.sql @@ -0,0 +1,10 @@ +{{ + config( + partition_by={ + "field": "date", + "data_type": "date", + "granularity": "day", + }, + alias="viagem_planejada", + ) +}} From 8095022a9ff452492d1459b970dd07605577dd88 Mon Sep 17 00:00:00 2001 From: Rafael Date: Wed, 23 Oct 2024 16:09:13 -0300 Subject: [PATCH 17/71] calendario gtfs --- queries/models/planejamento/calendario.sql | 159 ++++++++++++++++++ .../staging/aux_calendario_manual.sql | 21 +++ 2 files changed, 180 insertions(+) create mode 100644 queries/models/planejamento/calendario.sql create mode 100644 queries/models/planejamento/staging/aux_calendario_manual.sql diff --git a/queries/models/planejamento/calendario.sql b/queries/models/planejamento/calendario.sql new file mode 100644 index 000000000..4ca72ab65 --- /dev/null +++ b/queries/models/planejamento/calendario.sql @@ -0,0 +1,159 @@ +{{ + config( + partition_by={ + "field": "date", + "data_type": "date", + "granularity": "day", + }, + ) +}} + +{# {% set gtfs_feed_info = ref("feed_info_gtfs") %} #} +{% set gtfs_feed_info = "rj-smtr.gtfs.feed_info" %} + +{% if execute %} + {% if is_incremental() %} + {% set gtfs_feeds_query %} + select concat("'", feed_start_date, "'") + from {{ gtfs_feed_info }} + where + feed_start_date <= date("{{ var('date_range_end') }}") + and (feed_end_date IS NULL OR feed_end_date >= date("{{ var('date_range_end') }}")) + {% endset %} + + {% set gtfs_feeds = run_query(gtfs_feeds_query).columns[0].values() %} + {% endif %} +{% endif %} + +with + calendar as ( + select * + from `rj-smtr.gtfs.calendar` + where feed_start_date in ({{ gtfs_feeds | join(", ") }}) + ), + calendar_dates as ( + select + date as data, + case + when exception_type = '2' + then + regexp_replace( + service_id, + "^[U|S|D]_", + case + when extract(dayofweek from date) = 7 + then "S_" + when extract(dayofweek from date) = 1 + then "D_" + else "U_" + end + ) + else service_id + end as service_id, + exception_type, + feed_start_date, + from `rj-smtr.gtfs.calendar_dates` + where + feed_start_date in ({{ gtfs_feeds | join(", ") }}) + and date between date("{{ var('date_range_start') }}") and date( + "{{ var('date_range_end') }}" + ) + ), + datas as ( + select distinct data, extract(dayofweek from data) as dia_semana, + from + unnest( + generate_date_array( + date("{{ var('date_range_start') }}"), + date("{{ var('date_range_end') }}") + ) + ) as data + ), + datas_service as ( + select d.data, c.service_id, c.feed_version, c.feed_start_date + from datas d + join + calendar c + on d.data >= c.feed_start_date + and (d.data <= feed_end_date or feed_end_date is null) + and ( + (d.dia_semana = 1 and c.sunday = '1') + or (d.dia_semana = 2 and c.monday = '1') + or (d.dia_semana = 3 and c.tuesday = '1') + or (d.dia_semana = 4 and c.wednesday = '1') + or (d.dia_semana = 5 and c.thursday = '1') + or (d.dia_semana = 6 and c.friday = '1') + or (d.dia_semana = 7 and c.saturday = '1') + ) + ), + service_ids_retirados as ( + select d.*, c.service_id as c_service_id, + from datas_calendar d + left join + (select * from calendar_dates where exception_type = '2') c using ( + data, service_id + ) + where c.service_id is null + + ), + service_ids_adicionados as ( + select data, service_id, dia_semana, feed_start_date + from service_ids_retirados + + union distinct + + select data, service_id, dia_semana, feed_start_date + from calendar_dates + where exception_type = '1' + ), + service_id_agg as ( + select data, feed_start_date, array_agg(service_id) as service_ids + from service_ids_adicionados + group by 1, 2, 3 + ), + calendario_gtfs as ( + select + data, + case + when "D_REG" in unnest(service_ids) + then "Domingo" + when "S_REG" in unnest(service_ids) + then "Sábado" + when "U_REG" in unnest(service_ids) + then "Dia Útil" + end as tipo_dia, + "Regular" as tipo_os, + service_ids, + feed_start_date + from service_id_agg + ), + modificacoes_manuais as ( + select + c.data, + coalesce(m.tipo_dia, c.tipo_dia) as tipo_dia, + coalesce(m.tipo_os, c.tipo_os) as tipo_os, + service_ids, + coalesce(m.feed_start_date, c.feed_start_date) as feed_start_date + from calendario_gtfs c + left join {{ ref("aux_calendario_manual") }} m using (data) + ) +select + data, + tipo_dia, + case + when c.tipo_os = "Extraordinária - Verão" + then "Verão" + when c.tipo_os like "%Madonna%" + then "Madonna" + when c.tipo_os = "Regular" + then null + else c.tipo_os + end as subtipo_dia, + c.tipo_os, + c.service_ids, + i.feed_version + feed_start_date, + '{{ var("version") }}' as versao, + current_datetime("America/Sao_Paulo") as datetime_ultima_atualizacao +from modificacoes_manuais c +join `rj-smtr.gtfs.feed_info` i using (feed_start_date) diff --git a/queries/models/planejamento/staging/aux_calendario_manual.sql b/queries/models/planejamento/staging/aux_calendario_manual.sql new file mode 100644 index 000000000..56c16ef63 --- /dev/null +++ b/queries/models/planejamento/staging/aux_calendario_manual.sql @@ -0,0 +1,21 @@ +{{ config(materialized="ephemeral") }} + +select + data, + date(null) as feed_start_date, + cast(null as string) as tipo_dia, + case + when data between date(2024, 09, 14) and date(2024, 09, 15) + then "Verão + Rock in Rio" + when data between date(2024, 09, 19) and date(2024, 09, 22) + then "Rock in Rio" + when data = date(2024, 10, 06) + then "Eleição" + end as tipo_os +from + unnest( + generate_date_array( + date({{ var("data_versao_gtfs") }}), + date_add(date({{ var("data_versao_gtfs") }}), interval 1 year) + ) + ) as data From 1c31eadec197033c96b8511a23663460a70cf0af Mon Sep 17 00:00:00 2001 From: Rafael Date: Thu, 24 Oct 2024 18:23:46 -0300 Subject: [PATCH 18/71] cria tabela calendario --- queries/models/planejamento/calendario.sql | 169 +++++++++++------- .../staging/aux_calendario_manual.sql | 59 +++--- 2 files changed, 145 insertions(+), 83 deletions(-) diff --git a/queries/models/planejamento/calendario.sql b/queries/models/planejamento/calendario.sql index 4ca72ab65..40754ff2b 100644 --- a/queries/models/planejamento/calendario.sql +++ b/queries/models/planejamento/calendario.sql @@ -1,24 +1,36 @@ {{ config( partition_by={ - "field": "date", + "field": "data", "data_type": "date", "granularity": "day", }, + materialized="incremental", + incremental_strategy="insert_overwrite", ) }} {# {% set gtfs_feed_info = ref("feed_info_gtfs") %} #} {% set gtfs_feed_info = "rj-smtr.gtfs.feed_info" %} +{% set calendario_manual = ref("aux_calendario_manual") %} {% if execute %} {% if is_incremental() %} {% set gtfs_feeds_query %} - select concat("'", feed_start_date, "'") + select concat("'", feed_start_date, "'") as feed_start_date from {{ gtfs_feed_info }} where feed_start_date <= date("{{ var('date_range_end') }}") and (feed_end_date IS NULL OR feed_end_date >= date("{{ var('date_range_end') }}")) + + union distinct + + select distinct concat("'", feed_start_date, "'") as feed_start_date + from {{ calendario_manual }} + where + data between date("{{ var('date_range_start') }}") + and date("{{ var('date_range_end') }}") + and feed_start_date is not null {% endset %} {% set gtfs_feeds = run_query(gtfs_feeds_query).columns[0].values() %} @@ -29,53 +41,79 @@ with calendar as ( select * from `rj-smtr.gtfs.calendar` - where feed_start_date in ({{ gtfs_feeds | join(", ") }}) + {% if is_incremental() %} + where feed_start_date in ({{ gtfs_feeds | join(", ") }}) + {% endif %} + ), + datas as ( + select data, extract(dayofweek from data) as dia_semana, feed_start_date + from + {{ gtfs_feed_info }}, + unnest( + generate_date_array( + {% if is_incremental() %} + date("{{ var('date_range_start') }}"), + date("{{ var('date_range_end') }}") + {% else %}date("2024-09-01"), current_date("America/Sao_Paulo") + {% endif %} + ) + ) as data + where + data >= feed_start_date and (data <= feed_end_date or feed_end_date is null) + ), + modificacao_manual as ( + select + d.data, + d.dia_semana, + coalesce(m.feed_start_date, d.feed_start_date) as feed_start_date, + m.tipo_dia, + ifnull(m.tipo_os, "Regular") as tipo_os + from datas d + left join {{ calendario_manual }} m using (data) ), calendar_dates as ( select - date as data, + cd.date as data, + m.tipo_dia, + m.tipo_os, case - when exception_type = '2' + when cd.exception_type = '2' then regexp_replace( - service_id, + cd.service_id, "^[U|S|D]_", case - when extract(dayofweek from date) = 7 + when extract(dayofweek from cd.date) = 7 then "S_" - when extract(dayofweek from date) = 1 + when extract(dayofweek from cd.date) = 1 then "D_" else "U_" end ) - else service_id + else cd.service_id end as service_id, - exception_type, - feed_start_date, - from `rj-smtr.gtfs.calendar_dates` + cd.exception_type, + cd.feed_start_date, + from `rj-smtr.gtfs.calendar_dates` cd + join + modificacao_manual m + on cd.date = m.data + and cd.feed_start_date = m.feed_start_date where - feed_start_date in ({{ gtfs_feeds | join(", ") }}) - and date between date("{{ var('date_range_start') }}") and date( - "{{ var('date_range_end') }}" - ) - ), - datas as ( - select distinct data, extract(dayofweek from data) as dia_semana, - from - unnest( - generate_date_array( - date("{{ var('date_range_start') }}"), - date("{{ var('date_range_end') }}") + {% if is_incremental() %} + feed_start_date in ({{ gtfs_feeds | join(", ") }}) + and date between date("{{ var('date_range_start') }}") and date( + "{{ var('date_range_end') }}" ) - ) as data + {% else %} date <= current_date("America/Sao_Paulo") + {% endif %} ), - datas_service as ( - select d.data, c.service_id, c.feed_version, c.feed_start_date - from datas d + datas_service_id as ( + select d.data, d.tipo_dia, d.tipo_os, c.service_id, d.feed_start_date + from modificacao_manual d join calendar c - on d.data >= c.feed_start_date - and (d.data <= feed_end_date or feed_end_date is null) + on d.feed_start_date = c.feed_start_date and ( (d.dia_semana = 1 and c.sunday = '1') or (d.dia_semana = 2 and c.monday = '1') @@ -87,8 +125,8 @@ with ) ), service_ids_retirados as ( - select d.*, c.service_id as c_service_id, - from datas_calendar d + select d.* + from datas_service_id d left join (select * from calendar_dates where exception_type = '2') c using ( data, service_id @@ -97,49 +135,54 @@ with ), service_ids_adicionados as ( - select data, service_id, dia_semana, feed_start_date + select data, tipo_dia, tipo_os, service_id, feed_start_date from service_ids_retirados union distinct - select data, service_id, dia_semana, feed_start_date + select data, tipo_dia, tipo_os, service_id, feed_start_date from calendar_dates where exception_type = '1' ), - service_id_agg as ( - select data, feed_start_date, array_agg(service_id) as service_ids - from service_ids_adicionados - group by 1, 2, 3 - ), - calendario_gtfs as ( + service_id_corrigido as ( select data, + tipo_dia, + tipo_os, case - when "D_REG" in unnest(service_ids) - then "Domingo" - when "S_REG" in unnest(service_ids) - then "Sábado" - when "U_REG" in unnest(service_ids) - then "Dia Útil" - end as tipo_dia, - "Regular" as tipo_os, - service_ids, + when tipo_dia = "Domingo" + then regexp_replace(service_id, "^[U|S]_", "D_") + when tipo_dia = "Sábado" + then regexp_replace(service_id, "^[U|D]_", "S_") + when tipo_dia = "Dia Útil" + then regexp_replace(service_id, "^[S|D]_", "U_") + else service_id + end as service_id, feed_start_date - from service_id_agg + from service_ids_adicionados ), - modificacoes_manuais as ( + service_id_agg as ( select - c.data, - coalesce(m.tipo_dia, c.tipo_dia) as tipo_dia, - coalesce(m.tipo_os, c.tipo_os) as tipo_os, - service_ids, - coalesce(m.feed_start_date, c.feed_start_date) as feed_start_date - from calendario_gtfs c - left join {{ ref("aux_calendario_manual") }} m using (data) + data, + tipo_dia, + tipo_os, + feed_start_date, + array_agg(service_id) as service_ids + from service_id_corrigido + group by 1, 2, 3, 4 ) select data, - tipo_dia, + case + when c.tipo_dia is not null + then c.tipo_dia + when "D_REG" in unnest(c.service_ids) + then "Domingo" + when "S_REG" in unnest(c.service_ids) + then "Sábado" + when "U_REG" in unnest(c.service_ids) + then "Dia Útil" + end as tipo_dia, case when c.tipo_os = "Extraordinária - Verão" then "Verão" @@ -151,9 +194,9 @@ select end as subtipo_dia, c.tipo_os, c.service_ids, - i.feed_version - feed_start_date, + i.feed_version, + c.feed_start_date, '{{ var("version") }}' as versao, current_datetime("America/Sao_Paulo") as datetime_ultima_atualizacao -from modificacoes_manuais c -join `rj-smtr.gtfs.feed_info` i using (feed_start_date) +from service_id_agg c +join {{ gtfs_feed_info }} i using (feed_start_date) diff --git a/queries/models/planejamento/staging/aux_calendario_manual.sql b/queries/models/planejamento/staging/aux_calendario_manual.sql index 56c16ef63..1b02b19ed 100644 --- a/queries/models/planejamento/staging/aux_calendario_manual.sql +++ b/queries/models/planejamento/staging/aux_calendario_manual.sql @@ -1,21 +1,40 @@ -{{ config(materialized="ephemeral") }} +{{ + config( + partition_by={ + "field": "data", + "data_type": "date", + "granularity": "day", + }, + materialized="incremental", + incremental_strategy="insert_overwrite", + ) +}} -select - data, - date(null) as feed_start_date, - cast(null as string) as tipo_dia, - case - when data between date(2024, 09, 14) and date(2024, 09, 15) - then "Verão + Rock in Rio" - when data between date(2024, 09, 19) and date(2024, 09, 22) - then "Rock in Rio" - when data = date(2024, 10, 06) - then "Eleição" - end as tipo_os -from - unnest( - generate_date_array( - date({{ var("data_versao_gtfs") }}), - date_add(date({{ var("data_versao_gtfs") }}), interval 1 year) - ) - ) as data +with + datas as ( + select + data, + date(null) as feed_start_date, + cast(null as string) as tipo_dia, + case + when data between date(2024, 09, 14) and date(2024, 09, 15) + then "Verão + Rock in Rio" + when data between date(2024, 09, 19) and date(2024, 09, 22) + then "Rock in Rio" + when data = date(2024, 10, 06) + then "Eleição" + end as tipo_os + from + unnest( + generate_date_array( + {% if is_incremental() %} + date("{{ var('date_range_start') }}"), + date("{{ var('date_range_end') }}") + {% else %}date("2024-09-01"), current_date("America/Sao_Paulo") + {% endif %} + ) + ) as data + ) +select * +from datas +where feed_start_date is not null or tipo_dia is not null or tipo_os is not null From 46e32038b220e582c698d314691b02ed6dfed087 Mon Sep 17 00:00:00 2001 From: Rafael Date: Thu, 24 Oct 2024 18:24:04 -0300 Subject: [PATCH 19/71] cria tabela viagem_planejada --- .../viagem_planejada_planejamento.sql | 205 +++++++++++++++++- 1 file changed, 204 insertions(+), 1 deletion(-) diff --git a/queries/models/planejamento/viagem_planejada_planejamento.sql b/queries/models/planejamento/viagem_planejada_planejamento.sql index 1c39e4688..2b9eb8038 100644 --- a/queries/models/planejamento/viagem_planejada_planejamento.sql +++ b/queries/models/planejamento/viagem_planejada_planejamento.sql @@ -1,10 +1,213 @@ {{ config( partition_by={ - "field": "date", + "field": "data", "data_type": "date", "granularity": "day", }, alias="viagem_planejada", + incremental_strategy="merge", + unique_key="id_viagem", + incremental_predicates=[ + "DBT_INTERNAL_DEST.data between date('" + + var("date_range_start") + + "') and date_add(date('" + + var("date_range_end") + + "'), interval 1 day)" + ], ) }} + + +{# {% set gtfs_feed_info = ref("feed_info_gtfs") %} #} +{% set gtfs_feed_info = "rj-smtr.gtfs.feed_info" %} +{% set calendario = ref("calendario") %} + +{% if execute %} + {% if is_incremental() %} + {% set gtfs_feeds_query %} + select concat("'", feed_start_date, "'") as feed_start_date + from {{ gtfs_feed_info }} + where + feed_start_date <= date("{{ var('date_range_end') }}") + and (feed_end_date IS NULL OR feed_end_date >= date("{{ var('date_range_end') }}")) + + union distinct + + select distinct concat("'", feed_start_date, "'") as feed_start_date + from {{ calendario }} + where + data between date("{{ var('date_range_start') }}") + and date("{{ var('date_range_end') }}") + and feed_start_date is not null + {% endset %} + + {% set gtfs_feeds = run_query(gtfs_feeds_query).columns[0].values() %} + {% endif %} +{% endif %} + +with + calendario as ( + select * + from {{ calendario }} + {% if is_incremental() %} + where + data between date("{{ var('date_range_start') }}") and date( + "{{ var('date_range_end') }}" + ) + {% endif %} + ), + trips as ( + select * + from `rj-smtr.gtfs.trips` + {% if is_incremental() %} + where feed_start_date in ({{ gtfs_feeds | join(", ") }}) + {% endif %} + ), + frequencies as ( + select + *, + split(start_time, ":") as start_time_parts, + split(end_time, ":") as end_time_parts, + from `rj-smtr.gtfs.frequencies` + {% if is_incremental() %} + where feed_start_date in ({{ gtfs_feeds | join(", ") }}) + {% endif %} + ), + frequencies_tratada as ( + select + * except (start_time_parts, end_time_parts, start_time, end_time), + div(cast(start_time_parts[0] as integer), 24) days_to_add_start, + div(cast(end_time_parts[0] as integer), 24) days_to_add_end, + concat( + lpad( + cast( + if( + cast(start_time_parts[0] as integer) >= 24, + cast(start_time_parts[0] as integer) - 24, + cast(start_time_parts[0] as integer) + ) as string + ), + 2, + '0' + ), + ":", + start_time_parts[1], + ":", + start_time_parts[2] + ) start_time, + concat( + lpad( + cast( + if( + cast(end_time_parts[0] as integer) >= 24, + cast(end_time_parts[0] as integer) - 24, + cast(end_time_parts[0] as integer) + ) as string + ), + 2, + '0' + ), + ":", + end_time_parts[1], + ":", + end_time_parts[2] + ) end_time + from frequencies + ), + routes as ( + select + *, + case + when agency_id in ("22005", "22002", "22004", "22003") + then "Ônibus" + when agency_id = "20001" + then "BRT" + end as modo + from `rj-smtr.gtfs.routes` + {% if is_incremental() %} + where feed_start_date in ({{ gtfs_feeds | join(", ") }}) + {% endif %} + ), + trips_dia as ( + select + c.data, + t.trip_id, + r.modo, + t.route_id, + t.service_id, + r.route_short_name, + t.direction_id, + t.shape_id, + c.tipo_dia, + c.subtipo_dia, + c.tipo_os, + t.feed_version, + t.feed_start_date + from calendario c + join trips t using (feed_start_date) + join routes r using (feed_start_date, route_id) + where t.service_id in unnest(c.service_ids) + ), + trips_frequences_dia as ( + select + td.*, + timestamp( + concat( + cast(date_add(data, interval f.days_to_add_start day) as string), + ' ', + f.start_time + ), + "America/Sao_Paulo" + ) as start_timestamp, + timestamp( + concat( + cast(date_add(data, interval f.days_to_add_end day) as string), + ' ', + f.end_time + ), + "America/Sao_Paulo" + ) as end_timestamp, + f.headway_secs + from trips_dia td + join frequencies_tratada f using (feed_start_date, trip_id) + ), + viagens as ( + select *, datetime(partida, "America/Sao_Paulo") as datetime_partida + from + trips_frequences_dia, + unnest( + generate_timestamp_array( + start_timestamp, + timestamp_sub(end_timestamp, interval 1 second), + interval headway_secs second + ) + ) as partida + ) +select + date(datetime_partida) as data, + concat( + route_short_name, + "_", + direction_id, + "_", + shape_id, + "_", + format_timestamp("%Y%m%d%H%M%S", partida, "America/Sao_Paulo") + ) as id_viagem, + datetime_partida, + modo, + service_id, + trip_id, + route_id, + shape_id, + route_short_name as servico, + case when direction_id = '0' then "Ida" else "Volta" end as sentido, + tipo_dia, + subtipo_dia, + tipo_os, + feed_version, + feed_start_date, + '{{ var("version") }}' as versao, + current_datetime("America/Sao_Paulo") as datetime_ultima_atualizacao +from viagens From d61614767e50e3d3fae9828f78c325b0105a00f8 Mon Sep 17 00:00:00 2001 From: Rafael Date: Thu, 31 Oct 2024 12:10:17 -0300 Subject: [PATCH 20/71] ajusta tipo_dia sabado --- queries/models/planejamento/calendario.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/queries/models/planejamento/calendario.sql b/queries/models/planejamento/calendario.sql index 40754ff2b..68eb3936a 100644 --- a/queries/models/planejamento/calendario.sql +++ b/queries/models/planejamento/calendario.sql @@ -152,7 +152,7 @@ with case when tipo_dia = "Domingo" then regexp_replace(service_id, "^[U|S]_", "D_") - when tipo_dia = "Sábado" + when tipo_dia = "Sabado" then regexp_replace(service_id, "^[U|D]_", "S_") when tipo_dia = "Dia Útil" then regexp_replace(service_id, "^[S|D]_", "U_") From 64152736bf3e2ee5fc3348c8a977fd8f261e66f9 Mon Sep 17 00:00:00 2001 From: Rafael Date: Thu, 31 Oct 2024 12:10:39 -0300 Subject: [PATCH 21/71] adiciona dados da os --- .../viagem_planejada_planejamento.sql | 211 +++++++++++++++--- 1 file changed, 174 insertions(+), 37 deletions(-) diff --git a/queries/models/planejamento/viagem_planejada_planejamento.sql b/queries/models/planejamento/viagem_planejada_planejamento.sql index 2b9eb8038..bdfed710c 100644 --- a/queries/models/planejamento/viagem_planejada_planejamento.sql +++ b/queries/models/planejamento/viagem_planejada_planejamento.sql @@ -95,7 +95,7 @@ with start_time_parts[1], ":", start_time_parts[2] - ) start_time, + ) as start_time, concat( lpad( cast( @@ -112,7 +112,7 @@ with end_time_parts[1], ":", end_time_parts[2] - ) end_time + ) as end_time from frequencies ), routes as ( @@ -136,22 +136,23 @@ with r.modo, t.route_id, t.service_id, - r.route_short_name, + r.route_short_name as servico, t.direction_id, t.shape_id, c.tipo_dia, c.subtipo_dia, c.tipo_os, t.feed_version, - t.feed_start_date + t.feed_start_date, + regexp_extract(t.trip_headsign, r'\[.*?\]') as evento from calendario c - join trips t using (feed_start_date) - join routes r using (feed_start_date, route_id) + join trips t using (feed_start_date, feed_version) + join routes r using (feed_start_date, feed_version, route_id) where t.service_id in unnest(c.service_ids) ), trips_frequences_dia as ( select - td.*, + td.* except (evento), timestamp( concat( cast(date_add(data, interval f.days_to_add_start day) as string), @@ -170,12 +171,69 @@ with ) as end_timestamp, f.headway_secs from trips_dia td - join frequencies_tratada f using (feed_start_date, trip_id) + join frequencies_tratada f using (feed_start_date, feed_version, trip_id) + ), + end_time_tratado as ( + select + * except (end_timestamp), + case + when + end_timestamp > lead(start_timestamp) over ( + partition by + feed_start_date, feed_version, data, servico, direction_id + order by start_timestamp + ) + then + lead(start_timestamp) over ( + partition by + feed_start_date, feed_version, data, servico, direction_id + order by start_timestamp + ) + else end_timestamp + end as end_timestamp + from trips_frequences_dia + ), + os_trajetos_alternativos as ( + select * + from `rj-smtr.gtfs.ordem_servico_trajeto_alternativo` + {% if is_incremental() %} + where feed_start_date in ({{ gtfs_feeds | join(", ") }}) + {% endif %} + + ), + trips_alternativas as ( + select + data, + servico, + direction_id, + array_agg( + struct( + td.trip_id as trip_id, + td.shape_id as shape_id, + evento as evento, + case + when td.direction_id = '0' + then os.extensao_ida + when td.direction_id = '1' + then os.extensao_volta + end as extensao + ) + ) as trajetos_alternativos + from trips_dia td + join + os_trajetos_alternativos os using ( + feed_start_date, feed_version, tipo_os, servico, evento + ) + where td.trip_id not in (select trip_id from frequencies) + group by 1, 2, 3 ), viagens as ( - select *, datetime(partida, "America/Sao_Paulo") as datetime_partida + select + ett.*, + datetime(partida, "America/Sao_Paulo") as datetime_partida, + ta.trajetos_alternativos from - trips_frequences_dia, + end_time_tratado ett, unnest( generate_timestamp_array( start_timestamp, @@ -183,31 +241,110 @@ with interval headway_secs second ) ) as partida + left join trips_alternativas ta using (data, servico, direction_id) + ), + ordem_servico as ( + select + * except (horario_inicio, horario_fim), + parse_time("%H:%M:%S", lpad(horario_inicio, 8, '0')) as horario_inicio, + split(horario_fim, ":") horario_fim_parts + from `rj-smtr.gtfs.ordem_servico` + where + {% if is_incremental() %} feed_start_date in ({{ gtfs_feeds | join(", ") }}) + {% else %} feed_start_date >= "2024-09-01" + {% endif %} + ), + ordem_servico_tratada as ( + select + * except (horario_fim_parts), + div(cast(horario_fim_parts[0] as integer), 24) as dias_horario_fim, + parse_time( + "%H:%M:%S", + concat( + lpad( + cast( + if( + cast(horario_fim_parts[0] as integer) >= 24, + cast(horario_fim_parts[0] as integer) - 24, + cast(horario_fim_parts[0] as integer) + ) as string + ), + 2, + '0' + ), + ":", + horario_fim_parts[1], + ":", + horario_fim_parts[2] + ) + ) as horario_fim, + from ordem_servico + + ), + viagem_os as ( + select + v.*, + case + when v.direction_id = '0' + then os.extensao_ida + when v.direction_id = '1' + then os.extensao_volta + end as extensao + from viagens v + left join + ordem_servico_tratada os using ( + feed_start_date, feed_version, tipo_os, tipo_dia, servico + ) + where + (os.distancia_total_planejada is null or os.distancia_total_planejada > 0) + and ( + os.feed_start_date is null + or v.datetime_partida + between datetime(data, os.horario_inicio) and datetime( + date_add(data, interval os.dias_horario_fim day), os.horario_fim + ) + ) + ), + viagem_planejada as ( + select + date(datetime_partida) as data, + concat( + servico, + "_", + direction_id, + "_", + shape_id, + "_", + format_datetime("%Y%m%d%H%M%S", datetime_partida) + ) as id_viagem, + datetime_partida, + modo, + service_id, + trip_id, + route_id, + shape_id, + servico, + case when direction_id = '0' then "Ida" else "Volta" end as sentido, + extensao, + trajetos_alternativos, + data as data_referencia, + tipo_dia, + subtipo_dia, + tipo_os, + feed_version, + feed_start_date, + '{{ var("version") }}' as versao, + current_datetime("America/Sao_Paulo") as datetime_ultima_atualizacao + from viagem_os + ) +select * except (rn) +from + ( + select + *, + row_number() over ( + partition by id_viagem order by data_referencia desc + ) as rn + from viagem_planejada ) -select - date(datetime_partida) as data, - concat( - route_short_name, - "_", - direction_id, - "_", - shape_id, - "_", - format_timestamp("%Y%m%d%H%M%S", partida, "America/Sao_Paulo") - ) as id_viagem, - datetime_partida, - modo, - service_id, - trip_id, - route_id, - shape_id, - route_short_name as servico, - case when direction_id = '0' then "Ida" else "Volta" end as sentido, - tipo_dia, - subtipo_dia, - tipo_os, - feed_version, - feed_start_date, - '{{ var("version") }}' as versao, - current_datetime("America/Sao_Paulo") as datetime_ultima_atualizacao -from viagens +where rn = 1 From 2bc063a89665e17d876983aed7b6d1b3e6d4e0f9 Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 4 Nov 2024 09:53:14 -0300 Subject: [PATCH 22/71] altera variavel filtro / altera fonte para viagem_informada --- .../staging/gps_segmento_viagem.sql | 61 +++++++++++++----- .../monitoramento/staging/gps_viagem.sql | 62 +++++++++++++------ .../models/monitoramento/viagem_validacao.sql | 9 ++- .../models/planejamento/segmento_shape.sql | 3 +- .../viagem_planejada_planejamento.sql | 11 ---- 5 files changed, 96 insertions(+), 50 deletions(-) diff --git a/queries/models/monitoramento/staging/gps_segmento_viagem.sql b/queries/models/monitoramento/staging/gps_segmento_viagem.sql index f9e316467..3af1ebf7b 100644 --- a/queries/models/monitoramento/staging/gps_segmento_viagem.sql +++ b/queries/models/monitoramento/staging/gps_segmento_viagem.sql @@ -11,28 +11,52 @@ }} {% set incremental_filter %} - data = date_sub(date('{{ var("run_date") }}'), interval 1 day) + data between + date('{{ var("date_range_start") }}') + and date('{{ var("date_range_end") }}') {% endset %} -{# {% set data_versao_efetiva = ref("subsidio_data_versao_efetiva") %} #} -{% set data_versao_efetiva = ( - "rj-smtr.projeto_subsidio_sppo.subsidio_data_versao_efetiva" -) %} + +{% set calendario = ref("calendario") %} {% if execute %} - {% set data_versao_gtfs_query %} - select format_date("%Y-%m-%d", feed_start_date) from {{ data_versao_efetiva }} where {{ incremental_filter }} - {% endset %} + {% if is_incremental() %} + {% set gtfs_feeds_query %} + select distinct concat("'", feed_start_date, "'") as feed_start_date + from {{ calendario }} + where {{ incremental_filter }} + {% endset %} - {% set data_versao_gtfs = ( - run_query(data_versao_gtfs_query).columns[0].values()[0] - ) %} + {% set gtfs_feeds = run_query(gtfs_feeds_query).columns[0].values() %} + {% endif %} {% endif %} with + calendario as ( + select * + from {{ calendario }} + {% if is_incremental() %} + where + data between date("{{ var('date_range_start') }}") and date( + "{{ var('date_range_end') }}" + ) + {% endif %} + ), gps_viagem as ( - select id_viagem, shape_id, geo_point_gps - from {{ ref("gps_viagem") }} - where {{ incremental_filter }} + select + data, + gv.id_viagem, + gv.shape_id, + gv.geo_point_gps, + c.feed_version, + c.feed_start_date + from {{ ref("gps_viagem") }} gv + join calendario c using (data) + {% if is_incremental() %} + where + data between date_sub( + date('{{ var("date_range_start") }}'), interval 1 day + ) and date('{{ var("date_range_end") }}') + {% endif %} ), segmento as ( select @@ -44,14 +68,17 @@ with buffer, indicador_segmento_desconsiderado from {{ ref("segmento_shape") }} - where feed_start_date = '{{ data_versao_gtfs }}' + {% if is_incremental() %} + where feed_start_date in ({{ gtfs_feeds | join(", ") }}) + {% endif %} ), gps_segmento as ( select g.id_viagem, g.shape_id, s.id_segmento, count(*) as quantidade_gps from gps_viagem g join segmento s - on g.shape_id = s.shape_id + on g.feed_version = s.feed_version + and g.shape_id = s.shape_id and st_intersects(s.buffer, g.geo_point_gps) group by 1, 2, 3 ), @@ -68,7 +95,7 @@ with servico, sentido from {{ ref("viagem_informada_monitoramento") }} - where {{ incremental_filter }} + {% if is_incremental() %} where {{ incremental_filter }} {% endif %} ), viagem_segmento as ( select diff --git a/queries/models/monitoramento/staging/gps_viagem.sql b/queries/models/monitoramento/staging/gps_viagem.sql index 65341319b..e51546e6f 100644 --- a/queries/models/monitoramento/staging/gps_viagem.sql +++ b/queries/models/monitoramento/staging/gps_viagem.sql @@ -22,28 +22,53 @@ with route_id, shape_id, servico, - sentido - from {{ ref("viagem_informada_monitoramento") }} - where - - data = date_sub(date('{{ var("run_date") }}'), interval 1 day) - -- filtro de teste REMOVER - and date(datetime_partida) = date("2024-06-05") - and date(datetime_chegada) = date("2024-06-05") + sentido, + fonte_gps + {# from {{ ref("viagem_informada_monitoramento") }} #} + from `rj-smtr.monitoramento.viagem_informada` + {% if is_incremental() %} + where + data between date('{{ var("date_range_start") }}') and date( + '{{ var("date_range_end") }}' + ) + {% endif %} ), - gps as ( + gps_conecta as ( select data, timestamp_gps, servico, id_veiculo, latitude, longitude from `rj-smtr.br_rj_riodejaneiro_veiculos.gps_sppo` {# from {{ ref("gps_sppo") }} #} where - data - between date_sub(date('{{ var("run_date") }}'), interval 2 day) and date( - '{{ var("run_date") }}' - ) + {% if is_incremental() %} + data between date_sub( + date('{{ var("date_range_start") }}'), interval 1 day + ) and date_sub(date('{{ var("date_range_end") }}')) + {% else %} data >= date("2024-10-26") + {% endif %} + ), + gps_zirix as ( + select data, timestamp_gps, servico, id_veiculo, latitude, longitude + from `rj-smtr.br_rj_riodejaneiro_onibus_gps_zirix.gps_sppo` + {# from {{ ref("gps_sppo_zirix") }} #} + where + {% if is_incremental() %} + data between date_sub( + date('{{ var("date_range_start") }}'), interval 1 day + ) and date_sub(date('{{ var("date_range_end") }}')) + {% else %} data >= date("2024-10-26") + {% endif %} + ), + gps_union as ( + select *, 'conecta' as fornecedor + from gps_conecta + + union all + + select *, 'zirix' as fornecedor + from gps_zirix ) select - g.data, + v.data, g.timestamp_gps, g.id_veiculo, g.servico, @@ -57,14 +82,13 @@ select v.trip_id, v.route_id, v.shape_id, + v.fonte_gps, '{{ var("version") }}' as versao, current_datetime("America/Sao_Paulo") as datetime_ultima_atualizacao -from gps g +from gps_union g join viagem v - on g.timestamp_gps - between datetime_sub(v.datetime_partida, interval 10 minute) and datetime_add( - v.datetime_chegada, interval 10 minute - ) + on g.timestamp_gps between v.datetime_partida and v.datetime_chegada and g.id_veiculo = v.id_veiculo and g.servico = v.servico + and g.fornecedor = v.fonte_gps diff --git a/queries/models/monitoramento/viagem_validacao.sql b/queries/models/monitoramento/viagem_validacao.sql index 78938ff7f..305175d46 100644 --- a/queries/models/monitoramento/viagem_validacao.sql +++ b/queries/models/monitoramento/viagem_validacao.sql @@ -30,8 +30,13 @@ with feed_end_date from {{ ref("gps_segmento_viagem") }} where - data = date_sub(date('{{ var("run_date") }}'), interval 1 day) - and not indicador_segmento_desconsiderado + not indicador_segmento_desconsiderado + {% if is_incremental() %} + + and data between date('{{ var("date_range_start") }}') and date( + '{{ var("date_range_end") }}' + ) + {% endif %} group by data, id_viagem, diff --git a/queries/models/planejamento/segmento_shape.sql b/queries/models/planejamento/segmento_shape.sql index c13137e11..ef9b10432 100644 --- a/queries/models/planejamento/segmento_shape.sql +++ b/queries/models/planejamento/segmento_shape.sql @@ -110,7 +110,8 @@ from indicador_validacao_shape s.indicador_segmento_desconsiderado, s.versao from {{ this }} s - join {{ ref("feed_info_gtfs") }} fi using (feed_start_date) + {# join {{ ref("feed_info_gtfs") }} fi using (feed_start_date) #} + join `rj-smtr.gtfs.feed_info` fi using (feed_start_date) where feed_start_date = '{{ last_feed_version }}' {% endif %} diff --git a/queries/models/planejamento/viagem_planejada_planejamento.sql b/queries/models/planejamento/viagem_planejada_planejamento.sql index bdfed710c..3bd2cc564 100644 --- a/queries/models/planejamento/viagem_planejada_planejamento.sql +++ b/queries/models/planejamento/viagem_planejada_planejamento.sql @@ -19,27 +19,16 @@ }} -{# {% set gtfs_feed_info = ref("feed_info_gtfs") %} #} -{% set gtfs_feed_info = "rj-smtr.gtfs.feed_info" %} {% set calendario = ref("calendario") %} {% if execute %} {% if is_incremental() %} {% set gtfs_feeds_query %} - select concat("'", feed_start_date, "'") as feed_start_date - from {{ gtfs_feed_info }} - where - feed_start_date <= date("{{ var('date_range_end') }}") - and (feed_end_date IS NULL OR feed_end_date >= date("{{ var('date_range_end') }}")) - - union distinct - select distinct concat("'", feed_start_date, "'") as feed_start_date from {{ calendario }} where data between date("{{ var('date_range_start') }}") and date("{{ var('date_range_end') }}") - and feed_start_date is not null {% endset %} {% set gtfs_feeds = run_query(gtfs_feeds_query).columns[0].values() %} From bb9ed384d511236e9df57b54f22c180ed107dc4a Mon Sep 17 00:00:00 2001 From: Rafael Date: Wed, 6 Nov 2024 14:03:30 -0300 Subject: [PATCH 23/71] corrige viagens duplicadas --- .../staging/gps_segmento_viagem.sql | 31 ++++++++++--------- .../models/monitoramento/viagem_validacao.sql | 12 +++---- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/queries/models/monitoramento/staging/gps_segmento_viagem.sql b/queries/models/monitoramento/staging/gps_segmento_viagem.sql index 3af1ebf7b..a99b0605f 100644 --- a/queries/models/monitoramento/staging/gps_segmento_viagem.sql +++ b/queries/models/monitoramento/staging/gps_segmento_viagem.sql @@ -85,16 +85,19 @@ with viagem as ( select data, - id_viagem, - datetime_partida, - datetime_chegada, - id_veiculo, - trip_id, - route_id, - shape_id, - servico, - sentido - from {{ ref("viagem_informada_monitoramento") }} + v.id_viagem, + v.datetime_partida, + v.datetime_chegada, + v.id_veiculo, + v.trip_id, + v.route_id, + v.shape_id, + v.servico, + v.sentido, + c.feed_start_date, + c.feed_version + from {{ ref("viagem_informada_monitoramento") }} v + join calendario c using (data) {% if is_incremental() %} where {{ incremental_filter }} {% endif %} ), viagem_segmento as ( @@ -111,11 +114,10 @@ with s.indicador_segmento_desconsiderado, v.servico, v.sentido, - s.feed_version, - s.feed_start_date, - s.feed_end_date + feed_version, + feed_start_date from viagem v - join segmento s using (shape_id) + join segmento s using (shape_id, feed_version, feed_start_date) ) select v.data, @@ -133,7 +135,6 @@ select ifnull(g.quantidade_gps, 0) as quantidade_gps, v.feed_version, v.feed_start_date, - v.feed_end_date, '{{ var("version") }}' as versao, current_datetime("America/Sao_Paulo") as datetime_ultima_atualizacao from viagem_segmento v diff --git a/queries/models/monitoramento/viagem_validacao.sql b/queries/models/monitoramento/viagem_validacao.sql index 305175d46..9f11ce79b 100644 --- a/queries/models/monitoramento/viagem_validacao.sql +++ b/queries/models/monitoramento/viagem_validacao.sql @@ -26,8 +26,7 @@ with count(*) as quantidade_segmentos_verificados, countif(quantidade_gps > 0) as quantidade_segmentos_validos, feed_version, - feed_start_date, - feed_end_date + feed_start_date from {{ ref("gps_segmento_viagem") }} where not indicador_segmento_desconsiderado @@ -49,8 +48,7 @@ with servico, sentido, feed_version, - feed_start_date, - feed_end_date + feed_start_date ), indice as ( select @@ -69,8 +67,7 @@ with quantidade_segmentos_validos / quantidade_segmentos_verificados as indice_validacao, feed_version, - feed_start_date, - feed_end_date + feed_start_date from contagem ) select @@ -90,6 +87,5 @@ select indice_validacao >= {{ parametro_validacao }} as indicador_viagem_valida, {{ parametro_validacao }} as parametro_validacao, feed_version, - feed_start_date, - feed_end_date + feed_start_date from indice From e4ddad63015a19808741d26fe60b7b7dee8d18d4 Mon Sep 17 00:00:00 2001 From: Rafael Date: Thu, 7 Nov 2024 09:59:00 -0300 Subject: [PATCH 24/71] adiciona coluna modo / descomenta refs --- .../staging/gps_segmento_viagem.sql | 3 ++ .../monitoramento/staging/gps_viagem.sql | 1 + .../viagem_informada_monitoramento.sql | 49 +++++++++++++++++-- .../models/monitoramento/viagem_validacao.sql | 5 +- queries/models/planejamento/calendario.sql | 10 ++-- .../models/planejamento/segmento_shape.sql | 4 +- .../planejamento/shapes_geom_planejamento.sql | 5 +- .../viagem_planejada_planejamento.sql | 3 +- 8 files changed, 66 insertions(+), 14 deletions(-) diff --git a/queries/models/monitoramento/staging/gps_segmento_viagem.sql b/queries/models/monitoramento/staging/gps_segmento_viagem.sql index a99b0605f..f1e9fa509 100644 --- a/queries/models/monitoramento/staging/gps_segmento_viagem.sql +++ b/queries/models/monitoramento/staging/gps_segmento_viagem.sql @@ -88,6 +88,7 @@ with v.id_viagem, v.datetime_partida, v.datetime_chegada, + v.modo, v.id_veiculo, v.trip_id, v.route_id, @@ -106,6 +107,7 @@ with v.id_viagem, v.datetime_partida, v.datetime_chegada, + v.modo, v.id_veiculo, v.trip_id, v.route_id, @@ -124,6 +126,7 @@ select id_viagem, v.datetime_partida, v.datetime_chegada, + v.modo, v.id_veiculo, v.trip_id, v.route_id, diff --git a/queries/models/monitoramento/staging/gps_viagem.sql b/queries/models/monitoramento/staging/gps_viagem.sql index e51546e6f..2fd8804c0 100644 --- a/queries/models/monitoramento/staging/gps_viagem.sql +++ b/queries/models/monitoramento/staging/gps_viagem.sql @@ -70,6 +70,7 @@ with select v.data, g.timestamp_gps, + v.modo, g.id_veiculo, g.servico, v.sentido, diff --git a/queries/models/monitoramento/viagem_informada_monitoramento.sql b/queries/models/monitoramento/viagem_informada_monitoramento.sql index a87f3be38..8e861ae04 100644 --- a/queries/models/monitoramento/viagem_informada_monitoramento.sql +++ b/queries/models/monitoramento/viagem_informada_monitoramento.sql @@ -13,6 +13,8 @@ {% endset %} {% set staging_viagem_informada_rioonibus = ref("staging_viagem_informada_rioonibus") %} +{% set calendario = ref("calendario") %} + {% if execute %} {% if is_incremental() %} {% set partitions_query %} @@ -25,7 +27,16 @@ {% endset %} {% set partitions = run_query(partitions_query).columns[0].values() %} + + {% set gtfs_feeds_query %} + select distinct concat("'", feed_start_date, "'") as feed_start_date + from {{ calendario }} + where {{ incremental_filter }} + {% endset %} + + {% set gtfs_feeds = run_query(gtfs_feeds_query).columns[0].values() %} {% endif %} + {% endif %} with @@ -51,7 +62,7 @@ with end as sentido, fornecedor as fonte_gps, datetime_processamento, - timestamp_captura as datetime_captura, + timestamp_captura as datetime_captura from {{ staging_viagem_informada_rioonibus }} {% if is_incremental() %} where {{ incremental_filter }} {% endif %} ), @@ -62,7 +73,7 @@ with {% if is_incremental() and partitions | length > 0 %} union all - select * except (versao, datetime_ultima_atualizacao), 1 as priority + select * except (modo, versao, datetime_ultima_atualizacao), 1 as priority from {{ this }} where data in ({{ partitions | join(", ") }}) {% endif %} @@ -79,9 +90,41 @@ with from complete_partitions ) where rn = 1 + ), + calendario as ( + select * + from {{ calendario }} + {% if is_incremental() %} where {{ incremental_filter }} {% endif %} + ), + routes as ( + select * + from {{ ref("routes_gtfs") }} + {% if is_incremental() %} + where feed_start_date in ({{ gtfs_feeds | join(", ") }}) + {% endif %} + ), + viagem_modo as ( + select + data, + v.id_viagem, + v.datetime_partida, + v.datetime_chegada, + if(r.route_type = '200', 'Ônibus Executivo', 'Ônibus SPPO') as modo, + v.id_veiculo, + v.trip_id, + v.route_id, + v.shape_id, + v.servico, + v.sentido, + v.fonte_gps, + v.datetime_processamento, + v.datetime_captura + from deduplicado v + join calendario c using (data) + left join routes r using (route_id, feed_start_date, feed_info) ) select *, '{{ var("version") }}' as versao, current_datetime("America/Sao_Paulo") as datetime_ultima_atualizacao -from deduplicado +from viagem_modo diff --git a/queries/models/monitoramento/viagem_validacao.sql b/queries/models/monitoramento/viagem_validacao.sql index 9f11ce79b..1a2202102 100644 --- a/queries/models/monitoramento/viagem_validacao.sql +++ b/queries/models/monitoramento/viagem_validacao.sql @@ -17,6 +17,7 @@ with id_viagem, datetime_partida, datetime_chegada, + modo, id_veiculo, trip_id, route_id, @@ -31,7 +32,6 @@ with where not indicador_segmento_desconsiderado {% if is_incremental() %} - and data between date('{{ var("date_range_start") }}') and date( '{{ var("date_range_end") }}' ) @@ -41,6 +41,7 @@ with id_viagem, datetime_partida, datetime_chegada, + modo, id_veiculo, trip_id, route_id, @@ -56,6 +57,7 @@ with id_viagem, datetime_partida, datetime_chegada, + modo, id_veiculo, trip_id, route_id, @@ -75,6 +77,7 @@ select id_viagem, datetime_partida, datetime_chegada, + modo, id_veiculo, trip_id, route_id, diff --git a/queries/models/planejamento/calendario.sql b/queries/models/planejamento/calendario.sql index 68eb3936a..570f81380 100644 --- a/queries/models/planejamento/calendario.sql +++ b/queries/models/planejamento/calendario.sql @@ -10,8 +10,8 @@ ) }} -{# {% set gtfs_feed_info = ref("feed_info_gtfs") %} #} -{% set gtfs_feed_info = "rj-smtr.gtfs.feed_info" %} +{% set gtfs_feed_info = ref("feed_info_gtfs") %} +{# {% set gtfs_feed_info = "rj-smtr.gtfs.feed_info" %} #} {% set calendario_manual = ref("aux_calendario_manual") %} {% if execute %} @@ -40,7 +40,8 @@ with calendar as ( select * - from `rj-smtr.gtfs.calendar` + {# from `rj-smtr.gtfs.calendar` #} + from {{ ref("calendar_gtfs") }} {% if is_incremental() %} where feed_start_date in ({{ gtfs_feeds | join(", ") }}) {% endif %} @@ -94,7 +95,8 @@ with end as service_id, cd.exception_type, cd.feed_start_date, - from `rj-smtr.gtfs.calendar_dates` cd + {# from `rj-smtr.gtfs.calendar_dates` cd #} + from {{ ref("calendar_gtfs") }} cd join modificacao_manual m on cd.date = m.data diff --git a/queries/models/planejamento/segmento_shape.sql b/queries/models/planejamento/segmento_shape.sql index ef9b10432..79618eff4 100644 --- a/queries/models/planejamento/segmento_shape.sql +++ b/queries/models/planejamento/segmento_shape.sql @@ -110,8 +110,8 @@ from indicador_validacao_shape s.indicador_segmento_desconsiderado, s.versao from {{ this }} s - {# join {{ ref("feed_info_gtfs") }} fi using (feed_start_date) #} - join `rj-smtr.gtfs.feed_info` fi using (feed_start_date) + join {{ ref("feed_info_gtfs") }} fi using (feed_start_date) + {# join `rj-smtr.gtfs.feed_info` fi using (feed_start_date) #} where feed_start_date = '{{ last_feed_version }}' {% endif %} diff --git a/queries/models/planejamento/shapes_geom_planejamento.sql b/queries/models/planejamento/shapes_geom_planejamento.sql index c51b3c17a..0c61d8e0f 100644 --- a/queries/models/planejamento/shapes_geom_planejamento.sql +++ b/queries/models/planejamento/shapes_geom_planejamento.sql @@ -25,9 +25,8 @@ with shape_pt_sequence, st_geogpoint(shape_pt_lon, shape_pt_lat) as ponto_shape, concat(shape_pt_lon, " ", shape_pt_lat) as lon_lat, - from - {# {{ ref("shapes_gtfs") }} s #} - `rj-smtr.gtfs.shapes` s + from {{ ref("shapes_gtfs") }} s + {# `rj-smtr.gtfs.shapes` s #} {% if is_incremental() %} where feed_start_date diff --git a/queries/models/planejamento/viagem_planejada_planejamento.sql b/queries/models/planejamento/viagem_planejada_planejamento.sql index 3bd2cc564..b15c219d5 100644 --- a/queries/models/planejamento/viagem_planejada_planejamento.sql +++ b/queries/models/planejamento/viagem_planejada_planejamento.sql @@ -113,7 +113,8 @@ with when agency_id = "20001" then "BRT" end as modo - from `rj-smtr.gtfs.routes` + {# from `rj-smtr.gtfs.routes` #} + from {{ ref("routes_gtfs") }} {% if is_incremental() %} where feed_start_date in ({{ gtfs_feeds | join(", ") }}) {% endif %} From 2d0cbb2e764e2a4f8667089c2ae3f58fb90042b4 Mon Sep 17 00:00:00 2001 From: Rafael Date: Thu, 7 Nov 2024 10:14:19 -0300 Subject: [PATCH 25/71] altera filtro do gps --- queries/models/monitoramento/staging/gps_viagem.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/queries/models/monitoramento/staging/gps_viagem.sql b/queries/models/monitoramento/staging/gps_viagem.sql index 2fd8804c0..8fadecc0c 100644 --- a/queries/models/monitoramento/staging/gps_viagem.sql +++ b/queries/models/monitoramento/staging/gps_viagem.sql @@ -42,7 +42,7 @@ with data between date_sub( date('{{ var("date_range_start") }}'), interval 1 day ) and date_sub(date('{{ var("date_range_end") }}')) - {% else %} data >= date("2024-10-26") + {% else %} data >= date("2024-10-11") {% endif %} ), @@ -55,7 +55,7 @@ with data between date_sub( date('{{ var("date_range_start") }}'), interval 1 day ) and date_sub(date('{{ var("date_range_end") }}')) - {% else %} data >= date("2024-10-26") + {% else %} data >= date("2024-10-11") {% endif %} ), gps_union as ( From 7733cf7deb34e9e15f7cfa8f40ac36df056f291c Mon Sep 17 00:00:00 2001 From: Rafael Date: Thu, 7 Nov 2024 10:23:21 -0300 Subject: [PATCH 26/71] descomenta refs --- queries/models/monitoramento/staging/gps_viagem.sql | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/queries/models/monitoramento/staging/gps_viagem.sql b/queries/models/monitoramento/staging/gps_viagem.sql index 8fadecc0c..96d4df280 100644 --- a/queries/models/monitoramento/staging/gps_viagem.sql +++ b/queries/models/monitoramento/staging/gps_viagem.sql @@ -24,8 +24,8 @@ with servico, sentido, fonte_gps - {# from {{ ref("viagem_informada_monitoramento") }} #} - from `rj-smtr.monitoramento.viagem_informada` + from {{ ref("viagem_informada_monitoramento") }} + {# from `rj-smtr.monitoramento.viagem_informada` #} {% if is_incremental() %} where data between date('{{ var("date_range_start") }}') and date( @@ -35,8 +35,8 @@ with ), gps_conecta as ( select data, timestamp_gps, servico, id_veiculo, latitude, longitude - from `rj-smtr.br_rj_riodejaneiro_veiculos.gps_sppo` - {# from {{ ref("gps_sppo") }} #} + {# from `rj-smtr.br_rj_riodejaneiro_veiculos.gps_sppo` #} + from {{ ref("gps_sppo") }} where {% if is_incremental() %} data between date_sub( @@ -48,8 +48,8 @@ with ), gps_zirix as ( select data, timestamp_gps, servico, id_veiculo, latitude, longitude - from `rj-smtr.br_rj_riodejaneiro_onibus_gps_zirix.gps_sppo` - {# from {{ ref("gps_sppo_zirix") }} #} + {# from `rj-smtr.br_rj_riodejaneiro_onibus_gps_zirix.gps_sppo` #} + from {{ ref("gps_sppo_zirix") }} where {% if is_incremental() %} data between date_sub( From 24063b1160fb0d609c9cd17f3c0b845464bdfb56 Mon Sep 17 00:00:00 2001 From: Rafael Date: Thu, 7 Nov 2024 10:41:21 -0300 Subject: [PATCH 27/71] cria schema viagem_validacao --- queries/models/monitoramento/schema.yml | 89 ++++++++++++++++++++++++- 1 file changed, 88 insertions(+), 1 deletion(-) diff --git a/queries/models/monitoramento/schema.yml b/queries/models/monitoramento/schema.yml index f70306d6f..275e37e1a 100644 --- a/queries/models/monitoramento/schema.yml +++ b/queries/models/monitoramento/schema.yml @@ -18,6 +18,10 @@ models: description: Data e hora da chegada da viagem em GMT-3 data_type: datetime quote: true + - name: modo + description: "Tipo de transporte (BRT, Ônibus Executivo, ...)" + data_type: string + quote: true - name: id_veiculo description: Código identificador do veículo (número de ordem) data_type: string @@ -35,7 +39,7 @@ models: data_type: string quote: true - name: servico - description: "Nome curto da linha operada pelo veículo com variação de serviço (ex: 010, 011SN, ...) ou código da estação" + description: "Nome curto da linha operada pelo veículo com variação de serviço (ex: 010, 011SN, ...)" data_type: string quote: true - name: sentido @@ -62,3 +66,86 @@ models: description: "{{ doc('datetime_ultima_atualizacao') }}" data_type: datetime quote: true + - name: viagem_validacao + description: Tabela que informa se a viagem informada está ou não válida de acordo com os parâmetros definidos. + columns: + - name: data + description: Data da viagem (partição). + data_type: date + quote: true + - name: id_viagem + description: Identificador único da viagem. + data_type: string + quote: true + - name: datetime_partida + description: Data e hora da partida da viagem em GMT-3. + data_type: datetime + quote: true + - name: datetime_chegada + description: Data e hora da chegada da viagem em GMT-3. + data_type: datetime + quote: true + - name: modo + description: "Tipo de transporte (BRT, Ônibus Executivo, ...)" + data_type: string + quote: true + - name: id_veiculo + description: Código identificador do veículo (número de ordem). + data_type: string + quote: true + - name: trip_id + description: Identificador de uma viagem da tabela trips do GTFS. + data_type: int64 + quote: true + - name: route_id + description: Identificador de uma rota da tabela routes do GTFS. + data_type: string + quote: true + - name: shape_id + description: Identificador de um shape da tabela shapes do GTFS. + data_type: string + quote: true + - name: servico + description: "Nome curto da linha operada pelo veículo com variação de serviço (ex: 010, 011SN, ...)." + data_type: string + quote: true + - name: sentido + description: Sentido de operação do serviço. + data_type: string + quote: true + - name: quantidade_segmentos_verificados + description: Quantidade de segmentos considerados no validação. + data_type: int64 + quote: true + - name: quantidade_segmentos_validos + description: Quantidade de segmentos com pelo menos um sinal de gps. + data_type: int64 + quote: true + - name: indice_validacao + description: quantidade_segmentos_validos dividido por quantidade_segmentos_verificados + data_type: float64 + quote: true + - name: indicador_viagem_valida + description: Indica se a viagem está válida ou não. + data_type: boolean + quote: true + - name: parametro_validacao + description: Valor mínimo do indice_validacao para que a viagem seja validada. + data_type: float64 + quote: true + - name: feed_version + description: String que indica a versão atual do conjunto de dados GTFS. + data_type: string + quote: true + - name: feed_start_date + description: Data inicial do feed (versão). + data_type: date + quote: true + - name: versao + description: "{{ doc('versao') }}" + data_type: string + quote: true + - name: datetime_ultima_atualizacao + description: "{{ doc('datetime_ultima_atualizacao') }}" + data_type: datetime + quote: true From 167db6104c58a6075a744abef21fbb964d1c52e3 Mon Sep 17 00:00:00 2001 From: Rafael Date: Thu, 7 Nov 2024 10:41:29 -0300 Subject: [PATCH 28/71] add colunas de controle --- queries/models/monitoramento/viagem_validacao.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/queries/models/monitoramento/viagem_validacao.sql b/queries/models/monitoramento/viagem_validacao.sql index 1a2202102..190ad9ce4 100644 --- a/queries/models/monitoramento/viagem_validacao.sql +++ b/queries/models/monitoramento/viagem_validacao.sql @@ -90,5 +90,7 @@ select indice_validacao >= {{ parametro_validacao }} as indicador_viagem_valida, {{ parametro_validacao }} as parametro_validacao, feed_version, - feed_start_date + feed_start_date, + '{{ var("version") }}' as versao, + current_datetime("America/Sao_Paulo") as datetime_ultima_atualizacao from indice From 4b8aa10bc8a8aa3afe38bd761f77ceea2707a4d7 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 8 Nov 2024 09:25:06 -0300 Subject: [PATCH 29/71] cria selectors viagens 2 --- queries/selectors.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/queries/selectors.yml b/queries/selectors.yml index 6d28a8cbe..bc2e6f614 100644 --- a/queries/selectors.yml +++ b/queries/selectors.yml @@ -30,4 +30,22 @@ selectors: definition: method: file value: viagem_informada_monitoramento - parents: true \ No newline at end of file + parents: true + + - name: viagem_validacao + description: Materialização da tabela de validação das viagens informadas + definition: + union: + - method: fqn + value: gps_viagem + - method: fqn + value: gps_segmento_viagem + - method: fqn + value: viagem_validacao + + - name: planejamento_diario + description: Materialização das tabelas de planejamento que devem rodar diariamente + definition: + union: + - method: fqn + value: calendario From 30719b5ee0c53f896d60b7817d412f3d127cc959 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 8 Nov 2024 09:25:25 -0300 Subject: [PATCH 30/71] add segmento_shape e shape_geom --- queries/models/planejamento/schema.yml | 96 +++++++++++++++++++++++++- 1 file changed, 95 insertions(+), 1 deletion(-) diff --git a/queries/models/planejamento/schema.yml b/queries/models/planejamento/schema.yml index a352dc462..e5340d3c9 100644 --- a/queries/models/planejamento/schema.yml +++ b/queries/models/planejamento/schema.yml @@ -15,4 +15,98 @@ models: - name: partidas description: "{{ doc('partidas') }}" - name: quilometragem - description: "{{ doc('quilometragem') }}" \ No newline at end of file + description: "{{ doc('quilometragem') }}" + - name: segmento_shape + description: Tabela contendo os shapes segmentados usados na validação de viagens. + columns: + - name: feed_version + description: String que indica a versão atual do conjunto de dados GTFS. + data_type: string + quote: true + - name: feed_start_date + description: Partição) Data inicial do feed (versão). + data_type: date + quote: true + - name: feed_end_date + description: Data final do feed (versão). + data_type: date + quote: true + - name: shape_id + description: Identificador de shape. + data_type: string + quote: true + - name: id_segmento + description: Identificador do segmento. + data_type: string + quote: true + - name: segmento + description: Segmento em formato geográfico. + data_type: geography + quote: true + - name: wkt_segmento + description: Segmento em formato string WKT + data_type: string + quote: true + - name: comprimento_segmento + description: Comprimento do segmento emm metros. + data_type: float64 + quote: true + - name: buffer_completo + description: Área de 20m ao redor do segmento. + data_type: geography + quote: true + - name: buffer + description: Área de 20m ao redor do segmento com tratamento para não haver interseções entre ele e os segmentos posteriores. + data_type: geography + quote: true + - name: indicador_tunel + description: Indica se o segmento passa por um túnel. + data_type: boolean + quote: true + - name: indicador_area_prejudicada + description: Indica se o buffer teve a área reduzida em mais de 50% após o tratamento. + data_type: boolean + quote: true + - name: indicador_segmento_pequeno + description: Indica se o segmento é menor que 990 metros. + data_type: boolean + quote: true + - name: indicador_segmento_desconsiderado + description: Indica se o segmento deve ser desconsiderado na validação de viagens + data_type: boolean + quote: true + - name: versao + description: "{{ doc('versao') }}" + data_type: string + quote: true + - name: shapes_geom_planejamento + description: Tabela contendo shapes em formatos geográficos + columns: + - name: feed_version + description: String que indica a versão atual do conjunto de dados GTFS. + data_type: string + quote: true + - name: feed_start_date + description: (Partição) Data inicial do feed (versão). + data_type: date + quote: true + - name: feed_end_date + description: Data final do feed (versão). + data_type: date + quote: true + - name: shape_id + description: Identificador de shape. + data_type: string + quote: true + - name: shape + description: Shape em formato geográfico (usualmente LineString). + data_type: geography + quote: true + - name: wkt_shape + description: Shape em formato string WKT + data_type: string + quote: true + - name: versao + description: "{{ doc('versao') }}" + data_type: string + quote: true From 58c5029a0c55f4f9d0ea1c14848ff1b9ea00de0d Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 8 Nov 2024 09:25:45 -0300 Subject: [PATCH 31/71] add tipo_dia --- .../models/monitoramento/staging/gps_segmento_viagem.sql | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/queries/models/monitoramento/staging/gps_segmento_viagem.sql b/queries/models/monitoramento/staging/gps_segmento_viagem.sql index f1e9fa509..1a5c70144 100644 --- a/queries/models/monitoramento/staging/gps_segmento_viagem.sql +++ b/queries/models/monitoramento/staging/gps_segmento_viagem.sql @@ -11,9 +11,9 @@ }} {% set incremental_filter %} - data between - date('{{ var("date_range_start") }}') - and date('{{ var("date_range_end") }}') + data between + date('{{ var("date_range_start") }}') + and date('{{ var("date_range_end") }}') {% endset %} {% set calendario = ref("calendario") %} @@ -95,6 +95,7 @@ with v.shape_id, v.servico, v.sentido, + c.tipo_dia, c.feed_start_date, c.feed_version from {{ ref("viagem_informada_monitoramento") }} v @@ -116,6 +117,7 @@ with s.indicador_segmento_desconsiderado, v.servico, v.sentido, + v.tipo_dia, feed_version, feed_start_date from viagem v @@ -138,6 +140,7 @@ select ifnull(g.quantidade_gps, 0) as quantidade_gps, v.feed_version, v.feed_start_date, + v.tipo_dia, '{{ var("version") }}' as versao, current_datetime("America/Sao_Paulo") as datetime_ultima_atualizacao from viagem_segmento v From ae852598bbbbf32755c88608e07e737e4d2af4c8 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 8 Nov 2024 09:26:28 -0300 Subject: [PATCH 32/71] add modo --- queries/models/monitoramento/staging/gps_viagem.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/queries/models/monitoramento/staging/gps_viagem.sql b/queries/models/monitoramento/staging/gps_viagem.sql index 96d4df280..afb5879a4 100644 --- a/queries/models/monitoramento/staging/gps_viagem.sql +++ b/queries/models/monitoramento/staging/gps_viagem.sql @@ -17,6 +17,7 @@ with id_viagem, datetime_partida, datetime_chegada, + modo, id_veiculo, trip_id, route_id, From 45b2b6b4991a9c9d097943a5ea6c8cbfd1b399b2 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 8 Nov 2024 09:26:45 -0300 Subject: [PATCH 33/71] =?UTF-8?q?add=20indicador=20de=20servi=C3=A7o=20nao?= =?UTF-8?q?=20planejado?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../viagem_informada_monitoramento.sql | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/queries/models/monitoramento/viagem_informada_monitoramento.sql b/queries/models/monitoramento/viagem_informada_monitoramento.sql index 8e861ae04..ec04cbf22 100644 --- a/queries/models/monitoramento/viagem_informada_monitoramento.sql +++ b/queries/models/monitoramento/viagem_informada_monitoramento.sql @@ -28,13 +28,16 @@ {% set partitions = run_query(partitions_query).columns[0].values() %} - {% set gtfs_feeds_query %} + {% if partitions | length > 0 %} + {% set gtfs_feeds_query %} select distinct concat("'", feed_start_date, "'") as feed_start_date from {{ calendario }} - where {{ incremental_filter }} - {% endset %} + where data in ({{ partitions | join(", ") }}) + {% endset %} - {% set gtfs_feeds = run_query(gtfs_feeds_query).columns[0].values() %} + {% set gtfs_feeds = run_query(gtfs_feeds_query).columns[0].values() %} + {% else %} {% set gtfs_feeds = [] %} + {% endif %} {% endif %} {% endif %} @@ -121,7 +124,7 @@ with v.datetime_captura from deduplicado v join calendario c using (data) - left join routes r using (route_id, feed_start_date, feed_info) + left join routes r using (route_id, feed_start_date, feed_version) ) select *, From 6da7e1bc27a686895cf83f691cbf05938eeb6324 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 8 Nov 2024 09:27:13 -0300 Subject: [PATCH 34/71] adiciona indicador de viagem nao planejada --- .../models/monitoramento/viagem_validacao.sql | 49 ++++++++++++++++--- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/queries/models/monitoramento/viagem_validacao.sql b/queries/models/monitoramento/viagem_validacao.sql index 190ad9ce4..96aee7b97 100644 --- a/queries/models/monitoramento/viagem_validacao.sql +++ b/queries/models/monitoramento/viagem_validacao.sql @@ -8,6 +8,13 @@ ) }} +{% set incremental_filter %} + data between + date('{{ var("date_range_start") }}') + and date('{{ var("date_range_end") }}') +{% endset %} + + {% set parametro_validacao = 0.9 %} with @@ -26,16 +33,13 @@ with sentido, count(*) as quantidade_segmentos_verificados, countif(quantidade_gps > 0) as quantidade_segmentos_validos, + tipo_dia, feed_version, feed_start_date from {{ ref("gps_segmento_viagem") }} where not indicador_segmento_desconsiderado - {% if is_incremental() %} - and data between date('{{ var("date_range_start") }}') and date( - '{{ var("date_range_end") }}' - ) - {% endif %} + {% if is_incremental() %} {{ incremental_filter }} {% endif %} group by data, id_viagem, @@ -48,6 +52,7 @@ with shape_id, servico, sentido, + tipo_dia, feed_version, feed_start_date ), @@ -68,9 +73,35 @@ with quantidade_segmentos_validos, quantidade_segmentos_validos / quantidade_segmentos_verificados as indice_validacao, + tipo_dia, feed_version, feed_start_date from contagem + ), + trips as ( + select distinct + feed_start_date, + feed_version, + route_id, + case + when service_id like "D_%" + then "Domingo" + when service_id like "S_%" + then "Sabado" + when service_id like "U_%" + then "Dia Útil" + end as tipo_dia + from {{ ref("trips_gtfs") }} + where + (service_id like "D_%" or service_id like "S_%" or service_id like "U_%") + {% if is_incremental() %} + and feed_start_date in ({{ gtfs_feeds | join(", ") }}) + {% endif %} + ), + servicos_planejados as ( + select i.*, t.tipo_dia is not null as indicador_servico_planejado + from indice i + left join trips t using (feed_start_date, feed_version, route_id, tipo_dia) ) select data, @@ -87,10 +118,14 @@ select quantidade_segmentos_verificados, quantidade_segmentos_validos, indice_validacao, - indice_validacao >= {{ parametro_validacao }} as indicador_viagem_valida, + indice_validacao >= {{ parametro_validacao }} as indicador_trajeto_valido, + indicador_servico_planejado, + indice_validacao >= {{ parametro_validacao }} + and indicador_servico_planejado as indicador_viagem_valida, {{ parametro_validacao }} as parametro_validacao, + tipo_dia, feed_version, feed_start_date, '{{ var("version") }}' as versao, current_datetime("America/Sao_Paulo") as datetime_ultima_atualizacao -from indice +from servicos_planejados From a6a320142a7d7a2c1d4e6d6144406617a12bd23e Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 8 Nov 2024 09:27:41 -0300 Subject: [PATCH 35/71] altera data inicial --- queries/models/planejamento/calendario.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/queries/models/planejamento/calendario.sql b/queries/models/planejamento/calendario.sql index 570f81380..b51004930 100644 --- a/queries/models/planejamento/calendario.sql +++ b/queries/models/planejamento/calendario.sql @@ -55,7 +55,7 @@ with {% if is_incremental() %} date("{{ var('date_range_start') }}"), date("{{ var('date_range_end') }}") - {% else %}date("2024-09-01"), current_date("America/Sao_Paulo") + {% else %}date("2024-10-12"), current_date("America/Sao_Paulo") {% endif %} ) ) as data @@ -96,7 +96,7 @@ with cd.exception_type, cd.feed_start_date, {# from `rj-smtr.gtfs.calendar_dates` cd #} - from {{ ref("calendar_gtfs") }} cd + from {{ ref("calendar_dates_gtfs") }} cd join modificacao_manual m on cd.date = m.data @@ -181,7 +181,7 @@ select when "D_REG" in unnest(c.service_ids) then "Domingo" when "S_REG" in unnest(c.service_ids) - then "Sábado" + then "Sabado" when "U_REG" in unnest(c.service_ids) then "Dia Útil" end as tipo_dia, From 570a02e8bb900c6bf60bf0b6073a90a547593948 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 8 Nov 2024 09:27:56 -0300 Subject: [PATCH 36/71] altera ordem colunas --- queries/models/planejamento/segmento_shape.sql | 4 ++-- queries/models/planejamento/shapes_geom_planejamento.sql | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/queries/models/planejamento/segmento_shape.sql b/queries/models/planejamento/segmento_shape.sql index 79618eff4..d482fcff6 100644 --- a/queries/models/planejamento/segmento_shape.sql +++ b/queries/models/planejamento/segmento_shape.sql @@ -17,9 +17,9 @@ with aux_segmento as ( select - feed_version, feed_start_date, feed_end_date, + feed_version, shape_id, id_segmento, st_geogfromtext(wkt_segmento) as segmento, @@ -94,9 +94,9 @@ from indicador_validacao_shape union all select - s.feed_version, s.feed_start_date, fi.feed_end_date, + s.feed_version, s.shape_id, s.id_segmento, s.segmento, diff --git a/queries/models/planejamento/shapes_geom_planejamento.sql b/queries/models/planejamento/shapes_geom_planejamento.sql index 0c61d8e0f..46b976607 100644 --- a/queries/models/planejamento/shapes_geom_planejamento.sql +++ b/queries/models/planejamento/shapes_geom_planejamento.sql @@ -34,9 +34,9 @@ with {% endif %} ) select - feed_version, feed_start_date, feed_end_date, + feed_version, shape_id, st_makeline(array_agg(ponto_shape order by shape_pt_sequence)) as shape, concat( From 776b4281fd3c29b848cd9f4c7c27b1fb2b556c25 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 8 Nov 2024 09:28:24 -0300 Subject: [PATCH 37/71] cria pipeline validacao viagens --- .../treatment/monitoramento/CHANGELOG.md | 6 ++++ .../treatment/monitoramento/constants.py | 9 +++++- pipelines/treatment/monitoramento/flows.py | 29 +++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/pipelines/treatment/monitoramento/CHANGELOG.md b/pipelines/treatment/monitoramento/CHANGELOG.md index ee739f0c8..45d2e2800 100644 --- a/pipelines/treatment/monitoramento/CHANGELOG.md +++ b/pipelines/treatment/monitoramento/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog - monitoramento +## [1.1.0] - 2024-11-08 + +### Adicionado + +- Cria flow de tratamento de validação de viagens informadas (https://github.com/prefeitura-rio/pipelines_rj_smtr/pull/237) + ## [1.0.0] - 2024-10-21 ### Adicionado diff --git a/pipelines/treatment/monitoramento/constants.py b/pipelines/treatment/monitoramento/constants.py index 50873ecb2..50ade2a2e 100644 --- a/pipelines/treatment/monitoramento/constants.py +++ b/pipelines/treatment/monitoramento/constants.py @@ -6,7 +6,7 @@ from datetime import datetime from enum import Enum -from pipelines.schedules import cron_every_day_hour_7_minute_10 +from pipelines.schedules import cron_every_day_hour_6, cron_every_day_hour_7_minute_10 from pipelines.treatment.templates.utils import DBTSelector @@ -20,3 +20,10 @@ class constants(Enum): # pylint: disable=c0103 schedule_cron=cron_every_day_hour_7_minute_10, initial_datetime=datetime(2024, 10, 16, 0, 0, 0), ) + + VIAGEM_VALIDACAO_SELECTOR = DBTSelector( + name="viagem_validacao", + schedule_cron=cron_every_day_hour_6, + initial_datetime=datetime(2024, 10, 12, 0, 0, 0), + incremental_delay_hours=48, + ) diff --git a/pipelines/treatment/monitoramento/flows.py b/pipelines/treatment/monitoramento/flows.py index 80e591435..10a3d3c8f 100644 --- a/pipelines/treatment/monitoramento/flows.py +++ b/pipelines/treatment/monitoramento/flows.py @@ -9,6 +9,10 @@ constants as rioonibus_source_constants, ) from pipelines.constants import constants as smtr_constants +from pipelines.migration.br_rj_riodejaneiro_onibus_gps_zirix.constants import ( + constants as gps_zirix_constants, +) +from pipelines.schedules import cron_every_hour_minute_6 from pipelines.treatment.monitoramento.constants import constants from pipelines.treatment.templates.flows import create_default_materialization_flow @@ -18,3 +22,28 @@ agent_label=smtr_constants.RJ_SMTR_AGENT_LABEL.value, wait=[rioonibus_source_constants.VIAGEM_INFORMADA_SOURCE.value], ) + +VIAGEM_VALIDACAO_MATERIALIZACAO = create_default_materialization_flow( + flow_name="viagem_validacao - materializacao", + selector=constants.VIAGEM_VALIDACAO_SELECTOR.value, + agent_label=smtr_constants.RJ_SMTR_AGENT_LABEL.value, + wait=[ + VIAGEM_INFORMADA_MATERIALIZACAO, + { + "redis_key": f"{smtr_constants.GPS_SPPO_DATASET_ID.value}\ +.{smtr_constants.GPS_SPPO_TREATED_TABLE_ID.value}", + "dict_key": "last_run_timestamp", + "datetime_format": "%Y-%m-%dT%H:%M:%S", + "delay_hours": smtr_constants.GPS_SPPO_MATERIALIZE_DELAY_HOURS.value, + "schedule_cron": cron_every_hour_minute_6, + }, + { + "redis_key": f"{gps_zirix_constants.GPS_SPPO_ZIRIX_RAW_DATASET_ID.value}\ +.{smtr_constants.GPS_SPPO_TREATED_TABLE_ID.value}", + "dict_key": "last_run_timestamp", + "datetime_format": "%Y-%m-%dT%H:%M:%S", + "delay_hours": smtr_constants.GPS_SPPO_MATERIALIZE_DELAY_HOURS.value, + "schedule_cron": cron_every_hour_minute_6, + }, + ], +) From c882c1d7fb3d0b7aaaf55f2325147659d4972be7 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 8 Nov 2024 09:28:50 -0300 Subject: [PATCH 38/71] cria pipeline diario de planejamento --- pipelines/treatment/planejamento/CHANGELOG.md | 7 ++++++ pipelines/treatment/planejamento/__init__.py | 0 pipelines/treatment/planejamento/constants.py | 22 +++++++++++++++++++ pipelines/treatment/planejamento/flows.py | 14 ++++++++++++ 4 files changed, 43 insertions(+) create mode 100644 pipelines/treatment/planejamento/CHANGELOG.md create mode 100644 pipelines/treatment/planejamento/__init__.py create mode 100644 pipelines/treatment/planejamento/constants.py create mode 100644 pipelines/treatment/planejamento/flows.py diff --git a/pipelines/treatment/planejamento/CHANGELOG.md b/pipelines/treatment/planejamento/CHANGELOG.md new file mode 100644 index 000000000..0d7fecba7 --- /dev/null +++ b/pipelines/treatment/planejamento/CHANGELOG.md @@ -0,0 +1,7 @@ +# Changelog - planejamento + +## [1.0.0] - 2024-11-08 + +### Adicionado + +- Cria flow de tratamento diário de dados de planejamento (https://github.com/prefeitura-rio/pipelines_rj_smtr/pull/237) \ No newline at end of file diff --git a/pipelines/treatment/planejamento/__init__.py b/pipelines/treatment/planejamento/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pipelines/treatment/planejamento/constants.py b/pipelines/treatment/planejamento/constants.py new file mode 100644 index 000000000..641bf94da --- /dev/null +++ b/pipelines/treatment/planejamento/constants.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +""" +Valores constantes para materialização dos dados de planejamento +""" + +from datetime import datetime +from enum import Enum + +from pipelines.schedules import cron_every_day_hour_1 +from pipelines.treatment.templates.utils import DBTSelector + + +class constants(Enum): # pylint: disable=c0103 + """ + Valores constantes para materialização dos dados de planejamento + """ + + PLANEJAMENTO_DIARIO_SELECTOR = DBTSelector( + name="planejamento_diario", + schedule_cron=cron_every_day_hour_1, + initial_datetime=datetime(2024, 9, 1, 0, 0, 0), + ) diff --git a/pipelines/treatment/planejamento/flows.py b/pipelines/treatment/planejamento/flows.py new file mode 100644 index 000000000..c44e078a3 --- /dev/null +++ b/pipelines/treatment/planejamento/flows.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +""" +Flows de tratamento dos dados de planejamento +""" + +from pipelines.constants import constants as smtr_constants +from pipelines.treatment.planejamento.constants import constants +from pipelines.treatment.templates.flows import create_default_materialization_flow + +PLANEJAMENTO_DIARIO_MATERIALIZACAO = create_default_materialization_flow( + flow_name="planejamento_diario - materializacao", + selector=constants.PLANEJAMENTO_DIARIO_SELECTOR.value, + agent_label=smtr_constants.RJ_SMTR_AGENT_LABEL.value, +) From 7cd7f0426827cdbdeb1675b19bd2067c38ecfde4 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 8 Nov 2024 09:29:11 -0300 Subject: [PATCH 39/71] =?UTF-8?q?adiciona=20integra=C3=A7=C3=A3o=20com=20t?= =?UTF-8?q?abelas=20antigas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipelines/treatment/templates/CHANGELOG.md | 7 +++++++ pipelines/treatment/templates/tasks.py | 23 +++++++++++++++++++--- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/pipelines/treatment/templates/CHANGELOG.md b/pipelines/treatment/templates/CHANGELOG.md index 3050b5d77..140c1a162 100644 --- a/pipelines/treatment/templates/CHANGELOG.md +++ b/pipelines/treatment/templates/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog - treatment +## [1.0.1] - 2024-11-08 + +### Alterado + +- Adiciona lógica para verificar fontes de dados no padrão antigo (https://github.com/prefeitura-rio/pipelines_rj_smtr/pull/237) + + ## [1.0.0] - 2024-10-21 ### Adicionado diff --git a/pipelines/treatment/templates/tasks.py b/pipelines/treatment/templates/tasks.py index f31ec1045..6de89e83d 100644 --- a/pipelines/treatment/templates/tasks.py +++ b/pipelines/treatment/templates/tasks.py @@ -7,6 +7,7 @@ import requests from prefect import task from prefeitura_rio.pipelines_utils.logging import log +from prefeitura_rio.pipelines_utils.redis_pal import get_redis_client from pytz import timezone from pipelines.constants import constants @@ -19,7 +20,7 @@ from pipelines.utils.dataplex import DataQuality, DataQualityCheckArgs from pipelines.utils.gcp.bigquery import SourceTable from pipelines.utils.prefect import flow_is_running_local, rename_current_flow_run -from pipelines.utils.utils import convert_timezone +from pipelines.utils.utils import convert_timezone, cron_get_last_date # from pipelines.utils.utils import get_last_materialization_redis_key @@ -119,7 +120,7 @@ def wait_data_sources( env: str, datetime_start: datetime, datetime_end: datetime, - data_sources: list[Union[SourceTable, DBTSelector]], + data_sources: list[Union[SourceTable, DBTSelector, dict]], skip: bool, ): """ @@ -129,7 +130,7 @@ def wait_data_sources( env (str): prod ou dev datetime_start (datetime): Datetime inicial da materialização datetime_end (datetime): Datetime final da materialização - data_sources (list[Union[SourceTable, DBTSelector]]): Fontes de dados para esperar + data_sources (list[Union[SourceTable, DBTSelector, dict]]): Fontes de dados para esperar skip (bool): se a verificação deve ser pulada ou não """ if skip: @@ -150,6 +151,22 @@ def wait_data_sources( elif isinstance(ds, DBTSelector): name = f"{ds.name}" complete = ds.is_up_to_date(env=env, timestamp=datetime_end) + elif isinstance(ds, dict): + # source dicionário utilizado para compatibilização com flows antigos + name = ds["redis_key"] + redis_client = get_redis_client() + last_materialization = datetime.strptime( + redis_client.get(name)[ds["dict_key"]], + ds["datetime_format"], + ) + last_schedule = cron_get_last_date( + cron_expr=ds["schedule_cron"], + timestamp=datetime_end, + ) + complete = last_materialization >= last_schedule - timedelta( + hours=ds.get("delay_hours", 0) + ) + else: raise NotImplementedError(f"Espera por fontes do tipo {type(ds)} não implementada") From 4275b7fa6c22a29af44b3f184204773a1bf087d2 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 8 Nov 2024 09:29:22 -0300 Subject: [PATCH 40/71] add novos crons --- pipelines/schedules.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pipelines/schedules.py b/pipelines/schedules.py index b48c594d2..a8970d5c1 100644 --- a/pipelines/schedules.py +++ b/pipelines/schedules.py @@ -12,8 +12,11 @@ from pipelines.constants import constants from pipelines.constants import constants as emd_constants +cron_every_day_hour_1 = "0 1 * * *" +cron_every_day_hour_6 = "0 6 * * *" cron_every_day_hour_7 = "0 7 * * *" cron_every_day_hour_7_minute_10 = "10 7 * * *" +cron_every_hour_minute_6 = "6 * * * *" def generate_interval_schedule( From 82537ecd6500545e1481e997d6b1f496846eaac2 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 8 Nov 2024 09:29:36 -0300 Subject: [PATCH 41/71] importa flows planejamento --- pipelines/flows.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pipelines/flows.py b/pipelines/flows.py index 109e40f80..e84559977 100644 --- a/pipelines/flows.py +++ b/pipelines/flows.py @@ -21,3 +21,4 @@ from pipelines.migration.veiculo.flows import * # noqa from pipelines.serpro.flows import * # noqa from pipelines.treatment.monitoramento.flows import * # noqa +from pipelines.treatment.planejamento.flows import * # noqa From 19423b26de76a983f5e96cd0c2843c5b8583d0ce Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 8 Nov 2024 09:34:16 -0300 Subject: [PATCH 42/71] altera target para hmg --- queries/profiles.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/queries/profiles.yml b/queries/profiles.yml index e67088d2d..436d137af 100644 --- a/queries/profiles.yml +++ b/queries/profiles.yml @@ -34,7 +34,7 @@ queries: location: us method: service-account priority: interactive - project: rj-smtr-dev + project: rj-smtr threads: 1 type: bigquery @@ -78,4 +78,4 @@ queries: spark.executor.instances: "2" spark.driver.memory: 4g spark.driver.memoryOverhead: 1g - target: prod \ No newline at end of file + target: hmg \ No newline at end of file From bc021a6fbeaaba29415750dd66d7e3136b6a5fb4 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 8 Nov 2024 10:10:43 -0300 Subject: [PATCH 43/71] ajusta lock --- poetry.lock | 509 +++++++++++++++++++++++++++------------------------- 1 file changed, 267 insertions(+), 242 deletions(-) diff --git a/poetry.lock b/poetry.lock index 2d6363b94..ce6414a5a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -25,13 +25,13 @@ test = ["PyICU (>=2.4.2)", "coverage (>=3.7.1)", "cssselect (>=0.9.1)", "lxml (> [[package]] name = "async-timeout" -version = "4.0.3" +version = "5.0.1" description = "Timeout context manager for asyncio programs" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, - {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, + {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, + {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, ] [[package]] @@ -548,13 +548,13 @@ typing-extensions = ">=4.4,<5.0" [[package]] name = "dill" -version = "0.3.8" +version = "0.3.9" description = "serialize all of Python" optional = false python-versions = ">=3.8" files = [ - {file = "dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7"}, - {file = "dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca"}, + {file = "dill-0.3.9-py3-none-any.whl", hash = "sha256:468dff3b89520b474c0397703366b7b95eebe6303f108adf9b19da1f702be87a"}, + {file = "dill-0.3.9.tar.gz", hash = "sha256:81aa267dddf68cbfe8029c42ca9ec6a4ab3b22371d1c450abc54422577b4512c"}, ] [package.extras] @@ -563,13 +563,13 @@ profile = ["gprof2dot (>=2022.7.29)"] [[package]] name = "distlib" -version = "0.3.8" +version = "0.3.9" description = "Distribution utilities" optional = false python-versions = "*" files = [ - {file = "distlib-0.3.8-py2.py3-none-any.whl", hash = "sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784"}, - {file = "distlib-0.3.8.tar.gz", hash = "sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64"}, + {file = "distlib-0.3.9-py2.py3-none-any.whl", hash = "sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87"}, + {file = "distlib-0.3.9.tar.gz", hash = "sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403"}, ] [[package]] @@ -623,29 +623,29 @@ websockets = ["websocket-client (>=1.3.0)"] [[package]] name = "et-xmlfile" -version = "1.1.0" +version = "2.0.0" description = "An implementation of lxml.xmlfile for the standard library" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "et_xmlfile-1.1.0-py3-none-any.whl", hash = "sha256:a2ba85d1d6a74ef63837eed693bcb89c3f752169b0e3e7ae5b16ca5e1b3deada"}, - {file = "et_xmlfile-1.1.0.tar.gz", hash = "sha256:8eb9e2bc2f8c97e37a2dc85a09ecdcdec9d8a396530a6d5a33b30b9a92da0c5c"}, + {file = "et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa"}, + {file = "et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54"}, ] [[package]] name = "filelock" -version = "3.16.0" +version = "3.16.1" description = "A platform independent file lock." optional = false python-versions = ">=3.8" files = [ - {file = "filelock-3.16.0-py3-none-any.whl", hash = "sha256:f6ed4c963184f4c84dd5557ce8fece759a3724b37b80c6c4f20a2f63a4dc6609"}, - {file = "filelock-3.16.0.tar.gz", hash = "sha256:81de9eb8453c769b63369f87f11131a7ab04e367f8d97ad39dc230daa07e3bec"}, + {file = "filelock-3.16.1-py3-none-any.whl", hash = "sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0"}, + {file = "filelock-3.16.1.tar.gz", hash = "sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435"}, ] [package.extras] -docs = ["furo (>=2024.8.6)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4)"] -testing = ["covdefaults (>=2.3)", "coverage (>=7.6.1)", "diff-cover (>=9.1.1)", "pytest (>=8.3.2)", "pytest-asyncio (>=0.24)", "pytest-cov (>=5)", "pytest-mock (>=3.14)", "pytest-timeout (>=2.3.1)", "virtualenv (>=20.26.3)"] +docs = ["furo (>=2024.8.6)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4.1)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.6.1)", "diff-cover (>=9.2)", "pytest (>=8.3.3)", "pytest-asyncio (>=0.24)", "pytest-cov (>=5)", "pytest-mock (>=3.14)", "pytest-timeout (>=2.3.1)", "virtualenv (>=20.26.4)"] typing = ["typing-extensions (>=4.12.2)"] [[package]] @@ -1095,69 +1095,84 @@ files = [ [[package]] name = "greenlet" -version = "3.0.3" +version = "3.1.1" description = "Lightweight in-process concurrent programming" optional = false python-versions = ">=3.7" files = [ - {file = "greenlet-3.0.3-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:9da2bd29ed9e4f15955dd1595ad7bc9320308a3b766ef7f837e23ad4b4aac31a"}, - {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d353cadd6083fdb056bb46ed07e4340b0869c305c8ca54ef9da3421acbdf6881"}, - {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dca1e2f3ca00b84a396bc1bce13dd21f680f035314d2379c4160c98153b2059b"}, - {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3ed7fb269f15dc662787f4119ec300ad0702fa1b19d2135a37c2c4de6fadfd4a"}, - {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd4f49ae60e10adbc94b45c0b5e6a179acc1736cf7a90160b404076ee283cf83"}, - {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:73a411ef564e0e097dbe7e866bb2dda0f027e072b04da387282b02c308807405"}, - {file = "greenlet-3.0.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7f362975f2d179f9e26928c5b517524e89dd48530a0202570d55ad6ca5d8a56f"}, - {file = "greenlet-3.0.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:649dde7de1a5eceb258f9cb00bdf50e978c9db1b996964cd80703614c86495eb"}, - {file = "greenlet-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:68834da854554926fbedd38c76e60c4a2e3198c6fbed520b106a8986445caaf9"}, - {file = "greenlet-3.0.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:b1b5667cced97081bf57b8fa1d6bfca67814b0afd38208d52538316e9422fc61"}, - {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52f59dd9c96ad2fc0d5724107444f76eb20aaccb675bf825df6435acb7703559"}, - {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:afaff6cf5200befd5cec055b07d1c0a5a06c040fe5ad148abcd11ba6ab9b114e"}, - {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe754d231288e1e64323cfad462fcee8f0288654c10bdf4f603a39ed923bef33"}, - {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2797aa5aedac23af156bbb5a6aa2cd3427ada2972c828244eb7d1b9255846379"}, - {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7f009caad047246ed379e1c4dbcb8b020f0a390667ea74d2387be2998f58a22"}, - {file = "greenlet-3.0.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c5e1536de2aad7bf62e27baf79225d0d64360d4168cf2e6becb91baf1ed074f3"}, - {file = "greenlet-3.0.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:894393ce10ceac937e56ec00bb71c4c2f8209ad516e96033e4b3b1de270e200d"}, - {file = "greenlet-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:1ea188d4f49089fc6fb283845ab18a2518d279c7cd9da1065d7a84e991748728"}, - {file = "greenlet-3.0.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:70fb482fdf2c707765ab5f0b6655e9cfcf3780d8d87355a063547b41177599be"}, - {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4d1ac74f5c0c0524e4a24335350edad7e5f03b9532da7ea4d3c54d527784f2e"}, - {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:149e94a2dd82d19838fe4b2259f1b6b9957d5ba1b25640d2380bea9c5df37676"}, - {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:15d79dd26056573940fcb8c7413d84118086f2ec1a8acdfa854631084393efcc"}, - {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:881b7db1ebff4ba09aaaeae6aa491daeb226c8150fc20e836ad00041bcb11230"}, - {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fcd2469d6a2cf298f198f0487e0a5b1a47a42ca0fa4dfd1b6862c999f018ebbf"}, - {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1f672519db1796ca0d8753f9e78ec02355e862d0998193038c7073045899f305"}, - {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2516a9957eed41dd8f1ec0c604f1cdc86758b587d964668b5b196a9db5bfcde6"}, - {file = "greenlet-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:bba5387a6975598857d86de9eac14210a49d554a77eb8261cc68b7d082f78ce2"}, - {file = "greenlet-3.0.3-cp37-cp37m-macosx_11_0_universal2.whl", hash = "sha256:5b51e85cb5ceda94e79d019ed36b35386e8c37d22f07d6a751cb659b180d5274"}, - {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:daf3cb43b7cf2ba96d614252ce1684c1bccee6b2183a01328c98d36fcd7d5cb0"}, - {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99bf650dc5d69546e076f413a87481ee1d2d09aaaaaca058c9251b6d8c14783f"}, - {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dd6e660effd852586b6a8478a1d244b8dc90ab5b1321751d2ea15deb49ed414"}, - {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3391d1e16e2a5a1507d83e4a8b100f4ee626e8eca43cf2cadb543de69827c4c"}, - {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1f145462f1fa6e4a4ae3c0f782e580ce44d57c8f2c7aae1b6fa88c0b2efdb41"}, - {file = "greenlet-3.0.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1a7191e42732df52cb5f39d3527217e7ab73cae2cb3694d241e18f53d84ea9a7"}, - {file = "greenlet-3.0.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0448abc479fab28b00cb472d278828b3ccca164531daab4e970a0458786055d6"}, - {file = "greenlet-3.0.3-cp37-cp37m-win32.whl", hash = "sha256:b542be2440edc2d48547b5923c408cbe0fc94afb9f18741faa6ae970dbcb9b6d"}, - {file = "greenlet-3.0.3-cp37-cp37m-win_amd64.whl", hash = "sha256:01bc7ea167cf943b4c802068e178bbf70ae2e8c080467070d01bfa02f337ee67"}, - {file = "greenlet-3.0.3-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:1996cb9306c8595335bb157d133daf5cf9f693ef413e7673cb07e3e5871379ca"}, - {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ddc0f794e6ad661e321caa8d2f0a55ce01213c74722587256fb6566049a8b04"}, - {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c9db1c18f0eaad2f804728c67d6c610778456e3e1cc4ab4bbd5eeb8e6053c6fc"}, - {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7170375bcc99f1a2fbd9c306f5be8764eaf3ac6b5cb968862cad4c7057756506"}, - {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b66c9c1e7ccabad3a7d037b2bcb740122a7b17a53734b7d72a344ce39882a1b"}, - {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:098d86f528c855ead3479afe84b49242e174ed262456c342d70fc7f972bc13c4"}, - {file = "greenlet-3.0.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:81bb9c6d52e8321f09c3d165b2a78c680506d9af285bfccbad9fb7ad5a5da3e5"}, - {file = "greenlet-3.0.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fd096eb7ffef17c456cfa587523c5f92321ae02427ff955bebe9e3c63bc9f0da"}, - {file = "greenlet-3.0.3-cp38-cp38-win32.whl", hash = "sha256:d46677c85c5ba00a9cb6f7a00b2bfa6f812192d2c9f7d9c4f6a55b60216712f3"}, - {file = "greenlet-3.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:419b386f84949bf0e7c73e6032e3457b82a787c1ab4a0e43732898a761cc9dbf"}, - {file = "greenlet-3.0.3-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:da70d4d51c8b306bb7a031d5cff6cc25ad253affe89b70352af5f1cb68e74b53"}, - {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:086152f8fbc5955df88382e8a75984e2bb1c892ad2e3c80a2508954e52295257"}, - {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d73a9fe764d77f87f8ec26a0c85144d6a951a6c438dfe50487df5595c6373eac"}, - {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7dcbe92cc99f08c8dd11f930de4d99ef756c3591a5377d1d9cd7dd5e896da71"}, - {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1551a8195c0d4a68fac7a4325efac0d541b48def35feb49d803674ac32582f61"}, - {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64d7675ad83578e3fc149b617a444fab8efdafc9385471f868eb5ff83e446b8b"}, - {file = "greenlet-3.0.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b37eef18ea55f2ffd8f00ff8fe7c8d3818abd3e25fb73fae2ca3b672e333a7a6"}, - {file = "greenlet-3.0.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:77457465d89b8263bca14759d7c1684df840b6811b2499838cc5b040a8b5b113"}, - {file = "greenlet-3.0.3-cp39-cp39-win32.whl", hash = "sha256:57e8974f23e47dac22b83436bdcf23080ade568ce77df33159e019d161ce1d1e"}, - {file = "greenlet-3.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:c5ee858cfe08f34712f548c3c363e807e7186f03ad7a5039ebadb29e8c6be067"}, - {file = "greenlet-3.0.3.tar.gz", hash = "sha256:43374442353259554ce33599da8b692d5aa96f8976d567d4badf263371fbe491"}, + {file = "greenlet-3.1.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:0bbae94a29c9e5c7e4a2b7f0aae5c17e8e90acbfd3bf6270eeba60c39fce3563"}, + {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fde093fb93f35ca72a556cf72c92ea3ebfda3d79fc35bb19fbe685853869a83"}, + {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:36b89d13c49216cadb828db8dfa6ce86bbbc476a82d3a6c397f0efae0525bdd0"}, + {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:94b6150a85e1b33b40b1464a3f9988dcc5251d6ed06842abff82e42632fac120"}, + {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93147c513fac16385d1036b7e5b102c7fbbdb163d556b791f0f11eada7ba65dc"}, + {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da7a9bff22ce038e19bf62c4dd1ec8391062878710ded0a845bcf47cc0200617"}, + {file = "greenlet-3.1.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b2795058c23988728eec1f36a4e5e4ebad22f8320c85f3587b539b9ac84128d7"}, + {file = "greenlet-3.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ed10eac5830befbdd0c32f83e8aa6288361597550ba669b04c48f0f9a2c843c6"}, + {file = "greenlet-3.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:77c386de38a60d1dfb8e55b8c1101d68c79dfdd25c7095d51fec2dd800892b80"}, + {file = "greenlet-3.1.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:e4d333e558953648ca09d64f13e6d8f0523fa705f51cae3f03b5983489958c70"}, + {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09fc016b73c94e98e29af67ab7b9a879c307c6731a2c9da0db5a7d9b7edd1159"}, + {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d5e975ca70269d66d17dd995dafc06f1b06e8cb1ec1e9ed54c1d1e4a7c4cf26e"}, + {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b2813dc3de8c1ee3f924e4d4227999285fd335d1bcc0d2be6dc3f1f6a318ec1"}, + {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e347b3bfcf985a05e8c0b7d462ba6f15b1ee1c909e2dcad795e49e91b152c383"}, + {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e8f8c9cb53cdac7ba9793c276acd90168f416b9ce36799b9b885790f8ad6c0a"}, + {file = "greenlet-3.1.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:62ee94988d6b4722ce0028644418d93a52429e977d742ca2ccbe1c4f4a792511"}, + {file = "greenlet-3.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1776fd7f989fc6b8d8c8cb8da1f6b82c5814957264d1f6cf818d475ec2bf6395"}, + {file = "greenlet-3.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:48ca08c771c268a768087b408658e216133aecd835c0ded47ce955381105ba39"}, + {file = "greenlet-3.1.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:4afe7ea89de619adc868e087b4d2359282058479d7cfb94970adf4b55284574d"}, + {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f406b22b7c9a9b4f8aa9d2ab13d6ae0ac3e85c9a809bd590ad53fed2bf70dc79"}, + {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c3a701fe5a9695b238503ce5bbe8218e03c3bcccf7e204e455e7462d770268aa"}, + {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2846930c65b47d70b9d178e89c7e1a69c95c1f68ea5aa0a58646b7a96df12441"}, + {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99cfaa2110534e2cf3ba31a7abcac9d328d1d9f1b95beede58294a60348fba36"}, + {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1443279c19fca463fc33e65ef2a935a5b09bb90f978beab37729e1c3c6c25fe9"}, + {file = "greenlet-3.1.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b7cede291382a78f7bb5f04a529cb18e068dd29e0fb27376074b6d0317bf4dd0"}, + {file = "greenlet-3.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:23f20bb60ae298d7d8656c6ec6db134bca379ecefadb0b19ce6f19d1f232a942"}, + {file = "greenlet-3.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:7124e16b4c55d417577c2077be379514321916d5790fa287c9ed6f23bd2ffd01"}, + {file = "greenlet-3.1.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:05175c27cb459dcfc05d026c4232f9de8913ed006d42713cb8a5137bd49375f1"}, + {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:935e943ec47c4afab8965954bf49bfa639c05d4ccf9ef6e924188f762145c0ff"}, + {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:667a9706c970cb552ede35aee17339a18e8f2a87a51fba2ed39ceeeb1004798a"}, + {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b8a678974d1f3aa55f6cc34dc480169d58f2e6d8958895d68845fa4ab566509e"}, + {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efc0f674aa41b92da8c49e0346318c6075d734994c3c4e4430b1c3f853e498e4"}, + {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0153404a4bb921f0ff1abeb5ce8a5131da56b953eda6e14b88dc6bbc04d2049e"}, + {file = "greenlet-3.1.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:275f72decf9932639c1c6dd1013a1bc266438eb32710016a1c742df5da6e60a1"}, + {file = "greenlet-3.1.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:c4aab7f6381f38a4b42f269057aee279ab0fc7bf2e929e3d4abfae97b682a12c"}, + {file = "greenlet-3.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:b42703b1cf69f2aa1df7d1030b9d77d3e584a70755674d60e710f0af570f3761"}, + {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1695e76146579f8c06c1509c7ce4dfe0706f49c6831a817ac04eebb2fd02011"}, + {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7876452af029456b3f3549b696bb36a06db7c90747740c5302f74a9e9fa14b13"}, + {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4ead44c85f8ab905852d3de8d86f6f8baf77109f9da589cb4fa142bd3b57b475"}, + {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8320f64b777d00dd7ccdade271eaf0cad6636343293a25074cc5566160e4de7b"}, + {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6510bf84a6b643dabba74d3049ead221257603a253d0a9873f55f6a59a65f822"}, + {file = "greenlet-3.1.1-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:04b013dc07c96f83134b1e99888e7a79979f1a247e2a9f59697fa14b5862ed01"}, + {file = "greenlet-3.1.1-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:411f015496fec93c1c8cd4e5238da364e1da7a124bcb293f085bf2860c32c6f6"}, + {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:47da355d8687fd65240c364c90a31569a133b7b60de111c255ef5b606f2ae291"}, + {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:98884ecf2ffb7d7fe6bd517e8eb99d31ff7855a840fa6d0d63cd07c037f6a981"}, + {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f1d4aeb8891338e60d1ab6127af1fe45def5259def8094b9c7e34690c8858803"}, + {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db32b5348615a04b82240cc67983cb315309e88d444a288934ee6ceaebcad6cc"}, + {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dcc62f31eae24de7f8dce72134c8651c58000d3b1868e01392baea7c32c247de"}, + {file = "greenlet-3.1.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1d3755bcb2e02de341c55b4fca7a745a24a9e7212ac953f6b3a48d117d7257aa"}, + {file = "greenlet-3.1.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:b8da394b34370874b4572676f36acabac172602abf054cbc4ac910219f3340af"}, + {file = "greenlet-3.1.1-cp37-cp37m-win32.whl", hash = "sha256:a0dfc6c143b519113354e780a50381508139b07d2177cb6ad6a08278ec655798"}, + {file = "greenlet-3.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:54558ea205654b50c438029505def3834e80f0869a70fb15b871c29b4575ddef"}, + {file = "greenlet-3.1.1-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:346bed03fe47414091be4ad44786d1bd8bef0c3fcad6ed3dee074a032ab408a9"}, + {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfc59d69fc48664bc693842bd57acfdd490acafda1ab52c7836e3fc75c90a111"}, + {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d21e10da6ec19b457b82636209cbe2331ff4306b54d06fa04b7c138ba18c8a81"}, + {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:37b9de5a96111fc15418819ab4c4432e4f3c2ede61e660b1e33971eba26ef9ba"}, + {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ef9ea3f137e5711f0dbe5f9263e8c009b7069d8a1acea822bd5e9dae0ae49c8"}, + {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:85f3ff71e2e60bd4b4932a043fbbe0f499e263c628390b285cb599154a3b03b1"}, + {file = "greenlet-3.1.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:95ffcf719966dd7c453f908e208e14cde192e09fde6c7186c8f1896ef778d8cd"}, + {file = "greenlet-3.1.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:03a088b9de532cbfe2ba2034b2b85e82df37874681e8c470d6fb2f8c04d7e4b7"}, + {file = "greenlet-3.1.1-cp38-cp38-win32.whl", hash = "sha256:8b8b36671f10ba80e159378df9c4f15c14098c4fd73a36b9ad715f057272fbef"}, + {file = "greenlet-3.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:7017b2be767b9d43cc31416aba48aab0d2309ee31b4dbf10a1d38fb7972bdf9d"}, + {file = "greenlet-3.1.1-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:396979749bd95f018296af156201d6211240e7a23090f50a8d5d18c370084dc3"}, + {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca9d0ff5ad43e785350894d97e13633a66e2b50000e8a183a50a88d834752d42"}, + {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f6ff3b14f2df4c41660a7dec01045a045653998784bf8cfcb5a525bdffffbc8f"}, + {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:94ebba31df2aa506d7b14866fed00ac141a867e63143fe5bca82a8e503b36437"}, + {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73aaad12ac0ff500f62cebed98d8789198ea0e6f233421059fa68a5aa7220145"}, + {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63e4844797b975b9af3a3fb8f7866ff08775f5426925e1e0bbcfe7932059a12c"}, + {file = "greenlet-3.1.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7939aa3ca7d2a1593596e7ac6d59391ff30281ef280d8632fa03d81f7c5f955e"}, + {file = "greenlet-3.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d0028e725ee18175c6e422797c407874da24381ce0690d6b9396c204c7f7276e"}, + {file = "greenlet-3.1.1-cp39-cp39-win32.whl", hash = "sha256:5e06afd14cbaf9e00899fae69b24a32f2196c19de08fcb9f4779dd4f004e5e7c"}, + {file = "greenlet-3.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:3319aa75e0e0639bc15ff54ca327e8dc7a6fe404003496e3c6925cd3142e0e22"}, + {file = "greenlet-3.1.1.tar.gz", hash = "sha256:4ce3ac6cdb6adf7946475d7ef31777c26d94bccc377e070a7986bd2d5c515467"}, ] [package.extras] @@ -1278,13 +1293,13 @@ pyparsing = {version = ">=2.4.2,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.0.2 || >3.0 [[package]] name = "identify" -version = "2.6.0" +version = "2.6.1" description = "File identification library for Python" optional = false python-versions = ">=3.8" files = [ - {file = "identify-2.6.0-py2.py3-none-any.whl", hash = "sha256:e79ae4406387a9d300332b5fd366d8994f1525e8414984e1a59e058b2eda2dd0"}, - {file = "identify-2.6.0.tar.gz", hash = "sha256:cb171c685bdc31bcc4c1734698736a7d5b6c8bf2e0c15117f4d469c8640ae5cf"}, + {file = "identify-2.6.1-py2.py3-none-any.whl", hash = "sha256:53863bcac7caf8d2ed85bd20312ea5dcfc22226800f6d6881f232d861db5a8f0"}, + {file = "identify-2.6.1.tar.gz", hash = "sha256:91478c5fb7c3aac5ff7bf9b4344f803843dc586832d5f110d672b19aa1984c98"}, ] [package.extras] @@ -1784,13 +1799,13 @@ files = [ [[package]] name = "mslex" -version = "1.2.0" +version = "1.3.0" description = "shlex for windows" optional = false python-versions = ">=3.5" files = [ - {file = "mslex-1.2.0-py3-none-any.whl", hash = "sha256:c68ec637485ee3544c5847c1b4e78b02940b32708568fb1d8715491815aa2341"}, - {file = "mslex-1.2.0.tar.gz", hash = "sha256:79e2abc5a129dd71cdde58a22a2039abb7fa8afcbac498b723ba6e9b9fbacc14"}, + {file = "mslex-1.3.0-py3-none-any.whl", hash = "sha256:c7074b347201b3466fc077c5692fbce9b5f62a63a51f537a53fbbd02eff2eea4"}, + {file = "mslex-1.3.0.tar.gz", hash = "sha256:641c887d1d3db610eee2af37a8e5abda3f70b3006cdfd2d0d29dc0d1ae28a85d"}, ] [[package]] @@ -2169,13 +2184,13 @@ pytzdata = ">=2020.1" [[package]] name = "platformdirs" -version = "4.3.2" +version = "4.3.6" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.8" files = [ - {file = "platformdirs-4.3.2-py3-none-any.whl", hash = "sha256:eb1c8582560b34ed4ba105009a4badf7f6f85768b30126f351328507b2beb617"}, - {file = "platformdirs-4.3.2.tar.gz", hash = "sha256:9e5e27a08aa095dd127b9f2e764d74254f482fef22b0970773bfba79d091ab8c"}, + {file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"}, + {file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"}, ] [package.extras] @@ -2376,80 +2391,78 @@ test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] [[package]] name = "psycopg2-binary" -version = "2.9.9" +version = "2.9.10" description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "psycopg2-binary-2.9.9.tar.gz", hash = "sha256:7f01846810177d829c7692f1f5ada8096762d9172af1b1a28d4ab5b77c923c1c"}, - {file = "psycopg2_binary-2.9.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c2470da5418b76232f02a2fcd2229537bb2d5a7096674ce61859c3229f2eb202"}, - {file = "psycopg2_binary-2.9.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c6af2a6d4b7ee9615cbb162b0738f6e1fd1f5c3eda7e5da17861eacf4c717ea7"}, - {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:75723c3c0fbbf34350b46a3199eb50638ab22a0228f93fb472ef4d9becc2382b"}, - {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:83791a65b51ad6ee6cf0845634859d69a038ea9b03d7b26e703f94c7e93dbcf9"}, - {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0ef4854e82c09e84cc63084a9e4ccd6d9b154f1dbdd283efb92ecd0b5e2b8c84"}, - {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed1184ab8f113e8d660ce49a56390ca181f2981066acc27cf637d5c1e10ce46e"}, - {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d2997c458c690ec2bc6b0b7ecbafd02b029b7b4283078d3b32a852a7ce3ddd98"}, - {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b58b4710c7f4161b5e9dcbe73bb7c62d65670a87df7bcce9e1faaad43e715245"}, - {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:0c009475ee389757e6e34611d75f6e4f05f0cf5ebb76c6037508318e1a1e0d7e"}, - {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8dbf6d1bc73f1d04ec1734bae3b4fb0ee3cb2a493d35ede9badbeb901fb40f6f"}, - {file = "psycopg2_binary-2.9.9-cp310-cp310-win32.whl", hash = "sha256:3f78fd71c4f43a13d342be74ebbc0666fe1f555b8837eb113cb7416856c79682"}, - {file = "psycopg2_binary-2.9.9-cp310-cp310-win_amd64.whl", hash = "sha256:876801744b0dee379e4e3c38b76fc89f88834bb15bf92ee07d94acd06ec890a0"}, - {file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ee825e70b1a209475622f7f7b776785bd68f34af6e7a46e2e42f27b659b5bc26"}, - {file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1ea665f8ce695bcc37a90ee52de7a7980be5161375d42a0b6c6abedbf0d81f0f"}, - {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:143072318f793f53819048fdfe30c321890af0c3ec7cb1dfc9cc87aa88241de2"}, - {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c332c8d69fb64979ebf76613c66b985414927a40f8defa16cf1bc028b7b0a7b0"}, - {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7fc5a5acafb7d6ccca13bfa8c90f8c51f13d8fb87d95656d3950f0158d3ce53"}, - {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:977646e05232579d2e7b9c59e21dbe5261f403a88417f6a6512e70d3f8a046be"}, - {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b6356793b84728d9d50ead16ab43c187673831e9d4019013f1402c41b1db9b27"}, - {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bc7bb56d04601d443f24094e9e31ae6deec9ccb23581f75343feebaf30423359"}, - {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:77853062a2c45be16fd6b8d6de2a99278ee1d985a7bd8b103e97e41c034006d2"}, - {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:78151aa3ec21dccd5cdef6c74c3e73386dcdfaf19bced944169697d7ac7482fc"}, - {file = "psycopg2_binary-2.9.9-cp311-cp311-win32.whl", hash = "sha256:dc4926288b2a3e9fd7b50dc6a1909a13bbdadfc67d93f3374d984e56f885579d"}, - {file = "psycopg2_binary-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:b76bedd166805480ab069612119ea636f5ab8f8771e640ae103e05a4aae3e417"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e6f98446430fdf41bd36d4faa6cb409f5140c1c2cf58ce0bbdaf16af7d3f119"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c77e3d1862452565875eb31bdb45ac62502feabbd53429fdc39a1cc341d681ba"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe"}, - {file = "psycopg2_binary-2.9.9-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2293b001e319ab0d869d660a704942c9e2cce19745262a8aba2115ef41a0a42a"}, - {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03ef7df18daf2c4c07e2695e8cfd5ee7f748a1d54d802330985a78d2a5a6dca9"}, - {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a602ea5aff39bb9fac6308e9c9d82b9a35c2bf288e184a816002c9fae930b77"}, - {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8359bf4791968c5a78c56103702000105501adb557f3cf772b2c207284273984"}, - {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:275ff571376626195ab95a746e6a04c7df8ea34638b99fc11160de91f2fef503"}, - {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:f9b5571d33660d5009a8b3c25dc1db560206e2d2f89d3df1cb32d72c0d117d52"}, - {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:420f9bbf47a02616e8554e825208cb947969451978dceb77f95ad09c37791dae"}, - {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:4154ad09dac630a0f13f37b583eae260c6aa885d67dfbccb5b02c33f31a6d420"}, - {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a148c5d507bb9b4f2030a2025c545fccb0e1ef317393eaba42e7eabd28eb6041"}, - {file = "psycopg2_binary-2.9.9-cp37-cp37m-win32.whl", hash = "sha256:68fc1f1ba168724771e38bee37d940d2865cb0f562380a1fb1ffb428b75cb692"}, - {file = "psycopg2_binary-2.9.9-cp37-cp37m-win_amd64.whl", hash = "sha256:281309265596e388ef483250db3640e5f414168c5a67e9c665cafce9492eda2f"}, - {file = "psycopg2_binary-2.9.9-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:60989127da422b74a04345096c10d416c2b41bd7bf2a380eb541059e4e999980"}, - {file = "psycopg2_binary-2.9.9-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:246b123cc54bb5361588acc54218c8c9fb73068bf227a4a531d8ed56fa3ca7d6"}, - {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34eccd14566f8fe14b2b95bb13b11572f7c7d5c36da61caf414d23b91fcc5d94"}, - {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18d0ef97766055fec15b5de2c06dd8e7654705ce3e5e5eed3b6651a1d2a9a152"}, - {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d3f82c171b4ccd83bbaf35aa05e44e690113bd4f3b7b6cc54d2219b132f3ae55"}, - {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ead20f7913a9c1e894aebe47cccf9dc834e1618b7aa96155d2091a626e59c972"}, - {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:ca49a8119c6cbd77375ae303b0cfd8c11f011abbbd64601167ecca18a87e7cdd"}, - {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:323ba25b92454adb36fa425dc5cf6f8f19f78948cbad2e7bc6cdf7b0d7982e59"}, - {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:1236ed0952fbd919c100bc839eaa4a39ebc397ed1c08a97fc45fee2a595aa1b3"}, - {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:729177eaf0aefca0994ce4cffe96ad3c75e377c7b6f4efa59ebf003b6d398716"}, - {file = "psycopg2_binary-2.9.9-cp38-cp38-win32.whl", hash = "sha256:804d99b24ad523a1fe18cc707bf741670332f7c7412e9d49cb5eab67e886b9b5"}, - {file = "psycopg2_binary-2.9.9-cp38-cp38-win_amd64.whl", hash = "sha256:a6cdcc3ede532f4a4b96000b6362099591ab4a3e913d70bcbac2b56c872446f7"}, - {file = "psycopg2_binary-2.9.9-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:72dffbd8b4194858d0941062a9766f8297e8868e1dd07a7b36212aaa90f49472"}, - {file = "psycopg2_binary-2.9.9-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:30dcc86377618a4c8f3b72418df92e77be4254d8f89f14b8e8f57d6d43603c0f"}, - {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:31a34c508c003a4347d389a9e6fcc2307cc2150eb516462a7a17512130de109e"}, - {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:15208be1c50b99203fe88d15695f22a5bed95ab3f84354c494bcb1d08557df67"}, - {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1873aade94b74715be2246321c8650cabf5a0d098a95bab81145ffffa4c13876"}, - {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a58c98a7e9c021f357348867f537017057c2ed7f77337fd914d0bedb35dace7"}, - {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4686818798f9194d03c9129a4d9a702d9e113a89cb03bffe08c6cf799e053291"}, - {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ebdc36bea43063116f0486869652cb2ed7032dbc59fbcb4445c4862b5c1ecf7f"}, - {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:ca08decd2697fdea0aea364b370b1249d47336aec935f87b8bbfd7da5b2ee9c1"}, - {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ac05fb791acf5e1a3e39402641827780fe44d27e72567a000412c648a85ba860"}, - {file = "psycopg2_binary-2.9.9-cp39-cp39-win32.whl", hash = "sha256:9dba73be7305b399924709b91682299794887cbbd88e38226ed9f6712eabee90"}, - {file = "psycopg2_binary-2.9.9-cp39-cp39-win_amd64.whl", hash = "sha256:f7ae5d65ccfbebdfa761585228eb4d0df3a8b15cfb53bd953e713e09fbb12957"}, + {file = "psycopg2-binary-2.9.10.tar.gz", hash = "sha256:4b3df0e6990aa98acda57d983942eff13d824135fe2250e6522edaa782a06de2"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:0ea8e3d0ae83564f2fc554955d327fa081d065c8ca5cc6d2abb643e2c9c1200f"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:3e9c76f0ac6f92ecfc79516a8034a544926430f7b080ec5a0537bca389ee0906"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ad26b467a405c798aaa1458ba09d7e2b6e5f96b1ce0ac15d82fd9f95dc38a92"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:270934a475a0e4b6925b5f804e3809dd5f90f8613621d062848dd82f9cd62007"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:48b338f08d93e7be4ab2b5f1dbe69dc5e9ef07170fe1f86514422076d9c010d0"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f4152f8f76d2023aac16285576a9ecd2b11a9895373a1f10fd9db54b3ff06b4"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:32581b3020c72d7a421009ee1c6bf4a131ef5f0a968fab2e2de0c9d2bb4577f1"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:2ce3e21dc3437b1d960521eca599d57408a695a0d3c26797ea0f72e834c7ffe5"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:e984839e75e0b60cfe75e351db53d6db750b00de45644c5d1f7ee5d1f34a1ce5"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3c4745a90b78e51d9ba06e2088a2fe0c693ae19cc8cb051ccda44e8df8a6eb53"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-win32.whl", hash = "sha256:e5720a5d25e3b99cd0dc5c8a440570469ff82659bb09431c1439b92caf184d3b"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-win_amd64.whl", hash = "sha256:3c18f74eb4386bf35e92ab2354a12c17e5eb4d9798e4c0ad3a00783eae7cd9f1"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:04392983d0bb89a8717772a193cfaac58871321e3ec69514e1c4e0d4957b5aff"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:1a6784f0ce3fec4edc64e985865c17778514325074adf5ad8f80636cd029ef7c"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5f86c56eeb91dc3135b3fd8a95dc7ae14c538a2f3ad77a19645cf55bab1799c"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b3d2491d4d78b6b14f76881905c7a8a8abcf974aad4a8a0b065273a0ed7a2cb"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2286791ececda3a723d1910441c793be44625d86d1a4e79942751197f4d30341"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:512d29bb12608891e349af6a0cccedce51677725a921c07dba6342beaf576f9a"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5a507320c58903967ef7384355a4da7ff3f28132d679aeb23572753cbf2ec10b"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6d4fa1079cab9018f4d0bd2db307beaa612b0d13ba73b5c6304b9fe2fb441ff7"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:851485a42dbb0bdc1edcdabdb8557c09c9655dfa2ca0460ff210522e073e319e"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:35958ec9e46432d9076286dda67942ed6d968b9c3a6a2fd62b48939d1d78bf68"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-win32.whl", hash = "sha256:ecced182e935529727401b24d76634a357c71c9275b356efafd8a2a91ec07392"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-win_amd64.whl", hash = "sha256:ee0e8c683a7ff25d23b55b11161c2663d4b099770f6085ff0a20d4505778d6b4"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:880845dfe1f85d9d5f7c412efea7a08946a46894537e4e5d091732eb1d34d9a0"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9440fa522a79356aaa482aa4ba500b65f28e5d0e63b801abf6aa152a29bd842a"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3923c1d9870c49a2d44f795df0c889a22380d36ef92440ff618ec315757e539"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b2c956c028ea5de47ff3a8d6b3cc3330ab45cf0b7c3da35a2d6ff8420896526"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f758ed67cab30b9a8d2833609513ce4d3bd027641673d4ebc9c067e4d208eec1"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cd9b4f2cfab88ed4a9106192de509464b75a906462fb846b936eabe45c2063e"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dc08420625b5a20b53551c50deae6e231e6371194fa0651dbe0fb206452ae1f"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d7cd730dfa7c36dbe8724426bf5612798734bff2d3c3857f36f2733f5bfc7c00"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:155e69561d54d02b3c3209545fb08938e27889ff5a10c19de8d23eb5a41be8a5"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c3cc28a6fd5a4a26224007712e79b81dbaee2ffb90ff406256158ec4d7b52b47"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-win32.whl", hash = "sha256:ec8a77f521a17506a24a5f626cb2aee7850f9b69a0afe704586f63a464f3cd64"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-win_amd64.whl", hash = "sha256:18c5ee682b9c6dd3696dad6e54cc7ff3a1a9020df6a5c0f861ef8bfd338c3ca0"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:26540d4a9a4e2b096f1ff9cce51253d0504dca5a85872c7f7be23be5a53eb18d"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:e217ce4d37667df0bc1c397fdcd8de5e81018ef305aed9415c3b093faaeb10fb"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:245159e7ab20a71d989da00f280ca57da7641fa2cdcf71749c193cea540a74f7"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c4ded1a24b20021ebe677b7b08ad10bf09aac197d6943bfe6fec70ac4e4690d"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3abb691ff9e57d4a93355f60d4f4c1dd2d68326c968e7db17ea96df3c023ef73"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8608c078134f0b3cbd9f89b34bd60a943b23fd33cc5f065e8d5f840061bd0673"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:230eeae2d71594103cd5b93fd29d1ace6420d0b86f4778739cb1a5a32f607d1f"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:bb89f0a835bcfc1d42ccd5f41f04870c1b936d8507c6df12b7737febc40f0909"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f0c2d907a1e102526dd2986df638343388b94c33860ff3bbe1384130828714b1"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f8157bed2f51db683f31306aa497311b560f2265998122abe1dce6428bd86567"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:eb09aa7f9cecb45027683bb55aebaaf45a0df8bf6de68801a6afdc7947bb09d4"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b73d6d7f0ccdad7bc43e6d34273f70d587ef62f824d7261c4ae9b8b1b6af90e8"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce5ab4bf46a211a8e924d307c1b1fcda82368586a19d0a24f8ae166f5c784864"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:056470c3dc57904bbf63d6f534988bafc4e970ffd50f6271fc4ee7daad9498a5"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73aa0e31fa4bb82578f3a6c74a73c273367727de397a7a0f07bd83cbea696baa"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:8de718c0e1c4b982a54b41779667242bc630b2197948405b7bd8ce16bcecac92"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:5c370b1e4975df846b0277b4deba86419ca77dbc25047f535b0bb03d1a544d44"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:ffe8ed017e4ed70f68b7b371d84b7d4a790368db9203dfc2d222febd3a9c8863"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:8aecc5e80c63f7459a1a2ab2c64df952051df196294d9f739933a9f6687e86b3"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:7a813c8bdbaaaab1f078014b9b0b13f5de757e2b5d9be6403639b298a04d218b"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d00924255d7fc916ef66e4bf22f354a940c67179ad3fd7067d7a0a9c84d2fbfc"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7559bce4b505762d737172556a4e6ea8a9998ecac1e39b5233465093e8cee697"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e8b58f0a96e7a1e341fc894f62c1177a7c83febebb5ff9123b579418fdc8a481"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b269105e59ac96aba877c1707c600ae55711d9dcd3fc4b5012e4af68e30c648"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:79625966e176dc97ddabc142351e0409e28acf4660b88d1cf6adb876d20c490d"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:8aabf1c1a04584c168984ac678a668094d831f152859d06e055288fa515e4d30"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:19721ac03892001ee8fdd11507e6a2e01f4e37014def96379411ca99d78aeb2c"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7f5d859928e635fa3ce3477704acee0f667b3a3d3e4bb109f2b18d4005f38287"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-win32.whl", hash = "sha256:3216ccf953b3f267691c90c6fe742e45d890d8272326b4a8b20850a03d05b7b8"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-win_amd64.whl", hash = "sha256:30e34c4e97964805f715206c7b789d54a78b70f3ff19fbe590104b71c45600e5"}, ] [[package]] @@ -2820,25 +2833,29 @@ files = [ [[package]] name = "pywin32" -version = "306" +version = "308" description = "Python for Window Extensions" optional = false python-versions = "*" files = [ - {file = "pywin32-306-cp310-cp310-win32.whl", hash = "sha256:06d3420a5155ba65f0b72f2699b5bacf3109f36acbe8923765c22938a69dfc8d"}, - {file = "pywin32-306-cp310-cp310-win_amd64.whl", hash = "sha256:84f4471dbca1887ea3803d8848a1616429ac94a4a8d05f4bc9c5dcfd42ca99c8"}, - {file = "pywin32-306-cp311-cp311-win32.whl", hash = "sha256:e65028133d15b64d2ed8f06dd9fbc268352478d4f9289e69c190ecd6818b6407"}, - {file = "pywin32-306-cp311-cp311-win_amd64.whl", hash = "sha256:a7639f51c184c0272e93f244eb24dafca9b1855707d94c192d4a0b4c01e1100e"}, - {file = "pywin32-306-cp311-cp311-win_arm64.whl", hash = "sha256:70dba0c913d19f942a2db25217d9a1b726c278f483a919f1abfed79c9cf64d3a"}, - {file = "pywin32-306-cp312-cp312-win32.whl", hash = "sha256:383229d515657f4e3ed1343da8be101000562bf514591ff383ae940cad65458b"}, - {file = "pywin32-306-cp312-cp312-win_amd64.whl", hash = "sha256:37257794c1ad39ee9be652da0462dc2e394c8159dfd913a8a4e8eb6fd346da0e"}, - {file = "pywin32-306-cp312-cp312-win_arm64.whl", hash = "sha256:5821ec52f6d321aa59e2db7e0a35b997de60c201943557d108af9d4ae1ec7040"}, - {file = "pywin32-306-cp37-cp37m-win32.whl", hash = "sha256:1c73ea9a0d2283d889001998059f5eaaba3b6238f767c9cf2833b13e6a685f65"}, - {file = "pywin32-306-cp37-cp37m-win_amd64.whl", hash = "sha256:72c5f621542d7bdd4fdb716227be0dd3f8565c11b280be6315b06ace35487d36"}, - {file = "pywin32-306-cp38-cp38-win32.whl", hash = "sha256:e4c092e2589b5cf0d365849e73e02c391c1349958c5ac3e9d5ccb9a28e017b3a"}, - {file = "pywin32-306-cp38-cp38-win_amd64.whl", hash = "sha256:e8ac1ae3601bee6ca9f7cb4b5363bf1c0badb935ef243c4733ff9a393b1690c0"}, - {file = "pywin32-306-cp39-cp39-win32.whl", hash = "sha256:e25fd5b485b55ac9c057f67d94bc203f3f6595078d1fb3b458c9c28b7153a802"}, - {file = "pywin32-306-cp39-cp39-win_amd64.whl", hash = "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4"}, + {file = "pywin32-308-cp310-cp310-win32.whl", hash = "sha256:796ff4426437896550d2981b9c2ac0ffd75238ad9ea2d3bfa67a1abd546d262e"}, + {file = "pywin32-308-cp310-cp310-win_amd64.whl", hash = "sha256:4fc888c59b3c0bef905ce7eb7e2106a07712015ea1c8234b703a088d46110e8e"}, + {file = "pywin32-308-cp310-cp310-win_arm64.whl", hash = "sha256:a5ab5381813b40f264fa3495b98af850098f814a25a63589a8e9eb12560f450c"}, + {file = "pywin32-308-cp311-cp311-win32.whl", hash = "sha256:5d8c8015b24a7d6855b1550d8e660d8daa09983c80e5daf89a273e5c6fb5095a"}, + {file = "pywin32-308-cp311-cp311-win_amd64.whl", hash = "sha256:575621b90f0dc2695fec346b2d6302faebd4f0f45c05ea29404cefe35d89442b"}, + {file = "pywin32-308-cp311-cp311-win_arm64.whl", hash = "sha256:100a5442b7332070983c4cd03f2e906a5648a5104b8a7f50175f7906efd16bb6"}, + {file = "pywin32-308-cp312-cp312-win32.whl", hash = "sha256:587f3e19696f4bf96fde9d8a57cec74a57021ad5f204c9e627e15c33ff568897"}, + {file = "pywin32-308-cp312-cp312-win_amd64.whl", hash = "sha256:00b3e11ef09ede56c6a43c71f2d31857cf7c54b0ab6e78ac659497abd2834f47"}, + {file = "pywin32-308-cp312-cp312-win_arm64.whl", hash = "sha256:9b4de86c8d909aed15b7011182c8cab38c8850de36e6afb1f0db22b8959e3091"}, + {file = "pywin32-308-cp313-cp313-win32.whl", hash = "sha256:1c44539a37a5b7b21d02ab34e6a4d314e0788f1690d65b48e9b0b89f31abbbed"}, + {file = "pywin32-308-cp313-cp313-win_amd64.whl", hash = "sha256:fd380990e792eaf6827fcb7e187b2b4b1cede0585e3d0c9e84201ec27b9905e4"}, + {file = "pywin32-308-cp313-cp313-win_arm64.whl", hash = "sha256:ef313c46d4c18dfb82a2431e3051ac8f112ccee1a34f29c263c583c568db63cd"}, + {file = "pywin32-308-cp37-cp37m-win32.whl", hash = "sha256:1f696ab352a2ddd63bd07430080dd598e6369152ea13a25ebcdd2f503a38f1ff"}, + {file = "pywin32-308-cp37-cp37m-win_amd64.whl", hash = "sha256:13dcb914ed4347019fbec6697a01a0aec61019c1046c2b905410d197856326a6"}, + {file = "pywin32-308-cp38-cp38-win32.whl", hash = "sha256:5794e764ebcabf4ff08c555b31bd348c9025929371763b2183172ff4708152f0"}, + {file = "pywin32-308-cp38-cp38-win_amd64.whl", hash = "sha256:3b92622e29d651c6b783e368ba7d6722b1634b8e70bd376fd7610fe1992e19de"}, + {file = "pywin32-308-cp39-cp39-win32.whl", hash = "sha256:7873ca4dc60ab3287919881a7d4f88baee4a6e639aa6962de25a98ba6b193341"}, + {file = "pywin32-308-cp39-cp39-win_amd64.whl", hash = "sha256:71b3322d949b4cc20776436a9c9ba0eeedcbc9c650daa536df63f0ff111bb920"}, ] [[package]] @@ -3170,23 +3187,23 @@ tornado = ["tornado (>=5)"] [[package]] name = "setuptools" -version = "74.1.2" +version = "75.3.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-74.1.2-py3-none-any.whl", hash = "sha256:5f4c08aa4d3ebcb57a50c33b1b07e94315d7fc7230f7115e47fc99776c8ce308"}, - {file = "setuptools-74.1.2.tar.gz", hash = "sha256:95b40ed940a1c67eb70fc099094bd6e99c6ee7c23aa2306f4d2697ba7916f9c6"}, + {file = "setuptools-75.3.0-py3-none-any.whl", hash = "sha256:f2504966861356aa38616760c0f66568e535562374995367b4e69c7143cf6bcd"}, + {file = "setuptools-75.3.0.tar.gz", hash = "sha256:fba5dd4d766e97be1b1681d98712680ae8f2f26d7881245f2ce9e40714f1a686"}, ] [package.extras] check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.5.2)"] -core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.text (>=3.7)", "more-itertools (>=8.8)", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.collections", "jaraco.functools", "jaraco.text (>=3.7)", "more-itertools", "more-itertools (>=8.8)", "packaging", "packaging (>=24)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] enabler = ["pytest-enabler (>=2.2)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] -type = ["importlib-metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.11.*)", "pytest-mypy"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test (>=5.5)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib-metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.12.*)", "pytest-mypy"] [[package]] name = "six" @@ -3212,60 +3229,68 @@ files = [ [[package]] name = "sqlalchemy" -version = "2.0.34" +version = "2.0.36" description = "Database Abstraction Library" optional = false python-versions = ">=3.7" files = [ - {file = "SQLAlchemy-2.0.34-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:95d0b2cf8791ab5fb9e3aa3d9a79a0d5d51f55b6357eecf532a120ba3b5524db"}, - {file = "SQLAlchemy-2.0.34-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:243f92596f4fd4c8bd30ab8e8dd5965afe226363d75cab2468f2c707f64cd83b"}, - {file = "SQLAlchemy-2.0.34-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ea54f7300553af0a2a7235e9b85f4204e1fc21848f917a3213b0e0818de9a24"}, - {file = "SQLAlchemy-2.0.34-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:173f5f122d2e1bff8fbd9f7811b7942bead1f5e9f371cdf9e670b327e6703ebd"}, - {file = "SQLAlchemy-2.0.34-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:196958cde924a00488e3e83ff917be3b73cd4ed8352bbc0f2989333176d1c54d"}, - {file = "SQLAlchemy-2.0.34-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bd90c221ed4e60ac9d476db967f436cfcecbd4ef744537c0f2d5291439848768"}, - {file = "SQLAlchemy-2.0.34-cp310-cp310-win32.whl", hash = "sha256:3166dfff2d16fe9be3241ee60ece6fcb01cf8e74dd7c5e0b64f8e19fab44911b"}, - {file = "SQLAlchemy-2.0.34-cp310-cp310-win_amd64.whl", hash = "sha256:6831a78bbd3c40f909b3e5233f87341f12d0b34a58f14115c9e94b4cdaf726d3"}, - {file = "SQLAlchemy-2.0.34-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c7db3db284a0edaebe87f8f6642c2b2c27ed85c3e70064b84d1c9e4ec06d5d84"}, - {file = "SQLAlchemy-2.0.34-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:430093fce0efc7941d911d34f75a70084f12f6ca5c15d19595c18753edb7c33b"}, - {file = "SQLAlchemy-2.0.34-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79cb400c360c7c210097b147c16a9e4c14688a6402445ac848f296ade6283bbc"}, - {file = "SQLAlchemy-2.0.34-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb1b30f31a36c7f3fee848391ff77eebdd3af5750bf95fbf9b8b5323edfdb4ec"}, - {file = "SQLAlchemy-2.0.34-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8fddde2368e777ea2a4891a3fb4341e910a056be0bb15303bf1b92f073b80c02"}, - {file = "SQLAlchemy-2.0.34-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:80bd73ea335203b125cf1d8e50fef06be709619eb6ab9e7b891ea34b5baa2287"}, - {file = "SQLAlchemy-2.0.34-cp311-cp311-win32.whl", hash = "sha256:6daeb8382d0df526372abd9cb795c992e18eed25ef2c43afe518c73f8cccb721"}, - {file = "SQLAlchemy-2.0.34-cp311-cp311-win_amd64.whl", hash = "sha256:5bc08e75ed11693ecb648b7a0a4ed80da6d10845e44be0c98c03f2f880b68ff4"}, - {file = "SQLAlchemy-2.0.34-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:53e68b091492c8ed2bd0141e00ad3089bcc6bf0e6ec4142ad6505b4afe64163e"}, - {file = "SQLAlchemy-2.0.34-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bcd18441a49499bf5528deaa9dee1f5c01ca491fc2791b13604e8f972877f812"}, - {file = "SQLAlchemy-2.0.34-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:165bbe0b376541092bf49542bd9827b048357f4623486096fc9aaa6d4e7c59a2"}, - {file = "SQLAlchemy-2.0.34-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3330415cd387d2b88600e8e26b510d0370db9b7eaf984354a43e19c40df2e2b"}, - {file = "SQLAlchemy-2.0.34-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:97b850f73f8abbffb66ccbab6e55a195a0eb655e5dc74624d15cff4bfb35bd74"}, - {file = "SQLAlchemy-2.0.34-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7cee4c6917857fd6121ed84f56d1dc78eb1d0e87f845ab5a568aba73e78adf83"}, - {file = "SQLAlchemy-2.0.34-cp312-cp312-win32.whl", hash = "sha256:fbb034f565ecbe6c530dff948239377ba859420d146d5f62f0271407ffb8c580"}, - {file = "SQLAlchemy-2.0.34-cp312-cp312-win_amd64.whl", hash = "sha256:707c8f44931a4facd4149b52b75b80544a8d824162602b8cd2fe788207307f9a"}, - {file = "SQLAlchemy-2.0.34-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:24af3dc43568f3780b7e1e57c49b41d98b2d940c1fd2e62d65d3928b6f95f021"}, - {file = "SQLAlchemy-2.0.34-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e60ed6ef0a35c6b76b7640fe452d0e47acc832ccbb8475de549a5cc5f90c2c06"}, - {file = "SQLAlchemy-2.0.34-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:413c85cd0177c23e32dee6898c67a5f49296640041d98fddb2c40888fe4daa2e"}, - {file = "SQLAlchemy-2.0.34-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:25691f4adfb9d5e796fd48bf1432272f95f4bbe5f89c475a788f31232ea6afba"}, - {file = "SQLAlchemy-2.0.34-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:526ce723265643dbc4c7efb54f56648cc30e7abe20f387d763364b3ce7506c82"}, - {file = "SQLAlchemy-2.0.34-cp37-cp37m-win32.whl", hash = "sha256:13be2cc683b76977a700948411a94c67ad8faf542fa7da2a4b167f2244781cf3"}, - {file = "SQLAlchemy-2.0.34-cp37-cp37m-win_amd64.whl", hash = "sha256:e54ef33ea80d464c3dcfe881eb00ad5921b60f8115ea1a30d781653edc2fd6a2"}, - {file = "SQLAlchemy-2.0.34-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:43f28005141165edd11fbbf1541c920bd29e167b8bbc1fb410d4fe2269c1667a"}, - {file = "SQLAlchemy-2.0.34-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b68094b165a9e930aedef90725a8fcfafe9ef95370cbb54abc0464062dbf808f"}, - {file = "SQLAlchemy-2.0.34-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a1e03db964e9d32f112bae36f0cc1dcd1988d096cfd75d6a588a3c3def9ab2b"}, - {file = "SQLAlchemy-2.0.34-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:203d46bddeaa7982f9c3cc693e5bc93db476ab5de9d4b4640d5c99ff219bee8c"}, - {file = "SQLAlchemy-2.0.34-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:ae92bebca3b1e6bd203494e5ef919a60fb6dfe4d9a47ed2453211d3bd451b9f5"}, - {file = "SQLAlchemy-2.0.34-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:9661268415f450c95f72f0ac1217cc6f10256f860eed85c2ae32e75b60278ad8"}, - {file = "SQLAlchemy-2.0.34-cp38-cp38-win32.whl", hash = "sha256:895184dfef8708e15f7516bd930bda7e50ead069280d2ce09ba11781b630a434"}, - {file = "SQLAlchemy-2.0.34-cp38-cp38-win_amd64.whl", hash = "sha256:6e7cde3a2221aa89247944cafb1b26616380e30c63e37ed19ff0bba5e968688d"}, - {file = "SQLAlchemy-2.0.34-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:dbcdf987f3aceef9763b6d7b1fd3e4ee210ddd26cac421d78b3c206d07b2700b"}, - {file = "SQLAlchemy-2.0.34-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ce119fc4ce0d64124d37f66a6f2a584fddc3c5001755f8a49f1ca0a177ef9796"}, - {file = "SQLAlchemy-2.0.34-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a17d8fac6df9835d8e2b4c5523666e7051d0897a93756518a1fe101c7f47f2f0"}, - {file = "SQLAlchemy-2.0.34-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ebc11c54c6ecdd07bb4efbfa1554538982f5432dfb8456958b6d46b9f834bb7"}, - {file = "SQLAlchemy-2.0.34-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:2e6965346fc1491a566e019a4a1d3dfc081ce7ac1a736536367ca305da6472a8"}, - {file = "SQLAlchemy-2.0.34-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:220574e78ad986aea8e81ac68821e47ea9202b7e44f251b7ed8c66d9ae3f4278"}, - {file = "SQLAlchemy-2.0.34-cp39-cp39-win32.whl", hash = "sha256:b75b00083e7fe6621ce13cfce9d4469c4774e55e8e9d38c305b37f13cf1e874c"}, - {file = "SQLAlchemy-2.0.34-cp39-cp39-win_amd64.whl", hash = "sha256:c29d03e0adf3cc1a8c3ec62d176824972ae29b67a66cbb18daff3062acc6faa8"}, - {file = "SQLAlchemy-2.0.34-py3-none-any.whl", hash = "sha256:7286c353ee6475613d8beff83167374006c6b3e3f0e6491bfe8ca610eb1dec0f"}, - {file = "sqlalchemy-2.0.34.tar.gz", hash = "sha256:10d8f36990dd929690666679b0f42235c159a7051534adb135728ee52828dd22"}, + {file = "SQLAlchemy-2.0.36-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:59b8f3adb3971929a3e660337f5dacc5942c2cdb760afcabb2614ffbda9f9f72"}, + {file = "SQLAlchemy-2.0.36-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:37350015056a553e442ff672c2d20e6f4b6d0b2495691fa239d8aa18bb3bc908"}, + {file = "SQLAlchemy-2.0.36-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8318f4776c85abc3f40ab185e388bee7a6ea99e7fa3a30686580b209eaa35c08"}, + {file = "SQLAlchemy-2.0.36-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c245b1fbade9c35e5bd3b64270ab49ce990369018289ecfde3f9c318411aaa07"}, + {file = "SQLAlchemy-2.0.36-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:69f93723edbca7342624d09f6704e7126b152eaed3cdbb634cb657a54332a3c5"}, + {file = "SQLAlchemy-2.0.36-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f9511d8dd4a6e9271d07d150fb2f81874a3c8c95e11ff9af3a2dfc35fe42ee44"}, + {file = "SQLAlchemy-2.0.36-cp310-cp310-win32.whl", hash = "sha256:c3f3631693003d8e585d4200730616b78fafd5a01ef8b698f6967da5c605b3fa"}, + {file = "SQLAlchemy-2.0.36-cp310-cp310-win_amd64.whl", hash = "sha256:a86bfab2ef46d63300c0f06936bd6e6c0105faa11d509083ba8f2f9d237fb5b5"}, + {file = "SQLAlchemy-2.0.36-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fd3a55deef00f689ce931d4d1b23fa9f04c880a48ee97af488fd215cf24e2a6c"}, + {file = "SQLAlchemy-2.0.36-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4f5e9cd989b45b73bd359f693b935364f7e1f79486e29015813c338450aa5a71"}, + {file = "SQLAlchemy-2.0.36-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0ddd9db6e59c44875211bc4c7953a9f6638b937b0a88ae6d09eb46cced54eff"}, + {file = "SQLAlchemy-2.0.36-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2519f3a5d0517fc159afab1015e54bb81b4406c278749779be57a569d8d1bb0d"}, + {file = "SQLAlchemy-2.0.36-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:59b1ee96617135f6e1d6f275bbe988f419c5178016f3d41d3c0abb0c819f75bb"}, + {file = "SQLAlchemy-2.0.36-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:39769a115f730d683b0eb7b694db9789267bcd027326cccc3125e862eb03bfd8"}, + {file = "SQLAlchemy-2.0.36-cp311-cp311-win32.whl", hash = "sha256:66bffbad8d6271bb1cc2f9a4ea4f86f80fe5e2e3e501a5ae2a3dc6a76e604e6f"}, + {file = "SQLAlchemy-2.0.36-cp311-cp311-win_amd64.whl", hash = "sha256:23623166bfefe1487d81b698c423f8678e80df8b54614c2bf4b4cfcd7c711959"}, + {file = "SQLAlchemy-2.0.36-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7b64e6ec3f02c35647be6b4851008b26cff592a95ecb13b6788a54ef80bbdd4"}, + {file = "SQLAlchemy-2.0.36-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:46331b00096a6db1fdc052d55b101dbbfc99155a548e20a0e4a8e5e4d1362855"}, + {file = "SQLAlchemy-2.0.36-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdf3386a801ea5aba17c6410dd1dc8d39cf454ca2565541b5ac42a84e1e28f53"}, + {file = "SQLAlchemy-2.0.36-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac9dfa18ff2a67b09b372d5db8743c27966abf0e5344c555d86cc7199f7ad83a"}, + {file = "SQLAlchemy-2.0.36-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:90812a8933df713fdf748b355527e3af257a11e415b613dd794512461eb8a686"}, + {file = "SQLAlchemy-2.0.36-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1bc330d9d29c7f06f003ab10e1eaced295e87940405afe1b110f2eb93a233588"}, + {file = "SQLAlchemy-2.0.36-cp312-cp312-win32.whl", hash = "sha256:79d2e78abc26d871875b419e1fd3c0bca31a1cb0043277d0d850014599626c2e"}, + {file = "SQLAlchemy-2.0.36-cp312-cp312-win_amd64.whl", hash = "sha256:b544ad1935a8541d177cb402948b94e871067656b3a0b9e91dbec136b06a2ff5"}, + {file = "SQLAlchemy-2.0.36-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b5cc79df7f4bc3d11e4b542596c03826063092611e481fcf1c9dfee3c94355ef"}, + {file = "SQLAlchemy-2.0.36-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3c01117dd36800f2ecaa238c65365b7b16497adc1522bf84906e5710ee9ba0e8"}, + {file = "SQLAlchemy-2.0.36-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9bc633f4ee4b4c46e7adcb3a9b5ec083bf1d9a97c1d3854b92749d935de40b9b"}, + {file = "SQLAlchemy-2.0.36-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e46ed38affdfc95d2c958de328d037d87801cfcbea6d421000859e9789e61c2"}, + {file = "SQLAlchemy-2.0.36-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b2985c0b06e989c043f1dc09d4fe89e1616aadd35392aea2844f0458a989eacf"}, + {file = "SQLAlchemy-2.0.36-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4a121d62ebe7d26fec9155f83f8be5189ef1405f5973ea4874a26fab9f1e262c"}, + {file = "SQLAlchemy-2.0.36-cp313-cp313-win32.whl", hash = "sha256:0572f4bd6f94752167adfd7c1bed84f4b240ee6203a95e05d1e208d488d0d436"}, + {file = "SQLAlchemy-2.0.36-cp313-cp313-win_amd64.whl", hash = "sha256:8c78ac40bde930c60e0f78b3cd184c580f89456dd87fc08f9e3ee3ce8765ce88"}, + {file = "SQLAlchemy-2.0.36-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:be9812b766cad94a25bc63bec11f88c4ad3629a0cec1cd5d4ba48dc23860486b"}, + {file = "SQLAlchemy-2.0.36-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50aae840ebbd6cdd41af1c14590e5741665e5272d2fee999306673a1bb1fdb4d"}, + {file = "SQLAlchemy-2.0.36-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4557e1f11c5f653ebfdd924f3f9d5ebfc718283b0b9beebaa5dd6b77ec290971"}, + {file = "SQLAlchemy-2.0.36-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:07b441f7d03b9a66299ce7ccf3ef2900abc81c0db434f42a5694a37bd73870f2"}, + {file = "SQLAlchemy-2.0.36-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:28120ef39c92c2dd60f2721af9328479516844c6b550b077ca450c7d7dc68575"}, + {file = "SQLAlchemy-2.0.36-cp37-cp37m-win32.whl", hash = "sha256:b81ee3d84803fd42d0b154cb6892ae57ea6b7c55d8359a02379965706c7efe6c"}, + {file = "SQLAlchemy-2.0.36-cp37-cp37m-win_amd64.whl", hash = "sha256:f942a799516184c855e1a32fbc7b29d7e571b52612647866d4ec1c3242578fcb"}, + {file = "SQLAlchemy-2.0.36-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3d6718667da04294d7df1670d70eeddd414f313738d20a6f1d1f379e3139a545"}, + {file = "SQLAlchemy-2.0.36-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:72c28b84b174ce8af8504ca28ae9347d317f9dba3999e5981a3cd441f3712e24"}, + {file = "SQLAlchemy-2.0.36-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b11d0cfdd2b095e7b0686cf5fabeb9c67fae5b06d265d8180715b8cfa86522e3"}, + {file = "SQLAlchemy-2.0.36-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e32092c47011d113dc01ab3e1d3ce9f006a47223b18422c5c0d150af13a00687"}, + {file = "SQLAlchemy-2.0.36-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:6a440293d802d3011028e14e4226da1434b373cbaf4a4bbb63f845761a708346"}, + {file = "SQLAlchemy-2.0.36-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:c54a1e53a0c308a8e8a7dffb59097bff7facda27c70c286f005327f21b2bd6b1"}, + {file = "SQLAlchemy-2.0.36-cp38-cp38-win32.whl", hash = "sha256:1e0d612a17581b6616ff03c8e3d5eff7452f34655c901f75d62bd86449d9750e"}, + {file = "SQLAlchemy-2.0.36-cp38-cp38-win_amd64.whl", hash = "sha256:8958b10490125124463095bbdadda5aa22ec799f91958e410438ad6c97a7b793"}, + {file = "SQLAlchemy-2.0.36-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:dc022184d3e5cacc9579e41805a681187650e170eb2fd70e28b86192a479dcaa"}, + {file = "SQLAlchemy-2.0.36-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b817d41d692bf286abc181f8af476c4fbef3fd05e798777492618378448ee689"}, + {file = "SQLAlchemy-2.0.36-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a4e46a888b54be23d03a89be510f24a7652fe6ff660787b96cd0e57a4ebcb46d"}, + {file = "SQLAlchemy-2.0.36-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c4ae3005ed83f5967f961fd091f2f8c5329161f69ce8480aa8168b2d7fe37f06"}, + {file = "SQLAlchemy-2.0.36-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:03e08af7a5f9386a43919eda9de33ffda16b44eb11f3b313e6822243770e9763"}, + {file = "SQLAlchemy-2.0.36-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:3dbb986bad3ed5ceaf090200eba750b5245150bd97d3e67343a3cfed06feecf7"}, + {file = "SQLAlchemy-2.0.36-cp39-cp39-win32.whl", hash = "sha256:9fe53b404f24789b5ea9003fc25b9a3988feddebd7e7b369c8fac27ad6f52f28"}, + {file = "SQLAlchemy-2.0.36-cp39-cp39-win_amd64.whl", hash = "sha256:af148a33ff0349f53512a049c6406923e4e02bf2f26c5fb285f143faf4f0e46a"}, + {file = "SQLAlchemy-2.0.36-py3-none-any.whl", hash = "sha256:fddbe92b4760c6f5d48162aef14824add991aeda8ddadb3c31d56eb15ca69f8e"}, + {file = "sqlalchemy-2.0.36.tar.gz", hash = "sha256:7f2767680b6d2398aea7082e45a774b2b0767b5c8d8ffb9c8b683088ea9b29c5"}, ] [package.dependencies] @@ -3278,7 +3303,7 @@ aioodbc = ["aioodbc", "greenlet (!=0.4.17)"] aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"] asyncio = ["greenlet (!=0.4.17)"] asyncmy = ["asyncmy (>=0.2.3,!=0.2.4,!=0.2.6)", "greenlet (!=0.4.17)"] -mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2,!=1.1.5)"] +mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2,!=1.1.5,!=1.1.10)"] mssql = ["pyodbc"] mssql-pymssql = ["pymssql"] mssql-pyodbc = ["pyodbc"] @@ -3329,13 +3354,13 @@ widechars = ["wcwidth"] [[package]] name = "taskipy" -version = "1.13.0" +version = "1.14.0" description = "tasks runner for python projects" optional = false python-versions = "<4.0,>=3.6" files = [ - {file = "taskipy-1.13.0-py3-none-any.whl", hash = "sha256:56f42b7e508d9aed2c7b6365f8d3dab62dbd0c768c1ab606c819da4fc38421f7"}, - {file = "taskipy-1.13.0.tar.gz", hash = "sha256:2b52f0257958fed151f1340f7de93fcf0848f7a358ad62ba05c31c2ca04f89fe"}, + {file = "taskipy-1.14.0-py3-none-any.whl", hash = "sha256:29040d9a8038170602feb71792bdef5203720ed30f595304aee843625892452b"}, + {file = "taskipy-1.14.0.tar.gz", hash = "sha256:5d9631c29980481d59858f0a100ed3200cf7468ca8c0540ef19388586485532d"}, ] [package.dependencies] @@ -3379,13 +3404,13 @@ files = [ [[package]] name = "tomli" -version = "2.0.1" +version = "2.0.2" description = "A lil' TOML parser" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, + {file = "tomli-2.0.2-py3-none-any.whl", hash = "sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38"}, + {file = "tomli-2.0.2.tar.gz", hash = "sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed"}, ] [[package]] @@ -3522,13 +3547,13 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" -version = "20.26.4" +version = "20.27.1" description = "Virtual Python Environment builder" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "virtualenv-20.26.4-py3-none-any.whl", hash = "sha256:48f2695d9809277003f30776d155615ffc11328e6a0a8c1f0ec80188d7874a55"}, - {file = "virtualenv-20.26.4.tar.gz", hash = "sha256:c17f4e0f3e6036e9f26700446f85c76ab11df65ff6d8a9cbfad9f71aabfcf23c"}, + {file = "virtualenv-20.27.1-py3-none-any.whl", hash = "sha256:f11f1b8a29525562925f745563bfd48b189450f61fb34c4f9cc79dd5aa32a1f4"}, + {file = "virtualenv-20.27.1.tar.gz", hash = "sha256:142c6be10212543b32c6c45d3d3893dff89112cc588b7d0879ae5a1ec03a47ba"}, ] [package.dependencies] @@ -3686,4 +3711,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.11" -content-hash = "6b299f0c7ae418b422ff8a5fd5445985d249e4e432ef04efc9407869f65b7d05" +content-hash = "0875ccd822126eb1a69cfb6641038fa201b019885cde6139793d1559f08c2939" From 1d47abd029238760d6ba4e2ee78a7442c065d0e5 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 8 Nov 2024 10:32:15 -0300 Subject: [PATCH 44/71] corrige wait --- pipelines/treatment/monitoramento/flows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/treatment/monitoramento/flows.py b/pipelines/treatment/monitoramento/flows.py index 10a3d3c8f..4951bdb96 100644 --- a/pipelines/treatment/monitoramento/flows.py +++ b/pipelines/treatment/monitoramento/flows.py @@ -28,7 +28,7 @@ selector=constants.VIAGEM_VALIDACAO_SELECTOR.value, agent_label=smtr_constants.RJ_SMTR_AGENT_LABEL.value, wait=[ - VIAGEM_INFORMADA_MATERIALIZACAO, + constants.VIAGEM_INFORMADA_SELECTOR.value, { "redis_key": f"{smtr_constants.GPS_SPPO_DATASET_ID.value}\ .{smtr_constants.GPS_SPPO_TREATED_TABLE_ID.value}", From 4bdab15d9f6640dd33d396c7bec5fa2841cf4096 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 8 Nov 2024 10:53:56 -0300 Subject: [PATCH 45/71] add change log --- queries/models/monitoramento/CHANGELOG.md | 8 ++++++++ queries/models/planejamento/CHANGELOG.md | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/queries/models/monitoramento/CHANGELOG.md b/queries/models/monitoramento/CHANGELOG.md index 04bf81226..5e4c4f5c0 100644 --- a/queries/models/monitoramento/CHANGELOG.md +++ b/queries/models/monitoramento/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog - monitoramento +## [1.1.0] - 2024-11-08 + +### Adicionado +- Cria modelos de validação de viagens: `gps_viagem.sql`, `gps_segmento_viagem.sql` e `viagem_validacao.sql` (https://github.com/prefeitura-rio/pipelines_rj_smtr/pull/237) + +### Alterado +- Adiciona coluna `modo` no modelo `viagem_informada_monitoramento.sql` (https://github.com/prefeitura-rio/pipelines_rj_smtr/pull/237) + ## [1.0.1] - 2024-10-23 ### Corrigido diff --git a/queries/models/planejamento/CHANGELOG.md b/queries/models/planejamento/CHANGELOG.md index 3bd387d19..5efce68c4 100644 --- a/queries/models/planejamento/CHANGELOG.md +++ b/queries/models/planejamento/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog - planejamento +## [1.1.0] - 2024-11-08 + +### Adicionado + +- Cria modelos para tabela de calendario: `aux_calendario_manual.sql` e `calendario.sql` (https://github.com/prefeitura-rio/pipelines_rj_smtr/pull/237) +- Cria modelos para divisão de shapes: `aux_segmento_shape.py`, `aux_shapes_geom_filtrada.sql`, `shapes_geom_planejamento.sql` e `segmento_shape.sql` (https://github.com/prefeitura-rio/pipelines_rj_smtr/pull/237) + + ## [1.0.1] - 2024-10-30 ### Alterado From dc43be88fc6ac877ce01e8cae0809b0cf15588db Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 8 Nov 2024 10:54:06 -0300 Subject: [PATCH 46/71] remove viagem planejada --- .../viagem_planejada_planejamento.sql | 340 ------------------ 1 file changed, 340 deletions(-) delete mode 100644 queries/models/planejamento/viagem_planejada_planejamento.sql diff --git a/queries/models/planejamento/viagem_planejada_planejamento.sql b/queries/models/planejamento/viagem_planejada_planejamento.sql deleted file mode 100644 index b15c219d5..000000000 --- a/queries/models/planejamento/viagem_planejada_planejamento.sql +++ /dev/null @@ -1,340 +0,0 @@ -{{ - config( - partition_by={ - "field": "data", - "data_type": "date", - "granularity": "day", - }, - alias="viagem_planejada", - incremental_strategy="merge", - unique_key="id_viagem", - incremental_predicates=[ - "DBT_INTERNAL_DEST.data between date('" - + var("date_range_start") - + "') and date_add(date('" - + var("date_range_end") - + "'), interval 1 day)" - ], - ) -}} - - -{% set calendario = ref("calendario") %} - -{% if execute %} - {% if is_incremental() %} - {% set gtfs_feeds_query %} - select distinct concat("'", feed_start_date, "'") as feed_start_date - from {{ calendario }} - where - data between date("{{ var('date_range_start') }}") - and date("{{ var('date_range_end') }}") - {% endset %} - - {% set gtfs_feeds = run_query(gtfs_feeds_query).columns[0].values() %} - {% endif %} -{% endif %} - -with - calendario as ( - select * - from {{ calendario }} - {% if is_incremental() %} - where - data between date("{{ var('date_range_start') }}") and date( - "{{ var('date_range_end') }}" - ) - {% endif %} - ), - trips as ( - select * - from `rj-smtr.gtfs.trips` - {% if is_incremental() %} - where feed_start_date in ({{ gtfs_feeds | join(", ") }}) - {% endif %} - ), - frequencies as ( - select - *, - split(start_time, ":") as start_time_parts, - split(end_time, ":") as end_time_parts, - from `rj-smtr.gtfs.frequencies` - {% if is_incremental() %} - where feed_start_date in ({{ gtfs_feeds | join(", ") }}) - {% endif %} - ), - frequencies_tratada as ( - select - * except (start_time_parts, end_time_parts, start_time, end_time), - div(cast(start_time_parts[0] as integer), 24) days_to_add_start, - div(cast(end_time_parts[0] as integer), 24) days_to_add_end, - concat( - lpad( - cast( - if( - cast(start_time_parts[0] as integer) >= 24, - cast(start_time_parts[0] as integer) - 24, - cast(start_time_parts[0] as integer) - ) as string - ), - 2, - '0' - ), - ":", - start_time_parts[1], - ":", - start_time_parts[2] - ) as start_time, - concat( - lpad( - cast( - if( - cast(end_time_parts[0] as integer) >= 24, - cast(end_time_parts[0] as integer) - 24, - cast(end_time_parts[0] as integer) - ) as string - ), - 2, - '0' - ), - ":", - end_time_parts[1], - ":", - end_time_parts[2] - ) as end_time - from frequencies - ), - routes as ( - select - *, - case - when agency_id in ("22005", "22002", "22004", "22003") - then "Ônibus" - when agency_id = "20001" - then "BRT" - end as modo - {# from `rj-smtr.gtfs.routes` #} - from {{ ref("routes_gtfs") }} - {% if is_incremental() %} - where feed_start_date in ({{ gtfs_feeds | join(", ") }}) - {% endif %} - ), - trips_dia as ( - select - c.data, - t.trip_id, - r.modo, - t.route_id, - t.service_id, - r.route_short_name as servico, - t.direction_id, - t.shape_id, - c.tipo_dia, - c.subtipo_dia, - c.tipo_os, - t.feed_version, - t.feed_start_date, - regexp_extract(t.trip_headsign, r'\[.*?\]') as evento - from calendario c - join trips t using (feed_start_date, feed_version) - join routes r using (feed_start_date, feed_version, route_id) - where t.service_id in unnest(c.service_ids) - ), - trips_frequences_dia as ( - select - td.* except (evento), - timestamp( - concat( - cast(date_add(data, interval f.days_to_add_start day) as string), - ' ', - f.start_time - ), - "America/Sao_Paulo" - ) as start_timestamp, - timestamp( - concat( - cast(date_add(data, interval f.days_to_add_end day) as string), - ' ', - f.end_time - ), - "America/Sao_Paulo" - ) as end_timestamp, - f.headway_secs - from trips_dia td - join frequencies_tratada f using (feed_start_date, feed_version, trip_id) - ), - end_time_tratado as ( - select - * except (end_timestamp), - case - when - end_timestamp > lead(start_timestamp) over ( - partition by - feed_start_date, feed_version, data, servico, direction_id - order by start_timestamp - ) - then - lead(start_timestamp) over ( - partition by - feed_start_date, feed_version, data, servico, direction_id - order by start_timestamp - ) - else end_timestamp - end as end_timestamp - from trips_frequences_dia - ), - os_trajetos_alternativos as ( - select * - from `rj-smtr.gtfs.ordem_servico_trajeto_alternativo` - {% if is_incremental() %} - where feed_start_date in ({{ gtfs_feeds | join(", ") }}) - {% endif %} - - ), - trips_alternativas as ( - select - data, - servico, - direction_id, - array_agg( - struct( - td.trip_id as trip_id, - td.shape_id as shape_id, - evento as evento, - case - when td.direction_id = '0' - then os.extensao_ida - when td.direction_id = '1' - then os.extensao_volta - end as extensao - ) - ) as trajetos_alternativos - from trips_dia td - join - os_trajetos_alternativos os using ( - feed_start_date, feed_version, tipo_os, servico, evento - ) - where td.trip_id not in (select trip_id from frequencies) - group by 1, 2, 3 - ), - viagens as ( - select - ett.*, - datetime(partida, "America/Sao_Paulo") as datetime_partida, - ta.trajetos_alternativos - from - end_time_tratado ett, - unnest( - generate_timestamp_array( - start_timestamp, - timestamp_sub(end_timestamp, interval 1 second), - interval headway_secs second - ) - ) as partida - left join trips_alternativas ta using (data, servico, direction_id) - ), - ordem_servico as ( - select - * except (horario_inicio, horario_fim), - parse_time("%H:%M:%S", lpad(horario_inicio, 8, '0')) as horario_inicio, - split(horario_fim, ":") horario_fim_parts - from `rj-smtr.gtfs.ordem_servico` - where - {% if is_incremental() %} feed_start_date in ({{ gtfs_feeds | join(", ") }}) - {% else %} feed_start_date >= "2024-09-01" - {% endif %} - ), - ordem_servico_tratada as ( - select - * except (horario_fim_parts), - div(cast(horario_fim_parts[0] as integer), 24) as dias_horario_fim, - parse_time( - "%H:%M:%S", - concat( - lpad( - cast( - if( - cast(horario_fim_parts[0] as integer) >= 24, - cast(horario_fim_parts[0] as integer) - 24, - cast(horario_fim_parts[0] as integer) - ) as string - ), - 2, - '0' - ), - ":", - horario_fim_parts[1], - ":", - horario_fim_parts[2] - ) - ) as horario_fim, - from ordem_servico - - ), - viagem_os as ( - select - v.*, - case - when v.direction_id = '0' - then os.extensao_ida - when v.direction_id = '1' - then os.extensao_volta - end as extensao - from viagens v - left join - ordem_servico_tratada os using ( - feed_start_date, feed_version, tipo_os, tipo_dia, servico - ) - where - (os.distancia_total_planejada is null or os.distancia_total_planejada > 0) - and ( - os.feed_start_date is null - or v.datetime_partida - between datetime(data, os.horario_inicio) and datetime( - date_add(data, interval os.dias_horario_fim day), os.horario_fim - ) - ) - ), - viagem_planejada as ( - select - date(datetime_partida) as data, - concat( - servico, - "_", - direction_id, - "_", - shape_id, - "_", - format_datetime("%Y%m%d%H%M%S", datetime_partida) - ) as id_viagem, - datetime_partida, - modo, - service_id, - trip_id, - route_id, - shape_id, - servico, - case when direction_id = '0' then "Ida" else "Volta" end as sentido, - extensao, - trajetos_alternativos, - data as data_referencia, - tipo_dia, - subtipo_dia, - tipo_os, - feed_version, - feed_start_date, - '{{ var("version") }}' as versao, - current_datetime("America/Sao_Paulo") as datetime_ultima_atualizacao - from viagem_os - ) -select * except (rn) -from - ( - select - *, - row_number() over ( - partition by id_viagem order by data_referencia desc - ) as rn - from viagem_planejada - ) -where rn = 1 From 0f482e6c139009b1610ef6fac7816427cfc972d7 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 8 Nov 2024 10:56:28 -0300 Subject: [PATCH 47/71] =?UTF-8?q?remove=20execu=C3=A7=C3=A3o=20do=20calend?= =?UTF-8?q?ario=20junto=20com=20gtfs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipelines/migration/br_rj_riodejaneiro_gtfs/flows.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pipelines/migration/br_rj_riodejaneiro_gtfs/flows.py b/pipelines/migration/br_rj_riodejaneiro_gtfs/flows.py index 860a4f7ca..9ce6160e1 100644 --- a/pipelines/migration/br_rj_riodejaneiro_gtfs/flows.py +++ b/pipelines/migration/br_rj_riodejaneiro_gtfs/flows.py @@ -215,6 +215,7 @@ + " " + constants.PLANEJAMENTO_MATERIALIZACAO_DATASET_ID.value, _vars=dbt_vars, + exclude="calendario", ).set_upstream(task=wait_captura) wait_materialize_true = update_last_captured_os( From 3202631587c16fc82f8b0d24a44905c5f1a42726 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 8 Nov 2024 11:14:08 -0300 Subject: [PATCH 48/71] add conversao de tz --- pipelines/treatment/templates/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pipelines/treatment/templates/utils.py b/pipelines/treatment/templates/utils.py index f4ab5038d..fbbda2606 100644 --- a/pipelines/treatment/templates/utils.py +++ b/pipelines/treatment/templates/utils.py @@ -72,7 +72,7 @@ def get_last_materialized_datetime(self, env: str) -> datetime: ) ) - return last_datetime + return convert_timezone(timestamp=last_datetime) def get_datetime_end(self, timestamp: datetime) -> datetime: """ @@ -99,7 +99,9 @@ def is_up_to_date(self, env: str, timestamp: datetime) -> bool: """ last_materialization = self.get_last_materialized_datetime(env=env) last_schedule = cron_get_last_date(cron_expr=self.schedule_cron, timestamp=timestamp) - return last_materialization >= last_schedule - timedelta(hours=self.incremental_delay_hours) + return convert_timezone(timestamp=last_materialization) >= last_schedule - timedelta( + hours=self.incremental_delay_hours + ) def get_next_schedule_datetime(self, timestamp: datetime) -> datetime: """ From c7b9929256185f31d4db28c5535681d2b0a109fd Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 8 Nov 2024 11:14:45 -0300 Subject: [PATCH 49/71] remove conversao extra --- pipelines/treatment/templates/utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pipelines/treatment/templates/utils.py b/pipelines/treatment/templates/utils.py index fbbda2606..001c254c3 100644 --- a/pipelines/treatment/templates/utils.py +++ b/pipelines/treatment/templates/utils.py @@ -99,9 +99,7 @@ def is_up_to_date(self, env: str, timestamp: datetime) -> bool: """ last_materialization = self.get_last_materialized_datetime(env=env) last_schedule = cron_get_last_date(cron_expr=self.schedule_cron, timestamp=timestamp) - return convert_timezone(timestamp=last_materialization) >= last_schedule - timedelta( - hours=self.incremental_delay_hours - ) + return last_materialization >= last_schedule - timedelta(hours=self.incremental_delay_hours) def get_next_schedule_datetime(self, timestamp: datetime) -> datetime: """ From 8ee702b8adb833e5ad96a0f16df828182257451e Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 8 Nov 2024 11:15:43 -0300 Subject: [PATCH 50/71] converte tz --- pipelines/treatment/templates/tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pipelines/treatment/templates/tasks.py b/pipelines/treatment/templates/tasks.py index 9fc15b3b8..bcd07c9c1 100644 --- a/pipelines/treatment/templates/tasks.py +++ b/pipelines/treatment/templates/tasks.py @@ -167,9 +167,9 @@ def wait_data_sources( cron_expr=ds["schedule_cron"], timestamp=datetime_end, ) - complete = last_materialization >= last_schedule - timedelta( - hours=ds.get("delay_hours", 0) - ) + complete = convert_timezone( + timestamp=last_materialization + ) >= last_schedule - timedelta(hours=ds.get("delay_hours", 0)) else: raise NotImplementedError(f"Espera por fontes do tipo {type(ds)} não implementada") From f1d47e473c446b50b97a41ee4c606b182282465f Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 8 Nov 2024 11:42:45 -0300 Subject: [PATCH 51/71] teste --- pipelines/treatment/monitoramento/flows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/treatment/monitoramento/flows.py b/pipelines/treatment/monitoramento/flows.py index 4951bdb96..6be1ff6b3 100644 --- a/pipelines/treatment/monitoramento/flows.py +++ b/pipelines/treatment/monitoramento/flows.py @@ -28,7 +28,7 @@ selector=constants.VIAGEM_VALIDACAO_SELECTOR.value, agent_label=smtr_constants.RJ_SMTR_AGENT_LABEL.value, wait=[ - constants.VIAGEM_INFORMADA_SELECTOR.value, + # constants.VIAGEM_INFORMADA_SELECTOR.value, { "redis_key": f"{smtr_constants.GPS_SPPO_DATASET_ID.value}\ .{smtr_constants.GPS_SPPO_TREATED_TABLE_ID.value}", From cdfcfe1c705c4aa5cfdfdcf18a5fb5f2ec938ca5 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 8 Nov 2024 12:04:29 -0300 Subject: [PATCH 52/71] descomenta wait --- pipelines/treatment/monitoramento/flows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/treatment/monitoramento/flows.py b/pipelines/treatment/monitoramento/flows.py index 6be1ff6b3..4951bdb96 100644 --- a/pipelines/treatment/monitoramento/flows.py +++ b/pipelines/treatment/monitoramento/flows.py @@ -28,7 +28,7 @@ selector=constants.VIAGEM_VALIDACAO_SELECTOR.value, agent_label=smtr_constants.RJ_SMTR_AGENT_LABEL.value, wait=[ - # constants.VIAGEM_INFORMADA_SELECTOR.value, + constants.VIAGEM_INFORMADA_SELECTOR.value, { "redis_key": f"{smtr_constants.GPS_SPPO_DATASET_ID.value}\ .{smtr_constants.GPS_SPPO_TREATED_TABLE_ID.value}", From 4420c28fc9cc95bbcd5b7feb4bcabb94008064f9 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 8 Nov 2024 14:06:49 -0300 Subject: [PATCH 53/71] add variaveis --- queries/dbt_project.yml | 7 +++++++ queries/models/planejamento/segmento_shape.sql | 16 ++++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/queries/dbt_project.yml b/queries/dbt_project.yml index b527f3c78..eafcc737c 100644 --- a/queries/dbt_project.yml +++ b/queries/dbt_project.yml @@ -186,6 +186,13 @@ vars: ### Encontro de Contas ### encontro_contas_modo: "" + ### Viagens 2.0 ### + + limite_reducao_area_buffer: 0.5 + comprimento_minimo_segmento_shape: 990 + buffer_segmento_metros: 20 + buffer_tunel_metros: 50 + tests: rj_smtr: where: "DATA BETWEEN DATE('__date_range_start__') AND DATE('__date_range_end__')" diff --git a/queries/models/planejamento/segmento_shape.sql b/queries/models/planejamento/segmento_shape.sql index d482fcff6..6e9d42403 100644 --- a/queries/models/planejamento/segmento_shape.sql +++ b/queries/models/planejamento/segmento_shape.sql @@ -28,13 +28,19 @@ with from {{ ref("aux_segmento_shape") }} ), tunel as ( - select st_union_agg(st_buffer(geometry, 50)) as buffer_tunel + select + st_union_agg( + st_buffer(geometry, {{ var("buffer_tunel_metros") }}) + ) as buffer_tunel from {{ source("dados_mestres", "logradouro") }} where tipo = "Túnel" ), buffer_segmento as ( - select *, st_buffer(segmento, 20) as buffer_completo, from aux_segmento + select + *, + st_buffer(segmento, {{ var("buffer_segmento_metros") }}) as buffer_completo, + from aux_segmento ), intercessao_segmento as ( select @@ -64,8 +70,10 @@ with s.*, st_intersects(s.segmento, t.buffer_tunel) as indicador_tunel, st_area(s.buffer) / st_area(s.buffer_completo) - < 0.5 as indicador_area_prejudicada, - s.comprimento_segmento < 990 as indicador_segmento_pequeno, + < {{ var("limite_reducao_area_buffer") }} as indicador_area_prejudicada, + s.comprimento_segmento + < {{ var("comprimento_minimo_segmento_shape") }} + as indicador_segmento_pequeno, cast(id_segmento as integer) as id_segmento_int from buffer_segmento_recortado s cross join tunel t From d9fc0c55c21f512a9bc08c416a1655da268b4b66 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 8 Nov 2024 14:07:00 -0300 Subject: [PATCH 54/71] altera target --- queries/profiles.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/queries/profiles.yml b/queries/profiles.yml index 436d137af..c21cd60ac 100644 --- a/queries/profiles.yml +++ b/queries/profiles.yml @@ -78,4 +78,4 @@ queries: spark.executor.instances: "2" spark.driver.memory: 4g spark.driver.memoryOverhead: 1g - target: hmg \ No newline at end of file + target: prod \ No newline at end of file From c6c909f784c487e88dc80388cee10dae60bbb45d Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 8 Nov 2024 14:07:13 -0300 Subject: [PATCH 55/71] adiciona docs --- queries/models/monitoramento/schema.yml | 30 ++++++++++++------------- queries/models/planejamento/schema.yml | 4 ++-- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/queries/models/monitoramento/schema.yml b/queries/models/monitoramento/schema.yml index 275e37e1a..d0b7976ba 100644 --- a/queries/models/monitoramento/schema.yml +++ b/queries/models/monitoramento/schema.yml @@ -7,23 +7,23 @@ models: data_type: date quote: true - name: id_viagem - description: Identificador único da viagem + description: "{{ doc('id_viagem') }}" data_type: string quote: true - name: datetime_partida - description: Data e hora da partida da viagem em GMT-3 + description: "{{ doc('datetime_partida') }}" data_type: datetime quote: true - name: datetime_chegada - description: Data e hora da chegada da viagem em GMT-3 + description: "{{ doc('datetime_chegada') }}" data_type: datetime quote: true - name: modo - description: "Tipo de transporte (BRT, Ônibus Executivo, ...)" + description: "{{ doc('modo') }}" data_type: string quote: true - name: id_veiculo - description: Código identificador do veículo (número de ordem) + description: "{{ doc('id_veiculo') }}" data_type: string quote: true - name: trip_id @@ -39,11 +39,11 @@ models: data_type: string quote: true - name: servico - description: "Nome curto da linha operada pelo veículo com variação de serviço (ex: 010, 011SN, ...)" + description: "{{ doc('servico') }}" data_type: string quote: true - name: sentido - description: Sentido de operação do serviço + description: "{{ doc('sentido') }}" data_type: string quote: true - name: fonte_gps @@ -74,23 +74,23 @@ models: data_type: date quote: true - name: id_viagem - description: Identificador único da viagem. + description: "{{ doc('id_viagem') }}" data_type: string quote: true - name: datetime_partida - description: Data e hora da partida da viagem em GMT-3. + description: "{{ doc('datetime_partida') }}" data_type: datetime quote: true - name: datetime_chegada - description: Data e hora da chegada da viagem em GMT-3. + description: "{{ doc('datetime_chegada') }}" data_type: datetime quote: true - name: modo - description: "Tipo de transporte (BRT, Ônibus Executivo, ...)" + description: "{{ doc('modo') }}" data_type: string quote: true - name: id_veiculo - description: Código identificador do veículo (número de ordem). + description: "{{ doc('id_veiculo') }}" data_type: string quote: true - name: trip_id @@ -106,11 +106,11 @@ models: data_type: string quote: true - name: servico - description: "Nome curto da linha operada pelo veículo com variação de serviço (ex: 010, 011SN, ...)." + description: "{{ doc('servico') }}" data_type: string quote: true - name: sentido - description: Sentido de operação do serviço. + description: "{{ doc('sentido') }}" data_type: string quote: true - name: quantidade_segmentos_verificados @@ -138,7 +138,7 @@ models: data_type: string quote: true - name: feed_start_date - description: Data inicial do feed (versão). + description: "{{ doc('feed_start_date') }}" data_type: date quote: true - name: versao diff --git a/queries/models/planejamento/schema.yml b/queries/models/planejamento/schema.yml index e5340d3c9..f679d4b0e 100644 --- a/queries/models/planejamento/schema.yml +++ b/queries/models/planejamento/schema.yml @@ -24,7 +24,7 @@ models: data_type: string quote: true - name: feed_start_date - description: Partição) Data inicial do feed (versão). + description: "{{ doc('feed_start_date') }}" data_type: date quote: true - name: feed_end_date @@ -87,7 +87,7 @@ models: data_type: string quote: true - name: feed_start_date - description: (Partição) Data inicial do feed (versão). + description: "{{ doc('feed_start_date') }}" data_type: date quote: true - name: feed_end_date From 0aa6cbdfd295ee7644938273f78335c3228d21e2 Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 11 Nov 2024 10:28:51 -0300 Subject: [PATCH 56/71] corrige filtro incremental --- .../models/monitoramento/staging/gps_segmento_viagem.sql | 7 +------ queries/models/monitoramento/staging/gps_viagem.sql | 4 ++-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/queries/models/monitoramento/staging/gps_segmento_viagem.sql b/queries/models/monitoramento/staging/gps_segmento_viagem.sql index 1a5c70144..adb0267c1 100644 --- a/queries/models/monitoramento/staging/gps_segmento_viagem.sql +++ b/queries/models/monitoramento/staging/gps_segmento_viagem.sql @@ -51,12 +51,7 @@ with c.feed_start_date from {{ ref("gps_viagem") }} gv join calendario c using (data) - {% if is_incremental() %} - where - data between date_sub( - date('{{ var("date_range_start") }}'), interval 1 day - ) and date('{{ var("date_range_end") }}') - {% endif %} + {% if is_incremental() %} where {{ incremental_filter }} {% endif %} ), segmento as ( select diff --git a/queries/models/monitoramento/staging/gps_viagem.sql b/queries/models/monitoramento/staging/gps_viagem.sql index afb5879a4..01ce0062e 100644 --- a/queries/models/monitoramento/staging/gps_viagem.sql +++ b/queries/models/monitoramento/staging/gps_viagem.sql @@ -42,7 +42,7 @@ with {% if is_incremental() %} data between date_sub( date('{{ var("date_range_start") }}'), interval 1 day - ) and date_sub(date('{{ var("date_range_end") }}')) + ) and date_add(date('{{ var("date_range_end") }}'), interval 1 day) {% else %} data >= date("2024-10-11") {% endif %} @@ -55,7 +55,7 @@ with {% if is_incremental() %} data between date_sub( date('{{ var("date_range_start") }}'), interval 1 day - ) and date_sub(date('{{ var("date_range_end") }}')) + ) and date_add(date('{{ var("date_range_end") }}'), interval 1 day) {% else %} data >= date("2024-10-11") {% endif %} ), From 4f3f1453b5450fed2478d22db4b34d32bece878d Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 11 Nov 2024 14:21:14 -0300 Subject: [PATCH 57/71] altera logica viagem nao planejada --- .../staging/gps_segmento_viagem.sql | 3 ++ .../models/monitoramento/viagem_validacao.sql | 41 +++++++++---------- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/queries/models/monitoramento/staging/gps_segmento_viagem.sql b/queries/models/monitoramento/staging/gps_segmento_viagem.sql index adb0267c1..7900a060a 100644 --- a/queries/models/monitoramento/staging/gps_segmento_viagem.sql +++ b/queries/models/monitoramento/staging/gps_segmento_viagem.sql @@ -90,6 +90,7 @@ with v.shape_id, v.servico, v.sentido, + c.service_ids, c.tipo_dia, c.feed_start_date, c.feed_version @@ -112,6 +113,7 @@ with s.indicador_segmento_desconsiderado, v.servico, v.sentido, + v.service_ids, v.tipo_dia, feed_version, feed_start_date @@ -135,6 +137,7 @@ select ifnull(g.quantidade_gps, 0) as quantidade_gps, v.feed_version, v.feed_start_date, + v.service_ids, v.tipo_dia, '{{ var("version") }}' as versao, current_datetime("America/Sao_Paulo") as datetime_ultima_atualizacao diff --git a/queries/models/monitoramento/viagem_validacao.sql b/queries/models/monitoramento/viagem_validacao.sql index 96aee7b97..e1587e336 100644 --- a/queries/models/monitoramento/viagem_validacao.sql +++ b/queries/models/monitoramento/viagem_validacao.sql @@ -14,9 +14,6 @@ and date('{{ var("date_range_end") }}') {% endset %} - -{% set parametro_validacao = 0.9 %} - with contagem as ( select @@ -33,6 +30,7 @@ with sentido, count(*) as quantidade_segmentos_verificados, countif(quantidade_gps > 0) as quantidade_segmentos_validos, + service_ids, tipo_dia, feed_version, feed_start_date @@ -52,6 +50,7 @@ with shape_id, servico, sentido, + service_ids, tipo_dia, feed_version, feed_start_date @@ -73,6 +72,7 @@ with quantidade_segmentos_validos, quantidade_segmentos_validos / quantidade_segmentos_verificados as indice_validacao, + service_ids, tipo_dia, feed_version, feed_start_date @@ -83,25 +83,24 @@ with feed_start_date, feed_version, route_id, - case - when service_id like "D_%" - then "Domingo" - when service_id like "S_%" - then "Sabado" - when service_id like "U_%" - then "Dia Útil" - end as tipo_dia + array_agg(service_id) as service_ids, from {{ ref("trips_gtfs") }} - where - (service_id like "D_%" or service_id like "S_%" or service_id like "U_%") - {% if is_incremental() %} - and feed_start_date in ({{ gtfs_feeds | join(", ") }}) - {% endif %} + {% if is_incremental() %} + where feed_start_date in ({{ gtfs_feeds | join(", ") }}) + {% endif %} + group by 1, 2, 3 ), servicos_planejados as ( - select i.*, t.tipo_dia is not null as indicador_servico_planejado + select + i.*, + ( + select count(*) + from unnest(i.service_ids) as service_id + join unnest(t.service_ids) as service_id using (service_id) + ) + > 0 as indicador_servico_planejado from indice i - left join trips t using (feed_start_date, feed_version, route_id, tipo_dia) + left join trips t using (feed_start_date, feed_version, route_id) ) select data, @@ -118,11 +117,11 @@ select quantidade_segmentos_verificados, quantidade_segmentos_validos, indice_validacao, - indice_validacao >= {{ parametro_validacao }} as indicador_trajeto_valido, + indice_validacao >= {{ var("parametro_validacao") }} as indicador_trajeto_valido, indicador_servico_planejado, - indice_validacao >= {{ parametro_validacao }} + indice_validacao >= {{ var("parametro_validacao") }} and indicador_servico_planejado as indicador_viagem_valida, - {{ parametro_validacao }} as parametro_validacao, + {{ var("parametro_validacao") }} as parametro_validacao, tipo_dia, feed_version, feed_start_date, From 26e8cf1f74225455301f0f95231ce3521898b274 Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 11 Nov 2024 14:21:34 -0300 Subject: [PATCH 58/71] altera logica coluna modo --- .../monitoramento/viagem_informada_monitoramento.sql | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/queries/models/monitoramento/viagem_informada_monitoramento.sql b/queries/models/monitoramento/viagem_informada_monitoramento.sql index ec04cbf22..dea707085 100644 --- a/queries/models/monitoramento/viagem_informada_monitoramento.sql +++ b/queries/models/monitoramento/viagem_informada_monitoramento.sql @@ -112,7 +112,12 @@ with v.id_viagem, v.datetime_partida, v.datetime_chegada, - if(r.route_type = '200', 'Ônibus Executivo', 'Ônibus SPPO') as modo, + case + when r.route_type = '200' + then 'Ônibus Executivo' + when r.route_type = '700' + then 'Ônibus SPPO' + end as modo, v.id_veiculo, v.trip_id, v.route_id, From 83b5673afd8b109ad91fac0f7f14789eecb7ac10 Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 11 Nov 2024 14:21:46 -0300 Subject: [PATCH 59/71] add doc --- queries/models/monitoramento/schema.yml | 4 ++-- queries/models/planejamento/schema.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/queries/models/monitoramento/schema.yml b/queries/models/monitoramento/schema.yml index d0b7976ba..3725d605c 100644 --- a/queries/models/monitoramento/schema.yml +++ b/queries/models/monitoramento/schema.yml @@ -134,11 +134,11 @@ models: data_type: float64 quote: true - name: feed_version - description: String que indica a versão atual do conjunto de dados GTFS. + description: "{{ doc('feed_version') }}" data_type: string quote: true - name: feed_start_date - description: "{{ doc('feed_start_date') }}" + description: "Data inicial do feed (versão)." data_type: date quote: true - name: versao diff --git a/queries/models/planejamento/schema.yml b/queries/models/planejamento/schema.yml index f679d4b0e..3a25985af 100644 --- a/queries/models/planejamento/schema.yml +++ b/queries/models/planejamento/schema.yml @@ -20,7 +20,7 @@ models: description: Tabela contendo os shapes segmentados usados na validação de viagens. columns: - name: feed_version - description: String que indica a versão atual do conjunto de dados GTFS. + description: "{{ doc('feed_version') }}" data_type: string quote: true - name: feed_start_date @@ -83,7 +83,7 @@ models: description: Tabela contendo shapes em formatos geográficos columns: - name: feed_version - description: String que indica a versão atual do conjunto de dados GTFS. + description: "{{ doc('feed_version') }}" data_type: string quote: true - name: feed_start_date From f7ae526c2592e27f060dbd93211818f1877ceaea Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 11 Nov 2024 14:22:03 -0300 Subject: [PATCH 60/71] add docs feed_version --- queries/models/docs.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/queries/models/docs.md b/queries/models/docs.md index 724f6ebce..b471afcc8 100644 --- a/queries/models/docs.md +++ b/queries/models/docs.md @@ -200,6 +200,10 @@ Fim do período de operação planejado (Partição) Data inicial do feed (versão). {% enddocs %} +{% docs feed_version %} +String que indica a versão atual do conjunto de dados GTFS. +{% enddocs %} + {% docs linha %} Número da Linha {% enddocs %} From 43b455f3e43a93e94d847e17b32d1038ee66fdbd Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 11 Nov 2024 14:22:18 -0300 Subject: [PATCH 61/71] add parametro_validacao --- queries/dbt_project.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/queries/dbt_project.yml b/queries/dbt_project.yml index eafcc737c..8b7b4e278 100644 --- a/queries/dbt_project.yml +++ b/queries/dbt_project.yml @@ -192,6 +192,7 @@ vars: comprimento_minimo_segmento_shape: 990 buffer_segmento_metros: 20 buffer_tunel_metros: 50 + parametro_validacao: 0.9 tests: rj_smtr: From 331817bdcdd610a2cc1849e045ee6f3588bb145e Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 11 Nov 2024 14:22:28 -0300 Subject: [PATCH 62/71] altera project --- queries/profiles.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/queries/profiles.yml b/queries/profiles.yml index c21cd60ac..e67088d2d 100644 --- a/queries/profiles.yml +++ b/queries/profiles.yml @@ -34,7 +34,7 @@ queries: location: us method: service-account priority: interactive - project: rj-smtr + project: rj-smtr-dev threads: 1 type: bigquery From ad8109a3ac5f050495b93701c83baa7175616b5a Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 11 Nov 2024 15:47:48 -0300 Subject: [PATCH 63/71] corrige lock --- poetry.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index ce6414a5a..b808892a6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1293,13 +1293,13 @@ pyparsing = {version = ">=2.4.2,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.0.2 || >3.0 [[package]] name = "identify" -version = "2.6.1" +version = "2.6.2" description = "File identification library for Python" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "identify-2.6.1-py2.py3-none-any.whl", hash = "sha256:53863bcac7caf8d2ed85bd20312ea5dcfc22226800f6d6881f232d861db5a8f0"}, - {file = "identify-2.6.1.tar.gz", hash = "sha256:91478c5fb7c3aac5ff7bf9b4344f803843dc586832d5f110d672b19aa1984c98"}, + {file = "identify-2.6.2-py2.py3-none-any.whl", hash = "sha256:c097384259f49e372f4ea00a19719d95ae27dd5ff0fd77ad630aa891306b82f3"}, + {file = "identify-2.6.2.tar.gz", hash = "sha256:fab5c716c24d7a789775228823797296a2994b075fb6080ac83a102772a98cbd"}, ] [package.extras] @@ -3404,13 +3404,13 @@ files = [ [[package]] name = "tomli" -version = "2.0.2" +version = "2.1.0" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" files = [ - {file = "tomli-2.0.2-py3-none-any.whl", hash = "sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38"}, - {file = "tomli-2.0.2.tar.gz", hash = "sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed"}, + {file = "tomli-2.1.0-py3-none-any.whl", hash = "sha256:a5c57c3d1c56f5ccdf89f6523458f60ef716e210fc47c4cfb188c5ba473e0391"}, + {file = "tomli-2.1.0.tar.gz", hash = "sha256:3f646cae2aec94e17d04973e4249548320197cfabdf130015d023de4b74d8ab8"}, ] [[package]] From 216879cbb7cdd2f09e7b59d97166b8736fd2336b Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 11 Nov 2024 15:53:43 -0300 Subject: [PATCH 64/71] att lock --- poetry.lock | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index 5ffb39681..f62a65f5d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2883,7 +2883,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -3723,4 +3722,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.11" -content-hash = "0875ccd822126eb1a69cfb6641038fa201b019885cde6139793d1559f08c2939" +content-hash = "6511da8862567ed4397518e74ec8f84cfcdc02294a33e7b4b558a9da727e127a" From 75b9decb1754e2992822f61f8f5e479c16146b31 Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 11 Nov 2024 16:59:32 -0300 Subject: [PATCH 65/71] add 1 middle_index --- queries/models/planejamento/staging/aux_segmento_shape.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/queries/models/planejamento/staging/aux_segmento_shape.py b/queries/models/planejamento/staging/aux_segmento_shape.py index f579745a3..5ad5e8345 100644 --- a/queries/models/planejamento/staging/aux_segmento_shape.py +++ b/queries/models/planejamento/staging/aux_segmento_shape.py @@ -29,7 +29,7 @@ def cut(line, distance): dist_mod = line_len % distance dist_range = list(np.arange(0, line_len, distance)) - middle_index = len(dist_range) // 2 + middle_index = (len(dist_range) // 2) + 1 last_final_dist = 0 lines = [] From 363df3c50ca5960149e3f8538e0b5c201f586c47 Mon Sep 17 00:00:00 2001 From: Rafael Date: Tue, 12 Nov 2024 13:19:45 -0300 Subject: [PATCH 66/71] add indicadores --- queries/models/monitoramento/schema.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/queries/models/monitoramento/schema.yml b/queries/models/monitoramento/schema.yml index 3725d605c..6bcb0a811 100644 --- a/queries/models/monitoramento/schema.yml +++ b/queries/models/monitoramento/schema.yml @@ -122,9 +122,17 @@ models: data_type: int64 quote: true - name: indice_validacao - description: quantidade_segmentos_validos dividido por quantidade_segmentos_verificados + description: quantidade_segmentos_validos dividido por quantidade_segmentos_verificados. data_type: float64 quote: true + - name: indicador_trajeto_valido + description: Indica se o veículo cumpriu o trajeto (indice_validacao >= parametro_validacao). + data_type: boolean + quote: true + - name: indicador_servico_planejado + description: Indica se o serviço estava planejado para o tipo dia. + data_type: boolean + quote: true - name: indicador_viagem_valida description: Indica se a viagem está válida ou não. data_type: boolean From cad6335aa802fb84cce9834ad7ba640f8b450290 Mon Sep 17 00:00:00 2001 From: Rafael Date: Tue, 12 Nov 2024 13:20:28 -0300 Subject: [PATCH 67/71] add buffer e comprimento python --- queries/models/config.yml | 6 ++++ .../models/planejamento/segmento_shape.sql | 29 ++++++++----------- .../staging/aux_segmento_shape.py | 24 +++++++++++---- 3 files changed, 36 insertions(+), 23 deletions(-) create mode 100644 queries/models/config.yml diff --git a/queries/models/config.yml b/queries/models/config.yml new file mode 100644 index 000000000..c34b05c4f --- /dev/null +++ b/queries/models/config.yml @@ -0,0 +1,6 @@ +version: 2 + +models: + - name: aux_segmento_shape + config: + buffer_segmento_metros: "{{ var('buffer_segmento_metros') }}" \ No newline at end of file diff --git a/queries/models/planejamento/segmento_shape.sql b/queries/models/planejamento/segmento_shape.sql index 6e9d42403..eb140175f 100644 --- a/queries/models/planejamento/segmento_shape.sql +++ b/queries/models/planejamento/segmento_shape.sql @@ -24,7 +24,8 @@ with id_segmento, st_geogfromtext(wkt_segmento) as segmento, wkt_segmento, - round(st_length(st_geogfromtext(wkt_segmento)), 1) as comprimento_segmento + round(cast(comprimento_segmento as float64), 1) as comprimento_segmento, + st_geogfromtext(buffer_completo) as buffer_completo from {{ ref("aux_segmento_shape") }} ), tunel as ( @@ -36,33 +37,27 @@ with where tipo = "Túnel" ), - buffer_segmento as ( - select - *, - st_buffer(segmento, {{ var("buffer_segmento_metros") }}) as buffer_completo, - from aux_segmento - ), intercessao_segmento as ( select - b1.shape_id, - b1.id_segmento, - st_union(array_agg(b2.buffer_completo)) as buffer_segmento_posterior - from buffer_segmento b1 + s1.shape_id, + s1.id_segmento, + st_union(array_agg(s2.buffer_completo)) as buffer_segmento_posterior + from aux_segmento s1 join - buffer_segmento b2 - on b1.shape_id = b2.shape_id - and b1.id_segmento < b2.id_segmento - and st_intersects(b1.buffer_completo, b2.buffer_completo) + aux_segmento s2 + on s1.shape_id = s2.shape_id + and s1.id_segmento < s2.id_segmento + and st_intersects(s1.buffer_completo, s2.buffer_completo) group by 1, 2 ), buffer_segmento_recortado as ( select - b.*, + s.*, coalesce( st_difference(buffer_completo, i.buffer_segmento_posterior), buffer_completo ) as buffer - from buffer_segmento b + from aux_segmento s left join intercessao_segmento i using (shape_id, id_segmento) ), indicador_validacao_shape as ( diff --git a/queries/models/planejamento/staging/aux_segmento_shape.py b/queries/models/planejamento/staging/aux_segmento_shape.py index 5ad5e8345..88609f5af 100644 --- a/queries/models/planejamento/staging/aux_segmento_shape.py +++ b/queries/models/planejamento/staging/aux_segmento_shape.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import numpy as np import pyproj -from pyspark.sql.functions import col, explode, udf +from pyspark.sql.functions import col, explode, lit, udf from pyspark.sql.types import ArrayType, StringType from shapely import wkt @@ -24,7 +24,8 @@ def transform_projection(shape, from_utm=False): return transform(project, shape) -def cut(line, distance): +# calcular distancia e criar buffer aqui +def cut(line, distance, buffer_size): line_len = line.length dist_mod = line_len % distance @@ -40,17 +41,23 @@ def cut(line, distance): else: cut_distance = distance final_dist = last_final_dist + cut_distance + segment = substring(line, last_final_dist, final_dist) lines.append( - [str(i), transform_projection(substring(line, last_final_dist, final_dist), True).wkt] + [ + str(i), + transform_projection(segment, True).wkt, + segment.length, + transform_projection(segment.buffer(distance=buffer_size), True).wkt, + ] ) last_final_dist = final_dist return lines -def cut_udf(wkt_string): +def cut_udf(wkt_string, buffer_size): line = transform_projection(wkt.loads(wkt_string)) - return cut(line, distance=1000) + return cut(line, distance=1000, buffer_size=buffer_size) cut_udf = udf(cut_udf, ArrayType(ArrayType(StringType()))) @@ -62,7 +69,10 @@ def model(dbt, session): ) df = dbt.ref("aux_shapes_geom_filtrada") - df_segments = df.withColumn("shape_lists", cut_udf(col("wkt_shape"))) + df_segments = df.withColumn( + "shape_lists", + cut_udf(col("wkt_shape"), lit(dbt.config.get("buffer_segmento_metros"))), + ) df_exploded = ( df_segments.select( @@ -74,6 +84,8 @@ def model(dbt, session): ) .withColumn("id_segmento", col("shape_list").getItem(0)) .withColumn("wkt_segmento", col("shape_list").getItem(1)) + .withColumn("comprimento_segmento", col("shape_list").getItem(2)) + .withColumn("buffer_completo", col("shape_list").getItem(3)) .drop("shape_list") ) From 0ade9f334a2bcf60ff1f226777497e88a84a4ba2 Mon Sep 17 00:00:00 2001 From: Rafael Date: Tue, 12 Nov 2024 13:22:35 -0300 Subject: [PATCH 68/71] =?UTF-8?q?verifica=20altera=C3=A7=C3=B5es=20planeja?= =?UTF-8?q?mento?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipelines/treatment/monitoramento/flows.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pipelines/treatment/monitoramento/flows.py b/pipelines/treatment/monitoramento/flows.py index 4951bdb96..e3a220069 100644 --- a/pipelines/treatment/monitoramento/flows.py +++ b/pipelines/treatment/monitoramento/flows.py @@ -14,6 +14,9 @@ ) from pipelines.schedules import cron_every_hour_minute_6 from pipelines.treatment.monitoramento.constants import constants +from pipelines.treatment.planejamento.constants import ( + constants as planejamento_constants, +) from pipelines.treatment.templates.flows import create_default_materialization_flow VIAGEM_INFORMADA_MATERIALIZACAO = create_default_materialization_flow( @@ -29,6 +32,7 @@ agent_label=smtr_constants.RJ_SMTR_AGENT_LABEL.value, wait=[ constants.VIAGEM_INFORMADA_SELECTOR.value, + planejamento_constants.PLANEJAMENTO_DIARIO_SELECTOR.value, { "redis_key": f"{smtr_constants.GPS_SPPO_DATASET_ID.value}\ .{smtr_constants.GPS_SPPO_TREATED_TABLE_ID.value}", From b444c7cec7f4411cfbe5bcd3b10b9fec8c2f6d08 Mon Sep 17 00:00:00 2001 From: Rafael Date: Tue, 12 Nov 2024 20:10:43 -0300 Subject: [PATCH 69/71] add vars --- queries/dbt_project.yml | 3 + queries/models/config.yml | 5 +- .../staging/aux_segmento_shape.py | 109 +++++++++--------- 3 files changed, 60 insertions(+), 57 deletions(-) diff --git a/queries/dbt_project.yml b/queries/dbt_project.yml index c1e19628f..149d1f1ac 100644 --- a/queries/dbt_project.yml +++ b/queries/dbt_project.yml @@ -194,6 +194,9 @@ vars: buffer_segmento_metros: 20 buffer_tunel_metros: 50 parametro_validacao: 0.9 + comprimento_shape: 1000 + projecao_shapely: "EPSG:31983" + projecao_bq: "EPSG:4326" tests: rj_smtr: diff --git a/queries/models/config.yml b/queries/models/config.yml index c34b05c4f..2d77484d9 100644 --- a/queries/models/config.yml +++ b/queries/models/config.yml @@ -3,4 +3,7 @@ version: 2 models: - name: aux_segmento_shape config: - buffer_segmento_metros: "{{ var('buffer_segmento_metros') }}" \ No newline at end of file + buffer_segmento_metros: "{{ var('buffer_segmento_metros') }}" + comprimento_shape: "{{ var('comprimento_shape') }}" + projecao_shapely: "{{ var('projecao_shapely') }}" + projecao_bq: "{{ var('projecao_bq') }}" \ No newline at end of file diff --git a/queries/models/planejamento/staging/aux_segmento_shape.py b/queries/models/planejamento/staging/aux_segmento_shape.py index 88609f5af..603adfe1a 100644 --- a/queries/models/planejamento/staging/aux_segmento_shape.py +++ b/queries/models/planejamento/staging/aux_segmento_shape.py @@ -9,69 +9,66 @@ from shapely.ops import substring, transform -def transform_projection(shape, from_utm=False): - bq_projection = pyproj.CRS("EPSG:4326") - shapely_projection = pyproj.CRS("EPSG:31983") - if from_utm: - project = pyproj.Transformer.from_crs( - shapely_projection, bq_projection, always_xy=True - ).transform - else: - project = pyproj.Transformer.from_crs( - bq_projection, shapely_projection, always_xy=True - ).transform - - return transform(project, shape) - - -# calcular distancia e criar buffer aqui -def cut(line, distance, buffer_size): - line_len = line.length - - dist_mod = line_len % distance - dist_range = list(np.arange(0, line_len, distance)) - middle_index = (len(dist_range) // 2) + 1 - - last_final_dist = 0 - lines = [] - - for i, _ in enumerate(dist_range, start=1): - if i == middle_index: - cut_distance = dist_mod - else: - cut_distance = distance - final_dist = last_final_dist + cut_distance - segment = substring(line, last_final_dist, final_dist) - lines.append( - [ - str(i), - transform_projection(segment, True).wkt, - segment.length, - transform_projection(segment.buffer(distance=buffer_size), True).wkt, - ] - ) - last_final_dist = final_dist - - return lines - - -def cut_udf(wkt_string, buffer_size): - line = transform_projection(wkt.loads(wkt_string)) - return cut(line, distance=1000, buffer_size=buffer_size) - - -cut_udf = udf(cut_udf, ArrayType(ArrayType(StringType()))) - - def model(dbt, session): dbt.config( materialized="table", ) df = dbt.ref("aux_shapes_geom_filtrada") - + bq_projection = pyproj.CRS(dbt.config.get("projecao_bq")) + shapely_projection = pyproj.CRS(dbt.config.get("projecao_shapely")) + + def transform_projection(shape, from_shapely=False): + if from_shapely: + project = pyproj.Transformer.from_crs( + shapely_projection, bq_projection, always_xy=True + ).transform + else: + project = pyproj.Transformer.from_crs( + bq_projection, shapely_projection, always_xy=True + ).transform + + return transform(project, shape) + + def cut(line, distance, buffer_size): + line_len = line.length + dist_mod = line_len % distance + dist_range = list(np.arange(0, line_len, distance)) + middle_index = (len(dist_range) // 2) + 1 + + last_final_dist = 0 + lines = [] + + for i, _ in enumerate(dist_range, start=1): + if i == middle_index: + cut_distance = dist_mod + else: + cut_distance = distance + final_dist = last_final_dist + cut_distance + segment = substring(line, last_final_dist, final_dist) + lines.append( + [ + str(i), + transform_projection(segment, True).wkt, + segment.length, + transform_projection(segment.buffer(distance=buffer_size), True).wkt, + ] + ) + last_final_dist = final_dist + + return lines + + def cut_udf(wkt_string, distance, buffer_size): + line = transform_projection(wkt.loads(wkt_string)) + return cut(line, distance, buffer_size=buffer_size) + + cut_udf = udf(cut_udf, ArrayType(ArrayType(StringType()))) df_segments = df.withColumn( "shape_lists", - cut_udf(col("wkt_shape"), lit(dbt.config.get("buffer_segmento_metros"))), + cut_udf( + col("wkt_shape"), + lit(dbt.config.get("comprimento_shape")), + lit(dbt.config.get("buffer_segmento_metros")), + ), ) df_exploded = ( From 1614b9e5ce216c88f4e4d82361ae2088f80426d9 Mon Sep 17 00:00:00 2001 From: Rafael Date: Wed, 13 Nov 2024 10:51:47 -0300 Subject: [PATCH 70/71] altera round --- queries/models/planejamento/segmento_shape.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/queries/models/planejamento/segmento_shape.sql b/queries/models/planejamento/segmento_shape.sql index eb140175f..90c214704 100644 --- a/queries/models/planejamento/segmento_shape.sql +++ b/queries/models/planejamento/segmento_shape.sql @@ -24,7 +24,7 @@ with id_segmento, st_geogfromtext(wkt_segmento) as segmento, wkt_segmento, - round(cast(comprimento_segmento as float64), 1) as comprimento_segmento, + round(cast(comprimento_segmento as float64), 2) as comprimento_segmento, st_geogfromtext(buffer_completo) as buffer_completo from {{ ref("aux_segmento_shape") }} ), From 1d12aa693849cf982f68b1bfa50152e30272526b Mon Sep 17 00:00:00 2001 From: Rafael Date: Wed, 13 Nov 2024 10:52:16 -0300 Subject: [PATCH 71/71] altera nomes variaveis --- queries/dbt_project.yml | 4 ++-- queries/models/config.yml | 4 ++-- queries/models/planejamento/staging/aux_segmento_shape.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/queries/dbt_project.yml b/queries/dbt_project.yml index 149d1f1ac..961a0e797 100644 --- a/queries/dbt_project.yml +++ b/queries/dbt_project.yml @@ -195,8 +195,8 @@ vars: buffer_tunel_metros: 50 parametro_validacao: 0.9 comprimento_shape: 1000 - projecao_shapely: "EPSG:31983" - projecao_bq: "EPSG:4326" + projecao_sirgas_2000: "EPSG:31983" + projecao_wgs_84: "EPSG:4326" tests: rj_smtr: diff --git a/queries/models/config.yml b/queries/models/config.yml index 2d77484d9..182839e3e 100644 --- a/queries/models/config.yml +++ b/queries/models/config.yml @@ -5,5 +5,5 @@ models: config: buffer_segmento_metros: "{{ var('buffer_segmento_metros') }}" comprimento_shape: "{{ var('comprimento_shape') }}" - projecao_shapely: "{{ var('projecao_shapely') }}" - projecao_bq: "{{ var('projecao_bq') }}" \ No newline at end of file + projecao_sirgas_2000: "{{ var('projecao_sirgas_2000') }}" + projecao_wgs_84: "{{ var('projecao_wgs_84') }}" \ No newline at end of file diff --git a/queries/models/planejamento/staging/aux_segmento_shape.py b/queries/models/planejamento/staging/aux_segmento_shape.py index 603adfe1a..69745d7c0 100644 --- a/queries/models/planejamento/staging/aux_segmento_shape.py +++ b/queries/models/planejamento/staging/aux_segmento_shape.py @@ -14,8 +14,8 @@ def model(dbt, session): materialized="table", ) df = dbt.ref("aux_shapes_geom_filtrada") - bq_projection = pyproj.CRS(dbt.config.get("projecao_bq")) - shapely_projection = pyproj.CRS(dbt.config.get("projecao_shapely")) + bq_projection = pyproj.CRS(dbt.config.get("projecao_wgs_84")) + shapely_projection = pyproj.CRS(dbt.config.get("projecao_sirgas_2000")) def transform_projection(shape, from_shapely=False): if from_shapely: