diff --git a/jobbergate-api/CHANGELOG.md b/jobbergate-api/CHANGELOG.md index 8a3bd8e7..73eef3dd 100644 --- a/jobbergate-api/CHANGELOG.md +++ b/jobbergate-api/CHANGELOG.md @@ -6,6 +6,7 @@ This file keeps track of all notable changes to jobbergate-api ## Unreleased - Changed internals to avoid committing to the database when a GET request is made - Added extra settings to allow profiling and tracing on sentry +- Removed db-start from dev-tools ## 4.1.0a2 -- 2023-10-10 diff --git a/jobbergate-api/dev_tools/db.py b/jobbergate-api/dev_tools/db.py index 018c8646..d1929446 100644 --- a/jobbergate-api/dev_tools/db.py +++ b/jobbergate-api/dev_tools/db.py @@ -1,15 +1,13 @@ """Provide commands for interacting with local databases.""" -import json + import subprocess -import docker_gadgets import typer from loguru import logger from alembic.command import revision as sqla_migrate from alembic.command import upgrade as sqla_upgrade from alembic.config import Config -from jobbergate_api.config import settings from jobbergate_api.storage import build_db_url app = typer.Typer() @@ -23,43 +21,6 @@ def login(test: bool = typer.Option(False, help="Log into the test database.")): subprocess.run(["pgcli", url]) -@app.command() -def start(test: bool = typer.Option(False, help="Start a test database.")): - """Start a local postgres database for local development.""" - name = "dev-jobbergate-postgres" - kwargs = dict( - image="postgres:14.1", - env=dict( - POSTGRES_PASSWORD=settings.DATABASE_PSWD, - POSTGRES_DB=settings.DATABASE_NAME, - POSTGRES_USER=settings.DATABASE_USER, - ), - ports={"5432/tcp": settings.DATABASE_PORT}, - ) - if test: - kwargs.update( - env=dict( - POSTGRES_PASSWORD=settings.TEST_DATABASE_PSWD, - POSTGRES_DB=settings.TEST_DATABASE_NAME, - POSTGRES_USER=settings.TEST_DATABASE_USER, - ), - ports={"5432/tcp": settings.TEST_DATABASE_PORT}, - tmpfs={"/var/lib/postgresql/data": "rw"}, - ) - name = "test-jobbergate-postgres" - - logger.debug(f"Starting {name} with:\n {json.dumps(kwargs, indent=2)}") - - docker_gadgets.start_service(name, **kwargs) - - -@app.command() -def start_all(): - """Start all local databases.""" - start() - start(test=True) - - @app.command() def migrate( message: str = typer.Option("Unlabeled migration", help="The message to attach to the migration."), diff --git a/jobbergate-api/poetry.lock b/jobbergate-api/poetry.lock index 899d5906..636c6173 100644 --- a/jobbergate-api/poetry.lock +++ b/jobbergate-api/poetry.lock @@ -902,28 +902,6 @@ idna = ["idna (>=2.1,<4.0)"] trio = ["trio (>=0.14,<0.23)"] wmi = ["wmi (>=1.5.1,<2.0.0)"] -[[package]] -name = "docker" -version = "6.1.3" -description = "A Python library for the Docker Engine API." -category = "dev" -optional = false -python-versions = ">=3.7" -files = [ - {file = "docker-6.1.3-py3-none-any.whl", hash = "sha256:aecd2277b8bf8e506e484f6ab7aec39abe0038e29fa4a6d3ba86c3fe01844ed9"}, - {file = "docker-6.1.3.tar.gz", hash = "sha256:aa6d17830045ba5ef0168d5eaa34d37beeb113948c413affe1d5991fc11f9a20"}, -] - -[package.dependencies] -packaging = ">=14.0" -pywin32 = {version = ">=304", markers = "sys_platform == \"win32\""} -requests = ">=2.26.0" -urllib3 = ">=1.26.0" -websocket-client = ">=0.32.0" - -[package.extras] -ssh = ["paramiko (>=2.4.3)"] - [[package]] name = "ecdsa" version = "0.18.0" @@ -2021,23 +1999,6 @@ files = [ {file = "py_buzz-4.1.0.tar.gz", hash = "sha256:ac11dba4922b2af114126044597d2fcd15e8c6a74368bed91f3c6732c8f09812"}, ] -[[package]] -name = "py-docker-gadgets" -version = "0.1.3" -description = "Some convenience tools for managing docker containers in python" -category = "dev" -optional = false -python-versions = ">=3.8,<4.0" -files = [ - {file = "py_docker_gadgets-0.1.3-py3-none-any.whl", hash = "sha256:37cffcd451fb49096fe0687f85824b757319bdb7ad773eb8527e7a01b06b3f46"}, - {file = "py_docker_gadgets-0.1.3.tar.gz", hash = "sha256:5d360c9f057387c2ff19c6e1a7859a112254a4002b00c220f8693064292e716b"}, -] - -[package.dependencies] -docker = ">=5.0.3" -loguru = ">=0.6" -py-buzz = ">=3.0" - [[package]] name = "pyasn1" version = "0.5.0" @@ -2379,30 +2340,6 @@ files = [ {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, ] -[[package]] -name = "pywin32" -version = "306" -description = "Python for Window Extensions" -category = "dev" -optional = false -python-versions = "*" -files = [ - {file = "pywin32-306-cp310-cp310-win32.whl", hash = "sha256:06d3420a5155ba65f0b72f2699b5bacf3109f36acbe8923765c22938a69dfc8d"}, - {file = "pywin32-306-cp310-cp310-win_amd64.whl", hash = "sha256:84f4471dbca1887ea3803d8848a1616429ac94a4a8d05f4bc9c5dcfd42ca99c8"}, - {file = "pywin32-306-cp311-cp311-win32.whl", hash = "sha256:e65028133d15b64d2ed8f06dd9fbc268352478d4f9289e69c190ecd6818b6407"}, - {file = "pywin32-306-cp311-cp311-win_amd64.whl", hash = "sha256:a7639f51c184c0272e93f244eb24dafca9b1855707d94c192d4a0b4c01e1100e"}, - {file = "pywin32-306-cp311-cp311-win_arm64.whl", hash = "sha256:70dba0c913d19f942a2db25217d9a1b726c278f483a919f1abfed79c9cf64d3a"}, - {file = "pywin32-306-cp312-cp312-win32.whl", hash = "sha256:383229d515657f4e3ed1343da8be101000562bf514591ff383ae940cad65458b"}, - {file = "pywin32-306-cp312-cp312-win_amd64.whl", hash = "sha256:37257794c1ad39ee9be652da0462dc2e394c8159dfd913a8a4e8eb6fd346da0e"}, - {file = "pywin32-306-cp312-cp312-win_arm64.whl", hash = "sha256:5821ec52f6d321aa59e2db7e0a35b997de60c201943557d108af9d4ae1ec7040"}, - {file = "pywin32-306-cp37-cp37m-win32.whl", hash = "sha256:1c73ea9a0d2283d889001998059f5eaaba3b6238f767c9cf2833b13e6a685f65"}, - {file = "pywin32-306-cp37-cp37m-win_amd64.whl", hash = "sha256:72c5f621542d7bdd4fdb716227be0dd3f8565c11b280be6315b06ace35487d36"}, - {file = "pywin32-306-cp38-cp38-win32.whl", hash = "sha256:e4c092e2589b5cf0d365849e73e02c391c1349958c5ac3e9d5ccb9a28e017b3a"}, - {file = "pywin32-306-cp38-cp38-win_amd64.whl", hash = "sha256:e8ac1ae3601bee6ca9f7cb4b5363bf1c0badb935ef243c4733ff9a393b1690c0"}, - {file = "pywin32-306-cp39-cp39-win32.whl", hash = "sha256:e25fd5b485b55ac9c057f67d94bc203f3f6595078d1fb3b458c9c28b7153a802"}, - {file = "pywin32-306-cp39-cp39-win_amd64.whl", hash = "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4"}, -] - [[package]] name = "pyyaml" version = "6.0.1" @@ -3913,23 +3850,6 @@ files = [ {file = "wcwidth-0.2.8.tar.gz", hash = "sha256:8705c569999ffbb4f6a87c6d1b80f324bd6db952f5eb0b95bc07517f4c1813d4"}, ] -[[package]] -name = "websocket-client" -version = "1.6.3" -description = "WebSocket client for Python with low level API options" -category = "dev" -optional = false -python-versions = ">=3.8" -files = [ - {file = "websocket-client-1.6.3.tar.gz", hash = "sha256:3aad25d31284266bcfcfd1fd8a743f63282305a364b8d0948a43bd606acc652f"}, - {file = "websocket_client-1.6.3-py3-none-any.whl", hash = "sha256:6cfc30d051ebabb73a5fa246efdcc14c8fbebbd0330f8984ac3bb6d9edd2ad03"}, -] - -[package.extras] -docs = ["Sphinx (>=6.0)", "sphinx-rtd-theme (>=1.1.0)"] -optional = ["python-socks", "wsaccel"] -test = ["websockets"] - [[package]] name = "win32-setctime" version = "1.1.0" @@ -4121,4 +4041,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "f56b20c3ccbb0c245a66074f0ad16f681813c6c049167c34ead05664e391be18" +content-hash = "9edfa84e6ff2048a5ec5104d063ef4dd071f54c27c039ed9dec1dbf725cccb4d" diff --git a/jobbergate-api/pyproject.toml b/jobbergate-api/pyproject.toml index 90d98a32..b201ff07 100644 --- a/jobbergate-api/pyproject.toml +++ b/jobbergate-api/pyproject.toml @@ -58,7 +58,6 @@ nest_asyncio = "^1.3.3" pgcli = "^3.1.0" pre-commit = "^2.9.2" psycopg2 = "^2.9.5" -py-docker-gadgets = "^0.1.3" pyproject-flake8 = "^5.0.4" pytest = "^7" pytest-asyncio = "^0.21" diff --git a/jobbergate-docs/docs/source/developer_guide/dev_tools.md b/jobbergate-docs/docs/source/developer_guide/dev_tools.md index 1d6c423b..4b63a59c 100644 --- a/jobbergate-docs/docs/source/developer_guide/dev_tools.md +++ b/jobbergate-docs/docs/source/developer_guide/dev_tools.md @@ -52,71 +52,6 @@ poetry run dev-tools db --help ``` -### The `start` subcommand - -This command is used to start up a database in docker that is configured using the -values found in the execution environment for Jobbergate. - -To start a development database, invoke this command: - -```shell -poetry run dev-tools db start -``` - -Note that the values for the database name, host, port, and password will all be -gathered from the environment that the API is configured to use. The script should -produce some logging output that will indicate the status of the db: - -``` -2022-09-07 15:44:56.634 | DEBUG | dev_tools.db:start:57 - Starting dev-jobbergate-postgres with: -{ - "image": "postgres:14.1", - "env": { - "POSTGRES_PASSWORD": "compose-db-pswd", - "POSTGRES_DB": "compose-db-name", - "POSTGRES_USER": "compose-db-user" - }, - "ports": { - "5432/tcp": 5432 - } -} -2022-09-07 15:44:56.634 | DEBUG | docker_gadgets.gadgets:start_service:19 - Starting service 'dev-jobbergate-postgres' -2022-09-07 15:44:56.642 | DEBUG | docker_gadgets.gadgets:start_service:26 - Retrieving external image for dev-jobbergate-postgres using postgres:14.1 -2022-09-07 15:44:56.649 | DEBUG | docker_gadgets.helpers:get_image_external:43 - Pulling postgres:14.1 image (tag='14.1') -2022-09-07 15:45:03.190 | DEBUG | docker_gadgets.helpers:cleanup_container:13 - Checking for existing container: dev-jobbergate-postgres -2022-09-07 15:45:03.195 | DEBUG | docker_gadgets.helpers:cleanup_container:24 - No existing container found: dev-jobbergate-postgres -2022-09-07 15:45:03.196 | DEBUG | docker_gadgets.gadgets:start_service:31 - Checking if needed ports {'5432/tcp': 5432} are available -2022-09-07 15:45:03.217 | DEBUG | docker_gadgets.gadgets:start_service:34 - Starting container: dev-jobbergate-postgres -2022-09-07 15:45:03.647 | DEBUG | docker_gadgets.gadgets:start_service:48 - Started container: dev-jobbergate-postgres () -``` - -In this example, the container name is `dev-jobbergate-postgres` and it is running in -the docker container referenced by `e686e97595` with the port 5432 mapped from -the host machine to the container. - -!!!Note - - This command is also very convenient for starting up a database for unit testing. To - do so, you need the `--test` flag to the command. When you run the unit tests suite, - it will use the database started by this command. - - -### The `start-all` subcommand - -This command is just a convenient way of spinning up both a development database and a -test database in one command. It is equivalent to running the following two commands in -succession: - -```shell -poetry run dev-tools db start -``` - - -```shell -poetry run dev-tools db start --test -``` - - ### The `login` subcommand This command allows you to log in to the database that your Jobbergate API is configured @@ -277,7 +212,8 @@ The JSON output will look something like: ## The `dev-server` subcommand This command starts up a local development server for the Jobbergate API. It will -be created using the configuration set up in the environment settings. +be created using the configuration set up in the environment settings. This command is especially useful if +you want to run the API locally but connect to remote services such as a database and s3 hosted on AWS. To start the server, run: