From b6ff2e87b71220e14bd28508902e8875995d006e Mon Sep 17 00:00:00 2001 From: Daniil <94884910+Filienko@users.noreply.github.com> Date: Fri, 2 Aug 2024 18:04:09 -0700 Subject: [PATCH 01/15] Updating python to 3.11 --- .github/workflows/test.yaml | 2 +- Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d4de3c8..8b98690 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -6,7 +6,7 @@ on: [push] jobs: test: runs-on: ubuntu-latest - container: python:3.9 + container: python:3.11 steps: - uses: actions/checkout@v1 diff --git a/Dockerfile b/Dockerfile index a592d53..6b3b1cd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9 +FROM python:3.11 WORKDIR /opt/app From fe3ecf7867584cbd979c3de51ca49a568b2119b9 Mon Sep 17 00:00:00 2001 From: Daniil <94884910+Filienko@users.noreply.github.com> Date: Fri, 2 Aug 2024 21:16:45 -0700 Subject: [PATCH 02/15] Adding setup.cfg --- requirements.dev.txt | 145 +++++++++++++++++++++++++++++++++++++------ requirements.txt | 118 ++++++++++++++++++++++++++++------- setup.cfg | 55 ++++++++++++++++ setup.py | 3 + 4 files changed, 281 insertions(+), 40 deletions(-) create mode 100644 setup.cfg create mode 100644 setup.py diff --git a/requirements.dev.txt b/requirements.dev.txt index b7e7e19..89404ce 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -1,22 +1,129 @@ # -# This file is autogenerated by pip-compile -# To update, run: +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: # -# pip-compile +# pip-compile --extra=dev --output-file=requirements.dev.txt setup.cfg # ---requirement requirements.txt -attrs==21.2.0 # via pytest -importlib-metadata==4.8.1 # via pluggy, pytest -iniconfig==1.1.1 # via pytest -packaging==21.0 # via pytest -pluggy==1.0.0 # via pytest -psutil==5.8.0 # via mirakuru -py==1.10.0 # via pytest -pyparsing==2.4.7 # via packaging -pytest-datadir==1.3.1 # via pytest-datadir -pytest-mock==3.6.1 # via ml_services (setup.py) -pytest==6.2.5 # via pytest-mock, pytest-redis, ml_services (setup.py) -requests-mock==1.9.3 # via ml_services (setup.py) -toml==0.10.2 # via pytest -typing-extensions==3.10.0.2 # via importlib-metadata -zipp==3.5.0 # via importlib-metadata +attrs==23.2.0 + # via pytest +certifi==2024.7.4 + # via requests +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via flask +filelock==3.15.4 + # via + # huggingface-hub + # transformers +flask==1.1.2 + # via ml_services (setup.cfg) +fsspec==2024.6.1 + # via huggingface-hub +gunicorn==20.1.0 + # via ml_services (setup.cfg) +huggingface-hub==0.24.5 + # via transformers +idna==3.7 + # via requests +iniconfig==2.0.0 + # via pytest +itsdangerous==2.2.0 + # via + # flask + # requests-cache +jinja2==3.1.4 + # via flask +markupsafe==2.1.5 + # via + # jinja2 + # werkzeug +numpy==1.21.5 + # via + # ml_services (setup.cfg) + # pandas + # scipy + # transformers +nvidia-cublas-cu11==11.10.3.66 + # via + # nvidia-cudnn-cu11 + # torch +nvidia-cuda-nvrtc-cu11==11.7.99 + # via torch +nvidia-cuda-runtime-cu11==11.7.99 + # via torch +nvidia-cudnn-cu11==8.5.0.96 + # via torch +packaging==24.1 + # via + # huggingface-hub + # pytest + # transformers +pandas==1.3.5 + # via ml_services (setup.cfg) +pluggy==1.5.0 + # via pytest +py==1.11.0 + # via pytest +pytest==6.2.5 + # via + # ml_services (setup.cfg) + # pytest-datadir + # pytest-mock +pytest-datadir==1.3.1 + # via ml_services (setup.cfg) +pytest-mock==3.6.1 + # via ml_services (setup.cfg) +python-dateutil==2.9.0.post0 + # via pandas +pytz==2024.1 + # via pandas +pyyaml==6.0.1 + # via + # huggingface-hub + # transformers +regex==2024.7.24 + # via transformers +requests==2.32.3 + # via + # huggingface-hub + # requests-cache + # requests-mock + # transformers +requests-cache==0.6.4 + # via ml_services (setup.cfg) +requests-mock==1.9.3 + # via ml_services (setup.cfg) +scipy==1.7.3 + # via ml_services (setup.cfg) +six==1.16.0 + # via + # python-dateutil + # requests-mock + # url-normalize +tokenizers==0.13.3 + # via transformers +toml==0.10.2 + # via pytest +torch==1.13.0 + # via ml_services (setup.cfg) +tqdm==4.66.4 + # via + # huggingface-hub + # transformers +transformers==4.24.0 + # via ml_services (setup.cfg) +typing-extensions==4.12.2 + # via + # huggingface-hub + # torch +url-normalize==1.4.3 + # via requests-cache +urllib3==2.2.2 + # via requests +werkzeug==3.0.3 + # via flask +wheel==0.43.0 + # via + # nvidia-cublas-cu11 + # nvidia-cuda-runtime-cu11 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index d1c7db5..13dc51b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,23 +1,99 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements.txt -# -click==8.0.1 # via flask -flask==1.1.2 # via ml_services (setup.py) -gunicorn==20.1.0 # via ml_services (setup.py) -importlib-metadata==4.8.1 # via click -itsdangerous==2.0.1 # via flask -jinja2==3.0.1 # via flask -markupsafe==2.0.1 # via jinja2 -requests-cache==0.6.4 # via ml_services (setup.py) -six==1.16.0 # via ecdsa, flask-cors, url-normalize -typing-extensions==3.10.0.2 # via importlib-metadata -werkzeug==2.0.1 # via flask -zipp==3.5.0 # via importlib-metadata +certifi==2024.7.4 + # via requests +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via flask +filelock==3.15.4 + # via + # huggingface-hub + # transformers +flask==1.1.2 + # via ml_services (setup.py) +fsspec==2024.6.1 + # via huggingface-hub +gunicorn==20.1.0 + # via ml_services (setup.py) +huggingface-hub==0.24.5 + # via transformers +idna==3.7 + # via requests +itsdangerous==2.2.0 + # via + # flask + # requests-cache +jinja2==3.1.4 + # via flask +markupsafe==2.1.5 + # via + # jinja2 + # werkzeug +numpy==1.21.5 + # via + # ml_services (setup.py) + # pandas + # scipy + # transformers +nvidia-cublas-cu11==11.10.3.66 + # via + # nvidia-cudnn-cu11 + # torch +nvidia-cuda-nvrtc-cu11==11.7.99 + # via torch +nvidia-cuda-runtime-cu11==11.7.99 + # via torch +nvidia-cudnn-cu11==8.5.0.96 + # via torch +packaging==24.1 + # via + # huggingface-hub + # transformers +pandas==1.3.5 + # via ml_services (setup.py) +python-dateutil==2.9.0.post0 + # via pandas +pytz==2024.1 + # via pandas +pyyaml==6.0.1 + # via + # huggingface-hub + # transformers +regex==2024.7.24 + # via transformers +requests==2.32.3 + # via + # huggingface-hub + # requests-cache + # transformers +requests-cache==0.6.4 + # via ml_services (setup.py) +scipy==1.7.3 + # via ml_services (setup.py) +six==1.16.0 + # via + # python-dateutil + # url-normalize +tokenizers==0.13.3 + # via transformers torch==1.13.0 + # via ml_services (setup.py) +tqdm==4.66.4 + # via + # huggingface-hub + # transformers transformers==4.24.0 -scipy==1.7.3 -numpy==1.21.5 -pandas==1.3.5 \ No newline at end of file + # via ml_services (setup.py) +typing-extensions==4.12.2 + # via + # huggingface-hub + # torch +url-normalize==1.4.3 + # via requests-cache +urllib3==2.2.2 + # via requests +werkzeug==3.0.3 + # via flask +wheel==0.43.0 + # via + # nvidia-cublas-cu11 + # nvidia-cuda-runtime-cu11 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..4a717be --- /dev/null +++ b/setup.cfg @@ -0,0 +1,55 @@ +# pyup: ignore file +# setup.py configuration file +# https://setuptools.readthedocs.io/en/latest/setuptools.html#configuring-setup-using-setup-cfg-files + +[metadata] +name = patientsearch +version = attr: setuptools_scm.get_version + +[options] +packages = patientsearch +zip_safe = False +include_package_data = True +setup_requires = + setuptools_scm + +# abstract requirements; +# concrete requirements belong in requirements.txt +# https://caremad.io/posts/2013/07/setup-vs-requirement/ +install_requires = + flask + flask-jwt-extended + flask-session + gunicorn + requests + redis + redis-dict + jmespath + python-json-logger + +[options.extras_require] +dev = + pytest + pytest-flask + pytest-datadir + pytest-mock + +[flake8] +max-line-length=100 + +[tool:pytest] +addopts = --color yes --verbose +console_output_style = classic +filterwarnings = + # only print each warning once per module + module + + # Ignore testing-specific warning + ignore:^"localhost" is not a valid cookie domain, it must contain a.*:Warning + + # Ignore warnings on specific dependencies (already reported upstream) + ignore::DeprecationWarning:dogpile + ignore::DeprecationWarning:flask_user + + # error on bad assertions + error:.*assertion is always true.* diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..6068493 --- /dev/null +++ b/setup.py @@ -0,0 +1,3 @@ +from setuptools import setup + +setup() From 1817a8e50faec64f62dcade3bd58c3e7b661a3ba Mon Sep 17 00:00:00 2001 From: Daniil <94884910+Filienko@users.noreply.github.com> Date: Fri, 2 Aug 2024 21:17:33 -0700 Subject: [PATCH 03/15] New setup --- pyproject.toml | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++ setup.cfg | 50 ++++++++++++++++++--------------------- 2 files changed, 85 insertions(+), 28 deletions(-) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..36fe453 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,63 @@ +[project] +name = "isacc-ml" +description = "Flask-based microservice for machine learning utilities in the ISACC service" +version = "0.1.0" +authors = [ + { name="CIRG", email="cirg-adm@uw.edu" } +] +readme = "README.md" +requires-python = ">=3.7" +license = {file = "LICENSE"} + +[project.urls] +Homepage = "https://github.com/uwcirg/isacc-ml" +Repository = "https://github.com/uwcirg/isacc-ml.git" + +[build-system] +requires = ["setuptools>=45", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +packages = ["isacc_ml"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-v" + +[tool.mypy] +ignore_missing_imports = true +strict_optional = true + +[tool.poetry.dependencies] +python = "^3.7" +flask = "1.1.2" +gunicorn = "20.1.0" +requests-cache = "0.6.4" +torch = "1.13.0" +transformers = "4.24.0" +scipy = "1.7.3" +numpy = "1.21.5" +pandas = "1.3.5" +click = "8.0.1" +importlib-metadata = "4.8.1" +itsdangerous = "2.0.1" +jinja2 = "3.0.1" +markupsafe = "2.0.1" +six = "1.16.0" +typing-extensions = "3.10.0.2" +werkzeug = "2.0.1" +zipp = "3.5.0" + +[tool.poetry.dev-dependencies] +pytest = "6.2.5" +pytest-mock = "3.6.1" +pytest-datadir = "1.3.1" +requests-mock = "1.9.3" +attrs = "21.2.0" +iniconfig = "1.1.1" +packaging = "21.0" +pluggy = "1.0.0" +psutil = "5.8.0" +py = "1.10.0" +pyparsing = "2.4.7" +toml = "0.10.2" \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 4a717be..5702a0d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,38 +1,32 @@ -# pyup: ignore file -# setup.py configuration file -# https://setuptools.readthedocs.io/en/latest/setuptools.html#configuring-setup-using-setup-cfg-files - [metadata] -name = patientsearch -version = attr: setuptools_scm.get_version +name = isacc-ml +description = Flask-based microservice for NLP machine learning utilities in the ISACC service +author = CIRG +author_email = cirg-adm@uw.edu +url = https://github.com/uwcirg/isacc-ml +classifiers = + Programming Language :: Python :: 3.7 [options] -packages = patientsearch -zip_safe = False -include_package_data = True -setup_requires = - setuptools_scm - -# abstract requirements; -# concrete requirements belong in requirements.txt -# https://caremad.io/posts/2013/07/setup-vs-requirement/ +packages = ml_services +python_requires = >=3.7 install_requires = - flask - flask-jwt-extended - flask-session - gunicorn - requests - redis - redis-dict - jmespath - python-json-logger + flask==1.1.2 + gunicorn==20.1.0 + requests-cache==0.6.4 + numpy==1.21.5 + pandas==1.3.5 + scipy==1.7.3 + torch==1.13.0 + transformers==4.24.0 [options.extras_require] dev = - pytest - pytest-flask - pytest-datadir - pytest-mock + pytest==6.2.5 + pytest-mock==3.6.1 + pytest-datadir==1.3.1 + requests-mock==1.9.3 + [flake8] max-line-length=100 From a8b9233da2d9072b8d04db111d945b6b8d7c15aa Mon Sep 17 00:00:00 2001 From: Daniil <94884910+Filienko@users.noreply.github.com> Date: Fri, 2 Aug 2024 21:25:46 -0700 Subject: [PATCH 04/15] Updated libraries, removing pinned --- requirements.dev.txt | 8 +------- requirements.txt | 2 +- setup.cfg | 24 ++++++++++++------------ setup.py | 7 ++++++- 4 files changed, 20 insertions(+), 21 deletions(-) diff --git a/requirements.dev.txt b/requirements.dev.txt index 89404ce..1189cad 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -1,9 +1,3 @@ -# -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: -# -# pip-compile --extra=dev --output-file=requirements.dev.txt setup.cfg -# attrs==23.2.0 # via pytest certifi==2024.7.4 @@ -126,4 +120,4 @@ werkzeug==3.0.3 wheel==0.43.0 # via # nvidia-cublas-cu11 - # nvidia-cuda-runtime-cu11 \ No newline at end of file + # nvidia-cuda-runtime-cu11 diff --git a/requirements.txt b/requirements.txt index 13dc51b..32d9f58 100644 --- a/requirements.txt +++ b/requirements.txt @@ -96,4 +96,4 @@ werkzeug==3.0.3 wheel==0.43.0 # via # nvidia-cublas-cu11 - # nvidia-cuda-runtime-cu11 \ No newline at end of file + # nvidia-cuda-runtime-cu11 diff --git a/setup.cfg b/setup.cfg index 5702a0d..ddf9924 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,21 +11,21 @@ classifiers = packages = ml_services python_requires = >=3.7 install_requires = - flask==1.1.2 - gunicorn==20.1.0 - requests-cache==0.6.4 - numpy==1.21.5 - pandas==1.3.5 - scipy==1.7.3 - torch==1.13.0 - transformers==4.24.0 + flask + gunicorn + requests-cache + numpy + pandas + scipy + torch + transformers [options.extras_require] dev = - pytest==6.2.5 - pytest-mock==3.6.1 - pytest-datadir==1.3.1 - requests-mock==1.9.3 + pytest + pytest-mock + pytest-datadir + requests-mock [flake8] diff --git a/setup.py b/setup.py index 6068493..f63e95b 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,8 @@ from setuptools import setup -setup() +setup( + name="ml_services", + version="1.0", + package=["ml_services"], + include_package_data=True +) \ No newline at end of file From 9ccbea261c5ffaaa2bfc95c7c6ba69fb04ec56b9 Mon Sep 17 00:00:00 2001 From: Daniil <94884910+Filienko@users.noreply.github.com> Date: Tue, 6 Aug 2024 18:56:08 -0700 Subject: [PATCH 05/15] Updating the versions to ones generated by 3.9 pip-compile --- requirements.dev.txt | 124 ++++++++++++++++++++++++++++++------------- requirements.txt | 107 ++++++++++++++++++++++++++++--------- 2 files changed, 169 insertions(+), 62 deletions(-) diff --git a/requirements.dev.txt b/requirements.dev.txt index 1189cad..964240e 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -1,78 +1,122 @@ -attrs==23.2.0 - # via pytest +attrs==24.2.0 + # via + # cattrs + # requests-cache +blinker==1.8.2 + # via flask +cattrs==23.2.3 + # via requests-cache certifi==2024.7.4 # via requests charset-normalizer==3.3.2 # via requests click==8.1.7 # via flask +exceptiongroup==1.2.2 + # via + # cattrs + # pytest filelock==3.15.4 # via # huggingface-hub + # torch # transformers -flask==1.1.2 + # triton +flask==3.0.3 # via ml_services (setup.cfg) fsspec==2024.6.1 - # via huggingface-hub -gunicorn==20.1.0 + # via + # huggingface-hub + # torch +gunicorn==22.0.0 # via ml_services (setup.cfg) huggingface-hub==0.24.5 - # via transformers + # via + # tokenizers + # transformers idna==3.7 # via requests +importlib-metadata==8.2.0 + # via flask iniconfig==2.0.0 # via pytest itsdangerous==2.2.0 + # via flask +jinja2==3.1.4 # via # flask - # requests-cache -jinja2==3.1.4 - # via flask + # torch markupsafe==2.1.5 # via # jinja2 # werkzeug -numpy==1.21.5 +mpmath==1.3.0 + # via sympy +networkx==3.2.1 + # via torch +numpy==2.0.1 # via # ml_services (setup.cfg) # pandas # scipy # transformers -nvidia-cublas-cu11==11.10.3.66 +nvidia-cublas-cu12==12.1.3.1 # via - # nvidia-cudnn-cu11 + # nvidia-cudnn-cu12 + # nvidia-cusolver-cu12 # torch -nvidia-cuda-nvrtc-cu11==11.7.99 +nvidia-cuda-cupti-cu12==12.1.105 + # via torch +nvidia-cuda-nvrtc-cu12==12.1.105 + # via torch +nvidia-cuda-runtime-cu12==12.1.105 # via torch -nvidia-cuda-runtime-cu11==11.7.99 +nvidia-cudnn-cu12==9.1.0.70 # via torch -nvidia-cudnn-cu11==8.5.0.96 +nvidia-cufft-cu12==11.0.2.54 + # via torch +nvidia-curand-cu12==10.3.2.106 + # via torch +nvidia-cusolver-cu12==11.4.5.107 + # via torch +nvidia-cusparse-cu12==12.1.0.106 + # via + # nvidia-cusolver-cu12 + # torch +nvidia-nccl-cu12==2.20.5 + # via torch +nvidia-nvjitlink-cu12==12.6.20 + # via + # nvidia-cusolver-cu12 + # nvidia-cusparse-cu12 +nvidia-nvtx-cu12==12.1.105 # via torch packaging==24.1 # via + # gunicorn # huggingface-hub # pytest # transformers -pandas==1.3.5 +pandas==2.2.2 # via ml_services (setup.cfg) +platformdirs==4.2.2 + # via requests-cache pluggy==1.5.0 # via pytest -py==1.11.0 - # via pytest -pytest==6.2.5 +pytest==8.3.2 # via # ml_services (setup.cfg) # pytest-datadir # pytest-mock -pytest-datadir==1.3.1 +pytest-datadir==1.5.0 # via ml_services (setup.cfg) -pytest-mock==3.6.1 +pytest-mock==3.14.0 # via ml_services (setup.cfg) python-dateutil==2.9.0.post0 # via pandas pytz==2024.1 # via pandas -pyyaml==6.0.1 +pyyaml==6.0.2 # via # huggingface-hub # transformers @@ -84,40 +128,48 @@ requests==2.32.3 # requests-cache # requests-mock # transformers -requests-cache==0.6.4 +requests-cache==1.2.1 # via ml_services (setup.cfg) -requests-mock==1.9.3 +requests-mock==1.12.1 # via ml_services (setup.cfg) -scipy==1.7.3 +safetensors==0.4.4 + # via transformers +scipy==1.13.1 # via ml_services (setup.cfg) six==1.16.0 # via # python-dateutil - # requests-mock # url-normalize -tokenizers==0.13.3 +sympy==1.13.1 + # via torch +tokenizers==0.19.1 # via transformers -toml==0.10.2 +tomli==2.0.1 # via pytest -torch==1.13.0 +torch==2.4.0 # via ml_services (setup.cfg) -tqdm==4.66.4 +tqdm==4.66.5 # via # huggingface-hub # transformers -transformers==4.24.0 +transformers==4.44.0 # via ml_services (setup.cfg) +triton==3.0.0 + # via torch typing-extensions==4.12.2 # via + # cattrs # huggingface-hub # torch +tzdata==2024.1 + # via pandas url-normalize==1.4.3 # via requests-cache urllib3==2.2.2 - # via requests + # via + # requests + # requests-cache werkzeug==3.0.3 # via flask -wheel==0.43.0 - # via - # nvidia-cublas-cu11 - # nvidia-cuda-runtime-cu11 +zipp==3.19.2 + # via importlib-metadata diff --git a/requirements.txt b/requirements.txt index 32d9f58..3d28fac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,60 +1,106 @@ +attrs==24.2.0 + # via + # cattrs + # requests-cache +blinker==1.8.2 + # via flask +cattrs==23.2.3 + # via requests-cache certifi==2024.7.4 # via requests charset-normalizer==3.3.2 # via requests click==8.1.7 # via flask +exceptiongroup==1.2.2 + # via cattrs filelock==3.15.4 # via # huggingface-hub + # torch # transformers -flask==1.1.2 + # triton +flask==3.0.3 # via ml_services (setup.py) fsspec==2024.6.1 - # via huggingface-hub -gunicorn==20.1.0 + # via + # huggingface-hub + # torch +gunicorn==22.0.0 # via ml_services (setup.py) huggingface-hub==0.24.5 - # via transformers + # via + # tokenizers + # transformers idna==3.7 # via requests +importlib-metadata==8.2.0 + # via flask itsdangerous==2.2.0 + # via flask +jinja2==3.1.4 # via # flask - # requests-cache -jinja2==3.1.4 - # via flask + # torch markupsafe==2.1.5 # via # jinja2 # werkzeug -numpy==1.21.5 +mpmath==1.3.0 + # via sympy +networkx==3.2.1 + # via torch +numpy==2.0.1 # via # ml_services (setup.py) # pandas # scipy # transformers -nvidia-cublas-cu11==11.10.3.66 +nvidia-cublas-cu12==12.1.3.1 # via - # nvidia-cudnn-cu11 + # nvidia-cudnn-cu12 + # nvidia-cusolver-cu12 # torch -nvidia-cuda-nvrtc-cu11==11.7.99 +nvidia-cuda-cupti-cu12==12.1.105 # via torch -nvidia-cuda-runtime-cu11==11.7.99 +nvidia-cuda-nvrtc-cu12==12.1.105 # via torch -nvidia-cudnn-cu11==8.5.0.96 +nvidia-cuda-runtime-cu12==12.1.105 + # via torch +nvidia-cudnn-cu12==9.1.0.70 + # via torch +nvidia-cufft-cu12==11.0.2.54 + # via torch +nvidia-curand-cu12==10.3.2.106 + # via torch +nvidia-cusolver-cu12==11.4.5.107 + # via torch +nvidia-cusparse-cu12==12.1.0.106 + # via + # nvidia-cusolver-cu12 + # torch +nvidia-nccl-cu12==2.20.5 + # via torch +nvidia-nvjitlink-cu12==12.6.20 + # via + # nvidia-cusolver-cu12 + # nvidia-cusparse-cu12 +nvidia-nvtx-cu12==12.1.105 # via torch packaging==24.1 # via + # gunicorn # huggingface-hub # transformers -pandas==1.3.5 +pandas==2.2.2 # via ml_services (setup.py) +platformdirs==4.2.2 + # via requests-cache python-dateutil==2.9.0.post0 # via pandas pytz==2024.1 # via pandas -pyyaml==6.0.1 +pyyaml==6.0.2 # via # huggingface-hub # transformers @@ -65,35 +111,44 @@ requests==2.32.3 # huggingface-hub # requests-cache # transformers -requests-cache==0.6.4 +requests-cache==1.2.1 # via ml_services (setup.py) -scipy==1.7.3 +safetensors==0.4.4 + # via transformers +scipy==1.13.1 # via ml_services (setup.py) six==1.16.0 # via # python-dateutil # url-normalize -tokenizers==0.13.3 +sympy==1.13.1 + # via torch +tokenizers==0.19.1 # via transformers -torch==1.13.0 +torch==2.4.0 # via ml_services (setup.py) -tqdm==4.66.4 +tqdm==4.66.5 # via # huggingface-hub # transformers -transformers==4.24.0 +transformers==4.44.0 # via ml_services (setup.py) +triton==3.0.0 + # via torch typing-extensions==4.12.2 # via + # cattrs # huggingface-hub # torch +tzdata==2024.1 + # via pandas url-normalize==1.4.3 # via requests-cache urllib3==2.2.2 - # via requests + # via + # requests + # requests-cache werkzeug==3.0.3 # via flask -wheel==0.43.0 - # via - # nvidia-cublas-cu11 - # nvidia-cuda-runtime-cu11 +zipp==3.19.2 + # via importlib-metadata \ No newline at end of file From 6a48b7a1fb173f1adc408b2968505227c618644d Mon Sep 17 00:00:00 2001 From: Daniil <94884910+Filienko@users.noreply.github.com> Date: Tue, 6 Aug 2024 19:04:19 -0700 Subject: [PATCH 06/15] Removing pyproject, switching to setup to support older python --- pyproject.toml | 63 -------------------------------------------------- setup.cfg | 2 -- 2 files changed, 65 deletions(-) delete mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 36fe453..0000000 --- a/pyproject.toml +++ /dev/null @@ -1,63 +0,0 @@ -[project] -name = "isacc-ml" -description = "Flask-based microservice for machine learning utilities in the ISACC service" -version = "0.1.0" -authors = [ - { name="CIRG", email="cirg-adm@uw.edu" } -] -readme = "README.md" -requires-python = ">=3.7" -license = {file = "LICENSE"} - -[project.urls] -Homepage = "https://github.com/uwcirg/isacc-ml" -Repository = "https://github.com/uwcirg/isacc-ml.git" - -[build-system] -requires = ["setuptools>=45", "wheel"] -build-backend = "setuptools.build_meta" - -[tool.setuptools] -packages = ["isacc_ml"] - -[tool.pytest.ini_options] -testpaths = ["tests"] -addopts = "-v" - -[tool.mypy] -ignore_missing_imports = true -strict_optional = true - -[tool.poetry.dependencies] -python = "^3.7" -flask = "1.1.2" -gunicorn = "20.1.0" -requests-cache = "0.6.4" -torch = "1.13.0" -transformers = "4.24.0" -scipy = "1.7.3" -numpy = "1.21.5" -pandas = "1.3.5" -click = "8.0.1" -importlib-metadata = "4.8.1" -itsdangerous = "2.0.1" -jinja2 = "3.0.1" -markupsafe = "2.0.1" -six = "1.16.0" -typing-extensions = "3.10.0.2" -werkzeug = "2.0.1" -zipp = "3.5.0" - -[tool.poetry.dev-dependencies] -pytest = "6.2.5" -pytest-mock = "3.6.1" -pytest-datadir = "1.3.1" -requests-mock = "1.9.3" -attrs = "21.2.0" -iniconfig = "1.1.1" -packaging = "21.0" -pluggy = "1.0.0" -psutil = "5.8.0" -py = "1.10.0" -pyparsing = "2.4.7" -toml = "0.10.2" \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index ddf9924..871997e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,8 +4,6 @@ description = Flask-based microservice for NLP machine learning utilities in the author = CIRG author_email = cirg-adm@uw.edu url = https://github.com/uwcirg/isacc-ml -classifiers = - Programming Language :: Python :: 3.7 [options] packages = ml_services From b6a6dea4d7826f55726b4dba152bc4857a6cc46e Mon Sep 17 00:00:00 2001 From: Daniil <94884910+Filienko@users.noreply.github.com> Date: Tue, 6 Aug 2024 19:11:26 -0700 Subject: [PATCH 07/15] Removing version from setup --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index f63e95b..5da30f3 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,6 @@ setup( name="ml_services", - version="1.0", package=["ml_services"], include_package_data=True ) \ No newline at end of file From 2ff4e03c2243c61df3a6307419dd3f2d6010fc5f Mon Sep 17 00:00:00 2001 From: Daniil <94884910+Filienko@users.noreply.github.com> Date: Tue, 6 Aug 2024 19:21:09 -0700 Subject: [PATCH 08/15] Fixing formatting --- requirements.dev.txt | 6 ++++++ requirements.txt | 6 ++++++ setup.cfg | 1 - 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/requirements.dev.txt b/requirements.dev.txt index 964240e..9b6eb7e 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -1,3 +1,9 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --extra=dev --output-file=requirements.dev.txt setup.cfg +# attrs==24.2.0 # via # cattrs diff --git a/requirements.txt b/requirements.txt index 3d28fac..67ad804 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,9 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile +# attrs==24.2.0 # via # cattrs diff --git a/setup.cfg b/setup.cfg index 871997e..ee20d06 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,7 +25,6 @@ dev = pytest-datadir requests-mock - [flake8] max-line-length=100 From f8060fd6f0b8288a4a50575da361934eff89a3ad Mon Sep 17 00:00:00 2001 From: Daniil <94884910+Filienko@users.noreply.github.com> Date: Tue, 6 Aug 2024 19:40:14 -0700 Subject: [PATCH 09/15] Downgrading flask --- requirements.dev.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.dev.txt b/requirements.dev.txt index 9b6eb7e..a462780 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -28,7 +28,7 @@ filelock==3.15.4 # torch # transformers # triton -flask==3.0.3 +flask==2.0.1 # via ml_services (setup.cfg) fsspec==2024.6.1 # via @@ -46,7 +46,7 @@ importlib-metadata==8.2.0 # via flask iniconfig==2.0.0 # via pytest -itsdangerous==2.2.0 +itsdangerous==2.0.1 # via flask jinja2==3.1.4 # via @@ -175,7 +175,7 @@ urllib3==2.2.2 # via # requests # requests-cache -werkzeug==3.0.3 +werkzeug==2.0.1 # via flask zipp==3.19.2 # via importlib-metadata From 749c57ab9789d422a007441098062cef48e7d0fa Mon Sep 17 00:00:00 2001 From: Daniil <94884910+Filienko@users.noreply.github.com> Date: Tue, 6 Aug 2024 19:41:01 -0700 Subject: [PATCH 10/15] Returning to previous version --- requirements.dev.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.dev.txt b/requirements.dev.txt index a462780..9b6eb7e 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -28,7 +28,7 @@ filelock==3.15.4 # torch # transformers # triton -flask==2.0.1 +flask==3.0.3 # via ml_services (setup.cfg) fsspec==2024.6.1 # via @@ -46,7 +46,7 @@ importlib-metadata==8.2.0 # via flask iniconfig==2.0.0 # via pytest -itsdangerous==2.0.1 +itsdangerous==2.2.0 # via flask jinja2==3.1.4 # via @@ -175,7 +175,7 @@ urllib3==2.2.2 # via # requests # requests-cache -werkzeug==2.0.1 +werkzeug==3.0.3 # via flask zipp==3.19.2 # via importlib-metadata From 12becae7c0f53fbc9bd547273f8ede599d1092ee Mon Sep 17 00:00:00 2001 From: Daniil <94884910+Filienko@users.noreply.github.com> Date: Wed, 7 Aug 2024 18:39:30 -0700 Subject: [PATCH 11/15] WIP: Restoring the versions to ones generated by pip-compile --- requirements.dev.txt | 2 +- requirements.txt | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/requirements.dev.txt b/requirements.dev.txt index 9b6eb7e..aa8a2de 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -178,4 +178,4 @@ urllib3==2.2.2 werkzeug==3.0.3 # via flask zipp==3.19.2 - # via importlib-metadata + # via importlib-metadata \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 67ad804..84fd164 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,13 +27,13 @@ filelock==3.15.4 # transformers # triton flask==3.0.3 - # via ml_services (setup.py) + # via ml_services (setup.cfg) fsspec==2024.6.1 # via # huggingface-hub # torch gunicorn==22.0.0 - # via ml_services (setup.py) + # via ml_services (setup.cfg) huggingface-hub==0.24.5 # via # tokenizers @@ -58,7 +58,7 @@ networkx==3.2.1 # via torch numpy==2.0.1 # via - # ml_services (setup.py) + # ml_services (setup.cfg) # pandas # scipy # transformers @@ -99,7 +99,7 @@ packaging==24.1 # huggingface-hub # transformers pandas==2.2.2 - # via ml_services (setup.py) + # via ml_services (setup.cfg) platformdirs==4.2.2 # via requests-cache python-dateutil==2.9.0.post0 @@ -118,11 +118,11 @@ requests==2.32.3 # requests-cache # transformers requests-cache==1.2.1 - # via ml_services (setup.py) + # via ml_services (setup.cfg) safetensors==0.4.4 # via transformers scipy==1.13.1 - # via ml_services (setup.py) + # via ml_services (setup.cfg) six==1.16.0 # via # python-dateutil @@ -132,13 +132,13 @@ sympy==1.13.1 tokenizers==0.19.1 # via transformers torch==2.4.0 - # via ml_services (setup.py) + # via ml_services (setup.cfg) tqdm==4.66.5 # via # huggingface-hub # transformers transformers==4.44.0 - # via ml_services (setup.py) + # via ml_services (setup.cfg) triton==3.0.0 # via torch typing-extensions==4.12.2 From e4e7b4cd5b538f9eba1ef19b8f5671f60bfeeca2 Mon Sep 17 00:00:00 2001 From: Daniil <94884910+Filienko@users.noreply.github.com> Date: Tue, 20 Aug 2024 18:15:32 -0700 Subject: [PATCH 12/15] Updating the versions to latest --- requirements.dev.txt | 14 +++++++------- requirements.txt | 12 ++++++------ setup.cfg | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/requirements.dev.txt b/requirements.dev.txt index aa8a2de..7a0f774 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -34,15 +34,15 @@ fsspec==2024.6.1 # via # huggingface-hub # torch -gunicorn==22.0.0 +gunicorn==23.0.0 # via ml_services (setup.cfg) -huggingface-hub==0.24.5 +huggingface-hub==0.24.6 # via # tokenizers # transformers idna==3.7 # via requests -importlib-metadata==8.2.0 +importlib-metadata==8.4.0 # via flask iniconfig==2.0.0 # via pytest @@ -146,7 +146,7 @@ six==1.16.0 # via # python-dateutil # url-normalize -sympy==1.13.1 +sympy==1.13.2 # via torch tokenizers==0.19.1 # via transformers @@ -158,7 +158,7 @@ tqdm==4.66.5 # via # huggingface-hub # transformers -transformers==4.44.0 +transformers==4.44.1 # via ml_services (setup.cfg) triton==3.0.0 # via torch @@ -177,5 +177,5 @@ urllib3==2.2.2 # requests-cache werkzeug==3.0.3 # via flask -zipp==3.19.2 - # via importlib-metadata \ No newline at end of file +zipp==3.20.0 + # via importlib-metadata diff --git a/requirements.txt b/requirements.txt index 84fd164..7dccccb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,15 +32,15 @@ fsspec==2024.6.1 # via # huggingface-hub # torch -gunicorn==22.0.0 +gunicorn==23.0.0 # via ml_services (setup.cfg) -huggingface-hub==0.24.5 +huggingface-hub==0.24.6 # via # tokenizers # transformers idna==3.7 # via requests -importlib-metadata==8.2.0 +importlib-metadata==8.4.0 # via flask itsdangerous==2.2.0 # via flask @@ -127,7 +127,7 @@ six==1.16.0 # via # python-dateutil # url-normalize -sympy==1.13.1 +sympy==1.13.2 # via torch tokenizers==0.19.1 # via transformers @@ -137,7 +137,7 @@ tqdm==4.66.5 # via # huggingface-hub # transformers -transformers==4.44.0 +transformers==4.44.1 # via ml_services (setup.cfg) triton==3.0.0 # via torch @@ -156,5 +156,5 @@ urllib3==2.2.2 # requests-cache werkzeug==3.0.3 # via flask -zipp==3.19.2 +zipp==3.20.0 # via importlib-metadata \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index ee20d06..86544bf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,7 +7,7 @@ url = https://github.com/uwcirg/isacc-ml [options] packages = ml_services -python_requires = >=3.7 +python_requires = >=3.9 install_requires = flask gunicorn From 13daa0c326db4beb6b4f4756325d3bdb65d32c0e Mon Sep 17 00:00:00 2001 From: Daniil <94884910+Filienko@users.noreply.github.com> Date: Tue, 20 Aug 2024 18:21:58 -0700 Subject: [PATCH 13/15] WIP: trying to free up the space --- .github/workflows/test.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 8b98690..6124b83 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -9,6 +9,9 @@ jobs: container: python:3.11 steps: + - name: Delete huge unnecessary tools folder + run: rm -rf /opt/hostedtoolcache + - uses: actions/checkout@v1 - name: Install test runner From 645ef36ec7691729201991e18c8fc7d273d6cf3a Mon Sep 17 00:00:00 2001 From: Daniil <94884910+Filienko@users.noreply.github.com> Date: Thu, 22 Aug 2024 16:16:27 -0700 Subject: [PATCH 14/15] Deleting unnecessary folder --- .github/workflows/build-deliver.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-deliver.yaml b/.github/workflows/build-deliver.yaml index 08a1888..0844fdc 100644 --- a/.github/workflows/build-deliver.yaml +++ b/.github/workflows/build-deliver.yaml @@ -9,6 +9,9 @@ jobs: # needs: [test] runs-on: ubuntu-latest steps: + - name: Delete huge unnecessary tools folder + run: rm -rf /opt/hostedtoolcache + - name: Checkout git commit uses: actions/checkout@master with: From 971ab7ae36e0ef8b8b971673318763fe505a586e Mon Sep 17 00:00:00 2001 From: Daniil <94884910+Filienko@users.noreply.github.com> Date: Fri, 23 Aug 2024 15:42:23 -0700 Subject: [PATCH 15/15] Not deleting tools directories during tests --- .github/workflows/test.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6124b83..8b98690 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -9,9 +9,6 @@ jobs: container: python:3.11 steps: - - name: Delete huge unnecessary tools folder - run: rm -rf /opt/hostedtoolcache - - uses: actions/checkout@v1 - name: Install test runner