Merge pull request #309 from MITLibraries/maintenance-updates

Maintenance updates

.github/pull-request-template.md

@@ -1,38 +1,29 @@
-### What does this PR do?
-
-Describe the overall purpose of the PR changes. Doesn't need to be as specific as the
-individual commits.
-
-### Helpful background context
-
-Describe any additional context beyond what the PR accomplishes if it is likely to be
-useful to a reviewer.
-
-Delete this section if it isn't applicable to the PR.
+### Purpose and background context
+Describe the overall purpose of the PR changes and any useful background context.
 
 ### How can a reviewer manually see the effects of these changes?
-
 Explain how to see the proposed changes in the application if possible.
 
 Delete this section if it isn't applicable to the PR.
 
 ### Includes new or updated dependencies?
+YES | NO
 
+### Changes expectations for external applications?
 YES | NO
 
 ### What are the relevant tickets?
-
-Include links to Jira Software and/or Jira Service Management tickets here.
+- Include links to Jira Software and/or Jira Service Management tickets here.
 
 ### Developer
-
-- [ ] All new ENV is documented in README (or there is none)
+- [ ] All new ENV is documented in README
+- [ ] All new ENV has been added to staging and production environments
+- [ ] All related Jira tickets are linked in commit message(s)
 - [ ] Stakeholder approval has been confirmed (or is not needed)
 
-### Code Reviewer
-
-- [ ] The commit message is clear and follows our guidelines (not just this pull request message)
+### Code Reviewer(s)
+- [ ] The commit message is clear and follows our guidelines (not just this PR message)
 - [ ] There are appropriate tests covering any new functionality
-- [ ] The documentation has been updated or is unnecessary
-- [ ] The changes have been verified
+- [ ] The provided documentation is sufficient for understanding any new functionality introduced
+- [ ] Any manual tests have been performed **or** provided examples have been verified
 - [ ] New dependencies are appropriate or there were no changes

.pre-commit-config.yaml

@@ -0,0 +1,29 @@
+default_language_version:
+  python: python3.11 # set for project python version
+repos:
+  - repo: local
+    hooks:
+      - id: black-apply
+        name: black-apply
+        entry: pipenv run black
+        language: system
+        pass_filenames: true
+        types: ["python"]
+      - id: mypy
+        name: mypy
+        entry: pipenv run mypy
+        language: system
+        pass_filenames: true
+        types: ["python"]
+        exclude: "tests/"
+      - id: ruff-apply
+        name: ruff-apply
+        entry: pipenv run ruff check --fix
+        language: system
+        pass_filenames: true
+        types: ["python"]
+      - id: safety
+        name: safety
+        entry: pipenv check
+        language: system
+        pass_filenames: false

Makefile

@@ -6,67 +6,82 @@ ECR_URL_DEV:=222053980223.dkr.ecr.us-east-1.amazonaws.com/timdex-index-manager-d
 SHELL=/bin/bash
 DATETIME:=$(shell date -u +%Y%m%dT%H%M%SZ)
 
-### Dependency commands ###
+help: # preview Makefile commands
+	@awk 'BEGIN { FS = ":.*#"; print "Usage:  make <target>\n\nTargets:" } \
+/^[-_[:alpha:]]+:.?*#/ { printf "  %-15s%s\n", $$1, $$2 }' $(MAKEFILE_LIST)
 
-install: ## Install dependencies and CLI app
+## Dependency commands 
+
+install: # install Python dependencies and pre-commit hook
 	pipenv install --dev
+	pipenv run pre-commit install
 
-update: install ## Update all Python dependencies
+update: install # update Python dependencies
 	pipenv clean
 	pipenv update --dev
 
-### Test commands ###
+## Unit Test commands
 
-test: ## Run tests and print a coverage report
+test: # run tests and print a coverage report
 	pipenv run coverage run --source=tim -m pytest -vv
 	pipenv run coverage report -m
 
-coveralls: test
+coveralls: test # write coverage data to an LCOV report
 	pipenv run coverage lcov -o ./coverage/lcov.info
 
-### Code quality and safety commands ###
-
-lint: bandit black mypy pylama safety ## Run linting, code quality, and safety checks
+## Code quality and safety commands
 
-bandit:
-	pipenv run bandit -r tim
+lint: black mypy ruff safety # run linters
 
-black:
+black: # run 'black' linter and print a preview of suggested changes
 	pipenv run black --check --diff .
 
-mypy:
+mypy:  # run 'mypy' linter
 	pipenv run mypy tim
 
-pylama:
-	pipenv run pylama --options setup.cfg
+ruff: # run 'ruff' linter and print a preview of errors
+	pipenv run ruff check .
 
-safety:
+safety: # check for security vulnerabilities and verify Pipfile.lock is up-to-date
 	pipenv check
 	pipenv verify
 
-### Terraform-generated Developer Deploy Commands for Dev environment ###
-dist-dev: ## Build docker container (intended for developer-based manual build)
+lint-apply: # apply changes with 'black' and resolve fixable errors with 'ruff'
+	black-apply ruff-apply
+
+black-apply: # apply changes with 'black'
+	pipenv run black .
+
+ruff-apply: # resolve fixable errors with 'ruff'
+	pipenv run ruff check --fix .
+
+## Terraform-generated commands for container build and deployment in dev
+dist-dev: # build docker container (intended for developer-based manual build)
 	docker build --platform linux/amd64 \
 	    -t $(ECR_URL_DEV):latest \
 		-t $(ECR_URL_DEV):`git describe --always` \
 		-t $(ECR_NAME_DEV):latest .
 
-publish-dev: dist-dev ## Build, tag and push (intended for developer-based manual publish)
+publish-dev: dist-dev # build, tag and push (intended for developer-based manual publish)
 	docker login -u AWS -p $$(aws ecr get-login-password --region us-east-1) $(ECR_URL_DEV)
 	docker push $(ECR_URL_DEV):latest
 	docker push $(ECR_URL_DEV):`git describe --always`
 
-### Terraform-generated manual shortcuts for deploying to Stage ###
-### This requires that ECR_NAME_STAGE & ECR_URL_STAGE environment variables are set locally
-### by the developer and that the developer has authenticated to the correct AWS Account.
-### The values for the environment variables can be found in the stage_build.yml caller workflow.
-dist-stage: ## Only use in an emergency
+## Terraform-generated commands for container build and deployment in stage \
+This requires that ECR_NAME_STAGE and ECR_URL_STAGE environment variables \
+are set locally by the developer and that the developer has \
+authenticated to the correct AWS Account. The values for the environment \
+variables can be found in the stage_build.yml caller workflow. \
+While Stage should generally only be used in an emergency for most repos, \
+it is necessary for any testing requiring access to the Data Warehouse \
+because Cloud Connector is not enabled on Dev1.
+dist-stage:
 	docker build --platform linux/amd64 \
 	    -t $(ECR_URL_STAGE):latest \
 		-t $(ECR_URL_STAGE):`git describe --always` \
 		-t $(ECR_NAME_STAGE):latest .
 
-publish-stage: ## Only use in an emergency
+publish-stage:
 	docker login -u AWS -p $$(aws ecr get-login-password --region us-east-1) $(ECR_URL_STAGE)
 	docker push $(ECR_URL_STAGE):latest
 	docker push $(ECR_URL_STAGE):`git describe --always`

Pipfile

@@ -13,17 +13,16 @@ sentry-sdk = "*"
 smart-open = {extras = ["s3"], version = "*"}
 
 [dev-packages]
-bandit = "*"
 black = "*"
 boto3-stubs = "*"
-coverage = "*"
 coveralls = "*"
 freezegun = "*"
 mypy = "*"
+pre-commit = "*"
 pydocstyle = "*"
-pylama = {extras = ["all"], version = "*"}
 pytest = "*"
 vcrpy = "*"
+ruff = "*"
 
 [requires]
 python_version = "3.11"