From 3f0d62a27232e92c8400da2254f84042b3f50940 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Boisseau-Sierra?= <37387755+EBoisseauSierra@users.noreply.github.com> Date: Mon, 7 Oct 2024 14:24:23 +0200 Subject: [PATCH] ci: Add automation (#2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In this PR, we add GitHub Actions and configure pre-commit hooks to format the codebase explicitly. * ci: Update pre-commit hooks Replace flake8 & black with ruff, and bump dependencies Signed-off-by: Étienne Boisseau-Sierra * fix: Enforce new pre-commit Signed-off-by: Étienne Boisseau-Sierra * ci: Configure Dependabot Signed-off-by: Étienne Boisseau-Sierra * ci: Add GHA on PRs Signed-off-by: Étienne Boisseau-Sierra * ci: Drop pydocstyle as project is deprecated Signed-off-by: Étienne Boisseau-Sierra * ci: Drop bandit (use Ruff instead) Signed-off-by: Étienne Boisseau-Sierra * chore: Bump supported Python version to 3.12 Signed-off-by: Étienne Boisseau-Sierra * ci: Configure Ruff Signed-off-by: Étienne Boisseau-Sierra * chore: Fix codebase formatting Signed-off-by: Étienne Boisseau-Sierra * test: Add placeholder for now Signed-off-by: Étienne Boisseau-Sierra * ci: Allow 'assert' statements Signed-off-by: Étienne Boisseau-Sierra --------- Signed-off-by: Étienne Boisseau-Sierra --- .github/dependabot.yml | 17 ++++ .github/workflows/format_codebase.yml | 21 +++++ .../workflows/{pr-lint.yml => format_pr.yml} | 18 ++-- .github/workflows/run_python_tests.yml | 39 ++++++++ .pre-commit-config.yaml | 65 +++++-------- VERSION | 2 +- gsbparse/account_file.py | 34 +++---- gsbparse/account_section.py | 94 ++++++++++--------- gsbparse/transactions.py | 41 ++++++-- ruff.toml | 32 +++++++ setup.py | 12 +-- tests/test_placeholder.py | 2 + 12 files changed, 253 insertions(+), 124 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/format_codebase.yml rename .github/workflows/{pr-lint.yml => format_pr.yml} (59%) create mode 100644 .github/workflows/run_python_tests.yml create mode 100644 ruff.toml create mode 100644 tests/test_placeholder.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..8d8ffa5 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,17 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: weekly + - package-ecosystem: pip + directory: "/" + schedule: + interval: weekly + groups: + # Update minor and patch versions of Python packages together. + # Major version bumps are raised as separate PRs for each package. + python: + update-types: + - "minor" + - "patch" diff --git a/.github/workflows/format_codebase.yml b/.github/workflows/format_codebase.yml new file mode 100644 index 0000000..9e28a13 --- /dev/null +++ b/.github/workflows/format_codebase.yml @@ -0,0 +1,21 @@ +name: Lint & format codebase + +on: + pull_request: + workflow_dispatch: + push: + branches: + - main + - release/** + - pre-release/** + +jobs: + run_hooks: + name: Enforce pre-commit hooks + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Enforce pre-commits hooks + uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/pr-lint.yml b/.github/workflows/format_pr.yml similarity index 59% rename from .github/workflows/pr-lint.yml rename to .github/workflows/format_pr.yml index 728942e..80cf4e5 100644 --- a/.github/workflows/pr-lint.yml +++ b/.github/workflows/format_pr.yml @@ -1,4 +1,4 @@ -name: PR Lint +name: PR lint on: pull_request: @@ -10,15 +10,19 @@ on: jobs: check: + name: Validate PR title runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: morrisoncole/pr-lint-action@v1.5.1 + - name: Checkout + uses: actions/checkout@v4 + + - name: Validate PR title + uses: morrisoncole/pr-lint-action@v1.7.1 with: title-regex: "^(build|chore|ci|docs|feat|fix|perf|refactor|style|test|other)(\\(.+\\))?(\\!)?:\\s.+" on-failed-regex-fail-action: true - on-failed-regex-request-changes: false - on-failed-regex-create-review: false - on-failed-regex-comment: - "Please format your PR title to match: `%regex%`!" + on-failed-regex-request-changes: true + on-failed-regex-create-review: true + on-failed-regex-comment: "Please format your PR title to match: `%regex%`!" + on-succeeded-regex-dismiss-review-comment: "The PR title is now correctly formatted. Thanks!" repo-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/run_python_tests.yml b/.github/workflows/run_python_tests.yml new file mode 100644 index 0000000..7c8cb41 --- /dev/null +++ b/.github/workflows/run_python_tests.yml @@ -0,0 +1,39 @@ +name: Run Python unit tests + +on: + push: + branches: + - main + - release/** + - pre-release/** + pull_request: + types: [opened, edited, synchronize, reopened] + +jobs: + unit_tests: + name: Run unit tests + runs-on: ubuntu-latest + permissions: + # Allow the coverage report to be generated and posted + pull-requests: write + contents: write + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + id: python_setup + uses: actions/setup-python@v5 + with: + python-version: 3.12 + + - name: Install dependencies + id: dependencies + run: | + pip install --upgrade pip + pip install -e '.[test, build]' + + - name: Run unit tests + id: pytest + run: | + pytest tests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 178e8b9..0194ad9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,43 +1,30 @@ repos: -- repo: https://github.com/pre-commit/pre-commit-hooks + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-added-large-files + - id: check-json + - id: check-merge-conflict + - id: check-toml + - id: check-xml + - id: check-yaml + - id: debug-statements + - id: end-of-file-fixer + - id: mixed-line-ending + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] # allow markdown linebreak at EOL + - repo: https://github.com/charliermarsh/ruff-pre-commit + rev: "v0.6.9" + hooks: + - id: ruff + name: "Ruff linting" + args: + - --fix + - --exit-non-zero-on-fix + - id: ruff-format + name: "Ruff formatting" + - repo: https://github.com/compilerla/conventional-pre-commit rev: v3.4.0 hooks: - - id: check-yaml - - id: debug-statements - - id: end-of-file-fixer - - id: trailing-whitespace - args: [--markdown-linebreak-ext=md] # allow markdown linebreak at EOL -- repo: https://github.com/pre-commit/pygrep-hooks - rev: v1.7.0 - hooks: - - id: python-check-mock-methods - - id: python-use-type-annotations -- repo: https://github.com/PyCQA/bandit - rev: 1.7.0 - hooks: - - id: bandit - args: ["--skip", "B101"] # don't check `assert`, as all tests would then raise errors -- repo: https://github.com/pycqa/pydocstyle - rev: 5.1.1 - hooks: - - id: pydocstyle - exclude: ^tests/ # do not force detailed docstrings in tests - args: [--convention=google] -- repo: https://github.com/pycqa/isort - rev: 5.7.0 - hooks: - - id: isort - args: ["--profile", "black", "--line-length", "88"] # make isort compliant with our code style -- repo: https://github.com/psf/black - rev: 20.8b1 - hooks: - - id: black -- repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.4 - hooks: - - id: flake8 -- repo: https://github.com/compilerla/conventional-pre-commit - rev: v1.2.0 - hooks: - - id: conventional-pre-commit + - id: conventional-pre-commit stages: [commit-msg] diff --git a/VERSION b/VERSION index 9325c3c..0d91a54 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.0 \ No newline at end of file +0.3.0 diff --git a/gsbparse/account_file.py b/gsbparse/account_file.py index a07ae68..c508a9c 100644 --- a/gsbparse/account_file.py +++ b/gsbparse/account_file.py @@ -2,9 +2,9 @@ import logging from functools import cached_property -from typing import TextIO, Union +from typing import TextIO -import defusedxml.ElementTree as ET +import defusedxml.ElementTree as Et import pandas as pd from gsbparse.account_section import ( @@ -12,7 +12,7 @@ GsbSectionBudgetary, GsbSectionCategory, GsbSectionCurrency, - GsbSectionFinancial_year, + GsbSectionFinancialYear, GsbSectionParty, GsbSectionPayment, GsbSectionReconcile, @@ -27,7 +27,7 @@ class AccountFile: """Representation of a parsed `.gsb` file.""" - def __init__(self, source: Union[str, TextIO]) -> None: + def __init__(self, source: str | TextIO) -> None: """Initialize an AccountFile object, given its source (file object or path).""" self.source = source @@ -49,42 +49,42 @@ def sections(self) -> list[dict]: sections = {} # Read the .gsb XML content - tree = ET.parse(self.source) + tree = Et.parse(self.source) root = tree.getroot() # Instantiate each GsbSection with tags of the relevant type sections["Account"] = GsbSectionAccount( - [child.attrib for child in root if child.tag == "Account"] + [child.attrib for child in root if child.tag == "Account"], ) sections["Currency"] = GsbSectionCurrency( - [child.attrib for child in root if child.tag == "Currency"] + [child.attrib for child in root if child.tag == "Currency"], ) sections["Party"] = GsbSectionParty( - [child.attrib for child in root if child.tag == "Party"] + [child.attrib for child in root if child.tag == "Party"], ) sections["Category"] = GsbSectionCategory( - [child.attrib for child in root if child.tag == "Category"] + [child.attrib for child in root if child.tag == "Category"], ) sections["Sub_category"] = GsbSectionSubCategory( - [child.attrib for child in root if child.tag == "Sub_category"] + [child.attrib for child in root if child.tag == "Sub_category"], ) sections["Budgetary"] = GsbSectionBudgetary( - [child.attrib for child in root if child.tag == "Budgetary"] + [child.attrib for child in root if child.tag == "Budgetary"], ) sections["Sub_budgetary"] = GsbSectionSubBudgetary( - [child.attrib for child in root if child.tag == "Sub_budgetary"] + [child.attrib for child in root if child.tag == "Sub_budgetary"], ) sections["Transaction"] = GsbSectionTransaction( - [child.attrib for child in root if child.tag == "Transaction"] + [child.attrib for child in root if child.tag == "Transaction"], ) - sections["Financial_year"] = GsbSectionFinancial_year( - [child.attrib for child in root if child.tag == "Financial_year"] + sections["Financial_year"] = GsbSectionFinancialYear( + [child.attrib for child in root if child.tag == "Financial_year"], ) sections["Reconcile"] = GsbSectionReconcile( - [child.attrib for child in root if child.tag == "Reconcile"] + [child.attrib for child in root if child.tag == "Reconcile"], ) sections["Payment"] = GsbSectionPayment( - [child.attrib for child in root if child.tag == "Payment"] + [child.attrib for child in root if child.tag == "Payment"], ) return sections diff --git a/gsbparse/account_section.py b/gsbparse/account_section.py index b67c243..b7955f0 100644 --- a/gsbparse/account_section.py +++ b/gsbparse/account_section.py @@ -72,36 +72,42 @@ def __init__(self, records: list[dict]): def df(self): """Represent the list of records as a pd.DataFrame.""" # Create df - df = pd.DataFrame.from_records(self.records) + dataset = pd.DataFrame.from_records(self.records) # "Improve" df by casting correct columns dtype if self._idx_col is not None: - df.set_index(self._idx_col, inplace=True) + dataset = dataset.set_index(self._idx_col) if self._int_cols: - df[self._int_cols] = df[self._int_cols].apply( - pd.to_numeric, downcast="integer", errors="coerce" + dataset[self._int_cols] = dataset[self._int_cols].apply( + pd.to_numeric, + downcast="integer", + errors="coerce", ) if self._bool_cols: # Strings must be cast as integer first so that casting to bool # doesn't always yield True. # Cf. https://stackoverflow.com/q/52089711/5433628 - df[self._bool_cols] = ( - df[self._bool_cols] + dataset[self._bool_cols] = ( + dataset[self._bool_cols] .apply(pd.to_numeric, downcast="integer") .astype("bool") ) if self._currency_cols: - df[self._currency_cols] = df[self._currency_cols].apply(pd.to_numeric) + dataset[self._currency_cols] = dataset[self._currency_cols].apply( + pd.to_numeric, + ) if self._date_cols: - df[self._date_cols] = df[self._date_cols].apply( - pd.to_datetime, format="%m/%d/%Y", errors="coerce" + dataset[self._date_cols] = dataset[self._date_cols].apply( + pd.to_datetime, + format="%m/%d/%Y", + errors="coerce", ) - return df + return dataset class GsbSectionAccount(AccountSection): @@ -142,14 +148,14 @@ def _currency_cols(self): "Minimum_authorised_balance", ] - def __init__(self, XML_tags_attributes_values): + def __init__(self, xml_tags_attributes_values): """Build self.df from the list of XML tags attributes values. Args: - XML_tags_attributes_values (list(dict)): Values of "Account" tags + xml_tags_attributes_values (list(dict)): Values of "Account" tags attributes. """ - super(GsbSectionAccount, self).__init__(XML_tags_attributes_values) + super().__init__(xml_tags_attributes_values) class GsbSectionCurrency(AccountSection): @@ -173,14 +179,14 @@ def _name_col(self): def _int_cols(self): return ["Fl"] - def __init__(self, XML_tags_attributes_values): + def __init__(self, xml_tags_attributes_values): """Build self.df from the list of XML tags attributes values. Args: - XML_tags_attributes_values (list(dict)): Values of "Currency" tags + xml_tags_attributes_values (list(dict)): Values of "Currency" tags attributes. """ - super(GsbSectionCurrency, self).__init__(XML_tags_attributes_values) + super().__init__(xml_tags_attributes_values) class GsbSectionParty(AccountSection): @@ -204,14 +210,14 @@ def _name_col(self): def _bool_cols(self): return ["IgnCase", "UseRegex"] - def __init__(self, XML_tags_attributes_values): + def __init__(self, xml_tags_attributes_values): """Build self.df from the list of XML tags attributes values. Args: - XML_tags_attributes_values (list(dict)): Values of "Party" tags + xml_tags_attributes_values (list(dict)): Values of "Party" tags attributes. """ - super(GsbSectionParty, self).__init__(XML_tags_attributes_values) + super().__init__(xml_tags_attributes_values) class GsbSectionCategory(AccountSection): @@ -235,14 +241,14 @@ def _name_col(self): def _bool_cols(self): return ["Kd"] - def __init__(self, XML_tags_attributes_values): + def __init__(self, xml_tags_attributes_values): """Build self.df from the list of XML tags attributes values. Args: - XML_tags_attributes_values (list(dict)): Values of "Category" tags + xml_tags_attributes_values (list(dict)): Values of "Category" tags attributes. """ - super(GsbSectionCategory, self).__init__(XML_tags_attributes_values) + super().__init__(xml_tags_attributes_values) class GsbSectionSubCategory(AccountSection): @@ -262,14 +268,14 @@ def _idx_col(self): def _name_col(self): return ["Na"] - def __init__(self, XML_tags_attributes_values): + def __init__(self, xml_tags_attributes_values): """Build self.df from the list of XML tags attributes values. Args: - XML_tags_attributes_values (list(dict)): Values of "SubCategory" + xml_tags_attributes_values (list(dict)): Values of "SubCategory" tags attributes. """ - super(GsbSectionSubCategory, self).__init__(XML_tags_attributes_values) + super().__init__(xml_tags_attributes_values) class GsbSectionBudgetary(AccountSection): @@ -293,14 +299,14 @@ def _name_col(self): def _bool_cols(self): return ["Kd"] - def __init__(self, XML_tags_attributes_values): + def __init__(self, xml_tags_attributes_values): """Build self.df from the list of XML tags attributes values. Args: - XML_tags_attributes_values (list(dict)): Values of "Budgetary" tags + xml_tags_attributes_values (list(dict)): Values of "Budgetary" tags attributes. """ - super(GsbSectionBudgetary, self).__init__(XML_tags_attributes_values) + super().__init__(xml_tags_attributes_values) class GsbSectionSubBudgetary(AccountSection): @@ -320,14 +326,14 @@ def _idx_col(self): def _name_col(self): return ["Na"] - def __init__(self, XML_tags_attributes_values): + def __init__(self, xml_tags_attributes_values): """Build self.df from the list of XML tags attributes values. Args: - XML_tags_attributes_values (list(dict)): Values of "SubBudgetary" + xml_tags_attributes_values (list(dict)): Values of "SubBudgetary" tags attributes. """ - super(GsbSectionSubBudgetary, self).__init__(XML_tags_attributes_values) + super().__init__(xml_tags_attributes_values) class GsbSectionTransaction(AccountSection): @@ -359,14 +365,14 @@ def _currency_cols(self): def _date_cols(self): return ["Dt", "Dv"] - def __init__(self, XML_tags_attributes_values): + def __init__(self, xml_tags_attributes_values): """Build self.df from the list of XML tags attributes values. Args: - XML_tags_attributes_values (list(dict)): Values of "Transaction" + xml_tags_attributes_values (list(dict)): Values of "Transaction" tags attributes. """ - super(GsbSectionTransaction, self).__init__(XML_tags_attributes_values) + super().__init__(xml_tags_attributes_values) class GsbSectionPayment(AccountSection): @@ -398,17 +404,17 @@ def _bool_cols(self): "Automatic_number", ] - def __init__(self, XML_tags_attributes_values): + def __init__(self, xml_tags_attributes_values): """Build self.df from the list of XML tags attributes values. Args: - XML_tags_attributes_values (list(dict)): Values of "Payment" + xml_tags_attributes_values (list(dict)): Values of "Payment" tags attributes. """ - super(GsbSectionPayment, self).__init__(XML_tags_attributes_values) + super().__init__(xml_tags_attributes_values) -class GsbSectionFinancial_year(AccountSection): +class GsbSectionFinancialYear(AccountSection): """Represent the tags of a Grisbi file. Attributes: @@ -433,14 +439,14 @@ def _bool_cols(self): def _date_cols(self): return ["Bdte", "Edte"] - def __init__(self, XML_tags_attributes_values): + def __init__(self, xml_tags_attributes_values): """Build self.df from the list of XML tags attributes values. Args: - XML_tags_attributes_values (list(dict)): Values of "Financial_year" + xml_tags_attributes_values (list(dict)): Values of "Financial_year" tags attributes. """ - super(GsbSectionFinancial_year, self).__init__(XML_tags_attributes_values) + super().__init__(xml_tags_attributes_values) class GsbSectionReconcile(AccountSection): @@ -472,11 +478,11 @@ def _currency_cols(self): def _date_cols(self): return ["Idate", "Fdate"] - def __init__(self, XML_tags_attributes_values): + def __init__(self, xml_tags_attributes_values): """Build self.df from the list of XML tags attributes values. Args: - XML_tags_attributes_values (list(dict)): Values of "Reconcile" + xml_tags_attributes_values (list(dict)): Values of "Reconcile" tags attributes. """ - super(GsbSectionReconcile, self).__init__(XML_tags_attributes_values) + super().__init__(xml_tags_attributes_values) diff --git a/gsbparse/transactions.py b/gsbparse/transactions.py index 68e2682..b55a97d 100644 --- a/gsbparse/transactions.py +++ b/gsbparse/transactions.py @@ -1,7 +1,7 @@ """Define a representation of transactions.""" from functools import cached_property -from typing import TextIO, Union +from typing import TextIO import pandas as pd @@ -20,7 +20,7 @@ class Transactions: df (pd.DataFrame): Transactions of the Grisbi file """ - def __init__(self, source: Union[str, TextIO]) -> None: + def __init__(self, source: str | TextIO) -> None: """Create the user-friendly Transactions df from an AccountFile instance.""" self.source = source @@ -51,24 +51,30 @@ def _df(self) -> pd.DataFrame: def get_transactions( self, - columns: Union[list, dict, None] = None, + columns: list | dict | None = None, ignore_mother_transactions: bool = False, ): """Return all or a subset of the transactions.""" if ignore_mother_transactions: - df = self._df[~self._df[("Transaction", "Br")]] + dataset = self._df[~self._df[("Transaction", "Br")]] else: - df = self._df + dataset = self._df if columns is None: - return df - elif type(columns) == list: - return df[columns] - elif type(columns) == dict: + return dataset + if type(columns) is list: + return dataset[columns] + if type(columns) is dict: # “Columns name mapper doesn't relate with [index] level.” # Cf. https://stackoverflow.com/a/67458211/5433628 cols_rename_mapping = {key[1]: value for key, value in columns.items()} - return df[columns.keys()].rename(columns=cols_rename_mapping) + return dataset[columns.keys()].rename(columns=cols_rename_mapping) + + raise InvalidArgumentTypeError( + arg_name="columns", + expected_arg_type=list | dict | None, + actual_arg_type=type(columns), + ) @staticmethod def index_column_names( @@ -193,3 +199,18 @@ def _format_transactions_columns(self, transactions: pd.DataFrame) -> pd.DataFra df=self.account_file.sections["Transaction"].df, prefix=self.account_file.sections["Transaction"].section, ) + + +class InvalidArgumentTypeError(ValueError): + """Raised when an invalid argument type is passed to a method.""" + + def __init__( + self, + arg_name: str, + expected_arg_type: type, + actual_arg_type: type, + ) -> None: + super().__init__( + f"'{arg_name}' arg must be a {expected_arg_type}, " + f"got {actual_arg_type} instead.", + ) diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..b002992 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,32 @@ +target-version = "py312" + +[lint] +select = [ + "A", # builtins + "B", # bugbear + "C4", # comprehensions + "COM", # comma + # "D", # docstring conventions + "E", # all pydocstyle errors + "F", # flake8 + "I", # isort + "ICN", # import conventions + "ISC", # implicit string concatenation + "N", # PEP 8 naming conventions + "PD", # pandas + "PIE", + "PT", # pytest style + "Q", # quotes + "RET", # return statements + "RUF", # Ruff + "S", # bandit + "SIM", # simplify + "SLF", # self + "T20", + "TID", # tidy imports + "TRY", # Tryceratops + "UP", # use latest Python conventions +] +ignore = [ + "S101", # allow use of assert +] diff --git a/setup.py b/setup.py index 3326220..b791e9b 100644 --- a/setup.py +++ b/setup.py @@ -2,10 +2,10 @@ import setuptools -with open("README.md", "r") as fh: +with open("README.md") as fh: long_description = fh.read() -with open("VERSION", "r") as version_file: +with open("VERSION") as version_file: version = version_file.read().strip() setuptools.setup( @@ -23,15 +23,15 @@ package_data={"static": ["VERSION"]}, install_requires=["defusedxml", "pandas"], extras_require={ - "dev": ["black", "flake8", "pre-commit", "pylint"], - "test": ["pytest", "pytest-cov"], + "dev": ["ruff", "pre-commit"], + "test": ["pytest"], }, classifiers=[ "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.12", ], license="MIT", - python_requires=">=3.9", + python_requires=">=3.12", ) diff --git a/tests/test_placeholder.py b/tests/test_placeholder.py new file mode 100644 index 0000000..3ada1ee --- /dev/null +++ b/tests/test_placeholder.py @@ -0,0 +1,2 @@ +def test_placeholder(): + assert True