diff --git a/.ci/run b/.ci/run index b15eb1c..b2c184d 100755 --- a/.ci/run +++ b/.ci/run @@ -37,4 +37,4 @@ if ! command -v python3 &> /dev/null; then fi "$PY_BIN" -m pip install --user tox -"$PY_BIN" -m tox "$@" +"$PY_BIN" -m tox --parallel --parallel-live "$@" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b564673..71f4404 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,9 +22,12 @@ on: jobs: build: strategy: + fail-fast: false matrix: platform: [ubuntu-latest, macos-latest, windows-latest] python-version: ['3.8', '3.9', '3.10', '3.11'] + # FIXME doesn't work on '3.12' because instapaper lib is using python-oauth2, which is using distutils + # see https://github.com/rsgalloway/instapaper/issues/23 # vvv just an example of excluding stuff from matrix # exclude: [{platform: macos-latest, python-version: '3.6'}] @@ -33,9 +36,6 @@ jobs: steps: # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - - if: ${{ matrix.platform == 'macos-latest' && matrix.python-version == '3.11' }} - # hmm somehow only seems necessary for 3.11 on osx?? - run: echo "$HOME/Library/Python/${{ matrix.python-version }}/bin" >> $GITHUB_PATH - uses: actions/setup-python@v4 with: @@ -44,6 +44,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: recursive + fetch-depth: 0 # nicer to have all git history when debugging/for tests - uses: mxschmitt/action-tmate@v3 if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} @@ -56,3 +57,4 @@ jobs: with: name: .coverage.mypy_${{ matrix.platform }}_${{ matrix.python-version }} path: .coverage.mypy/ + diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d2a9879 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,62 @@ +# see https://github.com/karlicoss/pymplate for up-to-date reference +[project] +dynamic = ["version"] # version is managed by setuptools_scm +name = "instapexport" +dependencies = [ + # instapaper api (needed for export only) + # my version has some changes not in the upstream yet.. + "instapaper @ git+https://github.com/karlicoss/instapaper.git", +] +# TODO maybe split out DAL deps and export deps? might be nice + +## these need to be set if you're planning to upload to pypi +# description = "TODO" +# license = {file = "LICENSE"} +# authors = [ +# {name = "Dima Gerasimov (@karlicoss)", email = "karlicoss@gmail.com"}, +# ] +# maintainers = [ +# {name = "Dima Gerasimov (@karlicoss)", email = "karlicoss@gmail.com"}, +# ] +# +# [project.urls] +# Homepage = "https://github.com/karlicoss/pymplate" +## + +[project.optional-dependencies] +optional = [ + "orjson", + "colorlog", + "ijson", # faster iterative json processing +] +testing = [ + "pytest", + "ruff", + "mypy", + "lxml", # for mypy html coverage +] + + +[build-system] +requires = ["setuptools", "setuptools-scm"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +version_scheme = "python-simplified-semver" +local_scheme = "dirty-tag" + + +# nice things about pyproject.toml +# - zip_safe=False isn't neccessary anymore +# - correctly discovers namespace packages by defuilt? +# - correctly handles py.typed by default? +# - handles src layout automatically https://setuptools.pypa.io/en/latest/userguide/package_discovery.html#src-layout + +# things I'm not sure about yet +# - avoiding dupliation/variable reuse? +# - file/git dependencies? +# - unclear how to specify namespace package order https://github.com/seanbreckenridge/reorder_editable/issues/2 + +# TODO +# - maybe it has a nicer pypi upload system? not sure +# e.g. possibly use hatch/flit/pdb/poetry -- but not sure what's the benefit tbh diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..6b67ce9 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,29 @@ +ignore = [ +### too opinionated style checks + "E501", # too long lines + "E702", # Multiple statements on one line (semicolon) + "E731", # assigning lambda instead of using def + "E741", # Ambiguous variable name: `l` + "E742", # Ambiguous class name: `O + "E401", # Multiple imports on one line + "F403", # import *` used; unable to detect undefined names +### + +### + "E722", # Do not use bare `except` ## Sometimes it's useful for defensive imports and that sort of thing.. + "F811", # Redefinition of unused # this gets in the way of pytest fixtures (e.g. in cachew) + +## might be nice .. but later and I don't wanna make it strict + "E402", # Module level import not at top of file + +### maybe consider these soon +# sometimes it's useful to give a variable a name even if we don't use it as a documentation +# on the other hand, often is a sign of error + "F841", # Local variable `count` is assigned to but never used + "F401", # imported but unused +### +] + +exclude = [ + "src/hypexport/Hypothesis", +] diff --git a/setup.py b/setup.py deleted file mode 100644 index 81db244..0000000 --- a/setup.py +++ /dev/null @@ -1,27 +0,0 @@ -from setuptools import setup, find_namespace_packages # type: ignore - - -def main() -> None: - pkgs = find_namespace_packages('src') - pkg = min(pkgs) - return setup( - name=pkg, - zip_safe=False, - packages=pkgs, - package_dir={'': 'src'}, - package_data={pkg: ['py.typed']}, - - install_requires=[ - # my version has some changes not in the upstream yet.. - 'instapaper @ git+https://github.com/karlicoss/instapaper.git', - ], - extras_require={ - 'testing': ['pytest'], - 'linting': ['pytest', 'mypy', 'lxml'], # lxml for mypy coverage report - 'optional': ['orjson', 'colorlog', 'enlighten'], - }, - ) - - -if __name__ == '__main__': - main() diff --git a/src/instapexport/dal.py b/src/instapexport/dal.py index 0945fd8..c3317bc 100755 --- a/src/instapexport/dal.py +++ b/src/instapexport/dal.py @@ -7,7 +7,7 @@ from .exporthelpers.dal_helper import Json, PathIsh, pathify, datetime_aware -logger = logging_helper.makeLogger(__name__) +logger = logging_helper.make_logger(__name__) Bid = str Hid = str @@ -155,7 +155,7 @@ def pages(self) -> List[Page]: pages_ = [ Page( bookmark=bks[page_bid], - highlights=list(sorted(page_hls, key=lambda b: b.dt)) + highlights=list(sorted(page_hls, key=lambda b: b.dt)), ) for page_bid, page_hls in page2hls.items() ] @@ -170,6 +170,7 @@ def demo(dao: DAL) -> None: print(f"Parsed {len(pages)} pages") from collections import Counter + common = Counter({(x.url, x.title): len(x.highlights) for x in pages}).most_common(10) print("10 most highlighed pages:") for (url, title), count in common: diff --git a/src/instapexport/export.py b/src/instapexport/export.py index 440ef4c..fd955f9 100755 --- a/src/instapexport/export.py +++ b/src/instapexport/export.py @@ -1,22 +1,21 @@ #!/usr/bin/env python3 -import argparse import json from typing import List -from .exporthelpers.export_helper import Json +from .exporthelpers.export_helper import Json, setup_parser, Parser # NOTE: uses custom version (has some changes that are not in upstream yet) # https://github.com/karlicoss/instapaper -import instapaper # type: ignore[import] +import instapaper # type: ignore[import-untyped] instapaper._API_VERSION_ = "api/1.1" # see https://github.com/rsgalloway/instapaper/issues/11 def get_json( - oauth_id: str, - oauth_secret: str, - oauth_token: str, - oauth_token_secret: str, + oauth_id: str, + oauth_secret: str, + oauth_token: str, + oauth_token_secret: str, ) -> Json: LIMIT = 500 # default limit is something stupid @@ -27,7 +26,11 @@ def get_json( user_folders = api.folders() folders: List[str] = [ - 'unread', 'archive', 'starred', # default, as per api docs + ## default, as per api docs + 'unread', + 'archive', + 'starred', + ## *(str(f['folder_id']) for f in user_folders), ] bookmarks = {} @@ -40,12 +43,14 @@ def get_json( } -def login(): +def login() -> None: print("NOTE: You'll need your username/password once in order to get oauth_token") + # fmt: off oauth_id = input('Your ouath_id: ') oauth_secret = input('Your ouath_secret: ') username = input('Your username: ') password = input('Your password: ') + # fmt: on api = instapaper.Instapaper(oauth_id, oauth_secret) odata = api.login(username, password) @@ -53,7 +58,7 @@ def login(): print(odata) -def main(): +def main() -> None: parser = make_parser() args = parser.parse_args() @@ -70,10 +75,11 @@ def main(): def make_parser(): - from .exporthelpers.export_helper import setup_parser, Parser - parser = Parser(""" + parser = Parser( + """ Export your personal Instapaper data: bookmarked articles and highlights. -""") +""", + ) setup_parser( parser=parser, params=[ @@ -84,11 +90,16 @@ def make_parser(): ], extra_usage=''' You can also import ~instapexport.export~ as a module and call ~get_json~ function directly to get raw JSON. -''') - parser.add_argument('--login', action='store_true', help=''' +''', + ) + parser.add_argument( + '--login', + action='store_true', + help=''' Note: OAUTH_ID/OAUTH_SECRET have to be requrested by email https://www.instapaper.com/main/request_oauth_consumer_token - ''') + ''', + ) return parser diff --git a/src/instapexport/exporthelpers b/src/instapexport/exporthelpers index 86ff6c9..3b60c93 160000 --- a/src/instapexport/exporthelpers +++ b/src/instapexport/exporthelpers @@ -1 +1 @@ -Subproject commit 86ff6c956065f00b90d31764f44ba090669a2703 +Subproject commit 3b60c936b71cefac6f8007bb8823ba288b1369e3 diff --git a/tox.ini b/tox.ini index 1f774d3..8997ce5 100644 --- a/tox.ini +++ b/tox.ini @@ -1,26 +1,35 @@ [tox] -minversion = 3.5 +minversion = 3.21 # relies on the correct version of Python installed -envlist = tests,mypy +envlist = ruff,tests,mypy # https://github.com/tox-dev/tox/issues/20#issuecomment-247788333 # hack to prevent .tox from crapping to the project directory -toxworkdir={env:TOXWORKDIR_BASE:}{toxinidir}/.tox +toxworkdir = {env:TOXWORKDIR_BASE:}{toxinidir}/.tox [testenv] # TODO how to get package name from setuptools? package_name = "instapexport" passenv = # useful for tests to know they are running under ci - CI - CI_* + CI + CI_* # respect user's cache dirs to prevent tox from crapping into project dir - MYPY_CACHE_DIR - PYTHONPYCACHEPREFIX + PYTHONPYCACHEPREFIX + MYPY_CACHE_DIR + RUFF_CACHE_DIR +[testenv:ruff] +commands = + {envpython} -m pip install --use-pep517 -e .[testing] + {envpython} -m ruff src/ + + +# note: --use-pep517 here is necessary for tox --parallel flag to work properly +# otherwise it seems that it tries to modify .eggs dir in parallel and it fails [testenv:tests] commands = - {envpython} -m pip install -e .[testing] + {envpython} -m pip install --use-pep517 -e .[testing] # posargs allow test filtering, e.g. tox ... -- -k test_name {envpython} -m pytest \ --pyargs {[testenv]package_name} \ @@ -29,7 +38,7 @@ commands = [testenv:mypy] commands = - {envpython} -m pip install -e .[linting,optional] + {envpython} -m pip install --use-pep517 -e .[testing,optional] {envpython} -m mypy --install-types --non-interactive \ -p {[testenv]package_name} \ # txt report is a bit more convenient to view on CI