update exporthelpers & sync with pymplate

.ci/run

@@ -37,4 +37,4 @@ if ! command -v python3 &> /dev/null; then
 fi
 
 "$PY_BIN" -m pip install --user tox
-"$PY_BIN" -m tox "$@"
+"$PY_BIN" -m tox --parallel --parallel-live "$@"

.github/workflows/main.yml

@@ -22,9 +22,12 @@ on:
 jobs:
   build:
     strategy:
+      fail-fast: false
       matrix:
         platform: [ubuntu-latest, macos-latest, windows-latest]
         python-version: ['3.8', '3.9', '3.10', '3.11']
+        # FIXME doesn't work on '3.12' because instapaper lib is using python-oauth2, which is using distutils
+        # see https://github.com/rsgalloway/instapaper/issues/23
         # vvv just an example of excluding stuff from matrix
         # exclude: [{platform: macos-latest, python-version: '3.6'}]
 
@@ -33,9 +36,6 @@ jobs:
     steps:
     # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation
     - run: echo "$HOME/.local/bin" >> $GITHUB_PATH
-    - if: ${{ matrix.platform == 'macos-latest' && matrix.python-version == '3.11' }}
-      # hmm somehow only seems necessary for 3.11 on osx??
-      run: echo "$HOME/Library/Python/${{ matrix.python-version }}/bin" >> $GITHUB_PATH
 
     - uses: actions/setup-python@v4
       with:
@@ -44,6 +44,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: recursive
+        fetch-depth: 0  # nicer to have all git history when debugging/for tests
 
     - uses: mxschmitt/action-tmate@v3
       if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
@@ -56,3 +57,4 @@ jobs:
       with:
         name: .coverage.mypy_${{ matrix.platform }}_${{ matrix.python-version }}
         path: .coverage.mypy/
+

pyproject.toml

@@ -0,0 +1,62 @@
+# see https://github.com/karlicoss/pymplate for up-to-date reference
+[project]
+dynamic = ["version"]  # version is managed by setuptools_scm
+name = "instapexport"
+dependencies = [
+    # instapaper api (needed for export only)
+    # my version has some changes not in the upstream yet..
+    "instapaper @ git+https://github.com/karlicoss/instapaper.git",
+]
+# TODO maybe split out DAL deps and export deps? might be nice
+
+## these need to be set if you're planning to upload to pypi
+# description = "TODO"
+# license = {file = "LICENSE"}
+# authors = [
+#     {name = "Dima Gerasimov (@karlicoss)", email = "karlicoss@gmail.com"},
+# ]
+# maintainers = [
+#     {name = "Dima Gerasimov (@karlicoss)", email = "karlicoss@gmail.com"},
+# ]
+#
+# [project.urls]
+# Homepage = "https://github.com/karlicoss/pymplate"
+##
+
+[project.optional-dependencies]
+optional = [
+    "orjson",
+    "colorlog",
+    "ijson",  # faster iterative json processing
+]
+testing = [
+    "pytest",
+    "ruff",
+    "mypy",
+    "lxml",  # for mypy html coverage
+]
+
+
+[build-system]
+requires = ["setuptools", "setuptools-scm"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools_scm]
+version_scheme = "python-simplified-semver"
+local_scheme = "dirty-tag"
+
+
+# nice things about pyproject.toml
+# - zip_safe=False isn't neccessary anymore
+# - correctly discovers namespace packages by defuilt?
+# - correctly handles py.typed by default?
+# - handles src layout automatically https://setuptools.pypa.io/en/latest/userguide/package_discovery.html#src-layout
+
+# things I'm not sure about yet
+# - avoiding dupliation/variable reuse?
+# - file/git dependencies?
+# - unclear how to specify namespace package order https://github.com/seanbreckenridge/reorder_editable/issues/2
+
+# TODO
+# - maybe it has a nicer pypi upload system? not sure
+#   e.g. possibly use hatch/flit/pdb/poetry -- but not sure what's the benefit tbh

ruff.toml

@@ -0,0 +1,29 @@
+ignore = [
+### too opinionated style checks
+    "E501",  # too long lines
+    "E702",  # Multiple statements on one line (semicolon)
+    "E731",  # assigning lambda instead of using def
+    "E741",  # Ambiguous variable name: `l`
+    "E742",  # Ambiguous class name: `O
+    "E401",  # Multiple imports on one line
+    "F403",  # import *` used; unable to detect undefined names
+###
+
+###
+    "E722",  # Do not use bare `except` ## Sometimes it's useful for defensive imports and that sort of thing..
+    "F811",  # Redefinition of unused  # this gets in the way of pytest fixtures (e.g. in cachew)
+
+## might be nice .. but later and I don't wanna make it strict
+    "E402",  # Module level import not at top of file
+
+### maybe consider these soon
+# sometimes it's useful to give a variable a name even if we don't use it as a documentation
+# on the other hand, often is a sign of error
+    "F841",  # Local variable `count` is assigned to but never used
+    "F401",  # imported but unused
+###
+]
+
+exclude = [
+    "src/hypexport/Hypothesis",
+]

src/instapexport/dal.py

@@ -7,7 +7,7 @@
 from .exporthelpers.dal_helper import Json, PathIsh, pathify, datetime_aware
 
 
-logger = logging_helper.makeLogger(__name__)
+logger = logging_helper.make_logger(__name__)
 
 Bid = str
 Hid = str
@@ -155,7 +155,7 @@ def pages(self) -> List[Page]:
         pages_ = [
             Page(
                 bookmark=bks[page_bid],
-                highlights=list(sorted(page_hls, key=lambda b: b.dt))
+                highlights=list(sorted(page_hls, key=lambda b: b.dt)),
             )
             for page_bid, page_hls in page2hls.items()
         ]
@@ -170,6 +170,7 @@ def demo(dao: DAL) -> None:
     print(f"Parsed {len(pages)} pages")
 
     from collections import Counter
+
     common = Counter({(x.url, x.title): len(x.highlights) for x in pages}).most_common(10)
     print("10 most highlighed pages:")
     for (url, title), count in common:

src/instapexport/export.py

@@ -1,22 +1,21 @@
 #!/usr/bin/env python3
-import argparse
 import json
 from typing import List
 
-from .exporthelpers.export_helper import Json
+from .exporthelpers.export_helper import Json, setup_parser, Parser
 
 # NOTE: uses custom version (has some changes that are not in upstream yet)
 # https://github.com/karlicoss/instapaper
-import instapaper # type: ignore[import]
+import instapaper  # type: ignore[import-untyped]
 instapaper._API_VERSION_ = "api/1.1"
 # see https://github.com/rsgalloway/instapaper/issues/11
 
 
 def get_json(
-        oauth_id: str,
-        oauth_secret: str,
-        oauth_token: str,
-        oauth_token_secret: str,
+    oauth_id: str,
+    oauth_secret: str,
+    oauth_token: str,
+    oauth_token_secret: str,
 ) -> Json:
     LIMIT = 500
     # default limit is something stupid
@@ -27,7 +26,11 @@ def get_json(
 
     user_folders = api.folders()
     folders: List[str] = [
-        'unread', 'archive', 'starred', # default, as per api docs
+        ## default, as per api docs
+        'unread',
+        'archive',
+        'starred',
+        ##
         *(str(f['folder_id']) for f in user_folders),
     ]
     bookmarks = {}
@@ -40,20 +43,22 @@ def get_json(
     }
 
 
-def login():
+def login() -> None:
     print("NOTE: You'll need your username/password once in order to get oauth_token")
+    # fmt: off
     oauth_id     = input('Your ouath_id: ')
     oauth_secret = input('Your ouath_secret: ')
     username     = input('Your username: ')
     password     = input('Your password: ')
+    # fmt: on
 
     api = instapaper.Instapaper(oauth_id, oauth_secret)
     odata = api.login(username, password)
     print("Now paste this into your secrets file")
     print(odata)
 
 
-def main():
+def main() -> None:
     parser = make_parser()
     args = parser.parse_args()
 
@@ -70,10 +75,11 @@ def main():
 
 
 def make_parser():
-    from .exporthelpers.export_helper import setup_parser, Parser
-    parser = Parser("""
+    parser = Parser(
+        """
 Export your personal Instapaper data: bookmarked articles and highlights.
-""")
+""",
+    )
     setup_parser(
         parser=parser,
         params=[
@@ -84,11 +90,16 @@ def make_parser():
         ],
         extra_usage='''
 You can also import ~instapexport.export~ as a module and call ~get_json~ function directly to get raw JSON.
-''')
-    parser.add_argument('--login', action='store_true', help='''
+''',
+    )
+    parser.add_argument(
+        '--login',
+        action='store_true',
+        help='''
     Note: OAUTH_ID/OAUTH_SECRET have to be requrested by email
     https://www.instapaper.com/main/request_oauth_consumer_token
-    ''')
+    ''',
+    )
     return parser
 
 

tox.ini

@@ -1,26 +1,35 @@
 [tox]
-minversion = 3.5
+minversion = 3.21
 # relies on the correct version of Python installed
-envlist = tests,mypy
+envlist = ruff,tests,mypy
 # https://github.com/tox-dev/tox/issues/20#issuecomment-247788333
 # hack to prevent .tox from crapping to the project directory
-toxworkdir={env:TOXWORKDIR_BASE:}{toxinidir}/.tox
+toxworkdir = {env:TOXWORKDIR_BASE:}{toxinidir}/.tox
 
 [testenv]
 # TODO how to get package name from setuptools?
 package_name = "instapexport"
 passenv =
 # useful for tests to know they are running under ci
-  CI
-  CI_*
+    CI
+    CI_*
 # respect user's cache dirs to prevent tox from crapping into project dir
-  MYPY_CACHE_DIR
-  PYTHONPYCACHEPREFIX
+    PYTHONPYCACHEPREFIX
+    MYPY_CACHE_DIR
+    RUFF_CACHE_DIR
 
 
+[testenv:ruff]
+commands =
+    {envpython} -m pip install --use-pep517 -e .[testing]
+    {envpython} -m ruff src/
+
+
+# note: --use-pep517 here is necessary for tox --parallel flag to work properly
+# otherwise it seems that it tries to modify .eggs dir in parallel and it fails
 [testenv:tests]
 commands =
-    {envpython} -m pip install -e .[testing]
+    {envpython} -m pip install --use-pep517 -e .[testing]
     # posargs allow test filtering, e.g. tox ... -- -k test_name
     {envpython} -m pytest \
         --pyargs {[testenv]package_name} \
@@ -29,7 +38,7 @@ commands =
 
 [testenv:mypy]
 commands =
-    {envpython} -m pip install -e .[linting,optional]
+    {envpython} -m pip install --use-pep517 -e .[testing,optional]
     {envpython} -m mypy --install-types --non-interactive \
         -p {[testenv]package_name}       \
         # txt report is a bit more convenient to view on CI