test-and-publish.yaml

name: 'Test and Publish Suite'

on:
  push:
    branches:
      - main
    tags:
      - 'v*'
  workflow_call:
  workflow_dispatch:
    inputs:
      debug_enabled_os:
        type: choice
        description: Optionally, choose an OS to run the build with SSH debugging on (https://github.com/fawazahmed0/action-debug)
        required: false
        options:
        - ''
        - 'ubuntu-20.04'
        - 'macos-12'
        - 'macos-14'
        - 'windows-2022'
      debug_enabled_python:
        type: choice
        description: Choose a Python version to run the build with SSH debugging on
        required: false
        options:
        - ''
        - '3.8'
        - '3.9'
        - '3.10'
        - '3.11'
        - '3.12'
        - '3.13'
      build_type:
        type: choice
        description: 'Choose the build type to use'
        required: false
        default: 'Debug'
        options:
        - 'Debug'
        - 'Profile'
        - 'Sanitize'
        - 'DRelease'
        - 'Release'
        - 'None'
  pull_request:

env:
  # don't upgrade outdated brew packages because the process is too slow
  HOMEBREW_NO_INSTALL_UPGRADE: 1
  HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1

defaults:
  run:
    # run with Git Bash on Windows
    shell: bash

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build-spidermonkey-unix:
    strategy:
      fail-fast: false
      matrix:
        # Use Ubuntu 20.04 / macOS 13 x86_64 / macOS 14 arm64 + Python 3.10 to build SpiderMonkey
        os: [ 'ubuntu-20.04', 'macos-13', 'macos-14', 'pi' ] # macOS 14 runner exclusively runs on M1 hardwares
                                                             # see https://github.blog/changelog/2024-01-30-github-actions-macos-14-sonoma-is-now-available
        python_version: [ '3.10' ]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python_version }}
      - name: Read the mozilla-central commit hash to be used
        run: echo "MOZCENTRAL_VERSION=$(cat mozcentral.version)" >> $GITHUB_ENV
      - name: Cache spidermonkey build
        id: cache-spidermonkey
        uses: actions/cache@v4
        with:
          path: |
            ./_spidermonkey_install/*
          key: spidermonkey-${{ env.MOZCENTRAL_VERSION }}-${{ runner.os }}-${{ runner.arch }}
          lookup-only: true # skip download
      - name: Setup XCode
        if: ${{ matrix.os == 'macos-13' && steps.cache-spidermonkey.outputs.cache-hit != 'true' }}
        # SpiderMonkey requires XCode SDK version at least 13.3
        run: sudo xcode-select -switch /Applications/Xcode_14.3.app
      - name: Setup LLVM
        if: ${{ (matrix.os == 'macos-13' || matrix.os == 'macos-14') && steps.cache-spidermonkey.outputs.cache-hit != 'true' }}
        run: |
          brew install llvm@15
          ln -s $(brew --prefix llvm@15)/bin/lld /usr/local/bin/lld
          ln -s $(brew --prefix llvm@15)/bin/ld64.lld /usr/local/bin/ld64.lld
          ln -s $(brew --prefix llvm@15)/bin/llvm-install-name-tool /usr/local/bin/llvm-install-name-tool
          ld64.lld --version
          llvm-install-name-tool --version
      - name: Build spidermonkey
        if: ${{ steps.cache-spidermonkey.outputs.cache-hit != 'true' }}
        run: ./setup.sh
  build-spidermonkey-win:
    runs-on: windows-2022
      # SpiderMonkey requires Visual Studio 2022 or newer.
      # The Windows 2019 runner only has Visual Studio Enterprise 2019 installed.
    steps:
      - uses: actions/checkout@v4
      - name: Read the mozilla-central commit hash to be used
        run: echo "MOZCENTRAL_VERSION=$(cat mozcentral.version)" >> $GITHUB_ENV
      - name: Cache spidermonkey build
        id: cache-spidermonkey
        uses: actions/cache@v4
        with:
          path: |
            ./_spidermonkey_install/*
          key: spidermonkey-${{ env.MOZCENTRAL_VERSION }}-${{ runner.os }}-${{ runner.arch }}
          lookup-only: true # skip download
      - name: Install dependencies
        if: ${{ steps.cache-spidermonkey.outputs.cache-hit != 'true' }}
        shell: powershell
        run: |
          # Already installed in Github Actions runner
          # choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System' # add CMake to system PATH
          # choco install -y llvm gnuwin32-m4
          choco install -y wget make unzip
      - name: Install MozillaBuild
        if: ${{ steps.cache-spidermonkey.outputs.cache-hit != 'true' }}
        run: |
          wget -c -q https://ftp.mozilla.org/pub/mozilla/libraries/win32/MozillaBuildSetup-Latest.exe
          powershell -command 'Start-Process -Wait -FilePath "./MozillaBuildSetup-Latest.exe" -ArgumentList "/S"'
      - name: Build spidermonkey in MozillaBuild environment
        if: ${{ steps.cache-spidermonkey.outputs.cache-hit != 'true' }}
        env:
          # Preserve MozillaBuild v4.0.x behaviour
          # see https://groups.google.com/u/1/a/mozilla.org/g/dev-platform/c/hF51Q3j6ca8
          USE_MINTTY: 0
        run: /c/mozilla-build/start-shell.bat -use-full-path -here ./setup.sh
  build-and-test:
    needs: [build-spidermonkey-unix, build-spidermonkey-win]
    strategy:
      fail-fast: false
      matrix:
        os: [ 'ubuntu-20.04', 'macos-12', 'macos-14', 'windows-2022', 'pi' ]
        python_version: [ '3.8', '3.9', '3.10', '3.11', '3.12', '3.13' ]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
          fetch-depth: 0 # fetch all history for all branches and tags
          # poetry-dynamic-versioning needs git tags to produce the correct version number 
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python_version }}
      - name: Remove old poetry cache
        run: rm -rf ~/.cache/pypoetry
        if: ${{ matrix.os == 'pi' }}
      - name: Setup Poetry
        uses: snok/install-poetry@v1
        with:
          version: 1.5.1
      - name: Install Dependencies
        run: |
          echo "Installing Dependencies"
          if [[ "$OSTYPE" == "linux-gnu"* ]]; then # Linux
            sudo apt-get update -y
            sudo apt-get install -y cmake llvm
          elif [[ "$OSTYPE" == "darwin"* ]]; then # macOS
            brew update || true # allow failure
            brew install cmake pkg-config wget unzip coreutils # `coreutils` installs the `realpath` command
          fi
          echo "Installing python deps"
          poetry self add "poetry-dynamic-versioning[plugin]"
          poetry env use python$PYTHON_VERSION || poetry env use python3 # use the correct Python version we've set up
          poetry install --no-root --only=dev
          echo "Installed Dependencies"
        env:
          PYTHON_VERSION: ${{ matrix.python_version }}
      - name: Build Docs # only build docs once
        if: ${{ matrix.os == 'ubuntu-20.04' && matrix.python_version == '3.11' }}
        run: |
            sudo apt-get install -y graphviz 
            # the newest version in Ubuntu 20.04 repository is 1.8.17, but we need Doxygen 1.9 series
            wget -c -q https://www.doxygen.nl/files/doxygen-1.9.7.linux.bin.tar.gz
            tar xf doxygen-1.9.7.linux.bin.tar.gz
            cd doxygen-1.9.7 && sudo make install && cd -
            rm -rf doxygen-1.9.7 doxygen-1.9.7.linux.bin.tar.gz
            BUILD_DOCS=1 BUILD_TYPE=None poetry install
      - name: Upload Doxygen-generated docs as CI artifacts
        if: ${{ matrix.os == 'ubuntu-20.04' && matrix.python_version == '3.11' }}
        uses: actions/upload-artifact@v3
        with:
          name: docs-${{ github.run_id }}-${{ github.sha }}
          path: ./build/docs/html/
      - name: Read the mozilla-central commit hash to be used
        run: echo "MOZCENTRAL_VERSION=$(cat mozcentral.version)" >> $GITHUB_ENV
      - name: Use cached spidermonkey build
        uses: actions/cache@v4
        with:
          path: |
            ./_spidermonkey_install/*
          key: spidermonkey-${{ env.MOZCENTRAL_VERSION }}-${{ runner.os }}-${{ runner.arch }}
          fail-on-cache-miss: true # SpiderMonkey is expected to be cached in its dedicated job
      - name: Build pminit
        run: |
          cd python/pminit
          poetry build --format=sdist
          cd -
          mkdir -p ./dist/
          mv -v python/pminit/dist/* ./dist/
      - name: Build wheel
        run: |
          echo $(poetry run python --version)
          WORKFLOW_BUILD_TYPE=${{ inputs.build_type }}
          BUILD_TYPE=${WORKFLOW_BUILD_TYPE:-"Debug"} poetry build --format=wheel
          ls -lah ./dist/
      - name: Make the wheels we build also support lower versions of macOS
        if: ${{ matrix.os == 'macos-12' || matrix.os == 'macos-14' }}
        # Change the platform tag part of the wheel filename to `macosx_11_0_xxx` (means to support macOS 11.0 and above)
        # See https://packaging.python.org/en/latest/specifications/binary-distribution-format/#file-format
        # A wheel package file will only be selected by pip to install if the platform tag satisfies, regardless of whether the binary compatibility actually is.
        # Otherwise, pip would fallback to compile from the source distribution.
        run: |
          cd ./dist/
          for file in *.whl; do
            mv "$file" "$(echo "$file" | sed -E 's/macosx_[0-9]+_[0-9]+/macosx_11_0/')";
          done
      - name: Upload wheel as CI artifacts
        uses: actions/upload-artifact@v3
        with:
          name: wheel-${{ github.run_id }}-${{ github.sha }}
          path: ./dist/
      - name: Set cores to get stored in /cores
        if: ${{ matrix.os != 'windows-2022' }}
        # TODO (Caleb Aikens) figure out how to get Windows core dumps
        run: |
          sudo mkdir -p /cores
          sudo chmod 777 /cores
          # Core filenames will be of the form osname.pythonversion.executable.pid.timestamp:
          if [[ "$OSTYPE" == "linux-gnu"* ]]; then
            sudo bash -c 'echo "/cores/${OSTYPE}.$(poetry run python --version).%e.%p.%t" > /proc/sys/kernel/core_pattern'
          else
            sudo sysctl kern.corefile="/cores/${OSTYPE}.$(poetry run python --version).%e.%p.%y"
          fi
      - name: Run Python tests (pytest)
        run: |
          if [[ "$OSTYPE" == "linux-gnu"* || "$OSTYPE" == "darwin"* ]]; then
          # TODO (Caleb Aikens) figure out how to get Windows core dumps
            ulimit -c unlimited
          fi
          WORKFLOW_BUILD_TYPE=${{ inputs.build_type }}
          BUILD_TYPE=${WORKFLOW_BUILD_TYPE:-"Debug"} poetry run python -m pip install --force-reinstall --verbose ./dist/*
          poetry run python -m pytest tests/python
      - name: Run JS tests (peter-jr)
        if: ${{ (success() || failure()) }}
        run: |
          if [[ "$OSTYPE" == "linux-gnu"* || "$OSTYPE" == "darwin"* ]]; then
          # TODO (Caleb Aikens) figure out how to get Windows core dumps
            ulimit -c unlimited
          fi
          poetry run bash ./peter-jr ./tests/js/
      - name: SSH debug session
        if: ${{ (success() || failure()) && github.event_name == 'workflow_dispatch' && inputs.debug_enabled_os == matrix.os && inputs.debug_enabled_python == matrix.python_version}}
        uses: fawazahmed0/action-debug@main
        with:
          credentials: "admin:admin"
      - name: Upload core dumps as CI artifacts
        uses: actions/upload-artifact@v3
        if: ${{ matrix.os != 'windows-2022' && failure() }}
        # TODO (Caleb Aikens) figure out how to get Windows core dumps
        with:
          name: cores-${{ matrix.os }}-${{ matrix.python_version }}
          path: /cores
  sdist:
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: actions/setup-python@v5
        with:
          python-version: '3.9'
      - name: Setup Poetry
        uses: snok/install-poetry@v1
        with:
          version: 1.5.1
      - name: Build source distribution (sdist) file
        run: |
          poetry self add "poetry-dynamic-versioning[plugin]"
          poetry build --format=sdist
          ls -lah ./dist/
      - name: Upload sdist as CI artifacts
        uses: actions/upload-artifact@v3
        with:
          name: wheel-${{ github.run_id }}-${{ github.sha }}
          path: ./dist/
  check-install-from-sdist:
    needs: sdist
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/setup-python@v5
        with:
          python-version: '3.10'
      - name: Download wheels built
        uses: actions/download-artifact@v3
        with:
          name: wheel-${{ github.run_id }}-${{ github.sha }}
          path: ./dist/
      - run: pip install ./dist/pythonmonkey-*.tar.gz
  publish:
    needs: [build-and-test, sdist]
    runs-on: ubuntu-20.04
    if: ${{ success() && github.event_name == 'push' && github.ref_type == 'tag' }}
    steps:
      # no need to checkout
      - uses: actions/setup-python@v5
        with:
          python-version: '3.9'
      - run: pip install twine
      - name: Download wheels built
        uses: actions/download-artifact@v3
        with:
          name: wheel-${{ github.run_id }}-${{ github.sha }}
          path: ./dist/
      - run: ls -lah ./dist/
      - name: Publish package
        run: |
          twine upload dist/* \
            --non-interactive --skip-existing \
            --username __token__ --password ${{ secrets.PYPI_API_TOKEN }}
  publish-nightly:
    # Implement a very basic Python package repository (https://peps.python.org/pep-0503/)
    # and deploy the static files to GitHub Pages
    needs: [build-and-test, sdist]
    runs-on: ubuntu-20.04
    if: ${{ (success() || failure()) && (github.ref_name == 'main' || github.ref_type == 'tag') }} # publish nightly builds regardless of tests failure
    permissions: # grant GITHUB_TOKEN the permissions required to make a Pages deployment
      pages: write
      id-token: write
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    steps:
      # don't checkout
      - name: Download wheels built
        uses: actions/download-artifact@v3
        with:
          name: wheel-${{ github.run_id }}-${{ github.sha }}
          path: ./dist/
      - name: Download docs html generated by Doxygen
        uses: actions/download-artifact@v3
        with:
          name: docs-${{ github.run_id }}-${{ github.sha }}
          path: ./docs/
      - name: Move wheels to the correct repository project paths
        run: |
          mkdir -p ./pythonmonkey/ ./pminit/
          mv ./dist/pythonmonkey-* ./pythonmonkey/
          mv ./dist/pminit-* ./pminit/
      - name: Generate index page for the project
        run: |
          generate_index () {
            cd $1
            html="<html><head><title>PythonMonkey Nightly Builds</title></head><body>"
            for file in ./*; do # generate <a> tags for each file
              html+="<a href=\"$file\">$file</a><br/>"
            done
            html+="</body></html>"
            echo "$html" > ./index.html
            cd -
          }
          generate_index ./pythonmonkey/
          generate_index ./pminit/
      - name: Generate repository root page
        run: |
          html="<html><head><title>PythonMonkey Nightly Builds</title></head><body>"
          html+="<h1>PythonMonkey Nightly Builds</h1>"
          html+="<p>To install nightly builds, run</p>"
          html+="<pre>pip install --extra-index-url https://nightly.pythonmonkey.io/ --pre pythonmonkey</pre>"
          html+="<h3>Browse files:</h3>"
          html+="<li><a href="pythonmonkey/">pythonmonkey</a></li>"
          html+="<li><a href="pminit/">pminit</a></li>"
          html+="</body></html>"
          echo "$html" > ./index.html
      - uses: actions/upload-pages-artifact@v1
        with:
          path: ./
      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v2
  publish-archive:
    # Publish to ⊇istributive's archive server (https://archive.distributed.computer/releases/pythonmonkey/)
    needs: [build-and-test, sdist]
    runs-on: ubuntu-20.04
    if: ${{ (success() || failure()) && (github.ref_name == 'main' || github.ref_type == 'tag') }}
    environment:
      name: archive
      url: https://archive.distributed.computer/releases/pythonmonkey/${{ steps.get_path.outputs.ARCHIVE_PATH }}
    steps:
      # no need to checkout
      - name: Download wheels built
        uses: actions/download-artifact@v3
        with:
          name: wheel-${{ github.run_id }}-${{ github.sha }}
          path: ./
      - name: Download docs html generated by Doxygen
        uses: actions/download-artifact@v3
        with:
          name: docs-${{ github.run_id }}-${{ github.sha }}
          path: ./docs/
      - name: Get the pythonmonkey/pminit version number
        run: |
          file=$(ls ./pminit*.tar.gz | head -1)
          pm_version=$(basename "${file%.tar.gz}" | cut -d- -f2) # match /pminit-([^-]+).tar.gz/
          echo "PM_VERSION=$pm_version" >> $GITHUB_ENV
      - name: Get the archive type (nightly or releases) and path
        id: get_path
        run: |
          path="$ARCHIVE_TYPE/$PM_VERSION/"
          echo "$path"
          echo "ARCHIVE_PATH=$path" >> $GITHUB_OUTPUT
        env:
          ARCHIVE_TYPE: ${{ (github.ref_type == 'tag' && 'releases') || 'nightly' }}
      - name: SCP to the archive server
        uses: appleboy/scp-action@v0.1.7
        with:
          host: ${{ secrets.ARCHIVE_HOST }}
          username: pythonmonkey
          key: ${{ secrets.ARCHIVE_KEY }}
          source: ./*
          target: archive/${{ steps.get_path.outputs.ARCHIVE_PATH }}
          overwrite: true