calculate-times-bicycle #1075
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: calculate-times | |
| run-name: calculate-times-${{ inputs.mode }} | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| mode: | |
| required: true | |
| description: Mode of travel | |
| default: 'car' | |
| type: choice | |
| options: | |
| - car | |
| - bicycle | |
| - foot | |
| override_years: | |
| required: false | |
| description: | | |
| Comma-separated list of OSM data years to run e.g. 2020,2023. | |
| Will run all (see params.yaml) if null | |
| type: string | |
| override_states: | |
| required: false | |
| description: | | |
| Comma-separated state codes to run e.g. 01,06. | |
| Will run all (see params.yaml) if null | |
| type: string | |
| override_geographies: | |
| required: false | |
| description: | | |
| Comma-separated geographies to limit run e.g. county,tract. | |
| Will run all (see params.yaml) if null | |
| type: string | |
| env: | |
| AWS_DEFAULT_REGION: us-east-1 | |
| # See: https://github.com/aws/aws-cli/issues/5262#issuecomment-705832151 | |
| AWS_EC2_METADATA_DISABLED: true | |
| PYTHONUNBUFFERED: "1" | |
| jobs: | |
| setup-jobs: | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| years: ${{ steps.create-year-jobs.outputs.param }} | |
| states: ${{ steps.create-state-jobs.outputs.param }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Create year jobs | |
| id: create-year-jobs | |
| uses: ./.github/actions/parse-gh-input | |
| with: | |
| param_path: '.input.year' | |
| param_override: '${{ inputs.override_years }}' | |
| - name: Create state jobs | |
| id: create-state-jobs | |
| uses: ./.github/actions/parse-gh-input | |
| with: | |
| param_path: '.input.state' | |
| param_override: '${{ inputs.override_states }}' | |
| run-job: | |
| runs-on: ubuntu-24.04 | |
| needs: setup-jobs | |
| strategy: | |
| # Don't fail all chunks if one fails | |
| fail-fast: false | |
| matrix: | |
| year: ${{ fromJSON(needs.setup-jobs.outputs.years) }} | |
| state: ${{ fromJSON(needs.setup-jobs.outputs.states) }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Cloudflare credentials | |
| uses: ./.github/actions/setup-cloudflare-s3 | |
| with: | |
| CLOUDFLARE_S3_API_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_S3_API_ACCESS_KEY_ID }} | |
| CLOUDFLARE_S3_API_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_S3_API_SECRET_ACCESS_KEY }} | |
| - name: Remove unnecessary software and increase swap space | |
| uses: ./.github/actions/prep-disk-and-swap | |
| with: | |
| # Increase swapfile size to 10 GB | |
| swap_override: 10737418240 | |
| - name: Install DVC | |
| uses: ./.github/actions/setup-dvc | |
| - name: Fetch locations data | |
| uses: ./.github/actions/fetch-locations | |
| - name: Fetch OSM extract data | |
| shell: bash | |
| working-directory: 'data' | |
| run: | | |
| path=year=${{ matrix.year }}/geography=state/state=${{ matrix.state }}/${{ matrix.state }} | |
| uv run dvc pull --no-run-cache \ | |
| ./intermediate/osmextract/"$path".osm.pbf | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v5 | |
| with: | |
| enable-cache: true | |
| cache-suffix: "site-data" | |
| cache-dependency-glob: | | |
| pyproject.toml | |
| uv.lock | |
| - name: Install Python dependencies | |
| id: install-python-dependencies | |
| shell: bash | |
| run: | | |
| uv python install | |
| uv venv | |
| uv pip install ".[site,data]" | |
| - name: Create OSRM network files | |
| shell: bash | |
| working-directory: 'data' | |
| run: | | |
| ./src/create_osrmnetwork.sh ${{ inputs.mode }} \ | |
| ${{ matrix.year }} ${{ matrix.state }} | |
| - name: Create routing jobs per geography | |
| id: create-geo-jobs | |
| uses: ./.github/actions/parse-gh-input | |
| with: | |
| param_path: '.input.census.geography.all' | |
| param_override: '${{ inputs.override_geographies }}' | |
| - name: Run routing jobs | |
| shell: bash | |
| working-directory: 'data' | |
| run: | | |
| # Start the Docker backend before running jobs | |
| docker run --rm -d -p 5333:5000 -v "./build:/data" \ | |
| osrm/osrm-backend osrm-routed --algorithm ch \ | |
| --max-table-size 100000000 /data/${{ matrix.state }}.osrm | |
| geographies='${{ steps.create-geo-jobs.outputs.param }}' | |
| geographies_array=($(echo "$geographies" | jq -r '.[]')) | |
| for geo in "${geographies_array[@]}"; do | |
| echo "Starting job with parameters: mode=${{ inputs.mode }}, year=${{ matrix.year }}, geography=${geo}, state=${{ matrix.state }}" | |
| uv run ./src/calculate_times.py \ | |
| --mode ${{ inputs.mode }} --year ${{ matrix.year }} \ | |
| --geography "$geo" --state ${{ matrix.state }} \ | |
| --centroid-type weighted \ | |
| --write-to-s3 | |
| done |