feat: clean disk space less to make bot runs more efficient (#3388)

* feat: clean disk space less * fix: need to specify ci-service at cli
regro · Dec 12, 2024 · ec8068e · ec8068e
1 parent 6d85b70
commit ec8068e
Show file tree

Hide file tree

Showing 14 changed files with 78 additions and 58 deletions.
diff --git a/.github/workflows/bot-bot.yml b/.github/workflows/bot-bot.yml
@@ -44,7 +44,7 @@ jobs:
       - name: install bot code
         if: success() && ! env.CI_SKIP
         run: |
-          source cf-scripts/autotick-bot/install_bot_code.sh
+          source cf-scripts/autotick-bot/install_bot_code.sh --no-clean-disk-space
         env:
           BOT_TOKEN: ${{ secrets.AUTOTICK_BOT_TOKEN }}
 
@@ -76,8 +76,6 @@ jobs:
       - name: deploy
         if: github.ref == 'refs/heads/main' && always() && ! env.CI_SKIP
         run: |
-          sudo docker image prune --all --force
-
           pushd cf-graph
 
           export RUN_URL="https://github.com/regro/cf-scripts/actions/runs/${RUN_ID}"

diff --git a/.github/workflows/bot-feedstocks.yml b/.github/workflows/bot-feedstocks.yml
@@ -43,7 +43,7 @@ jobs:
       - name: install bot code
         if: success() && ! env.CI_SKIP
         run: |
-          source cf-scripts/autotick-bot/install_bot_code.sh
+          source cf-scripts/autotick-bot/install_bot_code.sh --no-clean-disk-space
         env:
           BOT_TOKEN: ${{ secrets.AUTOTICK_BOT_TOKEN }}
 
@@ -63,8 +63,6 @@ jobs:
       # - name: deploy
       #   if: github.ref == 'refs/heads/main' && ! cancelled() && ! env.CI_SKIP
       #   run: |
-      #     sudo docker image prune --all --force
-
       #     pushd cf-graph
 
       #     export RUN_URL="https://github.com/regro/cf-scripts/actions/runs/${RUN_ID}"

diff --git a/.github/workflows/bot-make-graph.yml b/.github/workflows/bot-make-graph.yml
@@ -44,7 +44,7 @@ jobs:
       - name: install bot code
         if: success() && ! env.CI_SKIP
         run: |
-          source cf-scripts/autotick-bot/install_bot_code.sh
+          source cf-scripts/autotick-bot/install_bot_code.sh --no-clean-disk-space
         env:
           BOT_TOKEN: ${{ secrets.AUTOTICK_BOT_TOKEN }}
 
@@ -66,8 +66,6 @@ jobs:
       # - name: deploy
       #   if: github.ref == 'refs/heads/main' && always() && ! env.CI_SKIP
       #   run: |
-      #     sudo docker image prune --all --force
-
       #     pushd cf-graph
 
       #     export RUN_URL="https://github.com/regro/cf-scripts/actions/runs/${RUN_ID}"

diff --git a/.github/workflows/bot-make-migrators.yml b/.github/workflows/bot-make-migrators.yml
@@ -44,7 +44,7 @@ jobs:
       - name: install bot code
         if: success() && ! env.CI_SKIP
         run: |
-          source cf-scripts/autotick-bot/install_bot_code.sh
+          source cf-scripts/autotick-bot/install_bot_code.sh --no-clean-disk-space
         env:
           BOT_TOKEN: ${{ secrets.AUTOTICK_BOT_TOKEN }}
 
@@ -66,8 +66,6 @@ jobs:
       # - name: deploy
       #   if: github.ref == 'refs/heads/main' && always() && ! env.CI_SKIP
       #   run: |
-      #     sudo docker image prune --all --force
-
       #     pushd cf-graph
 
       #     export RUN_URL="https://github.com/regro/cf-scripts/actions/runs/${RUN_ID}"

diff --git a/.github/workflows/bot-migrate-schema.yml b/.github/workflows/bot-migrate-schema.yml
@@ -48,7 +48,7 @@ jobs:
       - name: install bot code
         if: success() && ! env.CI_SKIP
         run: |
-          source cf-scripts/autotick-bot/install_bot_code.sh
+          source cf-scripts/autotick-bot/install_bot_code.sh --no-clean-disk-space
         env:
           BOT_TOKEN: ${{ secrets.AUTOTICK_BOT_TOKEN }}
 
@@ -71,8 +71,6 @@ jobs:
       - name: deploy
         if: github.ref == 'refs/heads/main' && always() && ! env.CI_SKIP
         run: |
-          sudo docker image prune --all --force
-
           pushd cf-graph
 
           export RUN_URL="https://github.com/regro/cf-scripts/actions/runs/${RUN_ID}"

diff --git a/.github/workflows/bot-prs.yml b/.github/workflows/bot-prs.yml
@@ -50,7 +50,7 @@ jobs:
       - name: install bot code
         if: success() && ! env.CI_SKIP
         run: |
-          source cf-scripts/autotick-bot/install_bot_code.sh
+          source cf-scripts/autotick-bot/install_bot_code.sh --no-clean-disk-space
         env:
           BOT_TOKEN: ${{ secrets.AUTOTICK_BOT_TOKEN }}
 
@@ -71,8 +71,6 @@ jobs:
       - name: deploy
         if: github.ref == 'refs/heads/main' && ! cancelled() && ! env.CI_SKIP
         run: |
-          sudo docker image prune --all --force
-
           pushd cf-graph
 
           export RUN_URL="https://github.com/regro/cf-scripts/actions/runs/${RUN_ID}"

diff --git a/.github/workflows/bot-pypi-mapping.yml b/.github/workflows/bot-pypi-mapping.yml
@@ -44,7 +44,7 @@ jobs:
       - name: install bot code
         if: success() && ! env.CI_SKIP
         run: |
-          source cf-scripts/autotick-bot/install_bot_code.sh
+          source cf-scripts/autotick-bot/install_bot_code.sh --no-clean-disk-space
         env:
           BOT_TOKEN: ${{ secrets.AUTOTICK_BOT_TOKEN }}
 
@@ -69,8 +69,6 @@ jobs:
       - name: deploy
         if: github.ref == 'refs/heads/main' && ! cancelled() && ! env.CI_SKIP
         run: |
-          sudo docker image prune --all --force
-
           pushd cf-graph
 
           export RUN_URL="https://github.com/regro/cf-scripts/actions/runs/${RUN_ID}"

diff --git a/.github/workflows/bot-update-nodes.yml b/.github/workflows/bot-update-nodes.yml
@@ -50,7 +50,7 @@ jobs:
       - name: install bot code
         if: success() && ! env.CI_SKIP
         run: |
-          source cf-scripts/autotick-bot/install_bot_code.sh
+          source cf-scripts/autotick-bot/install_bot_code.sh --no-clean-disk-space
         env:
           BOT_TOKEN: ${{ secrets.AUTOTICK_BOT_TOKEN }}
 
@@ -72,8 +72,6 @@ jobs:
       - name: deploy
         if: github.ref == 'refs/heads/main' && always() && ! env.CI_SKIP
         run: |
-          sudo docker image prune --all --force
-
           pushd cf-graph
 
           export RUN_URL="https://github.com/regro/cf-scripts/actions/runs/${RUN_ID}"

diff --git a/.github/workflows/bot-update-status-page.yml b/.github/workflows/bot-update-status-page.yml
@@ -43,7 +43,7 @@ jobs:
       - name: install bot code
         if: success() && ! env.CI_SKIP
         run: |
-          source cf-scripts/autotick-bot/install_bot_code.sh
+          source cf-scripts/autotick-bot/install_bot_code.sh --no-clean-disk-space
         env:
           BOT_TOKEN: ${{ secrets.AUTOTICK_BOT_TOKEN }}
 
@@ -59,8 +59,6 @@ jobs:
       - name: deploy
         if: github.ref == 'refs/heads/main' && ! cancelled() && ! env.CI_SKIP
         run: |
-          sudo docker image prune --all --force
-
           pushd cf-graph
 
           export RUN_URL="https://github.com/regro/cf-scripts/actions/runs/${RUN_ID}"

diff --git a/.github/workflows/bot-versions.yml b/.github/workflows/bot-versions.yml
@@ -48,7 +48,7 @@ jobs:
       - name: install bot code
         if: success() && ! env.CI_SKIP
         run: |
-          source cf-scripts/autotick-bot/install_bot_code.sh
+          source cf-scripts/autotick-bot/install_bot_code.sh --no-clean-disk-space
         env:
           BOT_TOKEN: ${{ secrets.AUTOTICK_BOT_TOKEN }}
 

diff --git a/autotick-bot/install_bot_code.sh b/autotick-bot/install_bot_code.sh
@@ -1,34 +1,4 @@
 #!/bin/bash
-clean_disk_space="true"
-for arg in "$@"; do
-  if [[ "$arg" == "--no-clean-disk-space" ]]; then
-    clean_disk_space="false"
-  fi
-done
-if [[ "${clean_disk_space}" == "true" ]]; then
-  # clean disk space
-  sudo mkdir -p /opt/empty_dir || true
-  for d in \
-  /opt/ghc \
-  /opt/hostedtoolcache \
-  /usr/lib/jvm \
-  /usr/local/.ghcup \
-  /usr/local/lib/android \
-  /usr/local/share/powershell \
-  /usr/share/dotnet \
-  /usr/share/swift \
-  ; do
-    sudo rsync --stats -a --delete /opt/empty_dir/ $d || true
-  done
-  sudo apt-get purge -y -f firefox \
-                          google-chrome-stable \
-                          microsoft-edge-stable
-  sudo apt-get autoremove -y >& /dev/null
-  sudo apt-get autoclean -y >& /dev/null
-  sudo docker image prune --all --force
-  df -h
-fi
-
 git config --global user.name regro-cf-autotick-bot
 git config --global user.email 36490558+regro-cf-autotick-bot@users.noreply.github.com
 git config --global pull.rebase false
@@ -42,6 +12,16 @@ pip install --no-deps --no-build-isolation -e .
 
 cd ..
 
+clean_disk_space="true"
+for arg in "$@"; do
+  if [[ "$arg" == "--no-clean-disk-space" ]]; then
+    clean_disk_space="false"
+  fi
+done
+if [[ "${clean_disk_space}" == "true" ]]; then
+  conda-forge-tick clean-disk-space --cli-service='github-actions'
+fi
+
 clone_graph="true"
 for arg in "$@"; do
   if [[ "$arg" == "--no-clone-graph" ]]; then

diff --git a/conda_forge_tick/cli.py b/conda_forge_tick/cli.py
@@ -293,6 +293,17 @@ def react_to_event(
     react_to_event(ctx, event, uid)
 
 
+@main.command(name="clean-disk-space")
+@click.option("--ci-service", required=True, type=click.Choice(["github-actions"]))
+def clean_disk_space(ci_service) -> None:
+    """
+    Clean up disk space on CI services.
+    """
+    from .os_utils import clean_disk_space
+
+    clean_disk_space(ci_service)
+
+
 if __name__ == "__main__":
     # This entrypoint can be used for debugging.
     # click will read the command line arguments and call the corresponding

diff --git a/conda_forge_tick/deploy.py b/conda_forge_tick/deploy.py
@@ -11,6 +11,7 @@
     CF_TICK_GRAPH_GITHUB_BACKEND_REPO,
     get_lazy_json_backends,
 )
+from .os_utils import clean_disk_space
 from .utils import (
     fold_log_lines,
     get_bot_run_url,
@@ -263,6 +264,8 @@ def deploy(ctx: CliContext, dirs_to_deploy: list[str] = None):
         do_git_ops = True
 
     if do_git_ops:
+        clean_disk_space()
+
         files_to_add = list((set(files_to_add) - files_done) | files_to_try_again)
         n_added = 0
         batch = 0

diff --git a/conda_forge_tick/os_utils.py b/conda_forge_tick/os_utils.py
@@ -3,6 +3,7 @@
 import logging
 import os
 import subprocess
+import tempfile
 
 logger = logging.getLogger(__name__)
 
@@ -52,3 +53,46 @@ def eval_cmd(cmd: list[str], **kwargs) -> str:
         c.check_returncode()
 
     return c.stdout.decode("utf-8")
+
+
+def clean_disk_space(ci_service: str = "github_actions") -> None:
+    """Clean up disk space on CI services.
+
+    Parameters
+    ----------
+    ci_service : str, optional
+        The CI service to clean up disk space for. Currently only "github-actions" is supported.
+        Default is "github-actions".
+    """
+    with tempfile.TempDirectory() as tempdir, pushd(tempdir):
+        with open("clean_disk.sh", "w") as f:
+            if ci_service == "github-actions":
+                f.write("""\
+  #!/bin/bash
+
+  # clean disk space
+  sudo mkdir -p /opt/empty_dir || true
+  for d in \
+  /opt/ghc \
+  /opt/hostedtoolcache \
+  /usr/lib/jvm \
+  /usr/local/.ghcup \
+  /usr/local/lib/android \
+  /usr/local/share/powershell \
+  /usr/share/dotnet \
+  /usr/share/swift \
+  ; do
+    sudo rsync --stats -a --delete /opt/empty_dir/ $d || true
+  done
+  sudo apt-get purge -y -f firefox \
+                          google-chrome-stable \
+                          microsoft-edge-stable
+  sudo apt-get autoremove -y >& /dev/null
+  sudo apt-get autoclean -y >& /dev/null
+  sudo docker image prune --all --force
+  df -h
+""")
+            else:
+                raise ValueError(f"Unknown CI service: {ci_service}")
+
+            subprocess.run(["bash", "clean_disk.sh"])