CI: Apply patch for IO

cburstedde · Jan 15, 2025 · 678d541 · 678d541
1 parent aa4aeff
commit 678d541
Show file tree

Hide file tree

Showing 4 changed files with 73 additions and 11 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -13,6 +13,12 @@ on:
       - ".github/workflows/composite-cmake/**"
   pull_request:
 
+env:
+  # Due to busy file system problems on the CI runners, we introduced a patch to
+  # sleep if the MPI count of an I/O operation is unexpectedly 0 and then retry
+  # the I/O operation once. The time is in microseconds.
+  IO_SLEEP_TIME: 2
+
 jobs:
 
   linux-multi:
@@ -29,6 +35,9 @@ jobs:
     - name: Checkout source code
       uses: actions/checkout@main
 
+    - name: Patch I/O for CI by sleeping if I/O intensity is too high
+      run: git apply doc/patch/patch-CI-IO.patch
+
     - name: Run bootstrap script
       run: ./bootstrap
 
@@ -37,23 +46,26 @@ jobs:
       run: |
         DIR="checkMPIdebug_shared" && mkdir -p "$DIR" && cd "$DIR"
         ../configure --enable-mpi --enable-debug --disable-shared \
-            CFLAGS="-O0 -g -Wall -Wextra -Wno-unused-parameter"
+            CFLAGS="-O0 -g -Wall -Wextra -Wno-unused-parameter" \
+            CPPFLAGS="-DSC_HACK_MPI_SLEEP=$IO_SLEEP_TIME"
         make -j V=0
         make -j check V=0
 
     - name: Make check with MPI, without debug
       shell: bash
       run: |
         DIR="checkMPI" && mkdir -p "$DIR" && cd "$DIR"
-        ../configure --enable-mpi CFLAGS="-O2"
+        ../configure --enable-mpi CFLAGS="-O2" \
+        CPPFLAGS="-DSC_HACK_MPI_SLEEP=$IO_SLEEP_TIME"
         make -j V=0
         make -j check V=0
 
     - name: Make check without MPI, with debug
       shell: bash
       run: |
         DIR="checkdebug" && mkdir -p "$DIR" && cd "$DIR"
-        ../configure --enable-debug CFLAGS="-O0 -g -Wall -Wno-uninitialized"
+        ../configure --enable-debug CFLAGS="-O0 -g -Wall -Wno-uninitialized" \
+        CPPFLAGS="-DSC_HACK_MPI_SLEEP=$IO_SLEEP_TIME"
         make -j V=0
         make -j check V=0
 
@@ -66,7 +78,8 @@ jobs:
       shell: bash
       run: |
         DIR="checkMPIdebugCXX" && mkdir -p "$DIR" && cd "$DIR"
-        ../configure --enable-mpi --enable-debug CFLAGS="-O0" CC=mpicxx
+        ../configure --enable-mpi --enable-debug CFLAGS="-O0" CC=mpicxx \
+        CPPFLAGS="-DSC_HACK_MPI_SLEEP=$IO_SLEEP_TIME"
         make -j V=0
         make -j check V=0
 
@@ -75,15 +88,16 @@ jobs:
       run: |
         DIR="checkOpenMPMPIdebug" && mkdir -p "$DIR" && cd "$DIR"
         ../configure --enable-openmp="-fopenmp" --enable-mpi \
-            --enable-debug CFLAGS="-O0"
+            --enable-debug CFLAGS="-O0" \
+            CPPFLAGS="-DSC_HACK_MPI_SLEEP=$IO_SLEEP_TIME"
         make -j V=0
         make -j check V=0
 
     - name: Make distcheck
       shell: bash
       run: |
         DIR="distcheck" && mkdir -p "$DIR" && cd "$DIR"
-        ../configure
+        ../configure CPPFLAGS="-DSC_HACK_MPI_SLEEP=$IO_SLEEP_TIME"
         make -j distcheck V=0
 
     - name: Upload log files
@@ -110,6 +124,9 @@ jobs:
     - name: Checkout source code
       uses: actions/checkout@main
 
+    - name: Patch I/O for CI by sleeping if I/O intensity is too high
+      run: git apply doc/patch/patch-CI-IO.patch
+
     - name: Run bootstrap script
       run: ./bootstrap
 
@@ -119,7 +136,8 @@ jobs:
         DIR="checkMPIdebug_valgrind" && mkdir -p "$DIR" && cd "$DIR"
         ../configure --enable-mpi --enable-debug \
                      --disable-shared --enable-valgrind \
-          CFLAGS="-O0 -g -Wall -Wextra -Wno-unused-parameter"
+          CFLAGS="-O0 -g -Wall -Wextra -Wno-unused-parameter" \
+          CPPFLAGS="-DSC_HACK_MPI_SLEEP=$IO_SLEEP_TIME"
         make -j V=0
         make -j check V=0
 
@@ -149,6 +167,9 @@ jobs:
       with:
         fetch-depth: 0
 
+    - name: Patch I/O for CI by sleeping if I/O intensity is too high
+      run: git apply doc/patch/patch-CI-IO.patch
+
     - name: Identify version
       shell: bash
       run: |
@@ -165,7 +186,8 @@ jobs:
         ../configure --enable-mpi --enable-debug \
             CFLAGS="-O0 -g -pedantic -Wall -Wextra -Werror \
                 -Wno-unused-parameter -Wno-builtin-declaration-mismatch \
-                -Wno-implicit-fallthrough"
+                -Wno-implicit-fallthrough" \
+                CPPFLAGS="-DSC_HACK_MPI_SLEEP=$IO_SLEEP_TIME"
         make -j V=0
         make -j check V=0
         make -j distcheck V=0

diff --git a/.github/workflows/ci_cmake.yml b/.github/workflows/ci_cmake.yml
@@ -20,6 +20,10 @@ env:
   CTEST_PARALLEL_LEVEL: 0
   CMAKE_INSTALL_PREFIX: ~/local
   CMAKE_PREFIX_PATH: ~/local
+  # Due to busy file system problems on the CI runners, we introduced a patch to
+  # sleep if the MPI count of an I/O operation is unexpectedly 0 and then retry
+  # the I/O operation once. The time is in milliseconds.
+  IO_SLEEP_TIME: 2
 
 jobs:
 
@@ -47,6 +51,9 @@ jobs:
     - uses: actions/checkout@v4
       name: Checkout source code
 
+    - name: Patch I/O for CI by sleeping if I/O intensity is too high
+      run: git apply doc/patch/patch-CI-IO.patch
+
     - name: Install system dependencies
       if: ${{ matrix.mpi }}
       run: |
@@ -76,6 +83,9 @@ jobs:
     - uses: actions/checkout@v4
       name: Checkout source code
 
+    - name: Patch I/O for CI by sleeping if I/O intensity is too high
+      run: git apply doc/patch/patch-CI-IO.patch
+
     - name: Install system dependencies
       if: ${{ matrix.mpi }}
       run: |
@@ -112,6 +122,14 @@ jobs:
     - name: Checkout source code
       uses: actions/checkout@v4
 
+    - name: Patch I/O for CI by sleeping if I/O intensity is too high
+      run: git apply doc/patch/patch-CI-IO.patch
+
+    - name: Set macro for I/O waiting
+      shell: bash
+      # use an enviroment variable to prepend the flag to the bulit-in compiler flags
+      run: echo "CFLAGS=-DSC_HACK_MPI_SLEEP=$IO_SLEEP_TIME" >> "$GITHUB_ENV"
+
     - name: CMake configure
       run: cmake --preset default -DSC_ENABLE_MPI:BOOL=${{ matrix.mpi }} -DSC_TEST_WITH_VALGRIND:BOOL=${{ matrix.valgrind }}
 
@@ -154,6 +172,9 @@ jobs:
     - uses: actions/checkout@v4
       name: Checkout source code
 
+    - name: Patch I/O for CI by sleeping if I/O intensity is too high
+      run: git apply doc/patch/patch-CI-IO.patch
+
     - name: Install system dependencies
       run: brew install open-mpi
 
@@ -177,6 +198,9 @@ jobs:
     - uses: actions/checkout@v4
       name: Checkout source code
 
+    - name: Patch I/O for CI by sleeping if I/O intensity is too high
+      run: git apply doc/patch/patch-CI-IO.patch
+
     - run: echo "CMAKE_INSTALL_PREFIX=$HOME/local" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
 
     - run: echo "CMAKE_PREFIX_PATH=$HOME/local" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append

diff --git a/.github/workflows/ci_darwin.yml b/.github/workflows/ci_darwin.yml
@@ -13,6 +13,12 @@ on:
       - ".github/workflows/composite-cmake/**"
   pull_request:
 
+env:
+  # Due to busy file system problems on the CI runners, we introduced a patch to
+  # sleep if the MPI count of an I/O operation is unexpectedly 0 and then retry
+  # the I/O operation once. The time is in milliseconds.
+  IO_SLEEP_TIME: 2
+
 jobs:
   darwin:
     runs-on: macos-latest
@@ -26,6 +32,9 @@ jobs:
      - uses: actions/checkout@main
        name: Checkout source code
 
+     - name: Patch I/O for CI by sleeping if I/O intensity is too high
+       run: git apply doc/patch/patch-CI-IO.patch
+
      - name: Install system dependencies
        run: brew install open-mpi libtool automake
 
@@ -36,23 +45,26 @@ jobs:
        run: |
           DIR="checkdebug" && mkdir -p "$DIR" && cd "$DIR"
           ../configure --enable-debug \
-            CFLAGS="-O0 -g -Wall -Wextra -Wno-unused-parameter"
+            CFLAGS="-O0 -g -Wall -Wextra -Wno-unused-parameter" \
+            CPPFLAGS="-DSC_HACK_MPI_SLEEP=$IO_SLEEP_TIME"
           make -j V=0
           make -j check V=0
 
      - name: Make check with MPI and debug
        run: |
           DIR="checkMPIdebug" && mkdir -p "$DIR" && cd "$DIR"
           ../configure --enable-mpi --enable-debug \
-            CFLAGS="-O0 -g -Wall -Wextra -Wno-unused-parameter"
+            CFLAGS="-O0 -g -Wall -Wextra -Wno-unused-parameter" \
+            CPPFLAGS="-DSC_HACK_MPI_SLEEP=$IO_SLEEP_TIME"
           make -j V=0
           make -j check V=0
 
      - name: Make check with MPI, debug and C++ compiler
        run: |
           DIR="checkMPIdebugCXX" && mkdir -p "$DIR" && cd "$DIR"
           ../configure --enable-mpi --enable-debug CC=mpicxx \
-            CFLAGS="-O0 -g -Wall -Wextra -Wno-unused-parameter"
+            CFLAGS="-O0 -g -Wall -Wextra -Wno-unused-parameter" \
+            CPPFLAGS="-DSC_HACK_MPI_SLEEP=$IO_SLEEP_TIME"
           make -j V=0
           make -j check V=0
 

diff --git a/.github/workflows/composite-cmake/action.yml b/.github/workflows/composite-cmake/action.yml
@@ -2,6 +2,10 @@ runs:
   using: "composite"
 
   steps:
+  - name: Set macro for I/O waiting
+    shell: bash
+    # use an enviroment variable to prepend the flag to the bulit-in compiler flags
+    run: echo "CFLAGS=-DSC_HACK_MPI_SLEEP=$IO_SLEEP_TIME" >> "$GITHUB_ENV"
   - name: CMake configure
     shell: bash
     run: >-