diff --git a/.github/workflows/switch.yml b/.github/workflows/switch.yml
index df6b3c81c..ec70c3d06 100644
--- a/.github/workflows/switch.yml
+++ b/.github/workflows/switch.yml
@@ -86,9 +86,16 @@ jobs:
           echo "${{ env.release_tag }}"
           echo "${{ env.release_name }}"
           echo "${{ env.release_pre }}"
-
-      - name: Cache cmake_build
-        id: cache-switch-cmake
+      - name: List build cache restore keys
+        shell : bash
+        run: |
+          # Look for the last 9 build caches (GitHub supports max 10 including current one)
+          for number in 1 2 3 4 5 6 7 8 9
+          do
+            id=$((${{ github.run_number }} - number))
+            echo "cache_$number=switch-${{ github.ref }}-$id" >> $GITHUB_ENV
+          done
+      - name: Handle build cache
         uses: actions/cache@v2
         with:
           # This is unnecessarily verbose and might break, but again ! seems broken
@@ -104,8 +111,17 @@ jobs:
             stk-code/lib/openal/cmake_build
             stk-code/lib/harfbuzz/cmake_build
           # Make sure PRs can't overwrite!
-          key: ${{ github.ref }}-switch-cmake-0
-
+          key: switch-${{ github.ref }}-${{ github.run_number }}
+          restore-keys: |
+            ${{ env.cache_1 }}
+            ${{ env.cache_2 }}
+            ${{ env.cache_3 }}
+            ${{ env.cache_4 }}
+            ${{ env.cache_5 }}
+            ${{ env.cache_6 }}
+            ${{ env.cache_7 }}
+            ${{ env.cache_8 }}
+            ${{ env.cache_9 }}
       - name: Run build script
         run: |
           cp -v stk-code/switch/pkgbuild-scripts/* "$DEVKITPRO/"
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
index cd9923fe2..e018bba4d 100644
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@@ -44,12 +44,31 @@ jobs:
           ${{ env.wget }} https://github.com/MestreLion/git-tools/archive/refs/heads/master.zip
           ${{ env.unzip }} master.zip
           python git-tools-master/git-restore-mtime
-      - name: Cache build
+      - name: List build cache restore keys
+        shell : bash
+        run: |
+          # Look for the last 9 build caches (GitHub supports max 10 including current one)
+          for number in 1 2 3 4 5 6 7 8 9
+          do
+            id=$((${{ github.run_number }} - number))
+            echo "cache_$number=windows-${{ github.ref }}-${{ matrix.arch }}-${{ matrix.os }}-$id" >> $GITHUB_ENV
+          done
+      - name: Handle build cache
         uses: actions/cache@v2
         with:
           path: |
             build
-          key: ${{ github.ref }}-${{ matrix.arch }}-${{ matrix.os }}-windows-build
+          key: windows-${{ github.ref }}-${{ matrix.arch }}-${{ matrix.os }}-${{ github.run_number }}
+          restore-keys: |
+            ${{ env.cache_1 }}
+            ${{ env.cache_2 }}
+            ${{ env.cache_3 }}
+            ${{ env.cache_4 }}
+            ${{ env.cache_5 }}
+            ${{ env.cache_6 }}
+            ${{ env.cache_7 }}
+            ${{ env.cache_8 }}
+            ${{ env.cache_9 }}
       - name: Download dependencies
         run: |
           ${{ env.wget }} https://github.com/supertuxkart/dependencies/releases/download/preview/dependencies-win-${{ matrix.arch }}.zip
@@ -105,22 +124,13 @@ jobs:
         run: |
           mkdir -p build
           cd build
-          # Always run with -DUSE_DIRECTX=ON breaks the cache
-          if [[ ! -f CMakeCache.txt ]]; then
-            cmake .. -DCMAKE_TOOLCHAIN_FILE=../toolchain.cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCHECK_ASSETS=OFF -DUSE_DIRECTX=ON
-          else
-            cmake ..
-          fi
+          cmake .. -DCMAKE_TOOLCHAIN_FILE=../toolchain.cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCHECK_ASSETS=OFF -DUSE_DIRECTX=ON
       - name: Configure bulid for MinGW (aarch64)
         if: ${{ matrix.os == 'ubuntu-latest' && matrix.arch == 'aarch64' }}
         run: |
           mkdir -p build
           cd build
-          if [[ ! -f CMakeCache.txt ]]; then
-            cmake .. -DLLVM_ARCH=aarch64 -DLLVM_PREFIX=/llvm-mingw -DCMAKE_TOOLCHAIN_FILE=../cmake/Toolchain-llvm-mingw.cmake -DCHECK_ASSETS=OFF -DUSE_DIRECTX=ON
-          else
-            cmake ..
-          fi
+          cmake .. -DLLVM_ARCH=aarch64 -DLLVM_PREFIX=/llvm-mingw -DCMAKE_TOOLCHAIN_FILE=../cmake/Toolchain-llvm-mingw.cmake -DCHECK_ASSETS=OFF -DUSE_DIRECTX=ON
       - name: Build for MSVC
         if: ${{ matrix.os == 'windows-latest' }}
         working-directory: build