From 3aeabb90f8378dbe185fe80c4f753064099044a7 Mon Sep 17 00:00:00 2001 From: Frostebite Date: Sun, 28 Dec 2025 16:47:47 +0000 Subject: [PATCH] pr feedback --- .github/workflows/cloud-runner-integrity.yml | 98 ++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/.github/workflows/cloud-runner-integrity.yml b/.github/workflows/cloud-runner-integrity.yml index a111ab7a..09986819 100644 --- a/.github/workflows/cloud-runner-integrity.yml +++ b/.github/workflows/cloud-runner-integrity.yml @@ -161,6 +161,28 @@ jobs: break fi done + - name: Ensure disk pressure cleared before test + timeout-minutes: 2 + run: | + echo "Ensuring disk pressure is cleared before test..." + rm -rf ./cloud-runner-cache/* || true + docker system prune -af --volumes || true + docker exec k3d-unity-builder-agent-0 sh -c "docker system prune -af --volumes 2>/dev/null || true" || true + # Wait for disk pressure taints to clear (with aggressive cleanup) + for i in {1..30}; do + if kubectl describe nodes 2>/dev/null | grep -q "node.kubernetes.io/disk-pressure"; then + echo "Disk pressure detected, cleaning up aggressively... ($i/30)" + docker exec k3d-unity-builder-agent-0 sh -c "docker system prune -af --volumes 2>/dev/null || true" || true + docker system prune -af --volumes || true + # Also clean up k3d internal images and containers + docker exec k3d-unity-builder-agent-0 sh -c "docker images -q | xargs -r docker rmi -f 2>/dev/null || true" || true + sleep 3 + else + echo "No disk pressure taints found, proceeding with test" + kubectl describe nodes | grep -i taint || echo "No taints found" + break + fi + done - name: Run cloud-runner-image test (validate image creation) timeout-minutes: 10 run: yarn run test "cloud-runner-image" --detectOpenHandles --forceExit --runInBand @@ -214,6 +236,25 @@ jobs: break fi done + - name: Ensure disk pressure cleared before test + timeout-minutes: 2 + run: | + echo "Ensuring disk pressure is cleared before test..." + rm -rf ./cloud-runner-cache/* || true + docker system prune -af --volumes || true + docker exec k3d-unity-builder-agent-0 sh -c "docker system prune -af --volumes 2>/dev/null || true" || true + for i in {1..30}; do + if kubectl describe nodes 2>/dev/null | grep -q "node.kubernetes.io/disk-pressure"; then + echo "Disk pressure detected, cleaning up aggressively... ($i/30)" + docker exec k3d-unity-builder-agent-0 sh -c "docker system prune -af --volumes 2>/dev/null || true" || true + docker system prune -af --volumes || true + docker exec k3d-unity-builder-agent-0 sh -c "docker images -q | xargs -r docker rmi -f 2>/dev/null || true" || true + sleep 3 + else + echo "No disk pressure taints found, proceeding with test" + break + fi + done - name: Run cloud-runner-kubernetes test (simple K8s build validation) timeout-minutes: 30 run: yarn run test "cloud-runner-kubernetes" --detectOpenHandles --forceExit --runInBand @@ -268,6 +309,25 @@ jobs: break fi done + - name: Ensure disk pressure cleared before test + timeout-minutes: 2 + run: | + echo "Ensuring disk pressure is cleared before test..." + rm -rf ./cloud-runner-cache/* || true + docker system prune -af --volumes || true + docker exec k3d-unity-builder-agent-0 sh -c "docker system prune -af --volumes 2>/dev/null || true" || true + for i in {1..30}; do + if kubectl describe nodes 2>/dev/null | grep -q "node.kubernetes.io/disk-pressure"; then + echo "Disk pressure detected, cleaning up aggressively... ($i/30)" + docker exec k3d-unity-builder-agent-0 sh -c "docker system prune -af --volumes 2>/dev/null || true" || true + docker system prune -af --volumes || true + docker exec k3d-unity-builder-agent-0 sh -c "docker images -q | xargs -r docker rmi -f 2>/dev/null || true" || true + sleep 3 + else + echo "No disk pressure taints found, proceeding with test" + break + fi + done - name: Run cloud-runner-s3-steps test (validate S3 operations with K8s) timeout-minutes: 30 run: yarn run test "cloud-runner-s3-steps" --detectOpenHandles --forceExit --runInBand @@ -322,6 +382,25 @@ jobs: break fi done + - name: Ensure disk pressure cleared before test + timeout-minutes: 2 + run: | + echo "Ensuring disk pressure is cleared before test..." + rm -rf ./cloud-runner-cache/* || true + docker system prune -af --volumes || true + docker exec k3d-unity-builder-agent-0 sh -c "docker system prune -af --volumes 2>/dev/null || true" || true + for i in {1..30}; do + if kubectl describe nodes 2>/dev/null | grep -q "node.kubernetes.io/disk-pressure"; then + echo "Disk pressure detected, cleaning up aggressively... ($i/30)" + docker exec k3d-unity-builder-agent-0 sh -c "docker system prune -af --volumes 2>/dev/null || true" || true + docker system prune -af --volumes || true + docker exec k3d-unity-builder-agent-0 sh -c "docker images -q | xargs -r docker rmi -f 2>/dev/null || true" || true + sleep 3 + else + echo "No disk pressure taints found, proceeding with test" + break + fi + done - name: Run cloud-runner-end2end-caching test timeout-minutes: 60 run: yarn run test "cloud-runner-end2end-caching" --detectOpenHandles --forceExit --runInBand @@ -376,6 +455,25 @@ jobs: break fi done + - name: Ensure disk pressure cleared before test + timeout-minutes: 2 + run: | + echo "Ensuring disk pressure is cleared before test..." + rm -rf ./cloud-runner-cache/* || true + docker system prune -af --volumes || true + docker exec k3d-unity-builder-agent-0 sh -c "docker system prune -af --volumes 2>/dev/null || true" || true + for i in {1..30}; do + if kubectl describe nodes 2>/dev/null | grep -q "node.kubernetes.io/disk-pressure"; then + echo "Disk pressure detected, cleaning up aggressively... ($i/30)" + docker exec k3d-unity-builder-agent-0 sh -c "docker system prune -af --volumes 2>/dev/null || true" || true + docker system prune -af --volumes || true + docker exec k3d-unity-builder-agent-0 sh -c "docker images -q | xargs -r docker rmi -f 2>/dev/null || true" || true + sleep 3 + else + echo "No disk pressure taints found, proceeding with test" + break + fi + done - name: Run cloud-runner-end2end-retaining test timeout-minutes: 60 run: yarn run test "cloud-runner-end2end-retaining" --detectOpenHandles --forceExit --runInBand