actions
/
unity-builder
mirror of https://github.com/game-ci/unity-builder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

PR feedback

cloud-runner-develop
Frostebite 2025-12-05 18:08:29 +00:00
parent 69731babfc
commit 956b2e4324
8 changed files with 432 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

417
.github/workflows/cloud-runner-integrity.yml vendored
View File

@ -85,58 +85,31 @@ jobs:
# Show available disk space
df -h
- run: yarn install --frozen-lockfile
- name: Run K8s tests sequentially
timeout-minutes: 180
- name: Clean up K8s test resources
run: |
# List of K8s tests to run sequentially
tests=(
"cloud-runner-end2end-caching"
"cloud-runner-end2end-retaining"
"cloud-runner-hooks"
)
failed_tests=()
for test in "${tests[@]}"; do
echo "========================================="
echo "Running test: $test"
echo "========================================="
# Clean up K8s resources before each test
echo "Cleaning up K8s resources before test..."
kubectl delete jobs --all --ignore-not-found=true --all-namespaces || true
kubectl delete pods --all --ignore-not-found=true --all-namespaces || true
kubectl delete pvc --all --ignore-not-found=true --all-namespaces || true
kubectl delete secrets --all --ignore-not-found=true --all-namespaces || true
# Clean up K8s resources before each test (only test resources, not system pods)
echo "Cleaning up K8s test resources..."
# Only clean up resources in default namespace and resources matching our test patterns
kubectl delete jobs --all --ignore-not-found=true -n default || true
# Delete completed/failed pods in default namespace (not system pods)
kubectl get pods -n default -o name 2>/dev/null | grep -E "(unity-builder-job-|helper-pod-)" | while read pod; do
kubectl delete "$pod" --ignore-not-found=true || true
done || true
# Only delete PVCs that match our naming pattern (unity-builder-pvc-*)
kubectl get pvc -n default -o name 2>/dev/null | grep "unity-builder-pvc-" | while read pvc; do
kubectl delete "$pvc" --ignore-not-found=true || true
done || true
# Only delete secrets that match our naming pattern (build-credentials-*)
kubectl get secrets -n default -o name 2>/dev/null | grep "build-credentials-" | while read secret; do
kubectl delete "$secret" --ignore-not-found=true || true
done || true
sleep 3
# Clean up disk space before each test
# Clean up disk space
rm -rf ./cloud-runner-cache/* || true
docker system prune -f || true
# Run the test
if yarn run test "$test" --detectOpenHandles --forceExit --runInBand; then
echo "✓ Test $test passed"
else
echo "✗ Test $test failed"
failed_tests+=("$test")
fi
echo ""
done
# Report results
if [ ${#failed_tests[@]} -eq 0 ]; then
echo "========================================="
echo "All tests passed!"
echo "========================================="
exit 0
else
echo "========================================="
echo "Failed tests: ${failed_tests[*]}"
echo "========================================="
exit 1
fi
- name: Run cloud-runner-end2end-caching test
timeout-minutes: 60
run: yarn run test "cloud-runner-end2end-caching" --detectOpenHandles --forceExit --runInBand
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
@ -160,6 +133,86 @@ jobs:
AWS_EC2_METADATA_DISABLED: 'true'
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up K8s test resources
run: |
kubectl delete jobs --all --ignore-not-found=true -n default || true
kubectl get pods -n default -o name 2>/dev/null | grep -E "(unity-builder-job-|helper-pod-)" | while read pod; do
kubectl delete "$pod" --ignore-not-found=true || true
done || true
kubectl get pvc -n default -o name 2>/dev/null | grep "unity-builder-pvc-" | while read pvc; do
kubectl delete "$pvc" --ignore-not-found=true || true
done || true
kubectl get secrets -n default -o name 2>/dev/null | grep "build-credentials-" | while read secret; do
kubectl delete "$secret" --ignore-not-found=true || true
done || true
sleep 3
rm -rf ./cloud-runner-cache/* || true
docker system prune -f || true
- name: Run cloud-runner-end2end-retaining test
timeout-minutes: 60
run: yarn run test "cloud-runner-end2end-retaining" --detectOpenHandles --forceExit --runInBand
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: k8s
containerCpu: '512'
containerMemory: '512'
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
AWS_S3_ENDPOINT: http://localhost:4566
AWS_ENDPOINT: http://localhost:4566
INPUT_AWSS3ENDPOINT: http://localhost:4566
INPUT_AWSENDPOINT: http://localhost:4566
AWS_S3_FORCE_PATH_STYLE: 'true'
AWS_EC2_METADATA_DISABLED: 'true'
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up K8s test resources
run: |
kubectl delete jobs --all --ignore-not-found=true -n default || true
kubectl get pods -n default -o name 2>/dev/null | grep -E "(unity-builder-job-|helper-pod-)" | while read pod; do
kubectl delete "$pod" --ignore-not-found=true || true
done || true
kubectl get pvc -n default -o name 2>/dev/null | grep "unity-builder-pvc-" | while read pvc; do
kubectl delete "$pvc" --ignore-not-found=true || true
done || true
kubectl get secrets -n default -o name 2>/dev/null | grep "build-credentials-" | while read secret; do
kubectl delete "$secret" --ignore-not-found=true || true
done || true
sleep 3
rm -rf ./cloud-runner-cache/* || true
docker system prune -f || true
- name: Run cloud-runner-hooks test
timeout-minutes: 60
run: yarn run test "cloud-runner-hooks" --detectOpenHandles --forceExit --runInBand
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: k8s
containerCpu: '512'
containerMemory: '512'
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
AWS_S3_ENDPOINT: http://localhost:4566
AWS_ENDPOINT: http://localhost:4566
INPUT_AWSS3ENDPOINT: http://localhost:4566
INPUT_AWSENDPOINT: http://localhost:4566
AWS_S3_FORCE_PATH_STYLE: 'true'
AWS_EC2_METADATA_DISABLED: 'true'
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
localstack:
name: Cloud Runner Tests (LocalStack)
runs-on: ubuntu-latest
@ -188,58 +241,230 @@ jobs:
# Show available disk space
df -h
- run: yarn install --frozen-lockfile
- name: Run LocalStack tests sequentially
timeout-minutes: 300
- name: Clean up disk space
run: |
# List of LocalStack tests to run sequentially
tests=(
"cloud-runner-end2end-locking"
"cloud-runner-end2end-caching"
"cloud-runner-end2end-retaining"
"cloud-runner-caching"
"cloud-runner-environment"
"cloud-runner-image"
"cloud-runner-hooks"
"cloud-runner-local-persistence"
"cloud-runner-locking-core"
"cloud-runner-locking-get-locked"
)
failed_tests=()
for test in "${tests[@]}"; do
echo "========================================="
echo "Running test: $test"
echo "========================================="
# Clean up disk space before each test
rm -rf ./cloud-runner-cache/* || true
docker system prune -f || true
df -h
# Run the test
if yarn run test "$test" --detectOpenHandles --forceExit --runInBand; then
echo "✓ Test $test passed"
else
echo "✗ Test $test failed"
failed_tests+=("$test")
fi
echo ""
done
# Report results
if [ ${#failed_tests[@]} -eq 0 ]; then
echo "========================================="
echo "All tests passed!"
echo "========================================="
exit 0
else
echo "========================================="
echo "Failed tests: ${failed_tests[*]}"
echo "========================================="
exit 1
fi
- name: Run cloud-runner-end2end-locking test
timeout-minutes: 60
run: yarn run test "cloud-runner-end2end-locking" --detectOpenHandles --forceExit --runInBand
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
AWS_ENDPOINT: http://localhost:4566
AWS_ENDPOINT_URL: http://localhost:4566
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space
run: |
rm -rf ./cloud-runner-cache/* || true
docker system prune -f || true
df -h
- name: Run cloud-runner-end2end-caching test
timeout-minutes: 60
run: yarn run test "cloud-runner-end2end-caching" --detectOpenHandles --forceExit --runInBand
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
AWS_ENDPOINT: http://localhost:4566
AWS_ENDPOINT_URL: http://localhost:4566
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space
run: |
rm -rf ./cloud-runner-cache/* || true
docker system prune -f || true
df -h
- name: Run cloud-runner-end2end-retaining test
timeout-minutes: 60
run: yarn run test "cloud-runner-end2end-retaining" --detectOpenHandles --forceExit --runInBand
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
AWS_ENDPOINT: http://localhost:4566
AWS_ENDPOINT_URL: http://localhost:4566
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space
run: |
rm -rf ./cloud-runner-cache/* || true
docker system prune -f || true
df -h
- name: Run cloud-runner-caching test
timeout-minutes: 60
run: yarn run test "cloud-runner-caching" --detectOpenHandles --forceExit --runInBand
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
AWS_ENDPOINT: http://localhost:4566
AWS_ENDPOINT_URL: http://localhost:4566
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space
run: |
rm -rf ./cloud-runner-cache/* || true
docker system prune -f || true
df -h
- name: Run cloud-runner-environment test
timeout-minutes: 60
run: yarn run test "cloud-runner-environment" --detectOpenHandles --forceExit --runInBand
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
AWS_ENDPOINT: http://localhost:4566
AWS_ENDPOINT_URL: http://localhost:4566
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space
run: |
rm -rf ./cloud-runner-cache/* || true
docker system prune -f || true
df -h
- name: Run cloud-runner-image test
timeout-minutes: 60
run: yarn run test "cloud-runner-image" --detectOpenHandles --forceExit --runInBand
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
AWS_ENDPOINT: http://localhost:4566
AWS_ENDPOINT_URL: http://localhost:4566
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space
run: |
rm -rf ./cloud-runner-cache/* || true
docker system prune -f || true
df -h
- name: Run cloud-runner-hooks test
timeout-minutes: 60
run: yarn run test "cloud-runner-hooks" --detectOpenHandles --forceExit --runInBand
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
AWS_ENDPOINT: http://localhost:4566
AWS_ENDPOINT_URL: http://localhost:4566
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space
run: |
rm -rf ./cloud-runner-cache/* || true
docker system prune -f || true
df -h
- name: Run cloud-runner-local-persistence test
timeout-minutes: 60
run: yarn run test "cloud-runner-local-persistence" --detectOpenHandles --forceExit --runInBand
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
AWS_ENDPOINT: http://localhost:4566
AWS_ENDPOINT_URL: http://localhost:4566
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space
run: |
rm -rf ./cloud-runner-cache/* || true
docker system prune -f || true
df -h
- name: Run cloud-runner-locking-core test
timeout-minutes: 60
run: yarn run test "cloud-runner-locking-core" --detectOpenHandles --forceExit --runInBand
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
AWS_ENDPOINT: http://localhost:4566
AWS_ENDPOINT_URL: http://localhost:4566
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space
run: |
rm -rf ./cloud-runner-cache/* || true
docker system prune -f || true
df -h
- name: Run cloud-runner-locking-get-locked test
timeout-minutes: 60
run: yarn run test "cloud-runner-locking-get-locked" --detectOpenHandles --forceExit --runInBand
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}

45
dist/index.js vendored
View File

@ -3861,6 +3861,7 @@ class KubernetesJobSpecFactory {
backoffLimit: 0,
template: {
spec: {
terminationGracePeriodSeconds: 90,
volumes: [
{
name: 'build-mount',
@ -3979,19 +3980,40 @@ class KubernetesPods {
if (conditions.length > 0) {
errorDetails.push(`Conditions: ${JSON.stringify(conditions.map((c) => ({ type: c.type, status: c.status, reason: c.reason, message: c.message })), undefined, 2)}`);
}
let containerExitCode;
let containerSucceeded = false;
if (containerStatuses.length > 0) {
containerStatuses.forEach((cs, idx) => {
if (cs.state?.waiting) {
errorDetails.push(`Container ${idx} (${cs.name}) waiting: ${cs.state.waiting.reason} - ${cs.state.waiting.message || ''}`);
}
if (cs.state?.terminated) {
errorDetails.push(`Container ${idx} (${cs.name}) terminated: ${cs.state.terminated.reason} - ${cs.state.terminated.message || ''} (exit code: ${cs.state.terminated.exitCode})`);
const exitCode = cs.state.terminated.exitCode;
containerExitCode = exitCode;
if (exitCode === 0) {
containerSucceeded = true;
}
errorDetails.push(`Container ${idx} (${cs.name}) terminated: ${cs.state.terminated.reason} - ${cs.state.terminated.message || ''} (exit code: ${exitCode})`);
}
});
}
if (events.length > 0) {
errorDetails.push(`Recent events: ${JSON.stringify(events.slice(-5), undefined, 2)}`);
}
// Check if only PreStopHook failed but container succeeded
const hasPreStopHookFailure = events.some((e) => e.reason === 'FailedPreStopHook');
if (containerSucceeded && containerExitCode === 0) {
// Container succeeded - PreStopHook failure is non-critical
if (hasPreStopHookFailure) {
cloud_runner_logger_1.default.logWarning(`Pod ${podName} marked as Failed due to PreStopHook failure, but container exited successfully (exit code 0). This is non-fatal.`);
}
else {
cloud_runner_logger_1.default.log(`Pod ${podName} container succeeded (exit code 0), but pod phase is Failed. Checking details...`);
}
cloud_runner_logger_1.default.log(`Pod details: ${errorDetails.join('\n')}`);
// Don't throw error - container succeeded, PreStopHook failure is non-critical
return false; // Pod is not running, but we don't treat it as a failure
}
const errorMessage = `K8s pod failed\n${errorDetails.join('\n')}`;
cloud_runner_logger_1.default.log(errorMessage);
throw new Error(errorMessage);
@ -6924,6 +6946,10 @@ class ContainerHookService {
if (step.image === undefined) {
step.image = `ubuntu`;
}
// Ensure allowFailure defaults to false if not explicitly set
if (step.allowFailure === undefined) {
step.allowFailure = false;
}
}
if (object === undefined) {
throw new Error(`Failed to parse ${steps}`);
@ -7296,7 +7322,22 @@ class CustomWorkflow {
// }
for (const step of steps) {
cloud_runner_logger_1.default.log(`Cloud Runner is running in custom job mode`);
output += await cloud_runner_1.default.Provider.runTaskInWorkflow(cloud_runner_1.default.buildParameters.buildGuid, step.image, step.commands, `/${cloud_runner_folders_1.CloudRunnerFolders.buildVolumeFolder}`, `/${cloud_runner_folders_1.CloudRunnerFolders.projectPathAbsolute}/`, environmentVariables, [...secrets, ...step.secrets]);
try {
const stepOutput = await cloud_runner_1.default.Provider.runTaskInWorkflow(cloud_runner_1.default.buildParameters.buildGuid, step.image, step.commands, `/${cloud_runner_folders_1.CloudRunnerFolders.buildVolumeFolder}`, `/${cloud_runner_folders_1.CloudRunnerFolders.projectPathAbsolute}/`, environmentVariables, [...secrets, ...step.secrets]);
output += stepOutput;
}
catch (error) {
const allowFailure = step.allowFailure === true;
const stepName = step.name || step.image || 'unknown';
if (allowFailure) {
cloud_runner_logger_1.default.logWarning(`Hook container "${stepName}" failed but allowFailure is true. Continuing build. Error: ${error?.message || error}`);
// Continue to next step
}
else {
cloud_runner_logger_1.default.log(`Hook container "${stepName}" failed and allowFailure is false (default). Stopping build.`);
throw error;
}
}
}
return output;
}

2
dist/index.js.map vendored
View File

File diff suppressed because one or more lines are too long

1
src/model/cloud-runner/providers/k8s/kubernetes-job-spec-factory.ts
View File

@ -61,6 +61,7 @@ class KubernetesJobSpecFactory {
backoffLimit: 0,
template: {
spec: {
terminationGracePeriodSeconds: 90, // Give PreStopHook (60s sleep) time to complete
volumes: [
{
name: 'build-mount',

29
src/model/cloud-runner/providers/k8s/kubernetes-pods.ts
View File

@ -32,6 +32,9 @@ class KubernetesPods {
);
}
let containerExitCode: number | undefined;
let containerSucceeded = false;
if (containerStatuses.length > 0) {
containerStatuses.forEach((cs, idx) => {
if (cs.state?.waiting) {
@ -40,10 +43,15 @@ class KubernetesPods {
);
}
if (cs.state?.terminated) {
const exitCode = cs.state.terminated.exitCode;
containerExitCode = exitCode;
if (exitCode === 0) {
containerSucceeded = true;
}
errorDetails.push(
`Container ${idx} (${cs.name}) terminated: ${cs.state.terminated.reason} - ${
cs.state.terminated.message || ''
} (exit code: ${cs.state.terminated.exitCode})`,
} (exit code: ${exitCode})`,
);
}
});
@ -53,6 +61,25 @@ class KubernetesPods {
errorDetails.push(`Recent events: ${JSON.stringify(events.slice(-5), undefined, 2)}`);
}
// Check if only PreStopHook failed but container succeeded
const hasPreStopHookFailure = events.some((e) => e.reason === 'FailedPreStopHook');
if (containerSucceeded && containerExitCode === 0) {
// Container succeeded - PreStopHook failure is non-critical
if (hasPreStopHookFailure) {
CloudRunnerLogger.logWarning(
`Pod ${podName} marked as Failed due to PreStopHook failure, but container exited successfully (exit code 0). This is non-fatal.`,
);
} else {
CloudRunnerLogger.log(
`Pod ${podName} container succeeded (exit code 0), but pod phase is Failed. Checking details...`,
);
}
CloudRunnerLogger.log(`Pod details: ${errorDetails.join('\n')}`);
// Don't throw error - container succeeded, PreStopHook failure is non-critical
return false; // Pod is not running, but we don't treat it as a failure
}
const errorMessage = `K8s pod failed\n${errorDetails.join('\n')}`;
CloudRunnerLogger.log(errorMessage);
throw new Error(errorMessage);

4
src/model/cloud-runner/services/hooks/container-hook-service.ts
View File

@ -334,6 +334,10 @@ export class ContainerHookService {
if (step.image === undefined) {
step.image = `ubuntu`;
}
// Ensure allowFailure defaults to false if not explicitly set
if (step.allowFailure === undefined) {
step.allowFailure = false;
}
}
if (object === undefined) {
throw new Error(`Failed to parse ${steps}`);

1
src/model/cloud-runner/services/hooks/container-hook.ts
View File

@ -6,4 +6,5 @@ export class ContainerHook {
public name!: string;
public image: string = `ubuntu`;
public hook!: string;
public allowFailure: boolean = false; // If true, hook failures won't stop the build
}

20
src/model/cloud-runner/workflows/custom-workflow.ts
View File

@ -32,7 +32,8 @@ export class CustomWorkflow {
// }
for (const step of steps) {
CloudRunnerLogger.log(`Cloud Runner is running in custom job mode`);
output += await CloudRunner.Provider.runTaskInWorkflow(
try {
const stepOutput = await CloudRunner.Provider.runTaskInWorkflow(
CloudRunner.buildParameters.buildGuid,
step.image,
step.commands,
@ -41,6 +42,23 @@ export class CustomWorkflow {
environmentVariables,
[...secrets, ...step.secrets],
);
output += stepOutput;
} catch (error: any) {
const allowFailure = step.allowFailure === true;
const stepName = step.name || step.image || 'unknown';
if (allowFailure) {
CloudRunnerLogger.logWarning(
`Hook container "${stepName}" failed but allowFailure is true. Continuing build. Error: ${error?.message || error}`,
);
// Continue to next step
} else {
CloudRunnerLogger.log(
`Hook container "${stepName}" failed and allowFailure is false (default). Stopping build.`,
);
throw error;
}
}
}
return output;