actions
/
unity-builder
mirror of https://github.com/game-ci/unity-builder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

PR feedback

cloud-runner-develop
Frostebite 2025-12-05 18:08:29 +00:00
parent 69731babfc
commit 956b2e4324
8 changed files with 432 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

429
.github/workflows/cloud-runner-integrity.yml vendored
View File

@ -85,58 +85,31 @@ jobs:
# Show available disk space # Show available disk space
df -h df -h
- run: yarn install --frozen-lockfile - run: yarn install --frozen-lockfile
- name: Run K8s tests sequentially - name: Clean up K8s test resources
timeout-minutes: 180
run: | run: |
# List of K8s tests to run sequentially # Clean up K8s resources before each test (only test resources, not system pods)
tests=( echo "Cleaning up K8s test resources..."
"cloud-runner-end2end-caching" # Only clean up resources in default namespace and resources matching our test patterns
"cloud-runner-end2end-retaining" kubectl delete jobs --all --ignore-not-found=true -n default || true
"cloud-runner-hooks" # Delete completed/failed pods in default namespace (not system pods)
) kubectl get pods -n default -o name 2>/dev/null | grep -E "(unity-builder-job-|helper-pod-)" | while read pod; do
kubectl delete "$pod" --ignore-not-found=true || true
failed_tests=() done || true
# Only delete PVCs that match our naming pattern (unity-builder-pvc-*)
for test in "${tests[@]}"; do kubectl get pvc -n default -o name 2>/dev/null | grep "unity-builder-pvc-" | while read pvc; do
echo "=========================================" kubectl delete "$pvc" --ignore-not-found=true || true
echo "Running test: $test" done || true
echo "=========================================" # Only delete secrets that match our naming pattern (build-credentials-*)
kubectl get secrets -n default -o name 2>/dev/null | grep "build-credentials-" | while read secret; do
# Clean up K8s resources before each test kubectl delete "$secret" --ignore-not-found=true || true
echo "Cleaning up K8s resources before test..." done || true
kubectl delete jobs --all --ignore-not-found=true --all-namespaces || true sleep 3
kubectl delete pods --all --ignore-not-found=true --all-namespaces || true # Clean up disk space
kubectl delete pvc --all --ignore-not-found=true --all-namespaces || true rm -rf ./cloud-runner-cache/* || true
kubectl delete secrets --all --ignore-not-found=true --all-namespaces || true docker system prune -f || true
sleep 3 - name: Run cloud-runner-end2end-caching test
timeout-minutes: 60
# Clean up disk space before each test run: yarn run test "cloud-runner-end2end-caching" --detectOpenHandles --forceExit --runInBand
rm -rf ./cloud-runner-cache/* || true
docker system prune -f || true
# Run the test
if yarn run test "$test" --detectOpenHandles --forceExit --runInBand; then
echo "✓ Test $test passed"
else
echo "✗ Test $test failed"
failed_tests+=("$test")
fi
echo ""
done
# Report results
if [ ${#failed_tests[@]} -eq 0 ]; then
echo "========================================="
echo "All tests passed!"
echo "========================================="
exit 0
else
echo "========================================="
echo "Failed tests: ${failed_tests[*]}"
echo "========================================="
exit 1
fi
env: env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
@ -160,6 +133,86 @@ jobs:
AWS_EC2_METADATA_DISABLED: 'true' AWS_EC2_METADATA_DISABLED: 'true'
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up K8s test resources
run: |
kubectl delete jobs --all --ignore-not-found=true -n default || true
kubectl get pods -n default -o name 2>/dev/null | grep -E "(unity-builder-job-|helper-pod-)" | while read pod; do
kubectl delete "$pod" --ignore-not-found=true || true
done || true
kubectl get pvc -n default -o name 2>/dev/null | grep "unity-builder-pvc-" | while read pvc; do
kubectl delete "$pvc" --ignore-not-found=true || true
done || true
kubectl get secrets -n default -o name 2>/dev/null | grep "build-credentials-" | while read secret; do
kubectl delete "$secret" --ignore-not-found=true || true
done || true
sleep 3
rm -rf ./cloud-runner-cache/* || true
docker system prune -f || true
- name: Run cloud-runner-end2end-retaining test
timeout-minutes: 60
run: yarn run test "cloud-runner-end2end-retaining" --detectOpenHandles --forceExit --runInBand
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: k8s
containerCpu: '512'
containerMemory: '512'
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
AWS_S3_ENDPOINT: http://localhost:4566
AWS_ENDPOINT: http://localhost:4566
INPUT_AWSS3ENDPOINT: http://localhost:4566
INPUT_AWSENDPOINT: http://localhost:4566
AWS_S3_FORCE_PATH_STYLE: 'true'
AWS_EC2_METADATA_DISABLED: 'true'
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up K8s test resources
run: |
kubectl delete jobs --all --ignore-not-found=true -n default || true
kubectl get pods -n default -o name 2>/dev/null | grep -E "(unity-builder-job-|helper-pod-)" | while read pod; do
kubectl delete "$pod" --ignore-not-found=true || true
done || true
kubectl get pvc -n default -o name 2>/dev/null | grep "unity-builder-pvc-" | while read pvc; do
kubectl delete "$pvc" --ignore-not-found=true || true
done || true
kubectl get secrets -n default -o name 2>/dev/null | grep "build-credentials-" | while read secret; do
kubectl delete "$secret" --ignore-not-found=true || true
done || true
sleep 3
rm -rf ./cloud-runner-cache/* || true
docker system prune -f || true
- name: Run cloud-runner-hooks test
timeout-minutes: 60
run: yarn run test "cloud-runner-hooks" --detectOpenHandles --forceExit --runInBand
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: k8s
containerCpu: '512'
containerMemory: '512'
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
AWS_S3_ENDPOINT: http://localhost:4566
AWS_ENDPOINT: http://localhost:4566
INPUT_AWSS3ENDPOINT: http://localhost:4566
INPUT_AWSENDPOINT: http://localhost:4566
AWS_S3_FORCE_PATH_STYLE: 'true'
AWS_EC2_METADATA_DISABLED: 'true'
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
localstack: localstack:
name: Cloud Runner Tests (LocalStack) name: Cloud Runner Tests (LocalStack)
runs-on: ubuntu-latest runs-on: ubuntu-latest
@ -188,58 +241,230 @@ jobs:
# Show available disk space # Show available disk space
df -h df -h
- run: yarn install --frozen-lockfile - run: yarn install --frozen-lockfile
- name: Run LocalStack tests sequentially - name: Clean up disk space
timeout-minutes: 300
run: | run: |
# List of LocalStack tests to run sequentially rm -rf ./cloud-runner-cache/* || true
tests=( docker system prune -f || true
"cloud-runner-end2end-locking" df -h
"cloud-runner-end2end-caching" - name: Run cloud-runner-end2end-locking test
"cloud-runner-end2end-retaining" timeout-minutes: 60
"cloud-runner-caching" run: yarn run test "cloud-runner-end2end-locking" --detectOpenHandles --forceExit --runInBand
"cloud-runner-environment" env:
"cloud-runner-image" UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
"cloud-runner-hooks" UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
"cloud-runner-local-persistence" UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
"cloud-runner-locking-core" PROJECT_PATH: test-project
"cloud-runner-locking-get-locked" TARGET_PLATFORM: StandaloneWindows64
) cloudRunnerTests: true
versioning: None
failed_tests=() KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws
for test in "${tests[@]}"; do AWS_ACCESS_KEY_ID: test
echo "=========================================" AWS_SECRET_ACCESS_KEY: test
echo "Running test: $test" AWS_ENDPOINT: http://localhost:4566
echo "=========================================" AWS_ENDPOINT_URL: http://localhost:4566
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
# Clean up disk space before each test GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
rm -rf ./cloud-runner-cache/* || true - name: Clean up disk space
docker system prune -f || true run: |
df -h rm -rf ./cloud-runner-cache/* || true
docker system prune -f || true
# Run the test df -h
if yarn run test "$test" --detectOpenHandles --forceExit --runInBand; then - name: Run cloud-runner-end2end-caching test
echo "✓ Test $test passed" timeout-minutes: 60
else run: yarn run test "cloud-runner-end2end-caching" --detectOpenHandles --forceExit --runInBand
echo "✗ Test $test failed" env:
failed_tests+=("$test") UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
fi UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
echo "" PROJECT_PATH: test-project
done TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
# Report results versioning: None
if [ ${#failed_tests[@]} -eq 0 ]; then KUBE_STORAGE_CLASS: local-path
echo "=========================================" PROVIDER_STRATEGY: aws
echo "All tests passed!" AWS_ACCESS_KEY_ID: test
echo "=========================================" AWS_SECRET_ACCESS_KEY: test
exit 0 AWS_ENDPOINT: http://localhost:4566
else AWS_ENDPOINT_URL: http://localhost:4566
echo "=========================================" GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
echo "Failed tests: ${failed_tests[*]}" GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
echo "=========================================" - name: Clean up disk space
exit 1 run: |
fi rm -rf ./cloud-runner-cache/* || true
docker system prune -f || true
df -h
- name: Run cloud-runner-end2end-retaining test
timeout-minutes: 60
run: yarn run test "cloud-runner-end2end-retaining" --detectOpenHandles --forceExit --runInBand
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
AWS_ENDPOINT: http://localhost:4566
AWS_ENDPOINT_URL: http://localhost:4566
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space
run: |
rm -rf ./cloud-runner-cache/* || true
docker system prune -f || true
df -h
- name: Run cloud-runner-caching test
timeout-minutes: 60
run: yarn run test "cloud-runner-caching" --detectOpenHandles --forceExit --runInBand
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
AWS_ENDPOINT: http://localhost:4566
AWS_ENDPOINT_URL: http://localhost:4566
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space
run: |
rm -rf ./cloud-runner-cache/* || true
docker system prune -f || true
df -h
- name: Run cloud-runner-environment test
timeout-minutes: 60
run: yarn run test "cloud-runner-environment" --detectOpenHandles --forceExit --runInBand
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
AWS_ENDPOINT: http://localhost:4566
AWS_ENDPOINT_URL: http://localhost:4566
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space
run: |
rm -rf ./cloud-runner-cache/* || true
docker system prune -f || true
df -h
- name: Run cloud-runner-image test
timeout-minutes: 60
run: yarn run test "cloud-runner-image" --detectOpenHandles --forceExit --runInBand
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
AWS_ENDPOINT: http://localhost:4566
AWS_ENDPOINT_URL: http://localhost:4566
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space
run: |
rm -rf ./cloud-runner-cache/* || true
docker system prune -f || true
df -h
- name: Run cloud-runner-hooks test
timeout-minutes: 60
run: yarn run test "cloud-runner-hooks" --detectOpenHandles --forceExit --runInBand
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
AWS_ENDPOINT: http://localhost:4566
AWS_ENDPOINT_URL: http://localhost:4566
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space
run: |
rm -rf ./cloud-runner-cache/* || true
docker system prune -f || true
df -h
- name: Run cloud-runner-local-persistence test
timeout-minutes: 60
run: yarn run test "cloud-runner-local-persistence" --detectOpenHandles --forceExit --runInBand
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
AWS_ENDPOINT: http://localhost:4566
AWS_ENDPOINT_URL: http://localhost:4566
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space
run: |
rm -rf ./cloud-runner-cache/* || true
docker system prune -f || true
df -h
- name: Run cloud-runner-locking-core test
timeout-minutes: 60
run: yarn run test "cloud-runner-locking-core" --detectOpenHandles --forceExit --runInBand
env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}
UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }}
PROJECT_PATH: test-project
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
KUBE_STORAGE_CLASS: local-path
PROVIDER_STRATEGY: aws
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
AWS_ENDPOINT: http://localhost:4566
AWS_ENDPOINT_URL: http://localhost:4566
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space
run: |
rm -rf ./cloud-runner-cache/* || true
docker system prune -f || true
df -h
- name: Run cloud-runner-locking-get-locked test
timeout-minutes: 60
run: yarn run test "cloud-runner-locking-get-locked" --detectOpenHandles --forceExit --runInBand
env: env:
UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }}
UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }}

45
dist/index.js vendored
View File

@ -3861,6 +3861,7 @@ class KubernetesJobSpecFactory {
backoffLimit: 0, backoffLimit: 0,
template: { template: {
spec: { spec: {
terminationGracePeriodSeconds: 90,
volumes: [ volumes: [
{ {
name: 'build-mount', name: 'build-mount',
@ -3979,19 +3980,40 @@ class KubernetesPods {
if (conditions.length > 0) { if (conditions.length > 0) {
errorDetails.push(`Conditions: ${JSON.stringify(conditions.map((c) => ({ type: c.type, status: c.status, reason: c.reason, message: c.message })), undefined, 2)}`); errorDetails.push(`Conditions: ${JSON.stringify(conditions.map((c) => ({ type: c.type, status: c.status, reason: c.reason, message: c.message })), undefined, 2)}`);
} }
let containerExitCode;
let containerSucceeded = false;
if (containerStatuses.length > 0) { if (containerStatuses.length > 0) {
containerStatuses.forEach((cs, idx) => { containerStatuses.forEach((cs, idx) => {
if (cs.state?.waiting) { if (cs.state?.waiting) {
errorDetails.push(`Container ${idx} (${cs.name}) waiting: ${cs.state.waiting.reason} - ${cs.state.waiting.message || ''}`); errorDetails.push(`Container ${idx} (${cs.name}) waiting: ${cs.state.waiting.reason} - ${cs.state.waiting.message || ''}`);
} }
if (cs.state?.terminated) { if (cs.state?.terminated) {
errorDetails.push(`Container ${idx} (${cs.name}) terminated: ${cs.state.terminated.reason} - ${cs.state.terminated.message || ''} (exit code: ${cs.state.terminated.exitCode})`); const exitCode = cs.state.terminated.exitCode;
containerExitCode = exitCode;
if (exitCode === 0) {
containerSucceeded = true;
}
errorDetails.push(`Container ${idx} (${cs.name}) terminated: ${cs.state.terminated.reason} - ${cs.state.terminated.message || ''} (exit code: ${exitCode})`);
} }
}); });
} }
if (events.length > 0) { if (events.length > 0) {
errorDetails.push(`Recent events: ${JSON.stringify(events.slice(-5), undefined, 2)}`); errorDetails.push(`Recent events: ${JSON.stringify(events.slice(-5), undefined, 2)}`);
} }
// Check if only PreStopHook failed but container succeeded
const hasPreStopHookFailure = events.some((e) => e.reason === 'FailedPreStopHook');
if (containerSucceeded && containerExitCode === 0) {
// Container succeeded - PreStopHook failure is non-critical
if (hasPreStopHookFailure) {
cloud_runner_logger_1.default.logWarning(`Pod ${podName} marked as Failed due to PreStopHook failure, but container exited successfully (exit code 0). This is non-fatal.`);
}
else {
cloud_runner_logger_1.default.log(`Pod ${podName} container succeeded (exit code 0), but pod phase is Failed. Checking details...`);
}
cloud_runner_logger_1.default.log(`Pod details: ${errorDetails.join('\n')}`);
// Don't throw error - container succeeded, PreStopHook failure is non-critical
return false; // Pod is not running, but we don't treat it as a failure
}
const errorMessage = `K8s pod failed\n${errorDetails.join('\n')}`; const errorMessage = `K8s pod failed\n${errorDetails.join('\n')}`;
cloud_runner_logger_1.default.log(errorMessage); cloud_runner_logger_1.default.log(errorMessage);
throw new Error(errorMessage); throw new Error(errorMessage);
@ -6924,6 +6946,10 @@ class ContainerHookService {
if (step.image === undefined) { if (step.image === undefined) {
step.image = `ubuntu`; step.image = `ubuntu`;
} }
// Ensure allowFailure defaults to false if not explicitly set
if (step.allowFailure === undefined) {
step.allowFailure = false;
}
} }
if (object === undefined) { if (object === undefined) {
throw new Error(`Failed to parse ${steps}`); throw new Error(`Failed to parse ${steps}`);
@ -7296,7 +7322,22 @@ class CustomWorkflow {
// } // }
for (const step of steps) { for (const step of steps) {
cloud_runner_logger_1.default.log(`Cloud Runner is running in custom job mode`); cloud_runner_logger_1.default.log(`Cloud Runner is running in custom job mode`);
output += await cloud_runner_1.default.Provider.runTaskInWorkflow(cloud_runner_1.default.buildParameters.buildGuid, step.image, step.commands, `/${cloud_runner_folders_1.CloudRunnerFolders.buildVolumeFolder}`, `/${cloud_runner_folders_1.CloudRunnerFolders.projectPathAbsolute}/`, environmentVariables, [...secrets, ...step.secrets]); try {
const stepOutput = await cloud_runner_1.default.Provider.runTaskInWorkflow(cloud_runner_1.default.buildParameters.buildGuid, step.image, step.commands, `/${cloud_runner_folders_1.CloudRunnerFolders.buildVolumeFolder}`, `/${cloud_runner_folders_1.CloudRunnerFolders.projectPathAbsolute}/`, environmentVariables, [...secrets, ...step.secrets]);
output += stepOutput;
}
catch (error) {
const allowFailure = step.allowFailure === true;
const stepName = step.name || step.image || 'unknown';
if (allowFailure) {
cloud_runner_logger_1.default.logWarning(`Hook container "${stepName}" failed but allowFailure is true. Continuing build. Error: ${error?.message || error}`);
// Continue to next step
}
else {
cloud_runner_logger_1.default.log(`Hook container "${stepName}" failed and allowFailure is false (default). Stopping build.`);
throw error;
}
}
} }
return output; return output;
} }

2
dist/index.js.map vendored
View File

File diff suppressed because one or more lines are too long

1
src/model/cloud-runner/providers/k8s/kubernetes-job-spec-factory.ts
View File

@ -61,6 +61,7 @@ class KubernetesJobSpecFactory {
backoffLimit: 0, backoffLimit: 0,
template: { template: {
spec: { spec: {
terminationGracePeriodSeconds: 90, // Give PreStopHook (60s sleep) time to complete
volumes: [ volumes: [
{ {
name: 'build-mount', name: 'build-mount',

29
src/model/cloud-runner/providers/k8s/kubernetes-pods.ts
View File

@ -32,6 +32,9 @@ class KubernetesPods {
); );
} }
let containerExitCode: number | undefined;
let containerSucceeded = false;
if (containerStatuses.length > 0) { if (containerStatuses.length > 0) {
containerStatuses.forEach((cs, idx) => { containerStatuses.forEach((cs, idx) => {
if (cs.state?.waiting) { if (cs.state?.waiting) {
@ -40,10 +43,15 @@ class KubernetesPods {
); );
} }
if (cs.state?.terminated) { if (cs.state?.terminated) {
const exitCode = cs.state.terminated.exitCode;
containerExitCode = exitCode;
if (exitCode === 0) {
containerSucceeded = true;
}
errorDetails.push( errorDetails.push(
`Container ${idx} (${cs.name}) terminated: ${cs.state.terminated.reason} - ${ `Container ${idx} (${cs.name}) terminated: ${cs.state.terminated.reason} - ${
cs.state.terminated.message || '' cs.state.terminated.message || ''
} (exit code: ${cs.state.terminated.exitCode})`, } (exit code: ${exitCode})`,
); );
} }
}); });
@ -53,6 +61,25 @@ class KubernetesPods {
errorDetails.push(`Recent events: ${JSON.stringify(events.slice(-5), undefined, 2)}`); errorDetails.push(`Recent events: ${JSON.stringify(events.slice(-5), undefined, 2)}`);
} }
// Check if only PreStopHook failed but container succeeded
const hasPreStopHookFailure = events.some((e) => e.reason === 'FailedPreStopHook');
if (containerSucceeded && containerExitCode === 0) {
// Container succeeded - PreStopHook failure is non-critical
if (hasPreStopHookFailure) {
CloudRunnerLogger.logWarning(
`Pod ${podName} marked as Failed due to PreStopHook failure, but container exited successfully (exit code 0). This is non-fatal.`,
);
} else {
CloudRunnerLogger.log(
`Pod ${podName} container succeeded (exit code 0), but pod phase is Failed. Checking details...`,
);
}
CloudRunnerLogger.log(`Pod details: ${errorDetails.join('\n')}`);
// Don't throw error - container succeeded, PreStopHook failure is non-critical
return false; // Pod is not running, but we don't treat it as a failure
}
const errorMessage = `K8s pod failed\n${errorDetails.join('\n')}`; const errorMessage = `K8s pod failed\n${errorDetails.join('\n')}`;
CloudRunnerLogger.log(errorMessage); CloudRunnerLogger.log(errorMessage);
throw new Error(errorMessage); throw new Error(errorMessage);

4
src/model/cloud-runner/services/hooks/container-hook-service.ts
View File

@ -334,6 +334,10 @@ export class ContainerHookService {
if (step.image === undefined) { if (step.image === undefined) {
step.image = `ubuntu`; step.image = `ubuntu`;
} }
// Ensure allowFailure defaults to false if not explicitly set
if (step.allowFailure === undefined) {
step.allowFailure = false;
}
} }
if (object === undefined) { if (object === undefined) {
throw new Error(`Failed to parse ${steps}`); throw new Error(`Failed to parse ${steps}`);

1
src/model/cloud-runner/services/hooks/container-hook.ts
View File

@ -6,4 +6,5 @@ export class ContainerHook {
public name!: string; public name!: string;
public image: string = `ubuntu`; public image: string = `ubuntu`;
public hook!: string; public hook!: string;
public allowFailure: boolean = false; // If true, hook failures won't stop the build
} }

36
src/model/cloud-runner/workflows/custom-workflow.ts
View File

@ -32,15 +32,33 @@ export class CustomWorkflow {
// } // }
for (const step of steps) { for (const step of steps) {
CloudRunnerLogger.log(`Cloud Runner is running in custom job mode`); CloudRunnerLogger.log(`Cloud Runner is running in custom job mode`);
output += await CloudRunner.Provider.runTaskInWorkflow( try {
CloudRunner.buildParameters.buildGuid, const stepOutput = await CloudRunner.Provider.runTaskInWorkflow(
step.image, CloudRunner.buildParameters.buildGuid,
step.commands, step.image,
`/${CloudRunnerFolders.buildVolumeFolder}`, step.commands,
`/${CloudRunnerFolders.projectPathAbsolute}/`, `/${CloudRunnerFolders.buildVolumeFolder}`,
environmentVariables, `/${CloudRunnerFolders.projectPathAbsolute}/`,
[...secrets, ...step.secrets], environmentVariables,
); [...secrets, ...step.secrets],
);
output += stepOutput;
} catch (error: any) {
const allowFailure = step.allowFailure === true;
const stepName = step.name || step.image || 'unknown';
if (allowFailure) {
CloudRunnerLogger.logWarning(
`Hook container "${stepName}" failed but allowFailure is true. Continuing build. Error: ${error?.message || error}`,
);
// Continue to next step
} else {
CloudRunnerLogger.log(
`Hook container "${stepName}" failed and allowFailure is false (default). Stopping build.`,
);
throw error;
}
}
} }
return output; return output;