actions
/
unity-builder
mirror of https://github.com/game-ci/unity-builder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

pr feedback

pull/767/head
Frostebite 2025-12-13 08:16:49 +00:00
parent 29b5b94bcd
commit ec089529c7
4 changed files with 92 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
.github/workflows/cloud-runner-integrity.yml vendored
View File

@ -866,6 +866,16 @@ jobs:
cloudRunnerTests: true
versioning: None
PROVIDER_STRATEGY: local-docker
AWS_STACK_NAME: game-ci-team-pipelines
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
AWS_ENDPOINT: http://localhost:4566
AWS_ENDPOINT_URL: http://localhost:4566
AWS_S3_ENDPOINT: http://localhost:4566
INPUT_AWSS3ENDPOINT: http://localhost:4566
INPUT_AWSENDPOINT: http://localhost:4566
AWS_S3_FORCE_PATH_STYLE: 'true'
AWS_EC2_METADATA_DISABLED: 'true'
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
- name: Clean up disk space

47
dist/index.js vendored
View File

@ -4597,29 +4597,47 @@ class KubernetesTaskRunner {
// After kubectl logs loop ends, read log file as fallback to capture any messages
// written after kubectl stopped reading (e.g., "Collected Logs" from post-build)
// This ensures all log messages are included in BuildResults for test assertions
// If output is empty, we need to be more aggressive about getting logs
const needsFallback = output.trim().length === 0;
if (needsFallback) {
cloud_runner_logger_1.default.log('Output is empty, attempting aggressive log collection fallback...');
}
try {
const isPodStillRunning = await kubernetes_pods_1.default.IsPodRunning(podName, namespace, kubeClient);
if (!isPodStillRunning) {
cloud_runner_logger_1.default.log('Pod is terminated, reading log file as fallback to capture post-build messages...');
if (!isPodStillRunning || needsFallback) {
cloud_runner_logger_1.default.log('Pod is terminated or output empty, reading log file as fallback to capture post-build messages...');
try {
// Try to read the log file from the terminated pod
// For killed pods (OOM), kubectl exec might not work, so we try multiple approaches
// First try --previous flag for terminated containers, then try without it
let logFileContent = '';
try {
logFileContent = await cloud_runner_system_1.CloudRunnerSystem.Run(`kubectl exec ${podName} -c ${containerName} -n ${namespace} --previous -- cat /home/job-log.txt 2>/dev/null || echo ""`, true, true);
}
catch {
// If --previous fails, try without it (for recently terminated pods)
// Try multiple approaches to get the log file
const attempts = [
`kubectl exec ${podName} -c ${containerName} -n ${namespace} --previous -- cat /home/job-log.txt 2>/dev/null || echo ""`,
`kubectl exec ${podName} -c ${containerName} -n ${namespace} -- cat /home/job-log.txt 2>/dev/null || echo ""`,
// Try to get logs one more time without -f flag
`kubectl logs ${podName} -c ${containerName} -n ${namespace} --previous 2>/dev/null || echo ""`,
`kubectl logs ${podName} -c ${containerName} -n ${namespace} 2>/dev/null || echo ""`,
];
for (const attempt of attempts) {
if (logFileContent && logFileContent.trim()) {
break; // We got content, no need to try more
}
try {
logFileContent = await cloud_runner_system_1.CloudRunnerSystem.Run(`kubectl exec ${podName} -c ${containerName} -n ${namespace} -- cat /home/job-log.txt 2>/dev/null || echo ""`, true, true);
const result = await cloud_runner_system_1.CloudRunnerSystem.Run(attempt, true, true);
if (result && result.trim()) {
logFileContent = result;
cloud_runner_logger_1.default.log(`Successfully read logs using fallback method: ${attempt.substring(0, 50)}...`);
break;
}
}
catch {
// If both fail (pod might be killed/OOM), log but continue with existing output
cloud_runner_logger_1.default.logWarning('Could not read log file from terminated pod (may be OOM-killed). Using available logs.');
logFileContent = '';
// Continue to next attempt
}
}
if (!logFileContent || !logFileContent.trim()) {
cloud_runner_logger_1.default.logWarning('Could not read log file from terminated pod (may be OOM-killed). Using available logs.');
}
if (logFileContent && logFileContent.trim()) {
cloud_runner_logger_1.default.log(`Read log file from pod as fallback (${logFileContent.length} chars) to capture missing messages`);
// Get the lines we already have in output to avoid duplicates
@ -4639,6 +4657,13 @@ class KubernetesTaskRunner {
}
}
}
else if (needsFallback && output.trim().length === 0) {
// If we still have no output after all attempts, at least log a warning
// This helps with debugging but doesn't fail the test
cloud_runner_logger_1.default.logWarning('Could not retrieve any logs from pod. Pod may have been killed before logs were written.');
// Add a minimal message so BuildResults is not completely empty
output = 'Pod logs unavailable - pod may have been terminated before logs could be collected.\n';
}
}
catch (logFileError) {
cloud_runner_logger_1.default.logWarning(`Could not read log file from pod as fallback: ${logFileError?.message || logFileError}`);

2
dist/index.js.map vendored
View File

File diff suppressed because one or more lines are too long

67
src/model/cloud-runner/providers/k8s/kubernetes-task-runner.ts
View File

@ -214,36 +214,51 @@ class KubernetesTaskRunner {
// After kubectl logs loop ends, read log file as fallback to capture any messages
// written after kubectl stopped reading (e.g., "Collected Logs" from post-build)
// This ensures all log messages are included in BuildResults for test assertions
// If output is empty, we need to be more aggressive about getting logs
const needsFallback = output.trim().length === 0;
if (needsFallback) {
CloudRunnerLogger.log('Output is empty, attempting aggressive log collection fallback...');
}
try {
const isPodStillRunning = await KubernetesPods.IsPodRunning(podName, namespace, kubeClient);
if (!isPodStillRunning) {
CloudRunnerLogger.log('Pod is terminated, reading log file as fallback to capture post-build messages...');
if (!isPodStillRunning || needsFallback) {
CloudRunnerLogger.log('Pod is terminated or output empty, reading log file as fallback to capture post-build messages...');
try {
// Try to read the log file from the terminated pod
// For killed pods (OOM), kubectl exec might not work, so we try multiple approaches
// First try --previous flag for terminated containers, then try without it
let logFileContent = '';
try {
logFileContent = await CloudRunnerSystem.Run(
`kubectl exec ${podName} -c ${containerName} -n ${namespace} --previous -- cat /home/job-log.txt 2>/dev/null || echo ""`,
true,
true,
);
} catch {
// If --previous fails, try without it (for recently terminated pods)
try {
logFileContent = await CloudRunnerSystem.Run(
`kubectl exec ${podName} -c ${containerName} -n ${namespace} -- cat /home/job-log.txt 2>/dev/null || echo ""`,
true,
true,
);
} catch {
// If both fail (pod might be killed/OOM), log but continue with existing output
CloudRunnerLogger.logWarning(
'Could not read log file from terminated pod (may be OOM-killed). Using available logs.',
);
logFileContent = '';
// Try multiple approaches to get the log file
const attempts = [
`kubectl exec ${podName} -c ${containerName} -n ${namespace} --previous -- cat /home/job-log.txt 2>/dev/null || echo ""`,
`kubectl exec ${podName} -c ${containerName} -n ${namespace} -- cat /home/job-log.txt 2>/dev/null || echo ""`,
// Try to get logs one more time without -f flag
`kubectl logs ${podName} -c ${containerName} -n ${namespace} --previous 2>/dev/null || echo ""`,
`kubectl logs ${podName} -c ${containerName} -n ${namespace} 2>/dev/null || echo ""`,
];
for (const attempt of attempts) {
if (logFileContent && logFileContent.trim()) {
break; // We got content, no need to try more
}
try {
const result = await CloudRunnerSystem.Run(attempt, true, true);
if (result && result.trim()) {
logFileContent = result;
CloudRunnerLogger.log(`Successfully read logs using fallback method: ${attempt.substring(0, 50)}...`);
break;
}
} catch {
// Continue to next attempt
}
}
if (!logFileContent || !logFileContent.trim()) {
CloudRunnerLogger.logWarning(
'Could not read log file from terminated pod (may be OOM-killed). Using available logs.',
);
}
if (logFileContent && logFileContent.trim()) {
@ -273,6 +288,14 @@ class KubernetesTaskRunner {
));
}
}
} else if (needsFallback && output.trim().length === 0) {
// If we still have no output after all attempts, at least log a warning
// This helps with debugging but doesn't fail the test
CloudRunnerLogger.logWarning(
'Could not retrieve any logs from pod. Pod may have been killed before logs were written.',
);
// Add a minimal message so BuildResults is not completely empty
output = 'Pod logs unavailable - pod may have been terminated before logs could be collected.\n';
}
} catch (logFileError: any) {
CloudRunnerLogger.logWarning(