pr feedback
parent
6f413e1f6a
commit
6a4ee1417d
|
|
@ -3968,10 +3968,26 @@ class KubernetesJobSpecFactory {
|
||||||
],
|
],
|
||||||
workingDir: `${workingDirectory}`,
|
workingDir: `${workingDirectory}`,
|
||||||
resources: {
|
resources: {
|
||||||
requests: {
|
requests: (() => {
|
||||||
memory: `${Number.parseInt(buildParameters.containerMemory) / 1024}G` || '750M',
|
// Use smaller resource requests for lightweight hook containers
|
||||||
cpu: Number.parseInt(buildParameters.containerCpu) / 1024 || '1',
|
// Hook containers typically use utility images like aws-cli, rclone, etc.
|
||||||
},
|
const lightweightImages = ['amazon/aws-cli', 'rclone/rclone', 'steamcmd/steamcmd', 'ubuntu'];
|
||||||
|
const isLightweightContainer = lightweightImages.some((lightImage) => image.includes(lightImage));
|
||||||
|
if (isLightweightContainer && process.env['cloudRunnerTests'] === 'true') {
|
||||||
|
// For test environments, use minimal resources for hook containers
|
||||||
|
return {
|
||||||
|
memory: '128Mi',
|
||||||
|
cpu: '100m', // 0.1 CPU
|
||||||
|
};
|
||||||
|
}
|
||||||
|
// For main build containers, use the configured resources
|
||||||
|
const memoryMB = Number.parseInt(buildParameters.containerMemory);
|
||||||
|
const cpuMB = Number.parseInt(buildParameters.containerCpu);
|
||||||
|
return {
|
||||||
|
memory: !isNaN(memoryMB) && memoryMB > 0 ? `${memoryMB / 1024}G` : '750M',
|
||||||
|
cpu: !isNaN(cpuMB) && cpuMB > 0 ? `${cpuMB / 1024}` : '1',
|
||||||
|
};
|
||||||
|
})(),
|
||||||
},
|
},
|
||||||
env: [
|
env: [
|
||||||
...adjustedEnvironment.map((x) => {
|
...adjustedEnvironment.map((x) => {
|
||||||
|
|
@ -4920,11 +4936,39 @@ class KubernetesTaskRunner {
|
||||||
const events = await kubeClient.listNamespacedEvent(namespace);
|
const events = await kubeClient.listNamespacedEvent(namespace);
|
||||||
const podEvents = events.body.items
|
const podEvents = events.body.items
|
||||||
.filter((x) => x.involvedObject?.name === podName)
|
.filter((x) => x.involvedObject?.name === podName)
|
||||||
.slice(-5)
|
.slice(-10)
|
||||||
.map((x) => `${x.type}: ${x.reason} - ${x.message}`);
|
.map((x) => `${x.type}: ${x.reason} - ${x.message}`);
|
||||||
if (podEvents.length > 0) {
|
if (podEvents.length > 0) {
|
||||||
message += `\n\nRecent Events:\n${podEvents.join('\n')}`;
|
message += `\n\nRecent Events:\n${podEvents.join('\n')}`;
|
||||||
}
|
}
|
||||||
|
// Get pod details to check for scheduling issues
|
||||||
|
try {
|
||||||
|
const podStatus = await kubeClient.readNamespacedPodStatus(podName, namespace);
|
||||||
|
const podSpec = podStatus.body.spec;
|
||||||
|
const podStatusDetails = podStatus.body.status;
|
||||||
|
// Check container resource requests
|
||||||
|
if (podSpec?.containers?.[0]?.resources?.requests) {
|
||||||
|
const requests = podSpec.containers[0].resources.requests;
|
||||||
|
message += `\n\nContainer Resource Requests:\n CPU: ${requests.cpu || 'not set'}\n Memory: ${requests.memory || 'not set'}\n Ephemeral Storage: ${requests['ephemeral-storage'] || 'not set'}`;
|
||||||
|
}
|
||||||
|
// Check node selector and tolerations
|
||||||
|
if (podSpec?.nodeSelector && Object.keys(podSpec.nodeSelector).length > 0) {
|
||||||
|
message += `\n\nNode Selector: ${JSON.stringify(podSpec.nodeSelector)}`;
|
||||||
|
}
|
||||||
|
if (podSpec?.tolerations && podSpec.tolerations.length > 0) {
|
||||||
|
message += `\n\nTolerations: ${JSON.stringify(podSpec.tolerations)}`;
|
||||||
|
}
|
||||||
|
// Check pod conditions for scheduling issues
|
||||||
|
if (podStatusDetails?.conditions) {
|
||||||
|
const unschedulable = podStatusDetails.conditions.find((c) => c.type === 'PodScheduled' && c.status === 'False');
|
||||||
|
if (unschedulable) {
|
||||||
|
message += `\n\nScheduling Issue: ${unschedulable.reason || 'Unknown'} - ${unschedulable.message || 'No message'}`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (podStatusError) {
|
||||||
|
// Ignore pod status fetch errors
|
||||||
|
}
|
||||||
}
|
}
|
||||||
catch {
|
catch {
|
||||||
// Ignore event fetch errors
|
// Ignore event fetch errors
|
||||||
|
|
|
||||||
File diff suppressed because one or more lines are too long
|
|
@ -90,10 +90,28 @@ class KubernetesJobSpecFactory {
|
||||||
|
|
||||||
workingDir: `${workingDirectory}`,
|
workingDir: `${workingDirectory}`,
|
||||||
resources: {
|
resources: {
|
||||||
requests: {
|
requests: (() => {
|
||||||
memory: `${Number.parseInt(buildParameters.containerMemory) / 1024}G` || '750M',
|
// Use smaller resource requests for lightweight hook containers
|
||||||
cpu: Number.parseInt(buildParameters.containerCpu) / 1024 || '1',
|
// Hook containers typically use utility images like aws-cli, rclone, etc.
|
||||||
},
|
const lightweightImages = ['amazon/aws-cli', 'rclone/rclone', 'steamcmd/steamcmd', 'ubuntu'];
|
||||||
|
const isLightweightContainer = lightweightImages.some((lightImage) => image.includes(lightImage));
|
||||||
|
|
||||||
|
if (isLightweightContainer && process.env['cloudRunnerTests'] === 'true') {
|
||||||
|
// For test environments, use minimal resources for hook containers
|
||||||
|
return {
|
||||||
|
memory: '128Mi',
|
||||||
|
cpu: '100m', // 0.1 CPU
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// For main build containers, use the configured resources
|
||||||
|
const memoryMB = Number.parseInt(buildParameters.containerMemory);
|
||||||
|
const cpuMB = Number.parseInt(buildParameters.containerCpu);
|
||||||
|
return {
|
||||||
|
memory: !isNaN(memoryMB) && memoryMB > 0 ? `${memoryMB / 1024}G` : '750M',
|
||||||
|
cpu: !isNaN(cpuMB) && cpuMB > 0 ? `${cpuMB / 1024}` : '1',
|
||||||
|
};
|
||||||
|
})(),
|
||||||
},
|
},
|
||||||
env: [
|
env: [
|
||||||
...adjustedEnvironment.map((x) => {
|
...adjustedEnvironment.map((x) => {
|
||||||
|
|
|
||||||
|
|
@ -544,11 +544,42 @@ class KubernetesTaskRunner {
|
||||||
const events = await kubeClient.listNamespacedEvent(namespace);
|
const events = await kubeClient.listNamespacedEvent(namespace);
|
||||||
const podEvents = events.body.items
|
const podEvents = events.body.items
|
||||||
.filter((x) => x.involvedObject?.name === podName)
|
.filter((x) => x.involvedObject?.name === podName)
|
||||||
.slice(-5)
|
.slice(-10)
|
||||||
.map((x) => `${x.type}: ${x.reason} - ${x.message}`);
|
.map((x) => `${x.type}: ${x.reason} - ${x.message}`);
|
||||||
if (podEvents.length > 0) {
|
if (podEvents.length > 0) {
|
||||||
message += `\n\nRecent Events:\n${podEvents.join('\n')}`;
|
message += `\n\nRecent Events:\n${podEvents.join('\n')}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Get pod details to check for scheduling issues
|
||||||
|
try {
|
||||||
|
const podStatus = await kubeClient.readNamespacedPodStatus(podName, namespace);
|
||||||
|
const podSpec = podStatus.body.spec;
|
||||||
|
const podStatusDetails = podStatus.body.status;
|
||||||
|
|
||||||
|
// Check container resource requests
|
||||||
|
if (podSpec?.containers?.[0]?.resources?.requests) {
|
||||||
|
const requests = podSpec.containers[0].resources.requests;
|
||||||
|
message += `\n\nContainer Resource Requests:\n CPU: ${requests.cpu || 'not set'}\n Memory: ${requests.memory || 'not set'}\n Ephemeral Storage: ${requests['ephemeral-storage'] || 'not set'}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check node selector and tolerations
|
||||||
|
if (podSpec?.nodeSelector && Object.keys(podSpec.nodeSelector).length > 0) {
|
||||||
|
message += `\n\nNode Selector: ${JSON.stringify(podSpec.nodeSelector)}`;
|
||||||
|
}
|
||||||
|
if (podSpec?.tolerations && podSpec.tolerations.length > 0) {
|
||||||
|
message += `\n\nTolerations: ${JSON.stringify(podSpec.tolerations)}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check pod conditions for scheduling issues
|
||||||
|
if (podStatusDetails?.conditions) {
|
||||||
|
const unschedulable = podStatusDetails.conditions.find((c: any) => c.type === 'PodScheduled' && c.status === 'False');
|
||||||
|
if (unschedulable) {
|
||||||
|
message += `\n\nScheduling Issue: ${unschedulable.reason || 'Unknown'} - ${unschedulable.message || 'No message'}`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (podStatusError) {
|
||||||
|
// Ignore pod status fetch errors
|
||||||
|
}
|
||||||
} catch {
|
} catch {
|
||||||
// Ignore event fetch errors
|
// Ignore event fetch errors
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue