pr feedback

cloud-runner-develop
Frostebite 2026-01-18 00:42:20 +00:00
parent 5eb19bd235
commit 7d014984cc
2 changed files with 64 additions and 9 deletions

View File

@ -49,26 +49,46 @@ class KubernetesStorage {
public static async watchUntilPVCNotPending(kubeClient: k8s.CoreV1Api, name: string, namespace: string) {
try {
CloudRunnerLogger.log(`watch Until PVC Not Pending ${name} ${namespace}`);
CloudRunnerLogger.log(`${await this.getPVCPhase(kubeClient, name, namespace)}`);
const initialPhase = await this.getPVCPhase(kubeClient, name, namespace);
CloudRunnerLogger.log(`Initial PVC phase: ${initialPhase}`);
// Wait until PVC is NOT Pending (i.e., Bound or Available)
await waitUntil(
async () => {
return (await this.getPVCPhase(kubeClient, name, namespace)) === 'Pending';
const phase = await this.getPVCPhase(kubeClient, name, namespace);
return phase !== 'Pending';
},
{
timeout: 750000,
intervalBetweenAttempts: 15000,
},
);
const finalPhase = await this.getPVCPhase(kubeClient, name, namespace);
CloudRunnerLogger.log(`PVC phase after wait: ${finalPhase}`);
if (finalPhase === 'Pending') {
throw new Error(`PVC ${name} is still Pending after timeout`);
}
} catch (error: any) {
core.error('Failed to watch PVC');
core.error(error.toString());
core.error(
`PVC Body: ${JSON.stringify(
(await kubeClient.readNamespacedPersistentVolumeClaim(name, namespace)).body,
undefined,
4,
)}`,
);
try {
const pvcBody = (await kubeClient.readNamespacedPersistentVolumeClaim(name, namespace)).body;
core.error(
`PVC Body: ${JSON.stringify(
{
phase: pvcBody.status?.phase,
conditions: pvcBody.status?.conditions,
message: pvcBody.status?.message,
},
undefined,
4,
)}`,
);
} catch {
// Ignore PVC read errors
}
throw error;
}
}

View File

@ -577,6 +577,11 @@ class KubernetesTaskRunner {
// Check pod conditions for scheduling issues
if (podStatusDetails?.conditions) {
const allConditions = podStatusDetails.conditions.map(
(c: any) => `${c.type}: ${c.status}${c.reason ? ` (${c.reason})` : ''}${c.message ? ` - ${c.message}` : ''}`,
);
message += `\n\nPod Conditions:\n${allConditions.join('\n')}`;
const unschedulable = podStatusDetails.conditions.find(
(c: any) => c.type === 'PodScheduled' && c.status === 'False',
);
@ -585,6 +590,36 @@ class KubernetesTaskRunner {
unschedulable.message || 'No message'
}`;
}
// Check if pod is assigned to a node
if (podStatusDetails.hostIP) {
message += `\n\nPod assigned to node: ${podStatusDetails.hostIP}`;
} else {
message += `\n\nPod not yet assigned to a node (scheduling pending)`;
}
}
// Check node resources if pod is assigned
if (podStatusDetails.hostIP) {
try {
const nodes = await kubeClient.listNode();
const assignedNode = nodes.body.items.find((n: any) =>
n.status.addresses?.some((a: any) => a.address === podStatusDetails.hostIP)
);
if (assignedNode) {
const allocatable = assignedNode.status.allocatable || {};
const capacity = assignedNode.status.capacity || {};
message += `\n\nNode Resources (${assignedNode.metadata.name}):\n Allocatable CPU: ${allocatable.cpu || 'unknown'}\n Allocatable Memory: ${allocatable.memory || 'unknown'}\n Allocatable Ephemeral Storage: ${allocatable['ephemeral-storage'] || 'unknown'}`;
// Check for taints that might prevent scheduling
if (assignedNode.spec.taints && assignedNode.spec.taints.length > 0) {
const taints = assignedNode.spec.taints.map((t: any) => `${t.key}=${t.value}:${t.effect}`).join(', ');
message += `\n Node Taints: ${taints}`;
}
}
} catch (nodeError) {
// Ignore node check errors
}
}
} catch (podStatusError) {
// Ignore pod status fetch errors