pr feedback
parent
16401bc381
commit
896e8fb7e8
|
|
@ -4578,18 +4578,37 @@ class KubernetesStorage {
|
||||||
static async watchUntilPVCNotPending(kubeClient, name, namespace) {
|
static async watchUntilPVCNotPending(kubeClient, name, namespace) {
|
||||||
try {
|
try {
|
||||||
cloud_runner_logger_1.default.log(`watch Until PVC Not Pending ${name} ${namespace}`);
|
cloud_runner_logger_1.default.log(`watch Until PVC Not Pending ${name} ${namespace}`);
|
||||||
cloud_runner_logger_1.default.log(`${await this.getPVCPhase(kubeClient, name, namespace)}`);
|
const initialPhase = await this.getPVCPhase(kubeClient, name, namespace);
|
||||||
|
cloud_runner_logger_1.default.log(`Initial PVC phase: ${initialPhase}`);
|
||||||
|
// Wait until PVC is NOT Pending (i.e., Bound or Available)
|
||||||
await (0, async_wait_until_1.waitUntil)(async () => {
|
await (0, async_wait_until_1.waitUntil)(async () => {
|
||||||
return (await this.getPVCPhase(kubeClient, name, namespace)) === 'Pending';
|
const phase = await this.getPVCPhase(kubeClient, name, namespace);
|
||||||
|
return phase !== 'Pending';
|
||||||
}, {
|
}, {
|
||||||
timeout: 750000,
|
timeout: 750000,
|
||||||
intervalBetweenAttempts: 15000,
|
intervalBetweenAttempts: 15000,
|
||||||
});
|
});
|
||||||
|
const finalPhase = await this.getPVCPhase(kubeClient, name, namespace);
|
||||||
|
cloud_runner_logger_1.default.log(`PVC phase after wait: ${finalPhase}`);
|
||||||
|
if (finalPhase === 'Pending') {
|
||||||
|
throw new Error(`PVC ${name} is still Pending after timeout`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
catch (error) {
|
catch (error) {
|
||||||
core.error('Failed to watch PVC');
|
core.error('Failed to watch PVC');
|
||||||
core.error(error.toString());
|
core.error(error.toString());
|
||||||
core.error(`PVC Body: ${JSON.stringify((await kubeClient.readNamespacedPersistentVolumeClaim(name, namespace)).body, undefined, 4)}`);
|
try {
|
||||||
|
const pvcBody = (await kubeClient.readNamespacedPersistentVolumeClaim(name, namespace)).body;
|
||||||
|
core.error(`PVC Body: ${JSON.stringify({
|
||||||
|
phase: pvcBody.status?.phase,
|
||||||
|
conditions: pvcBody.status?.conditions,
|
||||||
|
accessModes: pvcBody.spec?.accessModes,
|
||||||
|
storageClassName: pvcBody.spec?.storageClassName,
|
||||||
|
}, undefined, 4)}`);
|
||||||
|
}
|
||||||
|
catch {
|
||||||
|
// Ignore PVC read errors
|
||||||
|
}
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -5074,10 +5093,40 @@ class KubernetesTaskRunner {
|
||||||
}
|
}
|
||||||
// Check pod conditions for scheduling issues
|
// Check pod conditions for scheduling issues
|
||||||
if (podStatusDetails?.conditions) {
|
if (podStatusDetails?.conditions) {
|
||||||
|
const allConditions = podStatusDetails.conditions.map((c) => `${c.type}: ${c.status}${c.reason ? ` (${c.reason})` : ''}${c.message ? ` - ${c.message}` : ''}`);
|
||||||
|
message += `\n\nPod Conditions:\n${allConditions.join('\n')}`;
|
||||||
const unschedulable = podStatusDetails.conditions.find((c) => c.type === 'PodScheduled' && c.status === 'False');
|
const unschedulable = podStatusDetails.conditions.find((c) => c.type === 'PodScheduled' && c.status === 'False');
|
||||||
if (unschedulable) {
|
if (unschedulable) {
|
||||||
message += `\n\nScheduling Issue: ${unschedulable.reason || 'Unknown'} - ${unschedulable.message || 'No message'}`;
|
message += `\n\nScheduling Issue: ${unschedulable.reason || 'Unknown'} - ${unschedulable.message || 'No message'}`;
|
||||||
}
|
}
|
||||||
|
// Check if pod is assigned to a node
|
||||||
|
if (podStatusDetails?.hostIP) {
|
||||||
|
message += `\n\nPod assigned to node: ${podStatusDetails.hostIP}`;
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
message += `\n\nPod not yet assigned to a node (scheduling pending)`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Check node resources if pod is assigned
|
||||||
|
if (podStatusDetails?.hostIP) {
|
||||||
|
try {
|
||||||
|
const nodes = await kubeClient.listNode();
|
||||||
|
const hostIP = podStatusDetails.hostIP;
|
||||||
|
const assignedNode = nodes.body.items.find((n) => n.status?.addresses?.some((a) => a.address === hostIP));
|
||||||
|
if (assignedNode?.status && assignedNode.metadata?.name) {
|
||||||
|
const allocatable = assignedNode.status.allocatable || {};
|
||||||
|
const capacity = assignedNode.status.capacity || {};
|
||||||
|
message += `\n\nNode Resources (${assignedNode.metadata.name}):\n Allocatable CPU: ${allocatable.cpu || 'unknown'}\n Allocatable Memory: ${allocatable.memory || 'unknown'}\n Allocatable Ephemeral Storage: ${allocatable['ephemeral-storage'] || 'unknown'}`;
|
||||||
|
// Check for taints that might prevent scheduling
|
||||||
|
if (assignedNode.spec?.taints && assignedNode.spec.taints.length > 0) {
|
||||||
|
const taints = assignedNode.spec.taints.map((t) => `${t.key}=${t.value}:${t.effect}`).join(', ');
|
||||||
|
message += `\n Node Taints: ${taints}`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (nodeError) {
|
||||||
|
// Ignore node check errors
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
catch (podStatusError) {
|
catch (podStatusError) {
|
||||||
|
|
|
||||||
File diff suppressed because one or more lines are too long
|
|
@ -592,7 +592,7 @@ class KubernetesTaskRunner {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if pod is assigned to a node
|
// Check if pod is assigned to a node
|
||||||
if (podStatusDetails.hostIP) {
|
if (podStatusDetails?.hostIP) {
|
||||||
message += `\n\nPod assigned to node: ${podStatusDetails.hostIP}`;
|
message += `\n\nPod assigned to node: ${podStatusDetails.hostIP}`;
|
||||||
} else {
|
} else {
|
||||||
message += `\n\nPod not yet assigned to a node (scheduling pending)`;
|
message += `\n\nPod not yet assigned to a node (scheduling pending)`;
|
||||||
|
|
@ -600,19 +600,20 @@ class KubernetesTaskRunner {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check node resources if pod is assigned
|
// Check node resources if pod is assigned
|
||||||
if (podStatusDetails.hostIP) {
|
if (podStatusDetails?.hostIP) {
|
||||||
try {
|
try {
|
||||||
const nodes = await kubeClient.listNode();
|
const nodes = await kubeClient.listNode();
|
||||||
|
const hostIP = podStatusDetails.hostIP;
|
||||||
const assignedNode = nodes.body.items.find((n: any) =>
|
const assignedNode = nodes.body.items.find((n: any) =>
|
||||||
n.status.addresses?.some((a: any) => a.address === podStatusDetails.hostIP)
|
n.status?.addresses?.some((a: any) => a.address === hostIP)
|
||||||
);
|
);
|
||||||
if (assignedNode) {
|
if (assignedNode?.status && assignedNode.metadata?.name) {
|
||||||
const allocatable = assignedNode.status.allocatable || {};
|
const allocatable = assignedNode.status.allocatable || {};
|
||||||
const capacity = assignedNode.status.capacity || {};
|
const capacity = assignedNode.status.capacity || {};
|
||||||
message += `\n\nNode Resources (${assignedNode.metadata.name}):\n Allocatable CPU: ${allocatable.cpu || 'unknown'}\n Allocatable Memory: ${allocatable.memory || 'unknown'}\n Allocatable Ephemeral Storage: ${allocatable['ephemeral-storage'] || 'unknown'}`;
|
message += `\n\nNode Resources (${assignedNode.metadata.name}):\n Allocatable CPU: ${allocatable.cpu || 'unknown'}\n Allocatable Memory: ${allocatable.memory || 'unknown'}\n Allocatable Ephemeral Storage: ${allocatable['ephemeral-storage'] || 'unknown'}`;
|
||||||
|
|
||||||
// Check for taints that might prevent scheduling
|
// Check for taints that might prevent scheduling
|
||||||
if (assignedNode.spec.taints && assignedNode.spec.taints.length > 0) {
|
if (assignedNode.spec?.taints && assignedNode.spec.taints.length > 0) {
|
||||||
const taints = assignedNode.spec.taints.map((t: any) => `${t.key}=${t.value}:${t.effect}`).join(', ');
|
const taints = assignedNode.spec.taints.map((t: any) => `${t.key}=${t.value}:${t.effect}`).join(', ');
|
||||||
message += `\n Node Taints: ${taints}`;
|
message += `\n Node Taints: ${taints}`;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue