pr feedback
parent
5ff53ae347
commit
b98b1c7104
|
|
@ -5729,11 +5729,76 @@ class Caching {
|
|||
: latestInBranch;
|
||||
await cloud_runner_logger_1.default.log(`cache key ${cacheArtifactName} selection ${cacheSelection}`);
|
||||
if (await fileExists(`${cacheSelection}.tar${compressionSuffix}`)) {
|
||||
// Check disk space before extraction to prevent hangs
|
||||
let diskUsagePercent = 0;
|
||||
try {
|
||||
const diskCheckOutput = await cloud_runner_system_1.CloudRunnerSystem.Run(`df . 2>/dev/null || df /data 2>/dev/null || true`);
|
||||
const usageMatch = diskCheckOutput.match(/(\d+)%/);
|
||||
if (usageMatch) {
|
||||
diskUsagePercent = Number.parseInt(usageMatch[1], 10);
|
||||
}
|
||||
}
|
||||
catch {
|
||||
// Ignore disk check errors
|
||||
}
|
||||
// If disk is at 100%, skip cache extraction to prevent hangs
|
||||
if (diskUsagePercent >= 100) {
|
||||
const message = `Disk is at ${diskUsagePercent}% - skipping cache extraction to prevent hang. Cache may be incomplete or corrupted.`;
|
||||
cloud_runner_logger_1.default.logWarning(message);
|
||||
remote_client_logger_1.RemoteClientLogger.logWarning(message);
|
||||
// Continue without cache - build will proceed without cached Library
|
||||
process.chdir(startPath);
|
||||
return;
|
||||
}
|
||||
// Validate tar file integrity before extraction
|
||||
try {
|
||||
// Use tar -t to test the archive without extracting (fast check)
|
||||
// This will fail if the archive is corrupted
|
||||
await cloud_runner_system_1.CloudRunnerSystem.Run(`tar -tf ${cacheSelection}.tar${compressionSuffix} > /dev/null 2>&1 || (echo "Tar file validation failed" && exit 1)`);
|
||||
}
|
||||
catch (validationError) {
|
||||
const message = `Cache archive ${cacheSelection}.tar${compressionSuffix} appears to be corrupted or incomplete. Skipping cache extraction.`;
|
||||
cloud_runner_logger_1.default.logWarning(message);
|
||||
remote_client_logger_1.RemoteClientLogger.logWarning(message);
|
||||
// Continue without cache - build will proceed without cached Library
|
||||
process.chdir(startPath);
|
||||
return;
|
||||
}
|
||||
const resultsFolder = `results${cloud_runner_1.default.buildParameters.buildGuid}`;
|
||||
await cloud_runner_system_1.CloudRunnerSystem.Run(`mkdir -p ${resultsFolder}`);
|
||||
remote_client_logger_1.RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.tar${compressionSuffix}`);
|
||||
const fullResultsFolder = node_path_1.default.join(cacheFolder, resultsFolder);
|
||||
await cloud_runner_system_1.CloudRunnerSystem.Run(`tar -xf ${cacheSelection}.tar${compressionSuffix} -C ${fullResultsFolder}`);
|
||||
// Extract with timeout to prevent infinite hangs
|
||||
try {
|
||||
let tarExtractCommand = `tar -xf ${cacheSelection}.tar${compressionSuffix} -C ${fullResultsFolder}`;
|
||||
// Add timeout if available (600 seconds = 10 minutes)
|
||||
try {
|
||||
await cloud_runner_system_1.CloudRunnerSystem.Run(`which timeout > /dev/null 2>&1`, true, true);
|
||||
tarExtractCommand = `timeout 600 ${tarExtractCommand}`;
|
||||
}
|
||||
catch {
|
||||
// timeout command not available, use regular tar
|
||||
}
|
||||
await cloud_runner_system_1.CloudRunnerSystem.Run(tarExtractCommand);
|
||||
}
|
||||
catch (extractError) {
|
||||
const errorMessage = extractError?.message || extractError?.toString() || '';
|
||||
// Check for common tar errors that indicate corruption or disk issues
|
||||
if (errorMessage.includes('Unexpected EOF') ||
|
||||
errorMessage.includes('rmtlseek') ||
|
||||
errorMessage.includes('No space left') ||
|
||||
errorMessage.includes('timeout') ||
|
||||
errorMessage.includes('Terminated')) {
|
||||
const message = `Cache extraction failed (likely due to corrupted archive or disk space): ${errorMessage}. Continuing without cache.`;
|
||||
cloud_runner_logger_1.default.logWarning(message);
|
||||
remote_client_logger_1.RemoteClientLogger.logWarning(message);
|
||||
// Continue without cache - build will proceed without cached Library
|
||||
process.chdir(startPath);
|
||||
return;
|
||||
}
|
||||
// Re-throw other errors
|
||||
throw extractError;
|
||||
}
|
||||
remote_client_logger_1.RemoteClientLogger.log(`cache item extracted to ${fullResultsFolder}`);
|
||||
(0, node_console_1.assert)(await fileExists(fullResultsFolder), `cache extraction results folder exists`);
|
||||
const destinationParentFolder = node_path_1.default.resolve(destinationFolder, '..');
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -348,11 +348,82 @@ export class Caching {
|
|||
await CloudRunnerLogger.log(`cache key ${cacheArtifactName} selection ${cacheSelection}`);
|
||||
|
||||
if (await fileExists(`${cacheSelection}.tar${compressionSuffix}`)) {
|
||||
// Check disk space before extraction to prevent hangs
|
||||
let diskUsagePercent = 0;
|
||||
try {
|
||||
const diskCheckOutput = await CloudRunnerSystem.Run(`df . 2>/dev/null || df /data 2>/dev/null || true`);
|
||||
const usageMatch = diskCheckOutput.match(/(\d+)%/);
|
||||
if (usageMatch) {
|
||||
diskUsagePercent = Number.parseInt(usageMatch[1], 10);
|
||||
}
|
||||
} catch {
|
||||
// Ignore disk check errors
|
||||
}
|
||||
|
||||
// If disk is at 100%, skip cache extraction to prevent hangs
|
||||
if (diskUsagePercent >= 100) {
|
||||
const message = `Disk is at ${diskUsagePercent}% - skipping cache extraction to prevent hang. Cache may be incomplete or corrupted.`;
|
||||
CloudRunnerLogger.logWarning(message);
|
||||
RemoteClientLogger.logWarning(message);
|
||||
// Continue without cache - build will proceed without cached Library
|
||||
process.chdir(startPath);
|
||||
return;
|
||||
}
|
||||
|
||||
// Validate tar file integrity before extraction
|
||||
try {
|
||||
// Use tar -t to test the archive without extracting (fast check)
|
||||
// This will fail if the archive is corrupted
|
||||
await CloudRunnerSystem.Run(
|
||||
`tar -tf ${cacheSelection}.tar${compressionSuffix} > /dev/null 2>&1 || (echo "Tar file validation failed" && exit 1)`,
|
||||
);
|
||||
} catch (validationError) {
|
||||
const message = `Cache archive ${cacheSelection}.tar${compressionSuffix} appears to be corrupted or incomplete. Skipping cache extraction.`;
|
||||
CloudRunnerLogger.logWarning(message);
|
||||
RemoteClientLogger.logWarning(message);
|
||||
// Continue without cache - build will proceed without cached Library
|
||||
process.chdir(startPath);
|
||||
return;
|
||||
}
|
||||
|
||||
const resultsFolder = `results${CloudRunner.buildParameters.buildGuid}`;
|
||||
await CloudRunnerSystem.Run(`mkdir -p ${resultsFolder}`);
|
||||
RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.tar${compressionSuffix}`);
|
||||
const fullResultsFolder = path.join(cacheFolder, resultsFolder);
|
||||
await CloudRunnerSystem.Run(`tar -xf ${cacheSelection}.tar${compressionSuffix} -C ${fullResultsFolder}`);
|
||||
|
||||
// Extract with timeout to prevent infinite hangs
|
||||
try {
|
||||
let tarExtractCommand = `tar -xf ${cacheSelection}.tar${compressionSuffix} -C ${fullResultsFolder}`;
|
||||
// Add timeout if available (600 seconds = 10 minutes)
|
||||
try {
|
||||
await CloudRunnerSystem.Run(`which timeout > /dev/null 2>&1`, true, true);
|
||||
tarExtractCommand = `timeout 600 ${tarExtractCommand}`;
|
||||
} catch {
|
||||
// timeout command not available, use regular tar
|
||||
}
|
||||
|
||||
await CloudRunnerSystem.Run(tarExtractCommand);
|
||||
} catch (extractError: any) {
|
||||
const errorMessage = extractError?.message || extractError?.toString() || '';
|
||||
// Check for common tar errors that indicate corruption or disk issues
|
||||
if (
|
||||
errorMessage.includes('Unexpected EOF') ||
|
||||
errorMessage.includes('rmtlseek') ||
|
||||
errorMessage.includes('No space left') ||
|
||||
errorMessage.includes('timeout') ||
|
||||
errorMessage.includes('Terminated')
|
||||
) {
|
||||
const message = `Cache extraction failed (likely due to corrupted archive or disk space): ${errorMessage}. Continuing without cache.`;
|
||||
CloudRunnerLogger.logWarning(message);
|
||||
RemoteClientLogger.logWarning(message);
|
||||
// Continue without cache - build will proceed without cached Library
|
||||
process.chdir(startPath);
|
||||
return;
|
||||
}
|
||||
// Re-throw other errors
|
||||
throw extractError;
|
||||
}
|
||||
|
||||
RemoteClientLogger.log(`cache item extracted to ${fullResultsFolder}`);
|
||||
assert(await fileExists(fullResultsFolder), `cache extraction results folder exists`);
|
||||
const destinationParentFolder = path.resolve(destinationFolder, '..');
|
||||
|
|
|
|||
Loading…
Reference in New Issue