actions
/
unity-builder
mirror of https://github.com/game-ci/unity-builder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

transition zip usage in cache to uncompressed tar for speed

pull/387/head
Frostebite 2022-04-20 18:36:59 +01:00
parent 7fbfc6780d
commit a8c48b2fb6
4 changed files with 28 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
dist/index.js vendored
View File

@ -2642,15 +2642,15 @@ class Caching {
return typeof arguments_[number] != 'undefined' ? arguments_[number] : match; return typeof arguments_[number] != 'undefined' ? arguments_[number] : match;
}); });
}; };
yield cloud_runner_system_1.CloudRunnerSystem.Run(`zip -q -r ${cacheArtifactName}.zip ${path_1.default.basename(sourceFolder)}`); yield cloud_runner_system_1.CloudRunnerSystem.Run(`tar -cf ${cacheArtifactName}.tar -C ${path_1.default.basename(sourceFolder)}`);
console_1.assert(yield fileExists(`${cacheArtifactName}.zip`), 'cache zip exists'); console_1.assert(yield fileExists(`${cacheArtifactName}.tar`), 'cache archive exists');
console_1.assert(yield fileExists(path_1.default.basename(sourceFolder)), 'source folder exists'); console_1.assert(yield fileExists(path_1.default.basename(sourceFolder)), 'source folder exists');
if (cloud_runner_1.default.buildParameters.cachePushOverrideCommand) { if (cloud_runner_1.default.buildParameters.cachePushOverrideCommand) {
yield cloud_runner_system_1.CloudRunnerSystem.Run(formatFunction(cloud_runner_1.default.buildParameters.cachePushOverrideCommand)); yield cloud_runner_system_1.CloudRunnerSystem.Run(formatFunction(cloud_runner_1.default.buildParameters.cachePushOverrideCommand));
} }
yield cloud_runner_system_1.CloudRunnerSystem.Run(`mv ${cacheArtifactName}.zip ${cacheFolder}`); yield cloud_runner_system_1.CloudRunnerSystem.Run(`mv ${cacheArtifactName}.tar ${cacheFolder}`);
remote_client_logger_1.RemoteClientLogger.log(`moved ${cacheArtifactName}.zip to ${cacheFolder}`); remote_client_logger_1.RemoteClientLogger.log(`moved cache entry ${cacheArtifactName} to ${cacheFolder}`);
console_1.assert(yield fileExists(`${path_1.default.join(cacheFolder, cacheArtifactName)}.zip`), 'cache zip exists inside cache folder'); console_1.assert(yield fileExists(`${path_1.default.join(cacheFolder, cacheArtifactName)}.tar`), 'cache archive exists inside cache folder');
} }
catch (error) { catch (error) {
process.chdir(`${startPath}`); process.chdir(`${startPath}`);
@ -2671,11 +2671,11 @@ class Caching {
if (!(yield fileExists(destinationFolder))) { if (!(yield fileExists(destinationFolder))) {
yield fs_1.default.promises.mkdir(destinationFolder); yield fs_1.default.promises.mkdir(destinationFolder);
} }
const latestInBranch = yield (yield cloud_runner_system_1.CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .zip$ | head -1`)) const latestInBranch = yield (yield cloud_runner_system_1.CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .tar$ | head -1`))
.replace(/\n/g, ``) .replace(/\n/g, ``)
.replace('.zip', ''); .replace('.tar', '');
process.chdir(cacheFolder); process.chdir(cacheFolder);
const cacheSelection = cacheArtifactName !== `` && (yield fileExists(`${cacheArtifactName}.zip`)) ? cacheArtifactName : latestInBranch; const cacheSelection = cacheArtifactName !== `` && (yield fileExists(`${cacheArtifactName}.tar`)) ? cacheArtifactName : latestInBranch;
yield cloud_runner_logger_1.default.log(`cache key ${cacheArtifactName} selection ${cacheSelection}`); yield cloud_runner_logger_1.default.log(`cache key ${cacheArtifactName} selection ${cacheSelection}`);
// eslint-disable-next-line func-style // eslint-disable-next-line func-style
const formatFunction = function (format) { const formatFunction = function (format) {
@ -2687,12 +2687,12 @@ class Caching {
if (cloud_runner_1.default.buildParameters.cachePullOverrideCommand) { if (cloud_runner_1.default.buildParameters.cachePullOverrideCommand) {
yield cloud_runner_system_1.CloudRunnerSystem.Run(formatFunction(cloud_runner_1.default.buildParameters.cachePullOverrideCommand)); yield cloud_runner_system_1.CloudRunnerSystem.Run(formatFunction(cloud_runner_1.default.buildParameters.cachePullOverrideCommand));
} }
if (yield fileExists(`${cacheSelection}.zip`)) { if (yield fileExists(`${cacheSelection}.tar`)) {
const resultsFolder = `results${cloud_runner_1.default.buildParameters.buildGuid}`; const resultsFolder = `results${cloud_runner_1.default.buildParameters.buildGuid}`;
yield cloud_runner_system_1.CloudRunnerSystem.Run(`mkdir -p ${resultsFolder}`); yield cloud_runner_system_1.CloudRunnerSystem.Run(`mkdir -p ${resultsFolder}`);
remote_client_logger_1.RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.zip`); remote_client_logger_1.RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.tar`);
const fullResultsFolder = path_1.default.join(cacheFolder, resultsFolder); const fullResultsFolder = path_1.default.join(cacheFolder, resultsFolder);
yield cloud_runner_system_1.CloudRunnerSystem.Run(`unzip -q ${cacheSelection}.zip -d ${path_1.default.basename(resultsFolder)}`); yield cloud_runner_system_1.CloudRunnerSystem.Run(`tar -xf ${cacheSelection}.tar -C ${path_1.default.basename(resultsFolder)}`);
remote_client_logger_1.RemoteClientLogger.log(`cache item extracted to ${fullResultsFolder}`); remote_client_logger_1.RemoteClientLogger.log(`cache item extracted to ${fullResultsFolder}`);
console_1.assert(yield fileExists(fullResultsFolder), `cache extraction results folder exists`); console_1.assert(yield fileExists(fullResultsFolder), `cache extraction results folder exists`);
const destinationParentFolder = path_1.default.resolve(destinationFolder, '..'); const destinationParentFolder = path_1.default.resolve(destinationFolder, '..');
@ -2706,7 +2706,7 @@ class Caching {
else { else {
remote_client_logger_1.RemoteClientLogger.logWarning(`cache item ${cacheArtifactName} doesn't exist ${destinationFolder}`); remote_client_logger_1.RemoteClientLogger.logWarning(`cache item ${cacheArtifactName} doesn't exist ${destinationFolder}`);
if (cacheSelection !== ``) { if (cacheSelection !== ``) {
remote_client_logger_1.RemoteClientLogger.logWarning(`cache item ${cacheArtifactName}.zip doesn't exist ${destinationFolder}`); remote_client_logger_1.RemoteClientLogger.logWarning(`cache item ${cacheArtifactName}.tar doesn't exist ${destinationFolder}`);
throw new Error(`Failed to get cache item, but cache hit was found: ${cacheSelection}`); throw new Error(`Failed to get cache item, but cache hit was found: ${cacheSelection}`);
} }
} }
@ -3585,7 +3585,7 @@ class BuildAutomationWorkflow {
const buildHooks = cloud_runner_build_command_process_1.CloudRunnerBuildCommandProcessor.getHooks(cloud_runner_1.default.buildParameters.customJobHooks).filter((x) => x.step.includes(`build`)); const buildHooks = cloud_runner_build_command_process_1.CloudRunnerBuildCommandProcessor.getHooks(cloud_runner_1.default.buildParameters.customJobHooks).filter((x) => x.step.includes(`build`));
const builderPath = path_1.default.join(cloud_runner_folders_1.CloudRunnerFolders.builderPathAbsolute, 'dist', `index.js`).replace(/\\/g, `/`); const builderPath = path_1.default.join(cloud_runner_folders_1.CloudRunnerFolders.builderPathAbsolute, 'dist', `index.js`).replace(/\\/g, `/`);
return `apt-get update > /dev/null return `apt-get update > /dev/null
apt-get install -y zip tree npm git-lfs jq unzip git > /dev/null apt-get install -y tar tree npm git-lfs jq git > /dev/null
npm install -g n > /dev/null npm install -g n > /dev/null
n stable > /dev/null n stable > /dev/null
${setupHooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '} ${setupHooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}

2
dist/index.js.map vendored
View File

File diff suppressed because one or more lines are too long

26
src/model/cloud-runner/remote-client/caching.ts
View File

@ -70,17 +70,17 @@ export class Caching {
return typeof arguments_[number] != 'undefined' ? arguments_[number] : match; return typeof arguments_[number] != 'undefined' ? arguments_[number] : match;
}); });
}; };
await CloudRunnerSystem.Run(`zip -q -r ${cacheArtifactName}.zip ${path.basename(sourceFolder)}`); await CloudRunnerSystem.Run(`tar -cf ${cacheArtifactName}.tar -C ${path.basename(sourceFolder)}`);
assert(await fileExists(`${cacheArtifactName}.zip`), 'cache zip exists'); assert(await fileExists(`${cacheArtifactName}.tar`), 'cache archive exists');
assert(await fileExists(path.basename(sourceFolder)), 'source folder exists'); assert(await fileExists(path.basename(sourceFolder)), 'source folder exists');
if (CloudRunner.buildParameters.cachePushOverrideCommand) { if (CloudRunner.buildParameters.cachePushOverrideCommand) {
await CloudRunnerSystem.Run(formatFunction(CloudRunner.buildParameters.cachePushOverrideCommand)); await CloudRunnerSystem.Run(formatFunction(CloudRunner.buildParameters.cachePushOverrideCommand));
} }
await CloudRunnerSystem.Run(`mv ${cacheArtifactName}.zip ${cacheFolder}`); await CloudRunnerSystem.Run(`mv ${cacheArtifactName}.tar ${cacheFolder}`);
RemoteClientLogger.log(`moved ${cacheArtifactName}.zip to ${cacheFolder}`); RemoteClientLogger.log(`moved cache entry ${cacheArtifactName} to ${cacheFolder}`);
assert( assert(
await fileExists(`${path.join(cacheFolder, cacheArtifactName)}.zip`), await fileExists(`${path.join(cacheFolder, cacheArtifactName)}.tar`),
'cache zip exists inside cache folder', 'cache archive exists inside cache folder',
); );
} catch (error) { } catch (error) {
process.chdir(`${startPath}`); process.chdir(`${startPath}`);
@ -101,14 +101,14 @@ export class Caching {
await fs.promises.mkdir(destinationFolder); await fs.promises.mkdir(destinationFolder);
} }
const latestInBranch = await (await CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .zip$ | head -1`)) const latestInBranch = await (await CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .tar$ | head -1`))
.replace(/\n/g, ``) .replace(/\n/g, ``)
.replace('.zip', ''); .replace('.tar', '');
process.chdir(cacheFolder); process.chdir(cacheFolder);
const cacheSelection = const cacheSelection =
cacheArtifactName !== `` && (await fileExists(`${cacheArtifactName}.zip`)) ? cacheArtifactName : latestInBranch; cacheArtifactName !== `` && (await fileExists(`${cacheArtifactName}.tar`)) ? cacheArtifactName : latestInBranch;
await CloudRunnerLogger.log(`cache key ${cacheArtifactName} selection ${cacheSelection}`); await CloudRunnerLogger.log(`cache key ${cacheArtifactName} selection ${cacheSelection}`);
// eslint-disable-next-line func-style // eslint-disable-next-line func-style
@ -127,12 +127,12 @@ export class Caching {
await CloudRunnerSystem.Run(formatFunction(CloudRunner.buildParameters.cachePullOverrideCommand)); await CloudRunnerSystem.Run(formatFunction(CloudRunner.buildParameters.cachePullOverrideCommand));
} }
if (await fileExists(`${cacheSelection}.zip`)) { if (await fileExists(`${cacheSelection}.tar`)) {
const resultsFolder = `results${CloudRunner.buildParameters.buildGuid}`; const resultsFolder = `results${CloudRunner.buildParameters.buildGuid}`;
await CloudRunnerSystem.Run(`mkdir -p ${resultsFolder}`); await CloudRunnerSystem.Run(`mkdir -p ${resultsFolder}`);
RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.zip`); RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.tar`);
const fullResultsFolder = path.join(cacheFolder, resultsFolder); const fullResultsFolder = path.join(cacheFolder, resultsFolder);
await CloudRunnerSystem.Run(`unzip -q ${cacheSelection}.zip -d ${path.basename(resultsFolder)}`); await CloudRunnerSystem.Run(`tar -xf ${cacheSelection}.tar -C ${path.basename(resultsFolder)}`);
RemoteClientLogger.log(`cache item extracted to ${fullResultsFolder}`); RemoteClientLogger.log(`cache item extracted to ${fullResultsFolder}`);
assert(await fileExists(fullResultsFolder), `cache extraction results folder exists`); assert(await fileExists(fullResultsFolder), `cache extraction results folder exists`);
const destinationParentFolder = path.resolve(destinationFolder, '..'); const destinationParentFolder = path.resolve(destinationFolder, '..');
@ -152,7 +152,7 @@ export class Caching {
} else { } else {
RemoteClientLogger.logWarning(`cache item ${cacheArtifactName} doesn't exist ${destinationFolder}`); RemoteClientLogger.logWarning(`cache item ${cacheArtifactName} doesn't exist ${destinationFolder}`);
if (cacheSelection !== ``) { if (cacheSelection !== ``) {
RemoteClientLogger.logWarning(`cache item ${cacheArtifactName}.zip doesn't exist ${destinationFolder}`); RemoteClientLogger.logWarning(`cache item ${cacheArtifactName}.tar doesn't exist ${destinationFolder}`);
throw new Error(`Failed to get cache item, but cache hit was found: ${cacheSelection}`); throw new Error(`Failed to get cache item, but cache hit was found: ${cacheSelection}`);
} }
} }

2
src/model/cloud-runner/workflows/build-automation-workflow.ts
View File

@ -71,7 +71,7 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
const builderPath = path.join(CloudRunnerFolders.builderPathAbsolute, 'dist', `index.js`).replace(/\\/g, `/`); const builderPath = path.join(CloudRunnerFolders.builderPathAbsolute, 'dist', `index.js`).replace(/\\/g, `/`);
return `apt-get update > /dev/null return `apt-get update > /dev/null
apt-get install -y zip tree npm git-lfs jq unzip git > /dev/null apt-get install -y tar tree npm git-lfs jq git > /dev/null
npm install -g n > /dev/null npm install -g n > /dev/null
n stable > /dev/null n stable > /dev/null
${setupHooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '} ${setupHooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}