actions
/
unity-builder
mirror of https://github.com/game-ci/unity-builder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Cloud Runner 1.0 (#459) 

* Fix: post build caching, use linux path conversion

* Fix: post build caching via CLI

* Fix: post build caching via CLI

* Fix: post build caching via CLI

* Fix: post build caching via CLI

* Fix: post build caching via CLI

* Log if retained workspace option is present for testing

* Log if retained workspace option is present for testing

* Use retained workspace :O

* Use retained workspace :O

* Use retained workspace :O

* Use retained workspace :O

* Ignore garbage creating lock actions in test for now

* Lock workspace when using Get or Create Locked Workspace

* Lock workspace before creating workspace file to allow for an unblockable creation sequence with guarenteed lock for the original creator

* intuitive locking logs from the most important flow

* test naming

* consider lock folders without workspace file locked

* Use cache key to segment lock folders

* Use cache key to segment lock folders

* Use cache key to segment lock folders

* Use cache key to segment lock folders

* Use cache key to segment lock folders

* Skip all locking actions test as we now have two useful test flows

* Skip all locking actions test as we now have two useful test flows

* Skip all locking actions test as we now have two useful test flows

* Copy all of data folder to docker volume to enable local-docker retained workspace

* Fix: check for retained workspace

* Fix: check for retained workspace

* Fix: check for retained workspace

* Fix: check for retained workspace

* Fix: check for retained workspace

* Fix: check for retained workspace

* Fix: check for retained workspace

* Fix: check for retained workspace

* Skip main clone if game repo exists

* handle cloud runner git sync via sha not only branch

* handle cloud runner git sync via sha not only branch

* handle cloud runner git sync via sha not only branch

* handle cloud runner git sync via sha not only branch

* handle cloud runner git sync via sha not only branch

* handle cloud runner git sync via sha not only branch

* handle cloud runner git sync via sha not only branch

* handle cloud runner git sync via sha not only branch

* transfer locked workspace to static CloudRunner field

* transfer locked workspace to static CloudRunner field

* transfer locked workspace to static CloudRunner field

* custom hook files and test

* custom hook files and test

* custom hook files and test
pull/437/head
Frostebite 2022-10-06 20:42:33 +01:00 committed by GitHub
parent e56abbdd40
commit 7fcd51349b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 590 additions and 305 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

325
dist/index.js vendored
View File

@ -296,6 +296,9 @@ class BuildParameters {
readInputFromOverrideList: cloud_runner_options_1.default.readInputFromOverrideList(), readInputFromOverrideList: cloud_runner_options_1.default.readInputFromOverrideList(),
kubeStorageClass: cloud_runner_options_1.default.kubeStorageClass, kubeStorageClass: cloud_runner_options_1.default.kubeStorageClass,
cacheKey: cloud_runner_options_1.default.cacheKey, cacheKey: cloud_runner_options_1.default.cacheKey,
retainWorkspace: cloud_runner_options_1.default.retainWorkspaces,
useSharedLargePackages: cloud_runner_options_1.default.useSharedLargePackages,
useLZ4Compression: cloud_runner_options_1.default.useLZ4Compression,
}; };
}); });
} }
@ -485,6 +488,7 @@ const cloud_runner_options_reader_1 = __importDefault(__nccwpck_require__(3343))
const github_1 = __importDefault(__nccwpck_require__(83654)); const github_1 = __importDefault(__nccwpck_require__(83654));
const task_parameter_serializer_1 = __nccwpck_require__(35346); const task_parameter_serializer_1 = __nccwpck_require__(35346);
const cloud_runner_folders_1 = __nccwpck_require__(13527); const cloud_runner_folders_1 = __nccwpck_require__(13527);
const cloud_runner_system_1 = __nccwpck_require__(99393);
class Cli { class Cli {
static get isCliMode() { static get isCliMode() {
return Cli.options !== undefined && Cli.options.mode !== undefined && Cli.options.mode !== ''; return Cli.options !== undefined && Cli.options.mode !== undefined && Cli.options.mode !== '';
@ -539,6 +543,7 @@ class Cli {
${JSON.stringify(buildParameter, undefined, 4)} ${JSON.stringify(buildParameter, undefined, 4)}
`); `);
__1.CloudRunner.buildParameters = buildParameter; __1.CloudRunner.buildParameters = buildParameter;
__1.CloudRunner.lockedWorkspace = process.env.LOCKED_WORKSPACE;
return yield results.target[results.propertyKey](Cli.options); return yield results.target[results.propertyKey](Cli.options);
}); });
} }
@ -567,26 +572,13 @@ class Cli {
} }
static PostCLIBuild() { static PostCLIBuild() {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
const buildParameter = yield __1.BuildParameters.create();
/*
# LIBRARY CACHE
node ${builderPath} -m cache-push --cachePushFrom ${CloudRunnerFolders.ToLinuxFolder(
CloudRunnerFolders.libraryFolderAbsolute,
)} --artifactName lib-${guid} --cachePushTo ${CloudRunnerFolders.ToLinuxFolder(`${linuxCacheFolder}/Library`)}
echo "game ci cloud runner push build to cache"
# BUILD CACHE
node ${builderPath} -m cache-push --cachePushFrom ${CloudRunnerFolders.ToLinuxFolder(
CloudRunnerFolders.projectBuildFolderAbsolute,
)} --artifactName build-${guid} --cachePushTo ${`${CloudRunnerFolders.ToLinuxFolder(`${linuxCacheFolder}/build`)}`}
# RETAINED WORKSPACE CLEANUP
${BuildAutomationWorkflow.GetCleanupCommand(CloudRunnerFolders.projectPathAbsolute)}`;
*/
core.info(`Running POST build tasks`); core.info(`Running POST build tasks`);
caching_1.Caching.PushToCache(cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.libraryFolderAbsolute), cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(`${cloud_runner_folders_1.CloudRunnerFolders.cacheFolderFull}/Library`), `lib-${buildParameter.buildGuid}`); yield caching_1.Caching.PushToCache(cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(`${cloud_runner_folders_1.CloudRunnerFolders.cacheFolderFull}/Library`), cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.libraryFolderAbsolute), `lib-${__1.CloudRunner.buildParameters.buildGuid}`);
caching_1.Caching.PushToCache(cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.projectBuildFolderAbsolute), cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(`${cloud_runner_folders_1.CloudRunnerFolders.cacheFolderFull}/build`), `build-${buildParameter.buildGuid}`); yield caching_1.Caching.PushToCache(cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(`${cloud_runner_folders_1.CloudRunnerFolders.cacheFolderFull}/build`), cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.projectBuildFolderAbsolute), `build-${__1.CloudRunner.buildParameters.buildGuid}`);
if (!__1.CloudRunner.buildParameters.retainWorkspace) {
yield cloud_runner_system_1.CloudRunnerSystem.Run(`rm -r ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`);
}
yield remote_client_1.RemoteClient.runCustomHookFiles(`after-build`);
return new Promise((result) => result(``)); return new Promise((result) => result(``));
}); });
} }
@ -648,48 +640,123 @@ const fs = __importStar(__nccwpck_require__(57147));
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855)); const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
const cloud_runner_options_1 = __importDefault(__nccwpck_require__(96552)); const cloud_runner_options_1 = __importDefault(__nccwpck_require__(96552));
class SharedWorkspaceLocking { class SharedWorkspaceLocking {
static GetLockedWorkspace(workspaceIfCreated, runId) { static GetAllWorkspaces(buildParametersContext) {
return __awaiter(this, void 0, void 0, function* () {
return (yield SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`)).map((x) => x.replace(`/`, ``));
});
}
static DoesWorkspaceTopLevelExist(buildParametersContext) {
return __awaiter(this, void 0, void 0, function* () {
const results = (yield SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}`)).map((x) => x.replace(`/`, ``));
return results.includes(buildParametersContext.cacheKey);
});
}
static GetAllLocks(workspace, buildParametersContext) {
return __awaiter(this, void 0, void 0, function* () {
if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
throw new Error("Workspace doesn't exist, can't call get all locks");
}
return (yield SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/`)).map((x) => x.replace(`/`, ``));
});
}
static GetOrCreateLockedWorkspace(workspaceIfCreated, runId, buildParametersContext) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
if (!cloud_runner_options_1.default.retainWorkspaces) { if (!cloud_runner_options_1.default.retainWorkspaces) {
return; return;
} }
const workspaces = yield SharedWorkspaceLocking.GetFreeWorkspaces(); cloud_runner_logger_1.default.log(`run agent ${runId} is trying to access a workspace`);
for (const element of workspaces) { if (yield SharedWorkspaceLocking.DoesWorkspaceTopLevelExist(buildParametersContext)) {
if (yield SharedWorkspaceLocking.LockWorkspace(element, runId)) { const workspaces = yield SharedWorkspaceLocking.GetFreeWorkspaces(buildParametersContext);
return element; for (const element of workspaces) {
if (yield SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext)) {
cloud_runner_logger_1.default.log(`run agent ${runId} locked workspace: ${element}`);
return element;
}
} }
} }
return yield SharedWorkspaceLocking.CreateLockableWorkspace(workspaceIfCreated); const workspace = yield SharedWorkspaceLocking.CreateWorkspace(workspaceIfCreated, buildParametersContext, runId);
cloud_runner_logger_1.default.log(`run agent ${runId} didn't find a free workspace so created: ${workspace}`);
return workspace;
}); });
} }
static DoesWorkspaceExist(workspace) { static DoesWorkspaceExist(workspace, buildParametersContext) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
return (yield SharedWorkspaceLocking.GetAllWorkspaces()).includes(workspace); return (yield SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext)).includes(workspace);
}); });
} }
static GetFreeWorkspaces() { static HasWorkspaceLock(workspace, runId, buildParametersContext) {
return __awaiter(this, void 0, void 0, function* () {
if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
return false;
}
return ((yield SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext)).filter((x) => x.includes(runId))
.length > 0);
});
}
static GetFreeWorkspaces(buildParametersContext) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
const result = []; const result = [];
const workspaces = yield SharedWorkspaceLocking.GetAllWorkspaces(); const workspaces = yield SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
for (const element of workspaces) { for (const element of workspaces) {
if (!(yield SharedWorkspaceLocking.IsWorkspaceLocked(element))) { if (!(yield SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParametersContext))) {
result.push(element); result.push(element);
} }
} }
return result; return result;
}); });
} }
static GetAllWorkspaces() { static IsWorkspaceLocked(workspace, buildParametersContext) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
return (yield SharedWorkspaceLocking.ReadLines(`aws s3 ls s3://game-ci-test-storage/locks/`)).map((x) => x.replace(`/`, ``)); if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
return false;
}
const files = yield SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/`);
const workspaceFileDoesNotExists = files.filter((x) => {
return x.includes(`_workspace`);
}).length === 0;
const lockFilesExist = files.filter((x) => {
return x.includes(`_lock`);
}).length > 0;
return workspaceFileDoesNotExists || lockFilesExist;
}); });
} }
static GetAllLocks(workspace) { static CreateWorkspace(workspace, buildParametersContext, lockId = ``) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace))) { if (lockId !== ``) {
throw new Error("Workspace doesn't exist, can't call get all locks"); yield SharedWorkspaceLocking.LockWorkspace(workspace, lockId, buildParametersContext);
} }
return (yield SharedWorkspaceLocking.ReadLines(`aws s3 ls s3://game-ci-test-storage/locks/${workspace}/`)).map((x) => x.replace(`/`, ``)); const file = `${Date.now()}_workspace`;
fs.writeFileSync(file, '');
yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`, false, true);
fs.rmSync(file);
return workspace;
});
}
static LockWorkspace(workspace, runId, buildParametersContext) {
return __awaiter(this, void 0, void 0, function* () {
const file = `${Date.now()}_${runId}_lock`;
fs.writeFileSync(file, '');
yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`, false, true);
fs.rmSync(file);
return SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext);
});
}
static ReleaseWorkspace(workspace, runId, buildParametersContext) {
return __awaiter(this, void 0, void 0, function* () {
if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
return true;
}
const file = (yield SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext)).filter((x) => x.includes(`_${runId}_lock`));
cloud_runner_logger_1.default.log(`${JSON.stringify(yield SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext))}`);
cloud_runner_logger_1.default.log(`Deleting file ${file}`);
cloud_runner_logger_1.default.log(`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`);
yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`, false, true);
return !SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext);
});
}
static CleanupWorkspace(workspace, buildParametersContext) {
return __awaiter(this, void 0, void 0, function* () {
yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace} --recursive`, false, true);
}); });
} }
static ReadLines(command) { static ReadLines(command) {
@ -705,59 +772,9 @@ class SharedWorkspaceLocking {
}); });
}); });
} }
static LockWorkspace(workspace, runId) {
return __awaiter(this, void 0, void 0, function* () {
const file = `${Date.now()}_${runId}_lock`;
fs.writeFileSync(file, '');
yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 cp ./${file} s3://game-ci-test-storage/locks/${workspace}/${file}`, false, true);
fs.rmSync(file);
return SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId);
});
}
static ReleaseWorkspace(workspace, runId) {
return __awaiter(this, void 0, void 0, function* () {
if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace))) {
return true;
}
const file = (yield SharedWorkspaceLocking.GetAllLocks(workspace)).filter((x) => x.includes(`_${runId}_lock`));
cloud_runner_logger_1.default.log(`${JSON.stringify(yield SharedWorkspaceLocking.GetAllLocks(workspace))}`);
cloud_runner_logger_1.default.log(`Deleting file ${file}`);
cloud_runner_logger_1.default.log(`aws s3 rm s3://game-ci-test-storage/locks/${workspace}/${file}`);
yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 rm s3://game-ci-test-storage/locks/${workspace}/${file}`, false, true);
return !SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId);
});
}
static HasWorkspaceLock(workspace, runId) {
return __awaiter(this, void 0, void 0, function* () {
if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace))) {
return false;
}
return (yield SharedWorkspaceLocking.GetAllLocks(workspace)).filter((x) => x.includes(runId)).length > 0;
});
}
static IsWorkspaceLocked(workspace) {
return __awaiter(this, void 0, void 0, function* () {
if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace))) {
return false;
}
const files = yield SharedWorkspaceLocking.ReadLines(`aws s3 ls s3://game-ci-test-storage/locks/${workspace}/`);
// 1 Because we expect 1 workspace file to exist in every workspace folder
return files.length > 1;
});
}
static CreateLockableWorkspace(workspace) {
return __awaiter(this, void 0, void 0, function* () {
const file = `${Date.now()}_workspace`;
fs.writeFileSync(file, '');
yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 cp ./${file} s3://game-ci-test-storage/locks/${workspace}/${file}`, false, true);
fs.rmSync(file);
return workspace;
});
}
// eslint-disable-next-line no-unused-vars
static ReleaseLock(workspace) { }
} }
exports.SharedWorkspaceLocking = SharedWorkspaceLocking; exports.SharedWorkspaceLocking = SharedWorkspaceLocking;
SharedWorkspaceLocking.workspaceRoot = `s3://game-ci-test-storage/locks/`;
exports["default"] = SharedWorkspaceLocking; exports["default"] = SharedWorkspaceLocking;
@ -912,6 +929,12 @@ class CloudRunnerOptions {
static get retainWorkspacesMax() { static get retainWorkspacesMax() {
return Number(CloudRunnerOptions.getInput(`retainWorkspacesMax`)) || 5; return Number(CloudRunnerOptions.getInput(`retainWorkspacesMax`)) || 5;
} }
static get useSharedLargePackages() {
return CloudRunnerOptions.getInput(`useSharedLargePackages`) || false;
}
static get useLZ4Compression() {
return CloudRunnerOptions.getInput(`useLZ4Compression`) || true;
}
static ToEnvVarFormat(input) { static ToEnvVarFormat(input) {
if (input.toUpperCase() === input) { if (input.toUpperCase() === input) {
return input; return input;
@ -3672,7 +3695,6 @@ const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
const docker_1 = __importDefault(__nccwpck_require__(16934)); const docker_1 = __importDefault(__nccwpck_require__(16934));
const model_1 = __nccwpck_require__(41359); const model_1 = __nccwpck_require__(41359);
const fs_1 = __nccwpck_require__(57147); const fs_1 = __nccwpck_require__(57147);
// import * as core from '@actions/core';
class LocalDockerCloudRunner { class LocalDockerCloudRunner {
inspect() { inspect() {
throw new Error('Method not implemented.'); throw new Error('Method not implemented.');
@ -3707,6 +3729,7 @@ class LocalDockerCloudRunner {
this.buildParameters = buildParameters; this.buildParameters = buildParameters;
} }
runTask(buildGuid, image, commands, mountdir, workingdir, environment, secrets) { runTask(buildGuid, image, commands, mountdir, workingdir, environment, secrets) {
var _a;
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
cloud_runner_logger_1.default.log(buildGuid); cloud_runner_logger_1.default.log(buildGuid);
cloud_runner_logger_1.default.log(commands); cloud_runner_logger_1.default.log(commands);
@ -3730,6 +3753,7 @@ class LocalDockerCloudRunner {
} }
} }
let myOutput = ''; let myOutput = '';
const sharedFolder = ((_a = this.buildParameters) === null || _a === void 0 ? void 0 : _a.retainWorkspace) ? `/data/` : `/data/cache/`;
// core.info(JSON.stringify({ workspace, actionFolder, ...this.buildParameters, ...content }, undefined, 4)); // core.info(JSON.stringify({ workspace, actionFolder, ...this.buildParameters, ...content }, undefined, 4));
const entrypointFilePath = `start.sh`; const entrypointFilePath = `start.sh`;
fs_1.writeFileSync(`${workspace}/${entrypointFilePath}`, `#!/bin/bash fs_1.writeFileSync(`${workspace}/${entrypointFilePath}`, `#!/bin/bash
@ -3738,11 +3762,12 @@ class LocalDockerCloudRunner {
apt-get update > /dev/null && apt-get install -y tree> /dev/null apt-get update > /dev/null && apt-get install -y tree> /dev/null
mkdir -p /github/workspace/cloud-runner-cache mkdir -p /github/workspace/cloud-runner-cache
mkdir -p /data/cache mkdir -p /data/cache
cp -a /github/workspace/cloud-runner-cache/. /data/cache/ cp -a /github/workspace/cloud-runner-cache/. ${sharedFolder}
tree -L 2 /data/cache tree -L 3 ${sharedFolder}
${commands} ${commands}
cp -a /data/cache/. /github/workspace/cloud-runner-cache/ cp -a ${sharedFolder}. /github/workspace/cloud-runner-cache/
tree -L 2 /github/workspace/cloud-runner-cache tree -L 2 /github/workspace/cloud-runner-cache
tree -L 3 ${sharedFolder}
`, { `, {
flag: 'w', flag: 'w',
}); });
@ -3987,6 +4012,7 @@ class Caching {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
cacheArtifactName = cacheArtifactName.replace(' ', ''); cacheArtifactName = cacheArtifactName.replace(' ', '');
const startPath = process.cwd(); const startPath = process.cwd();
const compressionSuffix = cloud_runner_1.default.buildParameters.useLZ4Compression ? '.lz4' : '';
try { try {
if (!(yield fileExists(cacheFolder))) { if (!(yield fileExists(cacheFolder))) {
yield cloud_runner_system_1.CloudRunnerSystem.Run(`mkdir -p ${cacheFolder}`); yield cloud_runner_system_1.CloudRunnerSystem.Run(`mkdir -p ${cacheFolder}`);
@ -3998,7 +4024,7 @@ class Caching {
const contents = yield fs_1.default.promises.readdir(path_1.default.basename(sourceFolder)); const contents = yield fs_1.default.promises.readdir(path_1.default.basename(sourceFolder));
cloud_runner_logger_1.default.log(`There is ${contents.length} files/dir in the source folder ${path_1.default.basename(sourceFolder)}`); cloud_runner_logger_1.default.log(`There is ${contents.length} files/dir in the source folder ${path_1.default.basename(sourceFolder)}`);
if (cloud_runner_1.default.buildParameters.cloudRunnerIntegrationTests) { if (cloud_runner_1.default.buildParameters.cloudRunnerIntegrationTests) {
yield cloud_runner_system_1.CloudRunnerSystem.Run(`tree -L 2 ./..`); // await CloudRunnerSystem.Run(`tree -L 2 ./..`);
yield cloud_runner_system_1.CloudRunnerSystem.Run(`tree -L 2`); yield cloud_runner_system_1.CloudRunnerSystem.Run(`tree -L 2`);
} }
if (contents.length === 0) { if (contents.length === 0) {
@ -4006,13 +4032,13 @@ class Caching {
process.chdir(`${startPath}`); process.chdir(`${startPath}`);
return; return;
} }
yield cloud_runner_system_1.CloudRunnerSystem.Run(`tar -cf ${cacheArtifactName}.tar.lz4 ${path_1.default.basename(sourceFolder)}`); yield cloud_runner_system_1.CloudRunnerSystem.Run(`tar -cf ${cacheArtifactName}.tar${compressionSuffix} ${path_1.default.basename(sourceFolder)}`);
yield cloud_runner_system_1.CloudRunnerSystem.Run(`du ${cacheArtifactName}.tar.lz4`); yield cloud_runner_system_1.CloudRunnerSystem.Run(`du ${cacheArtifactName}.tar${compressionSuffix}`);
console_1.assert(yield fileExists(`${cacheArtifactName}.tar.lz4`), 'cache archive exists'); console_1.assert(yield fileExists(`${cacheArtifactName}.tar${compressionSuffix}`), 'cache archive exists');
console_1.assert(yield fileExists(path_1.default.basename(sourceFolder)), 'source folder exists'); console_1.assert(yield fileExists(path_1.default.basename(sourceFolder)), 'source folder exists');
yield cloud_runner_system_1.CloudRunnerSystem.Run(`mv ${cacheArtifactName}.tar.lz4 ${cacheFolder}`); yield cloud_runner_system_1.CloudRunnerSystem.Run(`mv ${cacheArtifactName}.tar${compressionSuffix} ${cacheFolder}`);
remote_client_logger_1.RemoteClientLogger.log(`moved cache entry ${cacheArtifactName} to ${cacheFolder}`); remote_client_logger_1.RemoteClientLogger.log(`moved cache entry ${cacheArtifactName} to ${cacheFolder}`);
console_1.assert(yield fileExists(`${path_1.default.join(cacheFolder, cacheArtifactName)}.tar.lz4`), 'cache archive exists inside cache folder'); console_1.assert(yield fileExists(`${path_1.default.join(cacheFolder, cacheArtifactName)}.tar${compressionSuffix}`), 'cache archive exists inside cache folder');
} }
catch (error) { catch (error) {
process.chdir(`${startPath}`); process.chdir(`${startPath}`);
@ -4024,6 +4050,7 @@ class Caching {
static PullFromCache(cacheFolder, destinationFolder, cacheArtifactName = ``) { static PullFromCache(cacheFolder, destinationFolder, cacheArtifactName = ``) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
cacheArtifactName = cacheArtifactName.replace(' ', ''); cacheArtifactName = cacheArtifactName.replace(' ', '');
const compressionSuffix = cloud_runner_1.default.buildParameters.useLZ4Compression ? '.lz4' : '';
const startPath = process.cwd(); const startPath = process.cwd();
remote_client_logger_1.RemoteClientLogger.log(`Caching for ${path_1.default.basename(destinationFolder)}`); remote_client_logger_1.RemoteClientLogger.log(`Caching for ${path_1.default.basename(destinationFolder)}`);
try { try {
@ -4033,20 +4060,20 @@ class Caching {
if (!(yield fileExists(destinationFolder))) { if (!(yield fileExists(destinationFolder))) {
yield fs_1.default.promises.mkdir(destinationFolder); yield fs_1.default.promises.mkdir(destinationFolder);
} }
const latestInBranch = yield (yield cloud_runner_system_1.CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .tar.lz4$ | head -1`)) const latestInBranch = yield (yield cloud_runner_system_1.CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .tar${compressionSuffix}$ | head -1`))
.replace(/\n/g, ``) .replace(/\n/g, ``)
.replace('.tar.lz4', ''); .replace(`.tar${compressionSuffix}`, '');
process.chdir(cacheFolder); process.chdir(cacheFolder);
const cacheSelection = cacheArtifactName !== `` && (yield fileExists(`${cacheArtifactName}.tar.lz4`)) const cacheSelection = cacheArtifactName !== `` && (yield fileExists(`${cacheArtifactName}.tar${compressionSuffix}`))
? cacheArtifactName ? cacheArtifactName
: latestInBranch; : latestInBranch;
yield cloud_runner_logger_1.default.log(`cache key ${cacheArtifactName} selection ${cacheSelection}`); yield cloud_runner_logger_1.default.log(`cache key ${cacheArtifactName} selection ${cacheSelection}`);
if (yield fileExists(`${cacheSelection}.tar.lz4`)) { if (yield fileExists(`${cacheSelection}.tar${compressionSuffix}`)) {
const resultsFolder = `results${cloud_runner_1.default.buildParameters.buildGuid}`; const resultsFolder = `results${cloud_runner_1.default.buildParameters.buildGuid}`;
yield cloud_runner_system_1.CloudRunnerSystem.Run(`mkdir -p ${resultsFolder}`); yield cloud_runner_system_1.CloudRunnerSystem.Run(`mkdir -p ${resultsFolder}`);
remote_client_logger_1.RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.tar.lz4`); remote_client_logger_1.RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.tar${compressionSuffix}`);
const fullResultsFolder = path_1.default.join(cacheFolder, resultsFolder); const fullResultsFolder = path_1.default.join(cacheFolder, resultsFolder);
yield cloud_runner_system_1.CloudRunnerSystem.Run(`tar -xf ${cacheSelection}.tar.lz4 -C ${fullResultsFolder}`); yield cloud_runner_system_1.CloudRunnerSystem.Run(`tar -xf ${cacheSelection}.tar${compressionSuffix} -C ${fullResultsFolder}`);
remote_client_logger_1.RemoteClientLogger.log(`cache item extracted to ${fullResultsFolder}`); remote_client_logger_1.RemoteClientLogger.log(`cache item extracted to ${fullResultsFolder}`);
console_1.assert(yield fileExists(fullResultsFolder), `cache extraction results folder exists`); console_1.assert(yield fileExists(fullResultsFolder), `cache extraction results folder exists`);
const destinationParentFolder = path_1.default.resolve(destinationFolder, '..'); const destinationParentFolder = path_1.default.resolve(destinationFolder, '..');
@ -4060,7 +4087,7 @@ class Caching {
else { else {
remote_client_logger_1.RemoteClientLogger.logWarning(`cache item ${cacheArtifactName} doesn't exist ${destinationFolder}`); remote_client_logger_1.RemoteClientLogger.logWarning(`cache item ${cacheArtifactName} doesn't exist ${destinationFolder}`);
if (cacheSelection !== ``) { if (cacheSelection !== ``) {
remote_client_logger_1.RemoteClientLogger.logWarning(`cache item ${cacheArtifactName}.tar.lz4 doesn't exist ${destinationFolder}`); remote_client_logger_1.RemoteClientLogger.logWarning(`cache item ${cacheArtifactName}.tar${compressionSuffix} doesn't exist ${destinationFolder}`);
throw new Error(`Failed to get cache item, but cache hit was found: ${cacheSelection}`); throw new Error(`Failed to get cache item, but cache hit was found: ${cacheSelection}`);
} }
} }
@ -4128,6 +4155,7 @@ const console_1 = __nccwpck_require__(96206);
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855)); const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
const cli_functions_repository_1 = __nccwpck_require__(85301); const cli_functions_repository_1 = __nccwpck_require__(85301);
const cloud_runner_system_1 = __nccwpck_require__(99393); const cloud_runner_system_1 = __nccwpck_require__(99393);
const yaml_1 = __importDefault(__nccwpck_require__(44603));
class RemoteClient { class RemoteClient {
static bootstrapRepository() { static bootstrapRepository() {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
@ -4167,8 +4195,16 @@ class RemoteClient {
} }
static cloneRepoWithoutLFSFiles() { static cloneRepoWithoutLFSFiles() {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
process.chdir(`${cloud_runner_folders_1.CloudRunnerFolders.repoPathAbsolute}`);
if (cloud_runner_1.default.buildParameters.cloudRunnerIntegrationTests) {
yield cloud_runner_system_1.CloudRunnerSystem.Run(`tree -L 2 ./..`);
}
if (fs_1.default.existsSync(path_1.default.join(cloud_runner_folders_1.CloudRunnerFolders.repoPathAbsolute, `.git`))) {
remote_client_logger_1.RemoteClientLogger.log(`${cloud_runner_folders_1.CloudRunnerFolders.repoPathAbsolute} repo exists - skipping clone - retained workspace mode ${cloud_runner_1.default.buildParameters.retainWorkspace}`);
yield cloud_runner_system_1.CloudRunnerSystem.Run(`git reset --hard ${cloud_runner_1.default.buildParameters.gitSha}`);
return;
}
try { try {
process.chdir(`${cloud_runner_folders_1.CloudRunnerFolders.repoPathAbsolute}`);
remote_client_logger_1.RemoteClientLogger.log(`Initializing source repository for cloning with caching of LFS files`); remote_client_logger_1.RemoteClientLogger.log(`Initializing source repository for cloning with caching of LFS files`);
yield cloud_runner_system_1.CloudRunnerSystem.Run(`git config --global advice.detachedHead false`); yield cloud_runner_system_1.CloudRunnerSystem.Run(`git config --global advice.detachedHead false`);
remote_client_logger_1.RemoteClientLogger.log(`Cloning the repository being built:`); remote_client_logger_1.RemoteClientLogger.log(`Cloning the repository being built:`);
@ -4179,6 +4215,7 @@ class RemoteClient {
console_1.assert(fs_1.default.existsSync(`.git`), 'git folder exists'); console_1.assert(fs_1.default.existsSync(`.git`), 'git folder exists');
remote_client_logger_1.RemoteClientLogger.log(`${cloud_runner_1.default.buildParameters.branch}`); remote_client_logger_1.RemoteClientLogger.log(`${cloud_runner_1.default.buildParameters.branch}`);
yield cloud_runner_system_1.CloudRunnerSystem.Run(`git checkout ${cloud_runner_1.default.buildParameters.branch}`); yield cloud_runner_system_1.CloudRunnerSystem.Run(`git checkout ${cloud_runner_1.default.buildParameters.branch}`);
yield cloud_runner_system_1.CloudRunnerSystem.Run(`git checkout ${cloud_runner_1.default.buildParameters.gitSha}`);
console_1.assert(fs_1.default.existsSync(path_1.default.join(`.git`, `lfs`)), 'LFS folder should not exist before caching'); console_1.assert(fs_1.default.existsSync(path_1.default.join(`.git`, `lfs`)), 'LFS folder should not exist before caching');
remote_client_logger_1.RemoteClientLogger.log(`Checked out ${cloud_runner_1.default.buildParameters.branch}`); remote_client_logger_1.RemoteClientLogger.log(`Checked out ${cloud_runner_1.default.buildParameters.branch}`);
} }
@ -4188,8 +4225,12 @@ class RemoteClient {
}); });
} }
static replaceLargePackageReferencesWithSharedReferences() { static replaceLargePackageReferencesWithSharedReferences() {
const manifest = fs_1.default.readFileSync(path_1.default.join(cloud_runner_folders_1.CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`), 'utf8');
if (cloud_runner_1.default.buildParameters.cloudRunnerIntegrationTests) { if (cloud_runner_1.default.buildParameters.cloudRunnerIntegrationTests) {
cloud_runner_logger_1.default.log(fs_1.default.readFileSync(path_1.default.join(cloud_runner_folders_1.CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`), 'utf8')); cloud_runner_logger_1.default.log(manifest);
}
if (cloud_runner_1.default.buildParameters.useSharedLargePackages) {
manifest.replace(/LargePackages/g, '../../LargePackages');
} }
} }
static pullLatestLFS() { static pullLatestLFS() {
@ -4204,12 +4245,34 @@ class RemoteClient {
} }
static runRemoteClientJob() { static runRemoteClientJob() {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
RemoteClient.handleRetainedWorkspace();
yield RemoteClient.bootstrapRepository(); yield RemoteClient.bootstrapRepository();
yield RemoteClient.runCustomHookFiles(`before-build`);
}); });
} }
static runCustomHookFiles(hookLifecycle) {
return __awaiter(this, void 0, void 0, function* () {
remote_client_logger_1.RemoteClientLogger.log(`RunCustomHookFiles: ${hookLifecycle}`);
const gameCiCustomHooksPath = path_1.default.join(cloud_runner_folders_1.CloudRunnerFolders.repoPathAbsolute, `game-ci`, `hooks`);
const files = fs_1.default.readdirSync(gameCiCustomHooksPath);
for (const file of files) {
const fileContents = fs_1.default.readFileSync(path_1.default.join(gameCiCustomHooksPath, file), `utf8`);
const fileContentsObject = yaml_1.default.parse(fileContents.toString());
if (fileContentsObject.hook === hookLifecycle) {
remote_client_logger_1.RemoteClientLogger.log(`Active Hook File ${file} contents: ${fileContents}`);
}
}
});
}
static handleRetainedWorkspace() {
if (!cloud_runner_1.default.buildParameters.retainWorkspace || !cloud_runner_1.default.lockedWorkspace) {
return;
}
remote_client_logger_1.RemoteClientLogger.log(`Retained Workspace: ${cloud_runner_1.default.lockedWorkspace}`);
}
} }
__decorate([ __decorate([
cli_functions_repository_1.CliFunction(`remote-cli`, `sets up a repository, usually before a game-ci build`) cli_functions_repository_1.CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
], RemoteClient, "runRemoteClientJob", null); ], RemoteClient, "runRemoteClientJob", null);
exports.RemoteClient = RemoteClient; exports.RemoteClient = RemoteClient;
@ -4334,7 +4397,9 @@ class CloudRunnerFolders {
} }
// Only the following paths that do not start a path.join with another "Full" suffixed property need to start with an absolute / // Only the following paths that do not start a path.join with another "Full" suffixed property need to start with an absolute /
static get uniqueCloudRunnerJobFolderAbsolute() { static get uniqueCloudRunnerJobFolderAbsolute() {
return path_1.default.join(`/`, CloudRunnerFolders.buildVolumeFolder, cloud_runner_1.default.buildParameters.buildGuid); return cloud_runner_1.default.buildParameters.retainWorkspace && cloud_runner_1.default.lockedWorkspace
? path_1.default.join(`/`, CloudRunnerFolders.buildVolumeFolder, cloud_runner_1.default.lockedWorkspace)
: path_1.default.join(`/`, CloudRunnerFolders.buildVolumeFolder, cloud_runner_1.default.buildParameters.buildGuid);
} }
static get cacheFolderFull() { static get cacheFolderFull() {
return path_1.default.join('/', CloudRunnerFolders.buildVolumeFolder, CloudRunnerFolders.cacheFolder, cloud_runner_1.default.buildParameters.cacheKey); return path_1.default.join('/', CloudRunnerFolders.buildVolumeFolder, CloudRunnerFolders.cacheFolder, cloud_runner_1.default.buildParameters.cacheKey);
@ -5042,21 +5107,23 @@ class BuildAutomationWorkflow {
try { try {
cloud_runner_logger_1.default.log(`Cloud Runner is running standard build automation`); cloud_runner_logger_1.default.log(`Cloud Runner is running standard build automation`);
if (cloud_runner_options_1.default.retainWorkspaces) { if (cloud_runner_options_1.default.retainWorkspaces) {
const workspace = (yield shared_workspace_locking_1.default.GetLockedWorkspace(`test-workspace-${cloud_runner_1.default.buildParameters.buildGuid}`, cloud_runner_1.default.buildParameters.buildGuid)) || cloud_runner_1.default.buildParameters.buildGuid; const workspace = (yield shared_workspace_locking_1.default.GetOrCreateLockedWorkspace(`test-workspace-${cloud_runner_1.default.buildParameters.buildGuid}`, cloud_runner_1.default.buildParameters.buildGuid, cloud_runner_1.default.buildParameters)) || cloud_runner_1.default.buildParameters.buildGuid;
process.env.LOCKED_WORKSPACE = workspace;
cloud_runner_1.default.lockedWorkspace = workspace;
cloud_runner_logger_1.default.logLine(`Using workspace ${workspace}`); cloud_runner_logger_1.default.logLine(`Using workspace ${workspace}`);
cloudRunnerStepState.environment = [ cloudRunnerStepState.environment = [
...cloudRunnerStepState.environment, ...cloudRunnerStepState.environment,
{ name: `LOCKED_WORKSPACE`, value: workspace }, { name: `LOCKED_WORKSPACE`, value: workspace },
]; ];
} }
if (!cloud_runner_1.default.buildParameters.isCliMode)
core.startGroup('pre build steps');
let output = ''; let output = '';
if (cloud_runner_1.default.buildParameters.preBuildSteps !== '') { if (cloud_runner_1.default.buildParameters.preBuildSteps !== '') {
if (!cloud_runner_1.default.buildParameters.isCliMode)
core.startGroup('pre build steps');
output += yield custom_workflow_1.CustomWorkflow.runCustomJob(cloud_runner_1.default.buildParameters.preBuildSteps, cloudRunnerStepState.environment, cloudRunnerStepState.secrets); output += yield custom_workflow_1.CustomWorkflow.runCustomJob(cloud_runner_1.default.buildParameters.preBuildSteps, cloudRunnerStepState.environment, cloudRunnerStepState.secrets);
if (!cloud_runner_1.default.buildParameters.isCliMode)
core.endGroup();
} }
if (!cloud_runner_1.default.buildParameters.isCliMode)
core.endGroup();
cloud_runner_logger_1.default.logWithTime('Configurable pre build step(s) time'); cloud_runner_logger_1.default.logWithTime('Configurable pre build step(s) time');
if (!cloud_runner_1.default.buildParameters.isCliMode) if (!cloud_runner_1.default.buildParameters.isCliMode)
core.startGroup('build'); core.startGroup('build');
@ -5067,16 +5134,17 @@ class BuildAutomationWorkflow {
if (!cloud_runner_1.default.buildParameters.isCliMode) if (!cloud_runner_1.default.buildParameters.isCliMode)
core.endGroup(); core.endGroup();
cloud_runner_logger_1.default.logWithTime('Build time'); cloud_runner_logger_1.default.logWithTime('Build time');
if (!cloud_runner_1.default.buildParameters.isCliMode)
core.startGroup('post build steps');
if (cloud_runner_1.default.buildParameters.postBuildSteps !== '') { if (cloud_runner_1.default.buildParameters.postBuildSteps !== '') {
if (!cloud_runner_1.default.buildParameters.isCliMode)
core.startGroup('post build steps');
output += yield custom_workflow_1.CustomWorkflow.runCustomJob(cloud_runner_1.default.buildParameters.postBuildSteps, cloudRunnerStepState.environment, cloudRunnerStepState.secrets); output += yield custom_workflow_1.CustomWorkflow.runCustomJob(cloud_runner_1.default.buildParameters.postBuildSteps, cloudRunnerStepState.environment, cloudRunnerStepState.secrets);
if (!cloud_runner_1.default.buildParameters.isCliMode)
core.endGroup();
} }
if (!cloud_runner_1.default.buildParameters.isCliMode)
core.endGroup();
cloud_runner_logger_1.default.logWithTime('Configurable post build step(s) time'); cloud_runner_logger_1.default.logWithTime('Configurable post build step(s) time');
if (cloud_runner_options_1.default.retainWorkspaces) { if (cloud_runner_options_1.default.retainWorkspaces) {
yield shared_workspace_locking_1.default.ReleaseWorkspace(`test-workspace-${cloud_runner_1.default.buildParameters.buildGuid}`, cloud_runner_1.default.buildParameters.buildGuid); yield shared_workspace_locking_1.default.ReleaseWorkspace(`test-workspace-${cloud_runner_1.default.buildParameters.buildGuid}`, cloud_runner_1.default.buildParameters.buildGuid, cloud_runner_1.default.buildParameters);
cloud_runner_1.default.lockedWorkspace = undefined;
} }
cloud_runner_logger_1.default.log(`Cloud Runner finished running standard build automation`); cloud_runner_logger_1.default.log(`Cloud Runner finished running standard build automation`);
return output; return output;
@ -5099,21 +5167,19 @@ class BuildAutomationWorkflow {
${BuildAutomationWorkflow.setupCommands(builderPath)} ${BuildAutomationWorkflow.setupCommands(builderPath)}
${setupHooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '} ${setupHooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
${buildHooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '} ${buildHooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
${BuildAutomationWorkflow.BuildCommands(builderPath, cloud_runner_1.default.buildParameters.buildGuid)} ${BuildAutomationWorkflow.BuildCommands(builderPath)}
${buildHooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}`; ${buildHooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}`;
} }
static setupCommands(builderPath) { static setupCommands(builderPath) {
const commands = `mkdir -p ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.builderPathAbsolute)} && git clone -q -b ${cloud_runner_1.default.buildParameters.cloudRunnerBranch} ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.unityBuilderRepoUrl)} "${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.builderPathAbsolute)}" && chmod +x ${builderPath}`;
return `export GIT_DISCOVERY_ACROSS_FILESYSTEM=1 return `export GIT_DISCOVERY_ACROSS_FILESYSTEM=1
echo "game ci cloud runner clone" echo "game ci cloud runner clone"
mkdir -p ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.builderPathAbsolute)} if [ -e "${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}" ]; then echo "Retained Workspace Already Exists!"; else ${commands}; fi
git clone -q -b ${cloud_runner_1.default.buildParameters.cloudRunnerBranch} ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.unityBuilderRepoUrl)} "${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.builderPathAbsolute)}"
chmod +x ${builderPath}
echo "game ci cloud runner bootstrap" echo "game ci cloud runner bootstrap"
node ${builderPath} -m remote-cli`; node ${builderPath} -m remote-cli-pre-build`;
} }
// ToDo: Replace with a very simple "node ${builderPath} -m build-cli" to run the scripts below without enlarging the request size // ToDo: Replace with a very simple "node ${builderPath} -m build-cli" to run the scripts below without enlarging the request size
static BuildCommands(builderPath, guid) { static BuildCommands(builderPath) {
const linuxCacheFolder = cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.cacheFolderFull);
const distFolder = path_1.default.join(cloud_runner_folders_1.CloudRunnerFolders.builderPathAbsolute, 'dist'); const distFolder = path_1.default.join(cloud_runner_folders_1.CloudRunnerFolders.builderPathAbsolute, 'dist');
const ubuntuPlatformsFolder = path_1.default.join(cloud_runner_folders_1.CloudRunnerFolders.builderPathAbsolute, 'dist', 'platforms', 'ubuntu'); const ubuntuPlatformsFolder = path_1.default.join(cloud_runner_folders_1.CloudRunnerFolders.builderPathAbsolute, 'dist', 'platforms', 'ubuntu');
return `echo "game ci cloud runner init" return `echo "game ci cloud runner init"
@ -5128,14 +5194,7 @@ class BuildAutomationWorkflow {
/entrypoint.sh /entrypoint.sh
echo "game ci cloud runner push library to cache" echo "game ci cloud runner push library to cache"
chmod +x ${builderPath} chmod +x ${builderPath}
# node ${builderPath} -m remote-cli-post node ${builderPath} -m remote-cli-post-build`;
node ${builderPath} -m cache-push --cachePushFrom ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.libraryFolderAbsolute)} --artifactName lib-${guid} --cachePushTo ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(`${linuxCacheFolder}/Library`)}
echo "game ci cloud runner push build to cache"
node ${builderPath} -m cache-push --cachePushFrom ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.projectBuildFolderAbsolute)} --artifactName build-${guid} --cachePushTo ${`${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(`${linuxCacheFolder}/build`)}`}
${BuildAutomationWorkflow.GetCleanupCommand(cloud_runner_folders_1.CloudRunnerFolders.projectPathAbsolute)}`;
}
static GetCleanupCommand(cleanupPath) {
return cloud_runner_options_1.default.retainWorkspaces ? `` : `rm -r ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cleanupPath)}`;
} }
} }
exports.BuildAutomationWorkflow = BuildAutomationWorkflow; exports.BuildAutomationWorkflow = BuildAutomationWorkflow;

2
dist/index.js.map vendored
View File

File diff suppressed because one or more lines are too long

3
game-ci/hooks/my-test-hook-post-build.yaml 100644
View File

@ -0,0 +1,3 @@
hook: post-build
run: |
post-build test!

3
game-ci/hooks/my-test-hook-pre-build.yaml 100644
View File

@ -0,0 +1,3 @@
hook: pre-build
run: |
pre-build test!

6
src/model/build-parameters.ts
View File

@ -63,6 +63,9 @@ class BuildParameters {
public cloudRunnerIntegrationTests!: boolean; public cloudRunnerIntegrationTests!: boolean;
public cloudRunnerBuilderPlatform!: string | undefined; public cloudRunnerBuilderPlatform!: string | undefined;
public isCliMode!: boolean; public isCliMode!: boolean;
public retainWorkspace!: boolean;
public useSharedLargePackages!: boolean;
public useLZ4Compression!: boolean;
static async create(): Promise<BuildParameters> { static async create(): Promise<BuildParameters> {
const buildFile = this.parseBuildFile(Input.buildName, Input.targetPlatform, Input.androidAppBundle); const buildFile = this.parseBuildFile(Input.buildName, Input.targetPlatform, Input.androidAppBundle);
@ -137,6 +140,9 @@ class BuildParameters {
readInputFromOverrideList: CloudRunnerOptions.readInputFromOverrideList(), readInputFromOverrideList: CloudRunnerOptions.readInputFromOverrideList(),
kubeStorageClass: CloudRunnerOptions.kubeStorageClass, kubeStorageClass: CloudRunnerOptions.kubeStorageClass,
cacheKey: CloudRunnerOptions.cacheKey, cacheKey: CloudRunnerOptions.cacheKey,
retainWorkspace: CloudRunnerOptions.retainWorkspaces,
useSharedLargePackages: CloudRunnerOptions.useSharedLargePackages,
useLZ4Compression: CloudRunnerOptions.useLZ4Compression,
}; };
} }

41
src/model/cli/cli.ts
View File

@ -13,6 +13,7 @@ import CloudRunnerOptionsReader from '../cloud-runner/services/cloud-runner-opti
import GitHub from '../github'; import GitHub from '../github';
import { TaskParameterSerializer } from '../cloud-runner/services/task-parameter-serializer'; import { TaskParameterSerializer } from '../cloud-runner/services/task-parameter-serializer';
import { CloudRunnerFolders } from '../cloud-runner/services/cloud-runner-folders'; import { CloudRunnerFolders } from '../cloud-runner/services/cloud-runner-folders';
import { CloudRunnerSystem } from '../cloud-runner/services/cloud-runner-system';
export class Cli { export class Cli {
public static options; public static options;
@ -77,6 +78,7 @@ export class Cli {
${JSON.stringify(buildParameter, undefined, 4)} ${JSON.stringify(buildParameter, undefined, 4)}
`); `);
CloudRunner.buildParameters = buildParameter; CloudRunner.buildParameters = buildParameter;
CloudRunner.lockedWorkspace = process.env.LOCKED_WORKSPACE;
return await results.target[results.propertyKey](Cli.options); return await results.target[results.propertyKey](Cli.options);
} }
@ -111,39 +113,28 @@ export class Cli {
@CliFunction(`remote-cli-post-build`, `runs a cloud runner build`) @CliFunction(`remote-cli-post-build`, `runs a cloud runner build`)
public static async PostCLIBuild(): Promise<string> { public static async PostCLIBuild(): Promise<string> {
const buildParameter = await BuildParameters.create();
/*
# LIBRARY CACHE
node ${builderPath} -m cache-push --cachePushFrom ${CloudRunnerFolders.ToLinuxFolder(
CloudRunnerFolders.libraryFolderAbsolute,
)} --artifactName lib-${guid} --cachePushTo ${CloudRunnerFolders.ToLinuxFolder(`${linuxCacheFolder}/Library`)}
echo "game ci cloud runner push build to cache"
# BUILD CACHE
node ${builderPath} -m cache-push --cachePushFrom ${CloudRunnerFolders.ToLinuxFolder(
CloudRunnerFolders.projectBuildFolderAbsolute,
)} --artifactName build-${guid} --cachePushTo ${`${CloudRunnerFolders.ToLinuxFolder(`${linuxCacheFolder}/build`)}`}
# RETAINED WORKSPACE CLEANUP
${BuildAutomationWorkflow.GetCleanupCommand(CloudRunnerFolders.projectPathAbsolute)}`;
*/
core.info(`Running POST build tasks`); core.info(`Running POST build tasks`);
Caching.PushToCache( await Caching.PushToCache(
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute),
CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderFull}/Library`), CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderFull}/Library`),
`lib-${buildParameter.buildGuid}`, CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute),
`lib-${CloudRunner.buildParameters.buildGuid}`,
); );
Caching.PushToCache( await Caching.PushToCache(
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.projectBuildFolderAbsolute),
CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderFull}/build`), CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderFull}/build`),
`build-${buildParameter.buildGuid}`, CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.projectBuildFolderAbsolute),
`build-${CloudRunner.buildParameters.buildGuid}`,
); );
if (!CloudRunner.buildParameters.retainWorkspace) {
await CloudRunnerSystem.Run(
`rm -r ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
);
}
await RemoteClient.runCustomHookFiles(`after-build`);
return new Promise((result) => result(``)); return new Promise((result) => result(``));
} }
} }

226
src/model/cli/shared-workspace-locking.ts
View File

@ -2,49 +2,179 @@ import { CloudRunnerSystem } from '../cloud-runner/services/cloud-runner-system'
import * as fs from 'fs'; import * as fs from 'fs';
import CloudRunnerLogger from '../cloud-runner/services/cloud-runner-logger'; import CloudRunnerLogger from '../cloud-runner/services/cloud-runner-logger';
import CloudRunnerOptions from '../cloud-runner/cloud-runner-options'; import CloudRunnerOptions from '../cloud-runner/cloud-runner-options';
import BuildParameters from '../build-parameters';
export class SharedWorkspaceLocking { export class SharedWorkspaceLocking {
public static async GetLockedWorkspace(workspaceIfCreated: string, runId: string) { private static readonly workspaceRoot = `s3://game-ci-test-storage/locks/`;
public static async GetAllWorkspaces(buildParametersContext: BuildParameters): Promise<string[]> {
return (
await SharedWorkspaceLocking.ReadLines(
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
)
).map((x) => x.replace(`/`, ``));
}
public static async DoesWorkspaceTopLevelExist(buildParametersContext: BuildParameters) {
const results = (await SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}`)).map(
(x) => x.replace(`/`, ``),
);
return results.includes(buildParametersContext.cacheKey);
}
public static async GetAllLocks(workspace: string, buildParametersContext: BuildParameters): Promise<string[]> {
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
throw new Error("Workspace doesn't exist, can't call get all locks");
}
return (
await SharedWorkspaceLocking.ReadLines(
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/`,
)
).map((x) => x.replace(`/`, ``));
}
public static async GetOrCreateLockedWorkspace(
workspaceIfCreated: string,
runId: string,
buildParametersContext: BuildParameters,
) {
if (!CloudRunnerOptions.retainWorkspaces) { if (!CloudRunnerOptions.retainWorkspaces) {
return; return;
} }
const workspaces = await SharedWorkspaceLocking.GetFreeWorkspaces(); CloudRunnerLogger.log(`run agent ${runId} is trying to access a workspace`);
for (const element of workspaces) {
if (await SharedWorkspaceLocking.LockWorkspace(element, runId)) { if (await SharedWorkspaceLocking.DoesWorkspaceTopLevelExist(buildParametersContext)) {
return element; const workspaces = await SharedWorkspaceLocking.GetFreeWorkspaces(buildParametersContext);
for (const element of workspaces) {
if (await SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext)) {
CloudRunnerLogger.log(`run agent ${runId} locked workspace: ${element}`);
return element;
}
} }
} }
return await SharedWorkspaceLocking.CreateLockableWorkspace(workspaceIfCreated); const workspace = await SharedWorkspaceLocking.CreateWorkspace(workspaceIfCreated, buildParametersContext, runId);
CloudRunnerLogger.log(`run agent ${runId} didn't find a free workspace so created: ${workspace}`);
return workspace;
} }
public static async DoesWorkspaceExist(workspace: string) { public static async DoesWorkspaceExist(workspace: string, buildParametersContext: BuildParameters) {
return (await SharedWorkspaceLocking.GetAllWorkspaces()).includes(workspace); return (await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext)).includes(workspace);
}
public static async HasWorkspaceLock(
workspace: string,
runId: string,
buildParametersContext: BuildParameters,
): Promise<boolean> {
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
return false;
}
return (
(await SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext)).filter((x) => x.includes(runId))
.length > 0
);
} }
public static async GetFreeWorkspaces(): Promise<string[]> { public static async GetFreeWorkspaces(buildParametersContext: BuildParameters): Promise<string[]> {
const result: string[] = []; const result: string[] = [];
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(); const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
for (const element of workspaces) { for (const element of workspaces) {
if (!(await SharedWorkspaceLocking.IsWorkspaceLocked(element))) { if (!(await SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParametersContext))) {
result.push(element); result.push(element);
} }
} }
return result; return result;
} }
public static async GetAllWorkspaces(): Promise<string[]> {
return (await SharedWorkspaceLocking.ReadLines(`aws s3 ls s3://game-ci-test-storage/locks/`)).map((x) => public static async IsWorkspaceLocked(workspace: string, buildParametersContext: BuildParameters): Promise<boolean> {
x.replace(`/`, ``), if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
return false;
}
const files = await SharedWorkspaceLocking.ReadLines(
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/`,
); );
const workspaceFileDoesNotExists =
files.filter((x) => {
return x.includes(`_workspace`);
}).length === 0;
const lockFilesExist =
files.filter((x) => {
return x.includes(`_lock`);
}).length > 0;
return workspaceFileDoesNotExists || lockFilesExist;
} }
public static async GetAllLocks(workspace: string): Promise<string[]> {
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace))) { public static async CreateWorkspace(workspace: string, buildParametersContext: BuildParameters, lockId: string = ``) {
throw new Error("Workspace doesn't exist, can't call get all locks"); if (lockId !== ``) {
await SharedWorkspaceLocking.LockWorkspace(workspace, lockId, buildParametersContext);
} }
return (await SharedWorkspaceLocking.ReadLines(`aws s3 ls s3://game-ci-test-storage/locks/${workspace}/`)).map( const file = `${Date.now()}_workspace`;
(x) => x.replace(`/`, ``), fs.writeFileSync(file, '');
await CloudRunnerSystem.Run(
`aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`,
false,
true,
);
fs.rmSync(file);
return workspace;
}
public static async LockWorkspace(
workspace: string,
runId: string,
buildParametersContext: BuildParameters,
): Promise<boolean> {
const file = `${Date.now()}_${runId}_lock`;
fs.writeFileSync(file, '');
await CloudRunnerSystem.Run(
`aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`,
false,
true,
);
fs.rmSync(file);
return SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext);
}
public static async ReleaseWorkspace(
workspace: string,
runId: string,
buildParametersContext: BuildParameters,
): Promise<boolean> {
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
return true;
}
const file = (await SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext)).filter((x) =>
x.includes(`_${runId}_lock`),
);
CloudRunnerLogger.log(
`${JSON.stringify(await SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext))}`,
);
CloudRunnerLogger.log(`Deleting file ${file}`);
CloudRunnerLogger.log(
`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`,
);
await CloudRunnerSystem.Run(
`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`,
false,
true,
);
return !SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext);
}
public static async CleanupWorkspace(workspace: string, buildParametersContext: BuildParameters) {
await CloudRunnerSystem.Run(
`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace} --recursive`,
false,
true,
); );
} }
@ -61,64 +191,6 @@ export class SharedWorkspaceLocking {
return lineValues[lineValues.length - 1]; return lineValues[lineValues.length - 1];
}); });
} }
public static async LockWorkspace(workspace: string, runId: string): Promise<boolean> {
const file = `${Date.now()}_${runId}_lock`;
fs.writeFileSync(file, '');
await CloudRunnerSystem.Run(
`aws s3 cp ./${file} s3://game-ci-test-storage/locks/${workspace}/${file}`,
false,
true,
);
fs.rmSync(file);
return SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId);
}
public static async ReleaseWorkspace(workspace: string, runId: string): Promise<boolean> {
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace))) {
return true;
}
const file = (await SharedWorkspaceLocking.GetAllLocks(workspace)).filter((x) => x.includes(`_${runId}_lock`));
CloudRunnerLogger.log(`${JSON.stringify(await SharedWorkspaceLocking.GetAllLocks(workspace))}`);
CloudRunnerLogger.log(`Deleting file ${file}`);
CloudRunnerLogger.log(`aws s3 rm s3://game-ci-test-storage/locks/${workspace}/${file}`);
await CloudRunnerSystem.Run(`aws s3 rm s3://game-ci-test-storage/locks/${workspace}/${file}`, false, true);
return !SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId);
}
public static async HasWorkspaceLock(workspace: string, runId: string): Promise<boolean> {
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace))) {
return false;
}
return (await SharedWorkspaceLocking.GetAllLocks(workspace)).filter((x) => x.includes(runId)).length > 0;
}
public static async IsWorkspaceLocked(workspace: string): Promise<boolean> {
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace))) {
return false;
}
const files = await SharedWorkspaceLocking.ReadLines(`aws s3 ls s3://game-ci-test-storage/locks/${workspace}/`);
// 1 Because we expect 1 workspace file to exist in every workspace folder
return files.length > 1;
}
public static async CreateLockableWorkspace(workspace: string) {
const file = `${Date.now()}_workspace`;
fs.writeFileSync(file, '');
await CloudRunnerSystem.Run(
`aws s3 cp ./${file} s3://game-ci-test-storage/locks/${workspace}/${file}`,
false,
true,
);
fs.rmSync(file);
return workspace;
}
// eslint-disable-next-line no-unused-vars
public static ReleaseLock(workspace: string) {}
} }
export default SharedWorkspaceLocking; export default SharedWorkspaceLocking;

8
src/model/cloud-runner/cloud-runner-options.ts
View File

@ -171,6 +171,14 @@ class CloudRunnerOptions {
return Number(CloudRunnerOptions.getInput(`retainWorkspacesMax`)) || 5; return Number(CloudRunnerOptions.getInput(`retainWorkspacesMax`)) || 5;
} }
public static get useSharedLargePackages(): boolean {
return CloudRunnerOptions.getInput(`useSharedLargePackages`) || false;
}
public static get useLZ4Compression(): boolean {
return CloudRunnerOptions.getInput(`useLZ4Compression`) || true;
}
public static ToEnvVarFormat(input: string) { public static ToEnvVarFormat(input: string) {
if (input.toUpperCase() === input) { if (input.toUpperCase() === input) {
return input; return input;

1
src/model/cloud-runner/cloud-runner.ts
View File

@ -20,6 +20,7 @@ class CloudRunner {
public static buildParameters: BuildParameters; public static buildParameters: BuildParameters;
private static defaultSecrets: CloudRunnerSecret[]; private static defaultSecrets: CloudRunnerSecret[];
private static cloudRunnerEnvironmentVariables: CloudRunnerEnvironmentVariable[]; private static cloudRunnerEnvironmentVariables: CloudRunnerEnvironmentVariable[];
static lockedWorkspace: string | undefined;
public static setup(buildParameters: BuildParameters) { public static setup(buildParameters: BuildParameters) {
CloudRunnerLogger.setup(); CloudRunnerLogger.setup();
CloudRunner.buildParameters = buildParameters; CloudRunner.buildParameters = buildParameters;

10
src/model/cloud-runner/providers/local-docker/index.ts
View File

@ -7,8 +7,6 @@ import Docker from '../../../docker';
import { Action } from '../../../../model'; import { Action } from '../../../../model';
import { writeFileSync } from 'fs'; import { writeFileSync } from 'fs';
// import * as core from '@actions/core';
class LocalDockerCloudRunner implements ProviderInterface { class LocalDockerCloudRunner implements ProviderInterface {
public buildParameters: BuildParameters | undefined; public buildParameters: BuildParameters | undefined;
@ -84,6 +82,7 @@ class LocalDockerCloudRunner implements ProviderInterface {
} }
} }
let myOutput = ''; let myOutput = '';
const sharedFolder = this.buildParameters?.retainWorkspace ? `/data/` : `/data/cache/`;
// core.info(JSON.stringify({ workspace, actionFolder, ...this.buildParameters, ...content }, undefined, 4)); // core.info(JSON.stringify({ workspace, actionFolder, ...this.buildParameters, ...content }, undefined, 4));
const entrypointFilePath = `start.sh`; const entrypointFilePath = `start.sh`;
@ -95,11 +94,12 @@ class LocalDockerCloudRunner implements ProviderInterface {
apt-get update > /dev/null && apt-get install -y tree> /dev/null apt-get update > /dev/null && apt-get install -y tree> /dev/null
mkdir -p /github/workspace/cloud-runner-cache mkdir -p /github/workspace/cloud-runner-cache
mkdir -p /data/cache mkdir -p /data/cache
cp -a /github/workspace/cloud-runner-cache/. /data/cache/ cp -a /github/workspace/cloud-runner-cache/. ${sharedFolder}
tree -L 2 /data/cache tree -L 3 ${sharedFolder}
${commands} ${commands}
cp -a /data/cache/. /github/workspace/cloud-runner-cache/ cp -a ${sharedFolder}. /github/workspace/cloud-runner-cache/
tree -L 2 /github/workspace/cloud-runner-cache tree -L 2 /github/workspace/cloud-runner-cache
tree -L 3 ${sharedFolder}
`, `,
{ {
flag: 'w', flag: 'w',

34
src/model/cloud-runner/remote-client/caching.ts
View File

@ -46,6 +46,7 @@ export class Caching {
public static async PushToCache(cacheFolder: string, sourceFolder: string, cacheArtifactName: string) { public static async PushToCache(cacheFolder: string, sourceFolder: string, cacheArtifactName: string) {
cacheArtifactName = cacheArtifactName.replace(' ', ''); cacheArtifactName = cacheArtifactName.replace(' ', '');
const startPath = process.cwd(); const startPath = process.cwd();
const compressionSuffix = CloudRunner.buildParameters.useLZ4Compression ? '.lz4' : '';
try { try {
if (!(await fileExists(cacheFolder))) { if (!(await fileExists(cacheFolder))) {
await CloudRunnerSystem.Run(`mkdir -p ${cacheFolder}`); await CloudRunnerSystem.Run(`mkdir -p ${cacheFolder}`);
@ -65,7 +66,7 @@ export class Caching {
); );
if (CloudRunner.buildParameters.cloudRunnerIntegrationTests) { if (CloudRunner.buildParameters.cloudRunnerIntegrationTests) {
await CloudRunnerSystem.Run(`tree -L 2 ./..`); // await CloudRunnerSystem.Run(`tree -L 2 ./..`);
await CloudRunnerSystem.Run(`tree -L 2`); await CloudRunnerSystem.Run(`tree -L 2`);
} }
@ -78,14 +79,16 @@ export class Caching {
return; return;
} }
await CloudRunnerSystem.Run(`tar -cf ${cacheArtifactName}.tar.lz4 ${path.basename(sourceFolder)}`); await CloudRunnerSystem.Run(
await CloudRunnerSystem.Run(`du ${cacheArtifactName}.tar.lz4`); `tar -cf ${cacheArtifactName}.tar${compressionSuffix} ${path.basename(sourceFolder)}`,
assert(await fileExists(`${cacheArtifactName}.tar.lz4`), 'cache archive exists'); );
await CloudRunnerSystem.Run(`du ${cacheArtifactName}.tar${compressionSuffix}`);
assert(await fileExists(`${cacheArtifactName}.tar${compressionSuffix}`), 'cache archive exists');
assert(await fileExists(path.basename(sourceFolder)), 'source folder exists'); assert(await fileExists(path.basename(sourceFolder)), 'source folder exists');
await CloudRunnerSystem.Run(`mv ${cacheArtifactName}.tar.lz4 ${cacheFolder}`); await CloudRunnerSystem.Run(`mv ${cacheArtifactName}.tar${compressionSuffix} ${cacheFolder}`);
RemoteClientLogger.log(`moved cache entry ${cacheArtifactName} to ${cacheFolder}`); RemoteClientLogger.log(`moved cache entry ${cacheArtifactName} to ${cacheFolder}`);
assert( assert(
await fileExists(`${path.join(cacheFolder, cacheArtifactName)}.tar.lz4`), await fileExists(`${path.join(cacheFolder, cacheArtifactName)}.tar${compressionSuffix}`),
'cache archive exists inside cache folder', 'cache archive exists inside cache folder',
); );
} catch (error) { } catch (error) {
@ -96,6 +99,7 @@ export class Caching {
} }
public static async PullFromCache(cacheFolder: string, destinationFolder: string, cacheArtifactName: string = ``) { public static async PullFromCache(cacheFolder: string, destinationFolder: string, cacheArtifactName: string = ``) {
cacheArtifactName = cacheArtifactName.replace(' ', ''); cacheArtifactName = cacheArtifactName.replace(' ', '');
const compressionSuffix = CloudRunner.buildParameters.useLZ4Compression ? '.lz4' : '';
const startPath = process.cwd(); const startPath = process.cwd();
RemoteClientLogger.log(`Caching for ${path.basename(destinationFolder)}`); RemoteClientLogger.log(`Caching for ${path.basename(destinationFolder)}`);
try { try {
@ -107,24 +111,26 @@ export class Caching {
await fs.promises.mkdir(destinationFolder); await fs.promises.mkdir(destinationFolder);
} }
const latestInBranch = await (await CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .tar.lz4$ | head -1`)) const latestInBranch = await (
await CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .tar${compressionSuffix}$ | head -1`)
)
.replace(/\n/g, ``) .replace(/\n/g, ``)
.replace('.tar.lz4', ''); .replace(`.tar${compressionSuffix}`, '');
process.chdir(cacheFolder); process.chdir(cacheFolder);
const cacheSelection = const cacheSelection =
cacheArtifactName !== `` && (await fileExists(`${cacheArtifactName}.tar.lz4`)) cacheArtifactName !== `` && (await fileExists(`${cacheArtifactName}.tar${compressionSuffix}`))
? cacheArtifactName ? cacheArtifactName
: latestInBranch; : latestInBranch;
await CloudRunnerLogger.log(`cache key ${cacheArtifactName} selection ${cacheSelection}`); await CloudRunnerLogger.log(`cache key ${cacheArtifactName} selection ${cacheSelection}`);
if (await fileExists(`${cacheSelection}.tar.lz4`)) { if (await fileExists(`${cacheSelection}.tar${compressionSuffix}`)) {
const resultsFolder = `results${CloudRunner.buildParameters.buildGuid}`; const resultsFolder = `results${CloudRunner.buildParameters.buildGuid}`;
await CloudRunnerSystem.Run(`mkdir -p ${resultsFolder}`); await CloudRunnerSystem.Run(`mkdir -p ${resultsFolder}`);
RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.tar.lz4`); RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.tar${compressionSuffix}`);
const fullResultsFolder = path.join(cacheFolder, resultsFolder); const fullResultsFolder = path.join(cacheFolder, resultsFolder);
await CloudRunnerSystem.Run(`tar -xf ${cacheSelection}.tar.lz4 -C ${fullResultsFolder}`); await CloudRunnerSystem.Run(`tar -xf ${cacheSelection}.tar${compressionSuffix} -C ${fullResultsFolder}`);
RemoteClientLogger.log(`cache item extracted to ${fullResultsFolder}`); RemoteClientLogger.log(`cache item extracted to ${fullResultsFolder}`);
assert(await fileExists(fullResultsFolder), `cache extraction results folder exists`); assert(await fileExists(fullResultsFolder), `cache extraction results folder exists`);
const destinationParentFolder = path.resolve(destinationFolder, '..'); const destinationParentFolder = path.resolve(destinationFolder, '..');
@ -144,7 +150,9 @@ export class Caching {
} else { } else {
RemoteClientLogger.logWarning(`cache item ${cacheArtifactName} doesn't exist ${destinationFolder}`); RemoteClientLogger.logWarning(`cache item ${cacheArtifactName} doesn't exist ${destinationFolder}`);
if (cacheSelection !== ``) { if (cacheSelection !== ``) {
RemoteClientLogger.logWarning(`cache item ${cacheArtifactName}.tar.lz4 doesn't exist ${destinationFolder}`); RemoteClientLogger.logWarning(
`cache item ${cacheArtifactName}.tar${compressionSuffix} doesn't exist ${destinationFolder}`,
);
throw new Error(`Failed to get cache item, but cache hit was found: ${cacheSelection}`); throw new Error(`Failed to get cache item, but cache hit was found: ${cacheSelection}`);
} }
} }

49
src/model/cloud-runner/remote-client/index.ts
View File

@ -9,6 +9,7 @@ import { assert } from 'console';
import CloudRunnerLogger from '../services/cloud-runner-logger'; import CloudRunnerLogger from '../services/cloud-runner-logger';
import { CliFunction } from '../../cli/cli-functions-repository'; import { CliFunction } from '../../cli/cli-functions-repository';
import { CloudRunnerSystem } from '../services/cloud-runner-system'; import { CloudRunnerSystem } from '../services/cloud-runner-system';
import YAML from 'yaml';
export class RemoteClient { export class RemoteClient {
public static async bootstrapRepository() { public static async bootstrapRepository() {
@ -61,8 +62,20 @@ export class RemoteClient {
} }
private static async cloneRepoWithoutLFSFiles() { private static async cloneRepoWithoutLFSFiles() {
process.chdir(`${CloudRunnerFolders.repoPathAbsolute}`);
if (CloudRunner.buildParameters.cloudRunnerIntegrationTests) {
await CloudRunnerSystem.Run(`tree -L 2 ./..`);
}
if (fs.existsSync(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`))) {
RemoteClientLogger.log(
`${CloudRunnerFolders.repoPathAbsolute} repo exists - skipping clone - retained workspace mode ${CloudRunner.buildParameters.retainWorkspace}`,
);
await CloudRunnerSystem.Run(`git reset --hard ${CloudRunner.buildParameters.gitSha}`);
return;
}
try { try {
process.chdir(`${CloudRunnerFolders.repoPathAbsolute}`);
RemoteClientLogger.log(`Initializing source repository for cloning with caching of LFS files`); RemoteClientLogger.log(`Initializing source repository for cloning with caching of LFS files`);
await CloudRunnerSystem.Run(`git config --global advice.detachedHead false`); await CloudRunnerSystem.Run(`git config --global advice.detachedHead false`);
RemoteClientLogger.log(`Cloning the repository being built:`); RemoteClientLogger.log(`Cloning the repository being built:`);
@ -78,6 +91,7 @@ export class RemoteClient {
assert(fs.existsSync(`.git`), 'git folder exists'); assert(fs.existsSync(`.git`), 'git folder exists');
RemoteClientLogger.log(`${CloudRunner.buildParameters.branch}`); RemoteClientLogger.log(`${CloudRunner.buildParameters.branch}`);
await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.branch}`); await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.branch}`);
await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.gitSha}`);
assert(fs.existsSync(path.join(`.git`, `lfs`)), 'LFS folder should not exist before caching'); assert(fs.existsSync(path.join(`.git`, `lfs`)), 'LFS folder should not exist before caching');
RemoteClientLogger.log(`Checked out ${CloudRunner.buildParameters.branch}`); RemoteClientLogger.log(`Checked out ${CloudRunner.buildParameters.branch}`);
} catch (error) { } catch (error) {
@ -86,10 +100,15 @@ export class RemoteClient {
} }
static replaceLargePackageReferencesWithSharedReferences() { static replaceLargePackageReferencesWithSharedReferences() {
const manifest = fs.readFileSync(
path.join(CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`),
'utf8',
);
if (CloudRunner.buildParameters.cloudRunnerIntegrationTests) { if (CloudRunner.buildParameters.cloudRunnerIntegrationTests) {
CloudRunnerLogger.log( CloudRunnerLogger.log(manifest);
fs.readFileSync(path.join(CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`), 'utf8'), }
); if (CloudRunner.buildParameters.useSharedLargePackages) {
manifest.replace(/LargePackages/g, '../../LargePackages');
} }
} }
@ -102,8 +121,28 @@ export class RemoteClient {
assert(fs.existsSync(CloudRunnerFolders.lfsFolderAbsolute)); assert(fs.existsSync(CloudRunnerFolders.lfsFolderAbsolute));
} }
@CliFunction(`remote-cli`, `sets up a repository, usually before a game-ci build`) @CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
static async runRemoteClientJob() { static async runRemoteClientJob() {
RemoteClient.handleRetainedWorkspace();
await RemoteClient.bootstrapRepository(); await RemoteClient.bootstrapRepository();
await RemoteClient.runCustomHookFiles(`before-build`);
}
static async runCustomHookFiles(hookLifecycle: string) {
RemoteClientLogger.log(`RunCustomHookFiles: ${hookLifecycle}`);
const gameCiCustomHooksPath = path.join(CloudRunnerFolders.repoPathAbsolute, `game-ci`, `hooks`);
const files = fs.readdirSync(gameCiCustomHooksPath);
for (const file of files) {
const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`);
const fileContentsObject = YAML.parse(fileContents.toString());
if (fileContentsObject.hook === hookLifecycle) {
RemoteClientLogger.log(`Active Hook File ${file} contents: ${fileContents}`);
}
}
}
static handleRetainedWorkspace() {
if (!CloudRunner.buildParameters.retainWorkspace || !CloudRunner.lockedWorkspace) {
return;
}
RemoteClientLogger.log(`Retained Workspace: ${CloudRunner.lockedWorkspace}`);
} }
} }

4
src/model/cloud-runner/services/cloud-runner-folders.ts
View File

@ -11,7 +11,9 @@ export class CloudRunnerFolders {
// Only the following paths that do not start a path.join with another "Full" suffixed property need to start with an absolute / // Only the following paths that do not start a path.join with another "Full" suffixed property need to start with an absolute /
public static get uniqueCloudRunnerJobFolderAbsolute(): string { public static get uniqueCloudRunnerJobFolderAbsolute(): string {
return path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.buildParameters.buildGuid); return CloudRunner.buildParameters.retainWorkspace && CloudRunner.lockedWorkspace
? path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.lockedWorkspace)
: path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.buildParameters.buildGuid);
} }
public static get cacheFolderFull(): string { public static get cacheFolderFull(): string {

49
src/model/cloud-runner/tests/cloud-runner-run-once-custom-hooks.test.ts 100644
View File

@ -0,0 +1,49 @@
import CloudRunner from '../cloud-runner';
import { BuildParameters, ImageTag } from '../..';
import UnityVersioning from '../../unity-versioning';
import { Cli } from '../../cli/cli';
import CloudRunnerLogger from '../services/cloud-runner-logger';
import { v4 as uuidv4 } from 'uuid';
import CloudRunnerOptions from '../cloud-runner-options';
import setups from './cloud-runner-suite.test';
async function CreateParameters(overrides) {
if (overrides) {
Cli.options = overrides;
}
return await BuildParameters.create();
}
describe('Cloud Runner Custom Hooks', () => {
it('Responds', () => {});
setups();
if (CloudRunnerOptions.cloudRunnerTests && CloudRunnerOptions.cloudRunnerCluster !== `k8s`) {
it('Run one build it should not already be retained, run subsequent build which should use retained workspace', async () => {
const overrides = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
retainWorkspaces: true,
};
const buildParameter2 = await CreateParameters(overrides);
const baseImage2 = new ImageTag(buildParameter2);
const results2 = await CloudRunner.run(buildParameter2, baseImage2.toString());
CloudRunnerLogger.log(`run 2 succeeded`);
const build2ContainsRetainedWorkspacePhrase = results2.includes(`Retained Workspace:`);
const build2ContainsWorkspaceExistsAlreadyPhrase = results2.includes(`Retained Workspace Already Exists!`);
const build2ContainsBuildSucceeded = results2.includes('Build succeeded');
const build2ContainsPreBuildHookMessage = results2.includes('pre-build test!');
const build2ContainsPostBuildHookMessage = results2.includes('post-build test!');
expect(build2ContainsRetainedWorkspacePhrase).toBeTruthy();
expect(build2ContainsWorkspaceExistsAlreadyPhrase).toBeTruthy();
expect(build2ContainsBuildSucceeded).toBeTruthy();
expect(build2ContainsPreBuildHookMessage).toBeTruthy();
expect(build2ContainsPostBuildHookMessage).toBeTruthy();
}, 10000000);
}
});

6
src/model/cloud-runner/tests/cloud-runner-run-twice-retaining.test.ts
View File

@ -19,7 +19,7 @@ describe('Cloud Runner Retain Workspace', () => {
it('Responds', () => {}); it('Responds', () => {});
setups(); setups();
if (CloudRunnerOptions.cloudRunnerTests && CloudRunnerOptions.cloudRunnerCluster !== `k8s`) { if (CloudRunnerOptions.cloudRunnerTests && CloudRunnerOptions.cloudRunnerCluster !== `k8s`) {
it('Run one build it should not use cache, run subsequent build which should use cache', async () => { it('Run one build it should not already be retained, run subsequent build which should use retained workspace', async () => {
const overrides = { const overrides = {
versioning: 'None', versioning: 'None',
projectPath: 'test-project', projectPath: 'test-project',
@ -51,6 +51,8 @@ describe('Cloud Runner Retain Workspace', () => {
const build2ContainsCacheKey = results2.includes(buildParameter.cacheKey); const build2ContainsCacheKey = results2.includes(buildParameter.cacheKey);
const build2ContainsBuildGuid1FromRetainedWorkspace = results2.includes(buildParameter.buildGuid); const build2ContainsBuildGuid1FromRetainedWorkspace = results2.includes(buildParameter.buildGuid);
const build2ContainsRetainedWorkspacePhrase = results2.includes(`Retained Workspace:`);
const build2ContainsWorkspaceExistsAlreadyPhrase = results2.includes(`Retained Workspace Already Exists!`);
const build2ContainsBuildSucceeded = results2.includes(buildSucceededString); const build2ContainsBuildSucceeded = results2.includes(buildSucceededString);
const build2NotContainsNoLibraryMessage = !results2.includes(libraryString); const build2NotContainsNoLibraryMessage = !results2.includes(libraryString);
const build2NotContainsZeroLibraryCacheFilesMessage = !results2.includes( const build2NotContainsZeroLibraryCacheFilesMessage = !results2.includes(
@ -61,6 +63,8 @@ describe('Cloud Runner Retain Workspace', () => {
); );
expect(build2ContainsCacheKey).toBeTruthy(); expect(build2ContainsCacheKey).toBeTruthy();
expect(build2ContainsRetainedWorkspacePhrase).toBeTruthy();
expect(build2ContainsWorkspaceExistsAlreadyPhrase).toBeTruthy();
expect(build2ContainsBuildGuid1FromRetainedWorkspace).toBeTruthy(); expect(build2ContainsBuildGuid1FromRetainedWorkspace).toBeTruthy();
expect(build2ContainsBuildSucceeded).toBeTruthy(); expect(build2ContainsBuildSucceeded).toBeTruthy();
expect(build2NotContainsZeroLibraryCacheFilesMessage).toBeTruthy(); expect(build2NotContainsZeroLibraryCacheFilesMessage).toBeTruthy();

77
src/model/cloud-runner/tests/shared-workspace-locking.test.ts
View File

@ -4,51 +4,94 @@ import setups from './cloud-runner-suite.test';
import CloudRunnerLogger from '../services/cloud-runner-logger'; import CloudRunnerLogger from '../services/cloud-runner-logger';
import { v4 as uuidv4 } from 'uuid'; import { v4 as uuidv4 } from 'uuid';
import CloudRunnerOptions from '../cloud-runner-options'; import CloudRunnerOptions from '../cloud-runner-options';
import UnityVersioning from '../../unity-versioning';
import BuildParameters from '../../build-parameters';
async function CreateParameters(overrides) {
if (overrides) {
Cli.options = overrides;
}
return await BuildParameters.create();
}
describe('Cloud Runner Locking', () => { describe('Cloud Runner Locking', () => {
setups(); setups();
it('Responds', () => {}); it('Responds', () => {});
if (CloudRunnerOptions.cloudRunnerTests) { if (CloudRunnerOptions.cloudRunnerTests) {
it(`simple locking flow`, async () => { it(`Simple Locking Flow`, async () => {
Cli.options.retainWorkspaces = true; Cli.options.retainWorkspaces = true;
const overrides: any = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
};
const buildParameters = await CreateParameters(overrides);
const newWorkspaceName = `test-workspace-${uuidv4()}`; const newWorkspaceName = `test-workspace-${uuidv4()}`;
const runId = uuidv4(); const runId = uuidv4();
await SharedWorkspaceLocking.CreateLockableWorkspace(newWorkspaceName); await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters);
const isExpectedUnlockedBeforeLocking = const isExpectedUnlockedBeforeLocking =
(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName)) === false; (await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === false;
expect(isExpectedUnlockedBeforeLocking).toBeTruthy(); expect(isExpectedUnlockedBeforeLocking).toBeTruthy();
await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId); await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters);
const isExpectedLockedAfterLocking = (await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName)) === true; const isExpectedLockedAfterLocking =
(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === true;
expect(isExpectedLockedAfterLocking).toBeTruthy(); expect(isExpectedLockedAfterLocking).toBeTruthy();
const locksBeforeRelease = await SharedWorkspaceLocking.GetAllLocks(newWorkspaceName); const locksBeforeRelease = await SharedWorkspaceLocking.GetAllLocks(newWorkspaceName, buildParameters);
CloudRunnerLogger.log(JSON.stringify(locksBeforeRelease, undefined, 4)); CloudRunnerLogger.log(JSON.stringify(locksBeforeRelease, undefined, 4));
expect(locksBeforeRelease.length > 1).toBeTruthy(); expect(locksBeforeRelease.length > 1).toBeTruthy();
await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId); await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters);
const locks = await SharedWorkspaceLocking.GetAllLocks(newWorkspaceName); const locks = await SharedWorkspaceLocking.GetAllLocks(newWorkspaceName, buildParameters);
expect(locks.length === 1).toBeTruthy(); expect(locks.length === 1).toBeTruthy();
const isExpectedLockedAfterReleasing = const isExpectedLockedAfterReleasing =
(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName)) === false; (await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === false;
expect(isExpectedLockedAfterReleasing).toBeTruthy(); expect(isExpectedLockedAfterReleasing).toBeTruthy();
}, 150000); }, 150000);
it('Locking', async () => { it.skip('All Locking Actions', async () => {
Cli.options.retainWorkspaces = true; Cli.options.retainWorkspaces = true;
CloudRunnerLogger.log(`GetAllWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetAllWorkspaces())}`); const overrides: any = {
CloudRunnerLogger.log(`GetFreeWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetFreeWorkspaces())}`); versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
};
const buildParameters = await CreateParameters(overrides);
CloudRunnerLogger.log( CloudRunnerLogger.log(
`IsWorkspaceLocked ${JSON.stringify(await SharedWorkspaceLocking.IsWorkspaceLocked('test-workspace'))}`, `GetAllWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetAllWorkspaces(buildParameters))}`,
); );
CloudRunnerLogger.log(`GetFreeWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetFreeWorkspaces())}`);
CloudRunnerLogger.log( CloudRunnerLogger.log(
`LockWorkspace ${JSON.stringify(await SharedWorkspaceLocking.LockWorkspace('test-workspace', uuidv4()))}`, `GetFreeWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetFreeWorkspaces(buildParameters))}`,
);
CloudRunnerLogger.log(
`IsWorkspaceLocked ${JSON.stringify(
await SharedWorkspaceLocking.IsWorkspaceLocked(`test-workspace-${uuidv4()}`, buildParameters),
)}`,
);
CloudRunnerLogger.log(
`GetFreeWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetFreeWorkspaces(buildParameters))}`,
);
CloudRunnerLogger.log(
`LockWorkspace ${JSON.stringify(
await SharedWorkspaceLocking.LockWorkspace(`test-workspace-${uuidv4()}`, uuidv4(), buildParameters),
)}`,
); );
CloudRunnerLogger.log( CloudRunnerLogger.log(
`CreateLockableWorkspace ${JSON.stringify( `CreateLockableWorkspace ${JSON.stringify(
await SharedWorkspaceLocking.CreateLockableWorkspace('test-workspace-2'), await SharedWorkspaceLocking.CreateWorkspace(`test-workspace-${uuidv4()}`, buildParameters),
)}`, )}`,
); );
CloudRunnerLogger.log( CloudRunnerLogger.log(
`GetLockedWorkspace ${JSON.stringify( `GetLockedWorkspace ${JSON.stringify(
await SharedWorkspaceLocking.GetLockedWorkspace('test-workspace-2', uuidv4()), await SharedWorkspaceLocking.GetOrCreateLockedWorkspace(
`test-workspace-${uuidv4()}`,
uuidv4(),
buildParameters,
),
)}`, )}`,
); );
}, 3000000); }, 3000000);

51
src/model/cloud-runner/workflows/build-automation-workflow.ts
View File

@ -26,11 +26,15 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
if (CloudRunnerOptions.retainWorkspaces) { if (CloudRunnerOptions.retainWorkspaces) {
const workspace = const workspace =
(await SharedWorkspaceLocking.GetLockedWorkspace( (await SharedWorkspaceLocking.GetOrCreateLockedWorkspace(
`test-workspace-${CloudRunner.buildParameters.buildGuid}`, `test-workspace-${CloudRunner.buildParameters.buildGuid}`,
CloudRunner.buildParameters.buildGuid, CloudRunner.buildParameters.buildGuid,
CloudRunner.buildParameters,
)) || CloudRunner.buildParameters.buildGuid; )) || CloudRunner.buildParameters.buildGuid;
process.env.LOCKED_WORKSPACE = workspace;
CloudRunner.lockedWorkspace = workspace;
CloudRunnerLogger.logLine(`Using workspace ${workspace}`); CloudRunnerLogger.logLine(`Using workspace ${workspace}`);
cloudRunnerStepState.environment = [ cloudRunnerStepState.environment = [
...cloudRunnerStepState.environment, ...cloudRunnerStepState.environment,
@ -38,16 +42,16 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
]; ];
} }
if (!CloudRunner.buildParameters.isCliMode) core.startGroup('pre build steps');
let output = ''; let output = '';
if (CloudRunner.buildParameters.preBuildSteps !== '') { if (CloudRunner.buildParameters.preBuildSteps !== '') {
if (!CloudRunner.buildParameters.isCliMode) core.startGroup('pre build steps');
output += await CustomWorkflow.runCustomJob( output += await CustomWorkflow.runCustomJob(
CloudRunner.buildParameters.preBuildSteps, CloudRunner.buildParameters.preBuildSteps,
cloudRunnerStepState.environment, cloudRunnerStepState.environment,
cloudRunnerStepState.secrets, cloudRunnerStepState.secrets,
); );
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
} }
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
CloudRunnerLogger.logWithTime('Configurable pre build step(s) time'); CloudRunnerLogger.logWithTime('Configurable pre build step(s) time');
if (!CloudRunner.buildParameters.isCliMode) core.startGroup('build'); if (!CloudRunner.buildParameters.isCliMode) core.startGroup('build');
@ -67,22 +71,24 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
if (!CloudRunner.buildParameters.isCliMode) core.endGroup(); if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
CloudRunnerLogger.logWithTime('Build time'); CloudRunnerLogger.logWithTime('Build time');
if (!CloudRunner.buildParameters.isCliMode) core.startGroup('post build steps');
if (CloudRunner.buildParameters.postBuildSteps !== '') { if (CloudRunner.buildParameters.postBuildSteps !== '') {
if (!CloudRunner.buildParameters.isCliMode) core.startGroup('post build steps');
output += await CustomWorkflow.runCustomJob( output += await CustomWorkflow.runCustomJob(
CloudRunner.buildParameters.postBuildSteps, CloudRunner.buildParameters.postBuildSteps,
cloudRunnerStepState.environment, cloudRunnerStepState.environment,
cloudRunnerStepState.secrets, cloudRunnerStepState.secrets,
); );
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
} }
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
CloudRunnerLogger.logWithTime('Configurable post build step(s) time'); CloudRunnerLogger.logWithTime('Configurable post build step(s) time');
if (CloudRunnerOptions.retainWorkspaces) { if (CloudRunnerOptions.retainWorkspaces) {
await SharedWorkspaceLocking.ReleaseWorkspace( await SharedWorkspaceLocking.ReleaseWorkspace(
`test-workspace-${CloudRunner.buildParameters.buildGuid}`, `test-workspace-${CloudRunner.buildParameters.buildGuid}`,
CloudRunner.buildParameters.buildGuid, CloudRunner.buildParameters.buildGuid,
CloudRunner.buildParameters,
); );
CloudRunner.lockedWorkspace = undefined;
} }
CloudRunnerLogger.log(`Cloud Runner finished running standard build automation`); CloudRunnerLogger.log(`Cloud Runner finished running standard build automation`);
@ -113,25 +119,28 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
${BuildAutomationWorkflow.setupCommands(builderPath)} ${BuildAutomationWorkflow.setupCommands(builderPath)}
${setupHooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '} ${setupHooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
${buildHooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '} ${buildHooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
${BuildAutomationWorkflow.BuildCommands(builderPath, CloudRunner.buildParameters.buildGuid)} ${BuildAutomationWorkflow.BuildCommands(builderPath)}
${buildHooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}`; ${buildHooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}`;
} }
private static setupCommands(builderPath) { private static setupCommands(builderPath) {
const commands = `mkdir -p ${CloudRunnerFolders.ToLinuxFolder(
CloudRunnerFolders.builderPathAbsolute,
)} && git clone -q -b ${CloudRunner.buildParameters.cloudRunnerBranch} ${CloudRunnerFolders.ToLinuxFolder(
CloudRunnerFolders.unityBuilderRepoUrl,
)} "${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.builderPathAbsolute)}" && chmod +x ${builderPath}`;
return `export GIT_DISCOVERY_ACROSS_FILESYSTEM=1 return `export GIT_DISCOVERY_ACROSS_FILESYSTEM=1
echo "game ci cloud runner clone" echo "game ci cloud runner clone"
mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.builderPathAbsolute)} if [ -e "${CloudRunnerFolders.ToLinuxFolder(
git clone -q -b ${CloudRunner.buildParameters.cloudRunnerBranch} ${CloudRunnerFolders.ToLinuxFolder( CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute,
CloudRunnerFolders.unityBuilderRepoUrl, )}" ]; then echo "Retained Workspace Already Exists!"; else ${commands}; fi
)} "${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.builderPathAbsolute)}"
chmod +x ${builderPath}
echo "game ci cloud runner bootstrap" echo "game ci cloud runner bootstrap"
node ${builderPath} -m remote-cli`; node ${builderPath} -m remote-cli-pre-build`;
} }
// ToDo: Replace with a very simple "node ${builderPath} -m build-cli" to run the scripts below without enlarging the request size // ToDo: Replace with a very simple "node ${builderPath} -m build-cli" to run the scripts below without enlarging the request size
private static BuildCommands(builderPath, guid) { private static BuildCommands(builderPath) {
const linuxCacheFolder = CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderFull);
const distFolder = path.join(CloudRunnerFolders.builderPathAbsolute, 'dist'); const distFolder = path.join(CloudRunnerFolders.builderPathAbsolute, 'dist');
const ubuntuPlatformsFolder = path.join(CloudRunnerFolders.builderPathAbsolute, 'dist', 'platforms', 'ubuntu'); const ubuntuPlatformsFolder = path.join(CloudRunnerFolders.builderPathAbsolute, 'dist', 'platforms', 'ubuntu');
@ -147,18 +156,6 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
/entrypoint.sh /entrypoint.sh
echo "game ci cloud runner push library to cache" echo "game ci cloud runner push library to cache"
chmod +x ${builderPath} chmod +x ${builderPath}
# node ${builderPath} -m remote-cli-post node ${builderPath} -m remote-cli-post-build`;
node ${builderPath} -m cache-push --cachePushFrom ${CloudRunnerFolders.ToLinuxFolder(
CloudRunnerFolders.libraryFolderAbsolute,
)} --artifactName lib-${guid} --cachePushTo ${CloudRunnerFolders.ToLinuxFolder(`${linuxCacheFolder}/Library`)}
echo "game ci cloud runner push build to cache"
node ${builderPath} -m cache-push --cachePushFrom ${CloudRunnerFolders.ToLinuxFolder(
CloudRunnerFolders.projectBuildFolderAbsolute,
)} --artifactName build-${guid} --cachePushTo ${`${CloudRunnerFolders.ToLinuxFolder(`${linuxCacheFolder}/build`)}`}
${BuildAutomationWorkflow.GetCleanupCommand(CloudRunnerFolders.projectPathAbsolute)}`;
}
private static GetCleanupCommand(cleanupPath: string) {
return CloudRunnerOptions.retainWorkspaces ? `` : `rm -r ${CloudRunnerFolders.ToLinuxFolder(cleanupPath)}`;
} }
} }