Cloud Runner 1.0 (#459)
* Fix: post build caching, use linux path conversion * Fix: post build caching via CLI * Fix: post build caching via CLI * Fix: post build caching via CLI * Fix: post build caching via CLI * Fix: post build caching via CLI * Log if retained workspace option is present for testing * Log if retained workspace option is present for testing * Use retained workspace :O * Use retained workspace :O * Use retained workspace :O * Use retained workspace :O * Ignore garbage creating lock actions in test for now * Lock workspace when using Get or Create Locked Workspace * Lock workspace before creating workspace file to allow for an unblockable creation sequence with guarenteed lock for the original creator * intuitive locking logs from the most important flow * test naming * consider lock folders without workspace file locked * Use cache key to segment lock folders * Use cache key to segment lock folders * Use cache key to segment lock folders * Use cache key to segment lock folders * Use cache key to segment lock folders * Skip all locking actions test as we now have two useful test flows * Skip all locking actions test as we now have two useful test flows * Skip all locking actions test as we now have two useful test flows * Copy all of data folder to docker volume to enable local-docker retained workspace * Fix: check for retained workspace * Fix: check for retained workspace * Fix: check for retained workspace * Fix: check for retained workspace * Fix: check for retained workspace * Fix: check for retained workspace * Fix: check for retained workspace * Fix: check for retained workspace * Skip main clone if game repo exists * handle cloud runner git sync via sha not only branch * handle cloud runner git sync via sha not only branch * handle cloud runner git sync via sha not only branch * handle cloud runner git sync via sha not only branch * handle cloud runner git sync via sha not only branch * handle cloud runner git sync via sha not only branch * handle cloud runner git sync via sha not only branch * handle cloud runner git sync via sha not only branch * transfer locked workspace to static CloudRunner field * transfer locked workspace to static CloudRunner field * transfer locked workspace to static CloudRunner field * custom hook files and test * custom hook files and test * custom hook files and testpull/437/head
Frostebite
GitHub
parent
e56abbdd40
commit
7fcd51349b
|
|
@ -296,6 +296,9 @@ class BuildParameters {
|
|||
readInputFromOverrideList: cloud_runner_options_1.default.readInputFromOverrideList(),
|
||||
kubeStorageClass: cloud_runner_options_1.default.kubeStorageClass,
|
||||
cacheKey: cloud_runner_options_1.default.cacheKey,
|
||||
retainWorkspace: cloud_runner_options_1.default.retainWorkspaces,
|
||||
useSharedLargePackages: cloud_runner_options_1.default.useSharedLargePackages,
|
||||
useLZ4Compression: cloud_runner_options_1.default.useLZ4Compression,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
|
@ -485,6 +488,7 @@ const cloud_runner_options_reader_1 = __importDefault(__nccwpck_require__(3343))
|
|||
const github_1 = __importDefault(__nccwpck_require__(83654));
|
||||
const task_parameter_serializer_1 = __nccwpck_require__(35346);
|
||||
const cloud_runner_folders_1 = __nccwpck_require__(13527);
|
||||
const cloud_runner_system_1 = __nccwpck_require__(99393);
|
||||
class Cli {
|
||||
static get isCliMode() {
|
||||
return Cli.options !== undefined && Cli.options.mode !== undefined && Cli.options.mode !== '';
|
||||
|
|
@ -539,6 +543,7 @@ class Cli {
|
|||
${JSON.stringify(buildParameter, undefined, 4)}
|
||||
`);
|
||||
__1.CloudRunner.buildParameters = buildParameter;
|
||||
__1.CloudRunner.lockedWorkspace = process.env.LOCKED_WORKSPACE;
|
||||
return yield results.target[results.propertyKey](Cli.options);
|
||||
});
|
||||
}
|
||||
|
|
@ -567,26 +572,13 @@ class Cli {
|
|||
}
|
||||
static PostCLIBuild() {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
const buildParameter = yield __1.BuildParameters.create();
|
||||
/*
|
||||
# LIBRARY CACHE
|
||||
node ${builderPath} -m cache-push --cachePushFrom ${CloudRunnerFolders.ToLinuxFolder(
|
||||
CloudRunnerFolders.libraryFolderAbsolute,
|
||||
)} --artifactName lib-${guid} --cachePushTo ${CloudRunnerFolders.ToLinuxFolder(`${linuxCacheFolder}/Library`)}
|
||||
|
||||
echo "game ci cloud runner push build to cache"
|
||||
|
||||
# BUILD CACHE
|
||||
node ${builderPath} -m cache-push --cachePushFrom ${CloudRunnerFolders.ToLinuxFolder(
|
||||
CloudRunnerFolders.projectBuildFolderAbsolute,
|
||||
)} --artifactName build-${guid} --cachePushTo ${`${CloudRunnerFolders.ToLinuxFolder(`${linuxCacheFolder}/build`)}`}
|
||||
|
||||
# RETAINED WORKSPACE CLEANUP
|
||||
${BuildAutomationWorkflow.GetCleanupCommand(CloudRunnerFolders.projectPathAbsolute)}`;
|
||||
*/
|
||||
core.info(`Running POST build tasks`);
|
||||
caching_1.Caching.PushToCache(cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.libraryFolderAbsolute), cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(`${cloud_runner_folders_1.CloudRunnerFolders.cacheFolderFull}/Library`), `lib-${buildParameter.buildGuid}`);
|
||||
caching_1.Caching.PushToCache(cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.projectBuildFolderAbsolute), cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(`${cloud_runner_folders_1.CloudRunnerFolders.cacheFolderFull}/build`), `build-${buildParameter.buildGuid}`);
|
||||
yield caching_1.Caching.PushToCache(cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(`${cloud_runner_folders_1.CloudRunnerFolders.cacheFolderFull}/Library`), cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.libraryFolderAbsolute), `lib-${__1.CloudRunner.buildParameters.buildGuid}`);
|
||||
yield caching_1.Caching.PushToCache(cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(`${cloud_runner_folders_1.CloudRunnerFolders.cacheFolderFull}/build`), cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.projectBuildFolderAbsolute), `build-${__1.CloudRunner.buildParameters.buildGuid}`);
|
||||
if (!__1.CloudRunner.buildParameters.retainWorkspace) {
|
||||
yield cloud_runner_system_1.CloudRunnerSystem.Run(`rm -r ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`);
|
||||
}
|
||||
yield remote_client_1.RemoteClient.runCustomHookFiles(`after-build`);
|
||||
return new Promise((result) => result(``));
|
||||
});
|
||||
}
|
||||
|
|
@ -648,48 +640,123 @@ const fs = __importStar(__nccwpck_require__(57147));
|
|||
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
|
||||
const cloud_runner_options_1 = __importDefault(__nccwpck_require__(96552));
|
||||
class SharedWorkspaceLocking {
|
||||
static GetLockedWorkspace(workspaceIfCreated, runId) {
|
||||
static GetAllWorkspaces(buildParametersContext) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
return (yield SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`)).map((x) => x.replace(`/`, ``));
|
||||
});
|
||||
}
|
||||
static DoesWorkspaceTopLevelExist(buildParametersContext) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
const results = (yield SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}`)).map((x) => x.replace(`/`, ``));
|
||||
return results.includes(buildParametersContext.cacheKey);
|
||||
});
|
||||
}
|
||||
static GetAllLocks(workspace, buildParametersContext) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
|
||||
throw new Error("Workspace doesn't exist, can't call get all locks");
|
||||
}
|
||||
return (yield SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/`)).map((x) => x.replace(`/`, ``));
|
||||
});
|
||||
}
|
||||
static GetOrCreateLockedWorkspace(workspaceIfCreated, runId, buildParametersContext) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
if (!cloud_runner_options_1.default.retainWorkspaces) {
|
||||
return;
|
||||
}
|
||||
const workspaces = yield SharedWorkspaceLocking.GetFreeWorkspaces();
|
||||
for (const element of workspaces) {
|
||||
if (yield SharedWorkspaceLocking.LockWorkspace(element, runId)) {
|
||||
return element;
|
||||
cloud_runner_logger_1.default.log(`run agent ${runId} is trying to access a workspace`);
|
||||
if (yield SharedWorkspaceLocking.DoesWorkspaceTopLevelExist(buildParametersContext)) {
|
||||
const workspaces = yield SharedWorkspaceLocking.GetFreeWorkspaces(buildParametersContext);
|
||||
for (const element of workspaces) {
|
||||
if (yield SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext)) {
|
||||
cloud_runner_logger_1.default.log(`run agent ${runId} locked workspace: ${element}`);
|
||||
return element;
|
||||
}
|
||||
}
|
||||
}
|
||||
return yield SharedWorkspaceLocking.CreateLockableWorkspace(workspaceIfCreated);
|
||||
const workspace = yield SharedWorkspaceLocking.CreateWorkspace(workspaceIfCreated, buildParametersContext, runId);
|
||||
cloud_runner_logger_1.default.log(`run agent ${runId} didn't find a free workspace so created: ${workspace}`);
|
||||
return workspace;
|
||||
});
|
||||
}
|
||||
static DoesWorkspaceExist(workspace) {
|
||||
static DoesWorkspaceExist(workspace, buildParametersContext) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
return (yield SharedWorkspaceLocking.GetAllWorkspaces()).includes(workspace);
|
||||
return (yield SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext)).includes(workspace);
|
||||
});
|
||||
}
|
||||
static GetFreeWorkspaces() {
|
||||
static HasWorkspaceLock(workspace, runId, buildParametersContext) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
|
||||
return false;
|
||||
}
|
||||
return ((yield SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext)).filter((x) => x.includes(runId))
|
||||
.length > 0);
|
||||
});
|
||||
}
|
||||
static GetFreeWorkspaces(buildParametersContext) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
const result = [];
|
||||
const workspaces = yield SharedWorkspaceLocking.GetAllWorkspaces();
|
||||
const workspaces = yield SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
|
||||
for (const element of workspaces) {
|
||||
if (!(yield SharedWorkspaceLocking.IsWorkspaceLocked(element))) {
|
||||
if (!(yield SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParametersContext))) {
|
||||
result.push(element);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
});
|
||||
}
|
||||
static GetAllWorkspaces() {
|
||||
static IsWorkspaceLocked(workspace, buildParametersContext) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
return (yield SharedWorkspaceLocking.ReadLines(`aws s3 ls s3://game-ci-test-storage/locks/`)).map((x) => x.replace(`/`, ``));
|
||||
if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
|
||||
return false;
|
||||
}
|
||||
const files = yield SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/`);
|
||||
const workspaceFileDoesNotExists = files.filter((x) => {
|
||||
return x.includes(`_workspace`);
|
||||
}).length === 0;
|
||||
const lockFilesExist = files.filter((x) => {
|
||||
return x.includes(`_lock`);
|
||||
}).length > 0;
|
||||
return workspaceFileDoesNotExists || lockFilesExist;
|
||||
});
|
||||
}
|
||||
static GetAllLocks(workspace) {
|
||||
static CreateWorkspace(workspace, buildParametersContext, lockId = ``) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace))) {
|
||||
throw new Error("Workspace doesn't exist, can't call get all locks");
|
||||
if (lockId !== ``) {
|
||||
yield SharedWorkspaceLocking.LockWorkspace(workspace, lockId, buildParametersContext);
|
||||
}
|
||||
return (yield SharedWorkspaceLocking.ReadLines(`aws s3 ls s3://game-ci-test-storage/locks/${workspace}/`)).map((x) => x.replace(`/`, ``));
|
||||
const file = `${Date.now()}_workspace`;
|
||||
fs.writeFileSync(file, '');
|
||||
yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`, false, true);
|
||||
fs.rmSync(file);
|
||||
return workspace;
|
||||
});
|
||||
}
|
||||
static LockWorkspace(workspace, runId, buildParametersContext) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
const file = `${Date.now()}_${runId}_lock`;
|
||||
fs.writeFileSync(file, '');
|
||||
yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`, false, true);
|
||||
fs.rmSync(file);
|
||||
return SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext);
|
||||
});
|
||||
}
|
||||
static ReleaseWorkspace(workspace, runId, buildParametersContext) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
|
||||
return true;
|
||||
}
|
||||
const file = (yield SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext)).filter((x) => x.includes(`_${runId}_lock`));
|
||||
cloud_runner_logger_1.default.log(`${JSON.stringify(yield SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext))}`);
|
||||
cloud_runner_logger_1.default.log(`Deleting file ${file}`);
|
||||
cloud_runner_logger_1.default.log(`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`);
|
||||
yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`, false, true);
|
||||
return !SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext);
|
||||
});
|
||||
}
|
||||
static CleanupWorkspace(workspace, buildParametersContext) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace} --recursive`, false, true);
|
||||
});
|
||||
}
|
||||
static ReadLines(command) {
|
||||
|
|
@ -705,59 +772,9 @@ class SharedWorkspaceLocking {
|
|||
});
|
||||
});
|
||||
}
|
||||
static LockWorkspace(workspace, runId) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
const file = `${Date.now()}_${runId}_lock`;
|
||||
fs.writeFileSync(file, '');
|
||||
yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 cp ./${file} s3://game-ci-test-storage/locks/${workspace}/${file}`, false, true);
|
||||
fs.rmSync(file);
|
||||
return SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId);
|
||||
});
|
||||
}
|
||||
static ReleaseWorkspace(workspace, runId) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace))) {
|
||||
return true;
|
||||
}
|
||||
const file = (yield SharedWorkspaceLocking.GetAllLocks(workspace)).filter((x) => x.includes(`_${runId}_lock`));
|
||||
cloud_runner_logger_1.default.log(`${JSON.stringify(yield SharedWorkspaceLocking.GetAllLocks(workspace))}`);
|
||||
cloud_runner_logger_1.default.log(`Deleting file ${file}`);
|
||||
cloud_runner_logger_1.default.log(`aws s3 rm s3://game-ci-test-storage/locks/${workspace}/${file}`);
|
||||
yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 rm s3://game-ci-test-storage/locks/${workspace}/${file}`, false, true);
|
||||
return !SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId);
|
||||
});
|
||||
}
|
||||
static HasWorkspaceLock(workspace, runId) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace))) {
|
||||
return false;
|
||||
}
|
||||
return (yield SharedWorkspaceLocking.GetAllLocks(workspace)).filter((x) => x.includes(runId)).length > 0;
|
||||
});
|
||||
}
|
||||
static IsWorkspaceLocked(workspace) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace))) {
|
||||
return false;
|
||||
}
|
||||
const files = yield SharedWorkspaceLocking.ReadLines(`aws s3 ls s3://game-ci-test-storage/locks/${workspace}/`);
|
||||
// 1 Because we expect 1 workspace file to exist in every workspace folder
|
||||
return files.length > 1;
|
||||
});
|
||||
}
|
||||
static CreateLockableWorkspace(workspace) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
const file = `${Date.now()}_workspace`;
|
||||
fs.writeFileSync(file, '');
|
||||
yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 cp ./${file} s3://game-ci-test-storage/locks/${workspace}/${file}`, false, true);
|
||||
fs.rmSync(file);
|
||||
return workspace;
|
||||
});
|
||||
}
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
static ReleaseLock(workspace) { }
|
||||
}
|
||||
exports.SharedWorkspaceLocking = SharedWorkspaceLocking;
|
||||
SharedWorkspaceLocking.workspaceRoot = `s3://game-ci-test-storage/locks/`;
|
||||
exports["default"] = SharedWorkspaceLocking;
|
||||
|
||||
|
||||
|
|
@ -912,6 +929,12 @@ class CloudRunnerOptions {
|
|||
static get retainWorkspacesMax() {
|
||||
return Number(CloudRunnerOptions.getInput(`retainWorkspacesMax`)) || 5;
|
||||
}
|
||||
static get useSharedLargePackages() {
|
||||
return CloudRunnerOptions.getInput(`useSharedLargePackages`) || false;
|
||||
}
|
||||
static get useLZ4Compression() {
|
||||
return CloudRunnerOptions.getInput(`useLZ4Compression`) || true;
|
||||
}
|
||||
static ToEnvVarFormat(input) {
|
||||
if (input.toUpperCase() === input) {
|
||||
return input;
|
||||
|
|
@ -3672,7 +3695,6 @@ const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
|
|||
const docker_1 = __importDefault(__nccwpck_require__(16934));
|
||||
const model_1 = __nccwpck_require__(41359);
|
||||
const fs_1 = __nccwpck_require__(57147);
|
||||
// import * as core from '@actions/core';
|
||||
class LocalDockerCloudRunner {
|
||||
inspect() {
|
||||
throw new Error('Method not implemented.');
|
||||
|
|
@ -3707,6 +3729,7 @@ class LocalDockerCloudRunner {
|
|||
this.buildParameters = buildParameters;
|
||||
}
|
||||
runTask(buildGuid, image, commands, mountdir, workingdir, environment, secrets) {
|
||||
var _a;
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
cloud_runner_logger_1.default.log(buildGuid);
|
||||
cloud_runner_logger_1.default.log(commands);
|
||||
|
|
@ -3730,6 +3753,7 @@ class LocalDockerCloudRunner {
|
|||
}
|
||||
}
|
||||
let myOutput = '';
|
||||
const sharedFolder = ((_a = this.buildParameters) === null || _a === void 0 ? void 0 : _a.retainWorkspace) ? `/data/` : `/data/cache/`;
|
||||
// core.info(JSON.stringify({ workspace, actionFolder, ...this.buildParameters, ...content }, undefined, 4));
|
||||
const entrypointFilePath = `start.sh`;
|
||||
fs_1.writeFileSync(`${workspace}/${entrypointFilePath}`, `#!/bin/bash
|
||||
|
|
@ -3738,11 +3762,12 @@ class LocalDockerCloudRunner {
|
|||
apt-get update > /dev/null && apt-get install -y tree> /dev/null
|
||||
mkdir -p /github/workspace/cloud-runner-cache
|
||||
mkdir -p /data/cache
|
||||
cp -a /github/workspace/cloud-runner-cache/. /data/cache/
|
||||
tree -L 2 /data/cache
|
||||
cp -a /github/workspace/cloud-runner-cache/. ${sharedFolder}
|
||||
tree -L 3 ${sharedFolder}
|
||||
${commands}
|
||||
cp -a /data/cache/. /github/workspace/cloud-runner-cache/
|
||||
cp -a ${sharedFolder}. /github/workspace/cloud-runner-cache/
|
||||
tree -L 2 /github/workspace/cloud-runner-cache
|
||||
tree -L 3 ${sharedFolder}
|
||||
`, {
|
||||
flag: 'w',
|
||||
});
|
||||
|
|
@ -3987,6 +4012,7 @@ class Caching {
|
|||
return __awaiter(this, void 0, void 0, function* () {
|
||||
cacheArtifactName = cacheArtifactName.replace(' ', '');
|
||||
const startPath = process.cwd();
|
||||
const compressionSuffix = cloud_runner_1.default.buildParameters.useLZ4Compression ? '.lz4' : '';
|
||||
try {
|
||||
if (!(yield fileExists(cacheFolder))) {
|
||||
yield cloud_runner_system_1.CloudRunnerSystem.Run(`mkdir -p ${cacheFolder}`);
|
||||
|
|
@ -3998,7 +4024,7 @@ class Caching {
|
|||
const contents = yield fs_1.default.promises.readdir(path_1.default.basename(sourceFolder));
|
||||
cloud_runner_logger_1.default.log(`There is ${contents.length} files/dir in the source folder ${path_1.default.basename(sourceFolder)}`);
|
||||
if (cloud_runner_1.default.buildParameters.cloudRunnerIntegrationTests) {
|
||||
yield cloud_runner_system_1.CloudRunnerSystem.Run(`tree -L 2 ./..`);
|
||||
// await CloudRunnerSystem.Run(`tree -L 2 ./..`);
|
||||
yield cloud_runner_system_1.CloudRunnerSystem.Run(`tree -L 2`);
|
||||
}
|
||||
if (contents.length === 0) {
|
||||
|
|
@ -4006,13 +4032,13 @@ class Caching {
|
|||
process.chdir(`${startPath}`);
|
||||
return;
|
||||
}
|
||||
yield cloud_runner_system_1.CloudRunnerSystem.Run(`tar -cf ${cacheArtifactName}.tar.lz4 ${path_1.default.basename(sourceFolder)}`);
|
||||
yield cloud_runner_system_1.CloudRunnerSystem.Run(`du ${cacheArtifactName}.tar.lz4`);
|
||||
console_1.assert(yield fileExists(`${cacheArtifactName}.tar.lz4`), 'cache archive exists');
|
||||
yield cloud_runner_system_1.CloudRunnerSystem.Run(`tar -cf ${cacheArtifactName}.tar${compressionSuffix} ${path_1.default.basename(sourceFolder)}`);
|
||||
yield cloud_runner_system_1.CloudRunnerSystem.Run(`du ${cacheArtifactName}.tar${compressionSuffix}`);
|
||||
console_1.assert(yield fileExists(`${cacheArtifactName}.tar${compressionSuffix}`), 'cache archive exists');
|
||||
console_1.assert(yield fileExists(path_1.default.basename(sourceFolder)), 'source folder exists');
|
||||
yield cloud_runner_system_1.CloudRunnerSystem.Run(`mv ${cacheArtifactName}.tar.lz4 ${cacheFolder}`);
|
||||
yield cloud_runner_system_1.CloudRunnerSystem.Run(`mv ${cacheArtifactName}.tar${compressionSuffix} ${cacheFolder}`);
|
||||
remote_client_logger_1.RemoteClientLogger.log(`moved cache entry ${cacheArtifactName} to ${cacheFolder}`);
|
||||
console_1.assert(yield fileExists(`${path_1.default.join(cacheFolder, cacheArtifactName)}.tar.lz4`), 'cache archive exists inside cache folder');
|
||||
console_1.assert(yield fileExists(`${path_1.default.join(cacheFolder, cacheArtifactName)}.tar${compressionSuffix}`), 'cache archive exists inside cache folder');
|
||||
}
|
||||
catch (error) {
|
||||
process.chdir(`${startPath}`);
|
||||
|
|
@ -4024,6 +4050,7 @@ class Caching {
|
|||
static PullFromCache(cacheFolder, destinationFolder, cacheArtifactName = ``) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
cacheArtifactName = cacheArtifactName.replace(' ', '');
|
||||
const compressionSuffix = cloud_runner_1.default.buildParameters.useLZ4Compression ? '.lz4' : '';
|
||||
const startPath = process.cwd();
|
||||
remote_client_logger_1.RemoteClientLogger.log(`Caching for ${path_1.default.basename(destinationFolder)}`);
|
||||
try {
|
||||
|
|
@ -4033,20 +4060,20 @@ class Caching {
|
|||
if (!(yield fileExists(destinationFolder))) {
|
||||
yield fs_1.default.promises.mkdir(destinationFolder);
|
||||
}
|
||||
const latestInBranch = yield (yield cloud_runner_system_1.CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .tar.lz4$ | head -1`))
|
||||
const latestInBranch = yield (yield cloud_runner_system_1.CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .tar${compressionSuffix}$ | head -1`))
|
||||
.replace(/\n/g, ``)
|
||||
.replace('.tar.lz4', '');
|
||||
.replace(`.tar${compressionSuffix}`, '');
|
||||
process.chdir(cacheFolder);
|
||||
const cacheSelection = cacheArtifactName !== `` && (yield fileExists(`${cacheArtifactName}.tar.lz4`))
|
||||
const cacheSelection = cacheArtifactName !== `` && (yield fileExists(`${cacheArtifactName}.tar${compressionSuffix}`))
|
||||
? cacheArtifactName
|
||||
: latestInBranch;
|
||||
yield cloud_runner_logger_1.default.log(`cache key ${cacheArtifactName} selection ${cacheSelection}`);
|
||||
if (yield fileExists(`${cacheSelection}.tar.lz4`)) {
|
||||
if (yield fileExists(`${cacheSelection}.tar${compressionSuffix}`)) {
|
||||
const resultsFolder = `results${cloud_runner_1.default.buildParameters.buildGuid}`;
|
||||
yield cloud_runner_system_1.CloudRunnerSystem.Run(`mkdir -p ${resultsFolder}`);
|
||||
remote_client_logger_1.RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.tar.lz4`);
|
||||
remote_client_logger_1.RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.tar${compressionSuffix}`);
|
||||
const fullResultsFolder = path_1.default.join(cacheFolder, resultsFolder);
|
||||
yield cloud_runner_system_1.CloudRunnerSystem.Run(`tar -xf ${cacheSelection}.tar.lz4 -C ${fullResultsFolder}`);
|
||||
yield cloud_runner_system_1.CloudRunnerSystem.Run(`tar -xf ${cacheSelection}.tar${compressionSuffix} -C ${fullResultsFolder}`);
|
||||
remote_client_logger_1.RemoteClientLogger.log(`cache item extracted to ${fullResultsFolder}`);
|
||||
console_1.assert(yield fileExists(fullResultsFolder), `cache extraction results folder exists`);
|
||||
const destinationParentFolder = path_1.default.resolve(destinationFolder, '..');
|
||||
|
|
@ -4060,7 +4087,7 @@ class Caching {
|
|||
else {
|
||||
remote_client_logger_1.RemoteClientLogger.logWarning(`cache item ${cacheArtifactName} doesn't exist ${destinationFolder}`);
|
||||
if (cacheSelection !== ``) {
|
||||
remote_client_logger_1.RemoteClientLogger.logWarning(`cache item ${cacheArtifactName}.tar.lz4 doesn't exist ${destinationFolder}`);
|
||||
remote_client_logger_1.RemoteClientLogger.logWarning(`cache item ${cacheArtifactName}.tar${compressionSuffix} doesn't exist ${destinationFolder}`);
|
||||
throw new Error(`Failed to get cache item, but cache hit was found: ${cacheSelection}`);
|
||||
}
|
||||
}
|
||||
|
|
@ -4128,6 +4155,7 @@ const console_1 = __nccwpck_require__(96206);
|
|||
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
|
||||
const cli_functions_repository_1 = __nccwpck_require__(85301);
|
||||
const cloud_runner_system_1 = __nccwpck_require__(99393);
|
||||
const yaml_1 = __importDefault(__nccwpck_require__(44603));
|
||||
class RemoteClient {
|
||||
static bootstrapRepository() {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
|
|
@ -4167,8 +4195,16 @@ class RemoteClient {
|
|||
}
|
||||
static cloneRepoWithoutLFSFiles() {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
process.chdir(`${cloud_runner_folders_1.CloudRunnerFolders.repoPathAbsolute}`);
|
||||
if (cloud_runner_1.default.buildParameters.cloudRunnerIntegrationTests) {
|
||||
yield cloud_runner_system_1.CloudRunnerSystem.Run(`tree -L 2 ./..`);
|
||||
}
|
||||
if (fs_1.default.existsSync(path_1.default.join(cloud_runner_folders_1.CloudRunnerFolders.repoPathAbsolute, `.git`))) {
|
||||
remote_client_logger_1.RemoteClientLogger.log(`${cloud_runner_folders_1.CloudRunnerFolders.repoPathAbsolute} repo exists - skipping clone - retained workspace mode ${cloud_runner_1.default.buildParameters.retainWorkspace}`);
|
||||
yield cloud_runner_system_1.CloudRunnerSystem.Run(`git reset --hard ${cloud_runner_1.default.buildParameters.gitSha}`);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
process.chdir(`${cloud_runner_folders_1.CloudRunnerFolders.repoPathAbsolute}`);
|
||||
remote_client_logger_1.RemoteClientLogger.log(`Initializing source repository for cloning with caching of LFS files`);
|
||||
yield cloud_runner_system_1.CloudRunnerSystem.Run(`git config --global advice.detachedHead false`);
|
||||
remote_client_logger_1.RemoteClientLogger.log(`Cloning the repository being built:`);
|
||||
|
|
@ -4179,6 +4215,7 @@ class RemoteClient {
|
|||
console_1.assert(fs_1.default.existsSync(`.git`), 'git folder exists');
|
||||
remote_client_logger_1.RemoteClientLogger.log(`${cloud_runner_1.default.buildParameters.branch}`);
|
||||
yield cloud_runner_system_1.CloudRunnerSystem.Run(`git checkout ${cloud_runner_1.default.buildParameters.branch}`);
|
||||
yield cloud_runner_system_1.CloudRunnerSystem.Run(`git checkout ${cloud_runner_1.default.buildParameters.gitSha}`);
|
||||
console_1.assert(fs_1.default.existsSync(path_1.default.join(`.git`, `lfs`)), 'LFS folder should not exist before caching');
|
||||
remote_client_logger_1.RemoteClientLogger.log(`Checked out ${cloud_runner_1.default.buildParameters.branch}`);
|
||||
}
|
||||
|
|
@ -4188,8 +4225,12 @@ class RemoteClient {
|
|||
});
|
||||
}
|
||||
static replaceLargePackageReferencesWithSharedReferences() {
|
||||
const manifest = fs_1.default.readFileSync(path_1.default.join(cloud_runner_folders_1.CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`), 'utf8');
|
||||
if (cloud_runner_1.default.buildParameters.cloudRunnerIntegrationTests) {
|
||||
cloud_runner_logger_1.default.log(fs_1.default.readFileSync(path_1.default.join(cloud_runner_folders_1.CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`), 'utf8'));
|
||||
cloud_runner_logger_1.default.log(manifest);
|
||||
}
|
||||
if (cloud_runner_1.default.buildParameters.useSharedLargePackages) {
|
||||
manifest.replace(/LargePackages/g, '../../LargePackages');
|
||||
}
|
||||
}
|
||||
static pullLatestLFS() {
|
||||
|
|
@ -4204,12 +4245,34 @@ class RemoteClient {
|
|||
}
|
||||
static runRemoteClientJob() {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
RemoteClient.handleRetainedWorkspace();
|
||||
yield RemoteClient.bootstrapRepository();
|
||||
yield RemoteClient.runCustomHookFiles(`before-build`);
|
||||
});
|
||||
}
|
||||
static runCustomHookFiles(hookLifecycle) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
remote_client_logger_1.RemoteClientLogger.log(`RunCustomHookFiles: ${hookLifecycle}`);
|
||||
const gameCiCustomHooksPath = path_1.default.join(cloud_runner_folders_1.CloudRunnerFolders.repoPathAbsolute, `game-ci`, `hooks`);
|
||||
const files = fs_1.default.readdirSync(gameCiCustomHooksPath);
|
||||
for (const file of files) {
|
||||
const fileContents = fs_1.default.readFileSync(path_1.default.join(gameCiCustomHooksPath, file), `utf8`);
|
||||
const fileContentsObject = yaml_1.default.parse(fileContents.toString());
|
||||
if (fileContentsObject.hook === hookLifecycle) {
|
||||
remote_client_logger_1.RemoteClientLogger.log(`Active Hook File ${file} contents: ${fileContents}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
static handleRetainedWorkspace() {
|
||||
if (!cloud_runner_1.default.buildParameters.retainWorkspace || !cloud_runner_1.default.lockedWorkspace) {
|
||||
return;
|
||||
}
|
||||
remote_client_logger_1.RemoteClientLogger.log(`Retained Workspace: ${cloud_runner_1.default.lockedWorkspace}`);
|
||||
}
|
||||
}
|
||||
__decorate([
|
||||
cli_functions_repository_1.CliFunction(`remote-cli`, `sets up a repository, usually before a game-ci build`)
|
||||
cli_functions_repository_1.CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
|
||||
], RemoteClient, "runRemoteClientJob", null);
|
||||
exports.RemoteClient = RemoteClient;
|
||||
|
||||
|
|
@ -4334,7 +4397,9 @@ class CloudRunnerFolders {
|
|||
}
|
||||
// Only the following paths that do not start a path.join with another "Full" suffixed property need to start with an absolute /
|
||||
static get uniqueCloudRunnerJobFolderAbsolute() {
|
||||
return path_1.default.join(`/`, CloudRunnerFolders.buildVolumeFolder, cloud_runner_1.default.buildParameters.buildGuid);
|
||||
return cloud_runner_1.default.buildParameters.retainWorkspace && cloud_runner_1.default.lockedWorkspace
|
||||
? path_1.default.join(`/`, CloudRunnerFolders.buildVolumeFolder, cloud_runner_1.default.lockedWorkspace)
|
||||
: path_1.default.join(`/`, CloudRunnerFolders.buildVolumeFolder, cloud_runner_1.default.buildParameters.buildGuid);
|
||||
}
|
||||
static get cacheFolderFull() {
|
||||
return path_1.default.join('/', CloudRunnerFolders.buildVolumeFolder, CloudRunnerFolders.cacheFolder, cloud_runner_1.default.buildParameters.cacheKey);
|
||||
|
|
@ -5042,21 +5107,23 @@ class BuildAutomationWorkflow {
|
|||
try {
|
||||
cloud_runner_logger_1.default.log(`Cloud Runner is running standard build automation`);
|
||||
if (cloud_runner_options_1.default.retainWorkspaces) {
|
||||
const workspace = (yield shared_workspace_locking_1.default.GetLockedWorkspace(`test-workspace-${cloud_runner_1.default.buildParameters.buildGuid}`, cloud_runner_1.default.buildParameters.buildGuid)) || cloud_runner_1.default.buildParameters.buildGuid;
|
||||
const workspace = (yield shared_workspace_locking_1.default.GetOrCreateLockedWorkspace(`test-workspace-${cloud_runner_1.default.buildParameters.buildGuid}`, cloud_runner_1.default.buildParameters.buildGuid, cloud_runner_1.default.buildParameters)) || cloud_runner_1.default.buildParameters.buildGuid;
|
||||
process.env.LOCKED_WORKSPACE = workspace;
|
||||
cloud_runner_1.default.lockedWorkspace = workspace;
|
||||
cloud_runner_logger_1.default.logLine(`Using workspace ${workspace}`);
|
||||
cloudRunnerStepState.environment = [
|
||||
...cloudRunnerStepState.environment,
|
||||
{ name: `LOCKED_WORKSPACE`, value: workspace },
|
||||
];
|
||||
}
|
||||
if (!cloud_runner_1.default.buildParameters.isCliMode)
|
||||
core.startGroup('pre build steps');
|
||||
let output = '';
|
||||
if (cloud_runner_1.default.buildParameters.preBuildSteps !== '') {
|
||||
if (!cloud_runner_1.default.buildParameters.isCliMode)
|
||||
core.startGroup('pre build steps');
|
||||
output += yield custom_workflow_1.CustomWorkflow.runCustomJob(cloud_runner_1.default.buildParameters.preBuildSteps, cloudRunnerStepState.environment, cloudRunnerStepState.secrets);
|
||||
if (!cloud_runner_1.default.buildParameters.isCliMode)
|
||||
core.endGroup();
|
||||
}
|
||||
if (!cloud_runner_1.default.buildParameters.isCliMode)
|
||||
core.endGroup();
|
||||
cloud_runner_logger_1.default.logWithTime('Configurable pre build step(s) time');
|
||||
if (!cloud_runner_1.default.buildParameters.isCliMode)
|
||||
core.startGroup('build');
|
||||
|
|
@ -5067,16 +5134,17 @@ class BuildAutomationWorkflow {
|
|||
if (!cloud_runner_1.default.buildParameters.isCliMode)
|
||||
core.endGroup();
|
||||
cloud_runner_logger_1.default.logWithTime('Build time');
|
||||
if (!cloud_runner_1.default.buildParameters.isCliMode)
|
||||
core.startGroup('post build steps');
|
||||
if (cloud_runner_1.default.buildParameters.postBuildSteps !== '') {
|
||||
if (!cloud_runner_1.default.buildParameters.isCliMode)
|
||||
core.startGroup('post build steps');
|
||||
output += yield custom_workflow_1.CustomWorkflow.runCustomJob(cloud_runner_1.default.buildParameters.postBuildSteps, cloudRunnerStepState.environment, cloudRunnerStepState.secrets);
|
||||
if (!cloud_runner_1.default.buildParameters.isCliMode)
|
||||
core.endGroup();
|
||||
}
|
||||
if (!cloud_runner_1.default.buildParameters.isCliMode)
|
||||
core.endGroup();
|
||||
cloud_runner_logger_1.default.logWithTime('Configurable post build step(s) time');
|
||||
if (cloud_runner_options_1.default.retainWorkspaces) {
|
||||
yield shared_workspace_locking_1.default.ReleaseWorkspace(`test-workspace-${cloud_runner_1.default.buildParameters.buildGuid}`, cloud_runner_1.default.buildParameters.buildGuid);
|
||||
yield shared_workspace_locking_1.default.ReleaseWorkspace(`test-workspace-${cloud_runner_1.default.buildParameters.buildGuid}`, cloud_runner_1.default.buildParameters.buildGuid, cloud_runner_1.default.buildParameters);
|
||||
cloud_runner_1.default.lockedWorkspace = undefined;
|
||||
}
|
||||
cloud_runner_logger_1.default.log(`Cloud Runner finished running standard build automation`);
|
||||
return output;
|
||||
|
|
@ -5099,21 +5167,19 @@ class BuildAutomationWorkflow {
|
|||
${BuildAutomationWorkflow.setupCommands(builderPath)}
|
||||
${setupHooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
|
||||
${buildHooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
|
||||
${BuildAutomationWorkflow.BuildCommands(builderPath, cloud_runner_1.default.buildParameters.buildGuid)}
|
||||
${BuildAutomationWorkflow.BuildCommands(builderPath)}
|
||||
${buildHooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}`;
|
||||
}
|
||||
static setupCommands(builderPath) {
|
||||
const commands = `mkdir -p ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.builderPathAbsolute)} && git clone -q -b ${cloud_runner_1.default.buildParameters.cloudRunnerBranch} ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.unityBuilderRepoUrl)} "${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.builderPathAbsolute)}" && chmod +x ${builderPath}`;
|
||||
return `export GIT_DISCOVERY_ACROSS_FILESYSTEM=1
|
||||
echo "game ci cloud runner clone"
|
||||
mkdir -p ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.builderPathAbsolute)}
|
||||
git clone -q -b ${cloud_runner_1.default.buildParameters.cloudRunnerBranch} ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.unityBuilderRepoUrl)} "${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.builderPathAbsolute)}"
|
||||
chmod +x ${builderPath}
|
||||
if [ -e "${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}" ]; then echo "Retained Workspace Already Exists!"; else ${commands}; fi
|
||||
echo "game ci cloud runner bootstrap"
|
||||
node ${builderPath} -m remote-cli`;
|
||||
node ${builderPath} -m remote-cli-pre-build`;
|
||||
}
|
||||
// ToDo: Replace with a very simple "node ${builderPath} -m build-cli" to run the scripts below without enlarging the request size
|
||||
static BuildCommands(builderPath, guid) {
|
||||
const linuxCacheFolder = cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.cacheFolderFull);
|
||||
static BuildCommands(builderPath) {
|
||||
const distFolder = path_1.default.join(cloud_runner_folders_1.CloudRunnerFolders.builderPathAbsolute, 'dist');
|
||||
const ubuntuPlatformsFolder = path_1.default.join(cloud_runner_folders_1.CloudRunnerFolders.builderPathAbsolute, 'dist', 'platforms', 'ubuntu');
|
||||
return `echo "game ci cloud runner init"
|
||||
|
|
@ -5128,14 +5194,7 @@ class BuildAutomationWorkflow {
|
|||
/entrypoint.sh
|
||||
echo "game ci cloud runner push library to cache"
|
||||
chmod +x ${builderPath}
|
||||
# node ${builderPath} -m remote-cli-post
|
||||
node ${builderPath} -m cache-push --cachePushFrom ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.libraryFolderAbsolute)} --artifactName lib-${guid} --cachePushTo ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(`${linuxCacheFolder}/Library`)}
|
||||
echo "game ci cloud runner push build to cache"
|
||||
node ${builderPath} -m cache-push --cachePushFrom ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.projectBuildFolderAbsolute)} --artifactName build-${guid} --cachePushTo ${`${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(`${linuxCacheFolder}/build`)}`}
|
||||
${BuildAutomationWorkflow.GetCleanupCommand(cloud_runner_folders_1.CloudRunnerFolders.projectPathAbsolute)}`;
|
||||
}
|
||||
static GetCleanupCommand(cleanupPath) {
|
||||
return cloud_runner_options_1.default.retainWorkspaces ? `` : `rm -r ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cleanupPath)}`;
|
||||
node ${builderPath} -m remote-cli-post-build`;
|
||||
}
|
||||
}
|
||||
exports.BuildAutomationWorkflow = BuildAutomationWorkflow;
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,3 @@
|
|||
hook: post-build
|
||||
run: |
|
||||
post-build test!
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
hook: pre-build
|
||||
run: |
|
||||
pre-build test!
|
||||
|
|
@ -63,6 +63,9 @@ class BuildParameters {
|
|||
public cloudRunnerIntegrationTests!: boolean;
|
||||
public cloudRunnerBuilderPlatform!: string | undefined;
|
||||
public isCliMode!: boolean;
|
||||
public retainWorkspace!: boolean;
|
||||
public useSharedLargePackages!: boolean;
|
||||
public useLZ4Compression!: boolean;
|
||||
|
||||
static async create(): Promise<BuildParameters> {
|
||||
const buildFile = this.parseBuildFile(Input.buildName, Input.targetPlatform, Input.androidAppBundle);
|
||||
|
|
@ -137,6 +140,9 @@ class BuildParameters {
|
|||
readInputFromOverrideList: CloudRunnerOptions.readInputFromOverrideList(),
|
||||
kubeStorageClass: CloudRunnerOptions.kubeStorageClass,
|
||||
cacheKey: CloudRunnerOptions.cacheKey,
|
||||
retainWorkspace: CloudRunnerOptions.retainWorkspaces,
|
||||
useSharedLargePackages: CloudRunnerOptions.useSharedLargePackages,
|
||||
useLZ4Compression: CloudRunnerOptions.useLZ4Compression,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import CloudRunnerOptionsReader from '../cloud-runner/services/cloud-runner-opti
|
|||
import GitHub from '../github';
|
||||
import { TaskParameterSerializer } from '../cloud-runner/services/task-parameter-serializer';
|
||||
import { CloudRunnerFolders } from '../cloud-runner/services/cloud-runner-folders';
|
||||
import { CloudRunnerSystem } from '../cloud-runner/services/cloud-runner-system';
|
||||
|
||||
export class Cli {
|
||||
public static options;
|
||||
|
|
@ -77,6 +78,7 @@ export class Cli {
|
|||
${JSON.stringify(buildParameter, undefined, 4)}
|
||||
`);
|
||||
CloudRunner.buildParameters = buildParameter;
|
||||
CloudRunner.lockedWorkspace = process.env.LOCKED_WORKSPACE;
|
||||
|
||||
return await results.target[results.propertyKey](Cli.options);
|
||||
}
|
||||
|
|
@ -111,39 +113,28 @@ export class Cli {
|
|||
|
||||
@CliFunction(`remote-cli-post-build`, `runs a cloud runner build`)
|
||||
public static async PostCLIBuild(): Promise<string> {
|
||||
const buildParameter = await BuildParameters.create();
|
||||
|
||||
/*
|
||||
# LIBRARY CACHE
|
||||
node ${builderPath} -m cache-push --cachePushFrom ${CloudRunnerFolders.ToLinuxFolder(
|
||||
CloudRunnerFolders.libraryFolderAbsolute,
|
||||
)} --artifactName lib-${guid} --cachePushTo ${CloudRunnerFolders.ToLinuxFolder(`${linuxCacheFolder}/Library`)}
|
||||
|
||||
echo "game ci cloud runner push build to cache"
|
||||
|
||||
# BUILD CACHE
|
||||
node ${builderPath} -m cache-push --cachePushFrom ${CloudRunnerFolders.ToLinuxFolder(
|
||||
CloudRunnerFolders.projectBuildFolderAbsolute,
|
||||
)} --artifactName build-${guid} --cachePushTo ${`${CloudRunnerFolders.ToLinuxFolder(`${linuxCacheFolder}/build`)}`}
|
||||
|
||||
# RETAINED WORKSPACE CLEANUP
|
||||
${BuildAutomationWorkflow.GetCleanupCommand(CloudRunnerFolders.projectPathAbsolute)}`;
|
||||
*/
|
||||
|
||||
core.info(`Running POST build tasks`);
|
||||
|
||||
Caching.PushToCache(
|
||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute),
|
||||
await Caching.PushToCache(
|
||||
CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderFull}/Library`),
|
||||
`lib-${buildParameter.buildGuid}`,
|
||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute),
|
||||
`lib-${CloudRunner.buildParameters.buildGuid}`,
|
||||
);
|
||||
|
||||
Caching.PushToCache(
|
||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.projectBuildFolderAbsolute),
|
||||
await Caching.PushToCache(
|
||||
CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderFull}/build`),
|
||||
`build-${buildParameter.buildGuid}`,
|
||||
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.projectBuildFolderAbsolute),
|
||||
`build-${CloudRunner.buildParameters.buildGuid}`,
|
||||
);
|
||||
|
||||
if (!CloudRunner.buildParameters.retainWorkspace) {
|
||||
await CloudRunnerSystem.Run(
|
||||
`rm -r ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
|
||||
);
|
||||
}
|
||||
|
||||
await RemoteClient.runCustomHookFiles(`after-build`);
|
||||
|
||||
return new Promise((result) => result(``));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,49 +2,179 @@ import { CloudRunnerSystem } from '../cloud-runner/services/cloud-runner-system'
|
|||
import * as fs from 'fs';
|
||||
import CloudRunnerLogger from '../cloud-runner/services/cloud-runner-logger';
|
||||
import CloudRunnerOptions from '../cloud-runner/cloud-runner-options';
|
||||
import BuildParameters from '../build-parameters';
|
||||
export class SharedWorkspaceLocking {
|
||||
public static async GetLockedWorkspace(workspaceIfCreated: string, runId: string) {
|
||||
private static readonly workspaceRoot = `s3://game-ci-test-storage/locks/`;
|
||||
public static async GetAllWorkspaces(buildParametersContext: BuildParameters): Promise<string[]> {
|
||||
return (
|
||||
await SharedWorkspaceLocking.ReadLines(
|
||||
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
|
||||
)
|
||||
).map((x) => x.replace(`/`, ``));
|
||||
}
|
||||
public static async DoesWorkspaceTopLevelExist(buildParametersContext: BuildParameters) {
|
||||
const results = (await SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}`)).map(
|
||||
(x) => x.replace(`/`, ``),
|
||||
);
|
||||
|
||||
return results.includes(buildParametersContext.cacheKey);
|
||||
}
|
||||
public static async GetAllLocks(workspace: string, buildParametersContext: BuildParameters): Promise<string[]> {
|
||||
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
|
||||
throw new Error("Workspace doesn't exist, can't call get all locks");
|
||||
}
|
||||
|
||||
return (
|
||||
await SharedWorkspaceLocking.ReadLines(
|
||||
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/`,
|
||||
)
|
||||
).map((x) => x.replace(`/`, ``));
|
||||
}
|
||||
public static async GetOrCreateLockedWorkspace(
|
||||
workspaceIfCreated: string,
|
||||
runId: string,
|
||||
buildParametersContext: BuildParameters,
|
||||
) {
|
||||
if (!CloudRunnerOptions.retainWorkspaces) {
|
||||
return;
|
||||
}
|
||||
|
||||
const workspaces = await SharedWorkspaceLocking.GetFreeWorkspaces();
|
||||
for (const element of workspaces) {
|
||||
if (await SharedWorkspaceLocking.LockWorkspace(element, runId)) {
|
||||
return element;
|
||||
CloudRunnerLogger.log(`run agent ${runId} is trying to access a workspace`);
|
||||
|
||||
if (await SharedWorkspaceLocking.DoesWorkspaceTopLevelExist(buildParametersContext)) {
|
||||
const workspaces = await SharedWorkspaceLocking.GetFreeWorkspaces(buildParametersContext);
|
||||
for (const element of workspaces) {
|
||||
if (await SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext)) {
|
||||
CloudRunnerLogger.log(`run agent ${runId} locked workspace: ${element}`);
|
||||
|
||||
return element;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return await SharedWorkspaceLocking.CreateLockableWorkspace(workspaceIfCreated);
|
||||
const workspace = await SharedWorkspaceLocking.CreateWorkspace(workspaceIfCreated, buildParametersContext, runId);
|
||||
CloudRunnerLogger.log(`run agent ${runId} didn't find a free workspace so created: ${workspace}`);
|
||||
|
||||
return workspace;
|
||||
}
|
||||
|
||||
public static async DoesWorkspaceExist(workspace: string) {
|
||||
return (await SharedWorkspaceLocking.GetAllWorkspaces()).includes(workspace);
|
||||
public static async DoesWorkspaceExist(workspace: string, buildParametersContext: BuildParameters) {
|
||||
return (await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext)).includes(workspace);
|
||||
}
|
||||
public static async HasWorkspaceLock(
|
||||
workspace: string,
|
||||
runId: string,
|
||||
buildParametersContext: BuildParameters,
|
||||
): Promise<boolean> {
|
||||
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return (
|
||||
(await SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext)).filter((x) => x.includes(runId))
|
||||
.length > 0
|
||||
);
|
||||
}
|
||||
|
||||
public static async GetFreeWorkspaces(): Promise<string[]> {
|
||||
public static async GetFreeWorkspaces(buildParametersContext: BuildParameters): Promise<string[]> {
|
||||
const result: string[] = [];
|
||||
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces();
|
||||
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
|
||||
for (const element of workspaces) {
|
||||
if (!(await SharedWorkspaceLocking.IsWorkspaceLocked(element))) {
|
||||
if (!(await SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParametersContext))) {
|
||||
result.push(element);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
public static async GetAllWorkspaces(): Promise<string[]> {
|
||||
return (await SharedWorkspaceLocking.ReadLines(`aws s3 ls s3://game-ci-test-storage/locks/`)).map((x) =>
|
||||
x.replace(`/`, ``),
|
||||
|
||||
public static async IsWorkspaceLocked(workspace: string, buildParametersContext: BuildParameters): Promise<boolean> {
|
||||
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
|
||||
return false;
|
||||
}
|
||||
const files = await SharedWorkspaceLocking.ReadLines(
|
||||
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/`,
|
||||
);
|
||||
|
||||
const workspaceFileDoesNotExists =
|
||||
files.filter((x) => {
|
||||
return x.includes(`_workspace`);
|
||||
}).length === 0;
|
||||
|
||||
const lockFilesExist =
|
||||
files.filter((x) => {
|
||||
return x.includes(`_lock`);
|
||||
}).length > 0;
|
||||
|
||||
return workspaceFileDoesNotExists || lockFilesExist;
|
||||
}
|
||||
public static async GetAllLocks(workspace: string): Promise<string[]> {
|
||||
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace))) {
|
||||
throw new Error("Workspace doesn't exist, can't call get all locks");
|
||||
|
||||
public static async CreateWorkspace(workspace: string, buildParametersContext: BuildParameters, lockId: string = ``) {
|
||||
if (lockId !== ``) {
|
||||
await SharedWorkspaceLocking.LockWorkspace(workspace, lockId, buildParametersContext);
|
||||
}
|
||||
|
||||
return (await SharedWorkspaceLocking.ReadLines(`aws s3 ls s3://game-ci-test-storage/locks/${workspace}/`)).map(
|
||||
(x) => x.replace(`/`, ``),
|
||||
const file = `${Date.now()}_workspace`;
|
||||
fs.writeFileSync(file, '');
|
||||
await CloudRunnerSystem.Run(
|
||||
`aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`,
|
||||
false,
|
||||
true,
|
||||
);
|
||||
fs.rmSync(file);
|
||||
|
||||
return workspace;
|
||||
}
|
||||
|
||||
public static async LockWorkspace(
|
||||
workspace: string,
|
||||
runId: string,
|
||||
buildParametersContext: BuildParameters,
|
||||
): Promise<boolean> {
|
||||
const file = `${Date.now()}_${runId}_lock`;
|
||||
fs.writeFileSync(file, '');
|
||||
await CloudRunnerSystem.Run(
|
||||
`aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`,
|
||||
false,
|
||||
true,
|
||||
);
|
||||
fs.rmSync(file);
|
||||
|
||||
return SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext);
|
||||
}
|
||||
|
||||
public static async ReleaseWorkspace(
|
||||
workspace: string,
|
||||
runId: string,
|
||||
buildParametersContext: BuildParameters,
|
||||
): Promise<boolean> {
|
||||
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
|
||||
return true;
|
||||
}
|
||||
const file = (await SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext)).filter((x) =>
|
||||
x.includes(`_${runId}_lock`),
|
||||
);
|
||||
CloudRunnerLogger.log(
|
||||
`${JSON.stringify(await SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext))}`,
|
||||
);
|
||||
CloudRunnerLogger.log(`Deleting file ${file}`);
|
||||
CloudRunnerLogger.log(
|
||||
`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`,
|
||||
);
|
||||
await CloudRunnerSystem.Run(
|
||||
`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`,
|
||||
false,
|
||||
true,
|
||||
);
|
||||
|
||||
return !SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext);
|
||||
}
|
||||
|
||||
public static async CleanupWorkspace(workspace: string, buildParametersContext: BuildParameters) {
|
||||
await CloudRunnerSystem.Run(
|
||||
`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace} --recursive`,
|
||||
false,
|
||||
true,
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -61,64 +191,6 @@ export class SharedWorkspaceLocking {
|
|||
return lineValues[lineValues.length - 1];
|
||||
});
|
||||
}
|
||||
|
||||
public static async LockWorkspace(workspace: string, runId: string): Promise<boolean> {
|
||||
const file = `${Date.now()}_${runId}_lock`;
|
||||
fs.writeFileSync(file, '');
|
||||
await CloudRunnerSystem.Run(
|
||||
`aws s3 cp ./${file} s3://game-ci-test-storage/locks/${workspace}/${file}`,
|
||||
false,
|
||||
true,
|
||||
);
|
||||
fs.rmSync(file);
|
||||
|
||||
return SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId);
|
||||
}
|
||||
|
||||
public static async ReleaseWorkspace(workspace: string, runId: string): Promise<boolean> {
|
||||
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace))) {
|
||||
return true;
|
||||
}
|
||||
const file = (await SharedWorkspaceLocking.GetAllLocks(workspace)).filter((x) => x.includes(`_${runId}_lock`));
|
||||
CloudRunnerLogger.log(`${JSON.stringify(await SharedWorkspaceLocking.GetAllLocks(workspace))}`);
|
||||
CloudRunnerLogger.log(`Deleting file ${file}`);
|
||||
CloudRunnerLogger.log(`aws s3 rm s3://game-ci-test-storage/locks/${workspace}/${file}`);
|
||||
await CloudRunnerSystem.Run(`aws s3 rm s3://game-ci-test-storage/locks/${workspace}/${file}`, false, true);
|
||||
|
||||
return !SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId);
|
||||
}
|
||||
public static async HasWorkspaceLock(workspace: string, runId: string): Promise<boolean> {
|
||||
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return (await SharedWorkspaceLocking.GetAllLocks(workspace)).filter((x) => x.includes(runId)).length > 0;
|
||||
}
|
||||
|
||||
public static async IsWorkspaceLocked(workspace: string): Promise<boolean> {
|
||||
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace))) {
|
||||
return false;
|
||||
}
|
||||
const files = await SharedWorkspaceLocking.ReadLines(`aws s3 ls s3://game-ci-test-storage/locks/${workspace}/`);
|
||||
|
||||
// 1 Because we expect 1 workspace file to exist in every workspace folder
|
||||
return files.length > 1;
|
||||
}
|
||||
|
||||
public static async CreateLockableWorkspace(workspace: string) {
|
||||
const file = `${Date.now()}_workspace`;
|
||||
fs.writeFileSync(file, '');
|
||||
await CloudRunnerSystem.Run(
|
||||
`aws s3 cp ./${file} s3://game-ci-test-storage/locks/${workspace}/${file}`,
|
||||
false,
|
||||
true,
|
||||
);
|
||||
fs.rmSync(file);
|
||||
|
||||
return workspace;
|
||||
}
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
public static ReleaseLock(workspace: string) {}
|
||||
}
|
||||
|
||||
export default SharedWorkspaceLocking;
|
||||
|
|
|
|||
|
|
@ -171,6 +171,14 @@ class CloudRunnerOptions {
|
|||
return Number(CloudRunnerOptions.getInput(`retainWorkspacesMax`)) || 5;
|
||||
}
|
||||
|
||||
public static get useSharedLargePackages(): boolean {
|
||||
return CloudRunnerOptions.getInput(`useSharedLargePackages`) || false;
|
||||
}
|
||||
|
||||
public static get useLZ4Compression(): boolean {
|
||||
return CloudRunnerOptions.getInput(`useLZ4Compression`) || true;
|
||||
}
|
||||
|
||||
public static ToEnvVarFormat(input: string) {
|
||||
if (input.toUpperCase() === input) {
|
||||
return input;
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ class CloudRunner {
|
|||
public static buildParameters: BuildParameters;
|
||||
private static defaultSecrets: CloudRunnerSecret[];
|
||||
private static cloudRunnerEnvironmentVariables: CloudRunnerEnvironmentVariable[];
|
||||
static lockedWorkspace: string | undefined;
|
||||
public static setup(buildParameters: BuildParameters) {
|
||||
CloudRunnerLogger.setup();
|
||||
CloudRunner.buildParameters = buildParameters;
|
||||
|
|
|
|||
|
|
@ -7,8 +7,6 @@ import Docker from '../../../docker';
|
|||
import { Action } from '../../../../model';
|
||||
import { writeFileSync } from 'fs';
|
||||
|
||||
// import * as core from '@actions/core';
|
||||
|
||||
class LocalDockerCloudRunner implements ProviderInterface {
|
||||
public buildParameters: BuildParameters | undefined;
|
||||
|
||||
|
|
@ -84,6 +82,7 @@ class LocalDockerCloudRunner implements ProviderInterface {
|
|||
}
|
||||
}
|
||||
let myOutput = '';
|
||||
const sharedFolder = this.buildParameters?.retainWorkspace ? `/data/` : `/data/cache/`;
|
||||
|
||||
// core.info(JSON.stringify({ workspace, actionFolder, ...this.buildParameters, ...content }, undefined, 4));
|
||||
const entrypointFilePath = `start.sh`;
|
||||
|
|
@ -95,11 +94,12 @@ class LocalDockerCloudRunner implements ProviderInterface {
|
|||
apt-get update > /dev/null && apt-get install -y tree> /dev/null
|
||||
mkdir -p /github/workspace/cloud-runner-cache
|
||||
mkdir -p /data/cache
|
||||
cp -a /github/workspace/cloud-runner-cache/. /data/cache/
|
||||
tree -L 2 /data/cache
|
||||
cp -a /github/workspace/cloud-runner-cache/. ${sharedFolder}
|
||||
tree -L 3 ${sharedFolder}
|
||||
${commands}
|
||||
cp -a /data/cache/. /github/workspace/cloud-runner-cache/
|
||||
cp -a ${sharedFolder}. /github/workspace/cloud-runner-cache/
|
||||
tree -L 2 /github/workspace/cloud-runner-cache
|
||||
tree -L 3 ${sharedFolder}
|
||||
`,
|
||||
{
|
||||
flag: 'w',
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ export class Caching {
|
|||
public static async PushToCache(cacheFolder: string, sourceFolder: string, cacheArtifactName: string) {
|
||||
cacheArtifactName = cacheArtifactName.replace(' ', '');
|
||||
const startPath = process.cwd();
|
||||
const compressionSuffix = CloudRunner.buildParameters.useLZ4Compression ? '.lz4' : '';
|
||||
try {
|
||||
if (!(await fileExists(cacheFolder))) {
|
||||
await CloudRunnerSystem.Run(`mkdir -p ${cacheFolder}`);
|
||||
|
|
@ -65,7 +66,7 @@ export class Caching {
|
|||
);
|
||||
|
||||
if (CloudRunner.buildParameters.cloudRunnerIntegrationTests) {
|
||||
await CloudRunnerSystem.Run(`tree -L 2 ./..`);
|
||||
// await CloudRunnerSystem.Run(`tree -L 2 ./..`);
|
||||
await CloudRunnerSystem.Run(`tree -L 2`);
|
||||
}
|
||||
|
||||
|
|
@ -78,14 +79,16 @@ export class Caching {
|
|||
return;
|
||||
}
|
||||
|
||||
await CloudRunnerSystem.Run(`tar -cf ${cacheArtifactName}.tar.lz4 ${path.basename(sourceFolder)}`);
|
||||
await CloudRunnerSystem.Run(`du ${cacheArtifactName}.tar.lz4`);
|
||||
assert(await fileExists(`${cacheArtifactName}.tar.lz4`), 'cache archive exists');
|
||||
await CloudRunnerSystem.Run(
|
||||
`tar -cf ${cacheArtifactName}.tar${compressionSuffix} ${path.basename(sourceFolder)}`,
|
||||
);
|
||||
await CloudRunnerSystem.Run(`du ${cacheArtifactName}.tar${compressionSuffix}`);
|
||||
assert(await fileExists(`${cacheArtifactName}.tar${compressionSuffix}`), 'cache archive exists');
|
||||
assert(await fileExists(path.basename(sourceFolder)), 'source folder exists');
|
||||
await CloudRunnerSystem.Run(`mv ${cacheArtifactName}.tar.lz4 ${cacheFolder}`);
|
||||
await CloudRunnerSystem.Run(`mv ${cacheArtifactName}.tar${compressionSuffix} ${cacheFolder}`);
|
||||
RemoteClientLogger.log(`moved cache entry ${cacheArtifactName} to ${cacheFolder}`);
|
||||
assert(
|
||||
await fileExists(`${path.join(cacheFolder, cacheArtifactName)}.tar.lz4`),
|
||||
await fileExists(`${path.join(cacheFolder, cacheArtifactName)}.tar${compressionSuffix}`),
|
||||
'cache archive exists inside cache folder',
|
||||
);
|
||||
} catch (error) {
|
||||
|
|
@ -96,6 +99,7 @@ export class Caching {
|
|||
}
|
||||
public static async PullFromCache(cacheFolder: string, destinationFolder: string, cacheArtifactName: string = ``) {
|
||||
cacheArtifactName = cacheArtifactName.replace(' ', '');
|
||||
const compressionSuffix = CloudRunner.buildParameters.useLZ4Compression ? '.lz4' : '';
|
||||
const startPath = process.cwd();
|
||||
RemoteClientLogger.log(`Caching for ${path.basename(destinationFolder)}`);
|
||||
try {
|
||||
|
|
@ -107,24 +111,26 @@ export class Caching {
|
|||
await fs.promises.mkdir(destinationFolder);
|
||||
}
|
||||
|
||||
const latestInBranch = await (await CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .tar.lz4$ | head -1`))
|
||||
const latestInBranch = await (
|
||||
await CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .tar${compressionSuffix}$ | head -1`)
|
||||
)
|
||||
.replace(/\n/g, ``)
|
||||
.replace('.tar.lz4', '');
|
||||
.replace(`.tar${compressionSuffix}`, '');
|
||||
|
||||
process.chdir(cacheFolder);
|
||||
|
||||
const cacheSelection =
|
||||
cacheArtifactName !== `` && (await fileExists(`${cacheArtifactName}.tar.lz4`))
|
||||
cacheArtifactName !== `` && (await fileExists(`${cacheArtifactName}.tar${compressionSuffix}`))
|
||||
? cacheArtifactName
|
||||
: latestInBranch;
|
||||
await CloudRunnerLogger.log(`cache key ${cacheArtifactName} selection ${cacheSelection}`);
|
||||
|
||||
if (await fileExists(`${cacheSelection}.tar.lz4`)) {
|
||||
if (await fileExists(`${cacheSelection}.tar${compressionSuffix}`)) {
|
||||
const resultsFolder = `results${CloudRunner.buildParameters.buildGuid}`;
|
||||
await CloudRunnerSystem.Run(`mkdir -p ${resultsFolder}`);
|
||||
RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.tar.lz4`);
|
||||
RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.tar${compressionSuffix}`);
|
||||
const fullResultsFolder = path.join(cacheFolder, resultsFolder);
|
||||
await CloudRunnerSystem.Run(`tar -xf ${cacheSelection}.tar.lz4 -C ${fullResultsFolder}`);
|
||||
await CloudRunnerSystem.Run(`tar -xf ${cacheSelection}.tar${compressionSuffix} -C ${fullResultsFolder}`);
|
||||
RemoteClientLogger.log(`cache item extracted to ${fullResultsFolder}`);
|
||||
assert(await fileExists(fullResultsFolder), `cache extraction results folder exists`);
|
||||
const destinationParentFolder = path.resolve(destinationFolder, '..');
|
||||
|
|
@ -144,7 +150,9 @@ export class Caching {
|
|||
} else {
|
||||
RemoteClientLogger.logWarning(`cache item ${cacheArtifactName} doesn't exist ${destinationFolder}`);
|
||||
if (cacheSelection !== ``) {
|
||||
RemoteClientLogger.logWarning(`cache item ${cacheArtifactName}.tar.lz4 doesn't exist ${destinationFolder}`);
|
||||
RemoteClientLogger.logWarning(
|
||||
`cache item ${cacheArtifactName}.tar${compressionSuffix} doesn't exist ${destinationFolder}`,
|
||||
);
|
||||
throw new Error(`Failed to get cache item, but cache hit was found: ${cacheSelection}`);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import { assert } from 'console';
|
|||
import CloudRunnerLogger from '../services/cloud-runner-logger';
|
||||
import { CliFunction } from '../../cli/cli-functions-repository';
|
||||
import { CloudRunnerSystem } from '../services/cloud-runner-system';
|
||||
import YAML from 'yaml';
|
||||
|
||||
export class RemoteClient {
|
||||
public static async bootstrapRepository() {
|
||||
|
|
@ -61,8 +62,20 @@ export class RemoteClient {
|
|||
}
|
||||
|
||||
private static async cloneRepoWithoutLFSFiles() {
|
||||
process.chdir(`${CloudRunnerFolders.repoPathAbsolute}`);
|
||||
if (CloudRunner.buildParameters.cloudRunnerIntegrationTests) {
|
||||
await CloudRunnerSystem.Run(`tree -L 2 ./..`);
|
||||
}
|
||||
|
||||
if (fs.existsSync(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`))) {
|
||||
RemoteClientLogger.log(
|
||||
`${CloudRunnerFolders.repoPathAbsolute} repo exists - skipping clone - retained workspace mode ${CloudRunner.buildParameters.retainWorkspace}`,
|
||||
);
|
||||
await CloudRunnerSystem.Run(`git reset --hard ${CloudRunner.buildParameters.gitSha}`);
|
||||
|
||||
return;
|
||||
}
|
||||
try {
|
||||
process.chdir(`${CloudRunnerFolders.repoPathAbsolute}`);
|
||||
RemoteClientLogger.log(`Initializing source repository for cloning with caching of LFS files`);
|
||||
await CloudRunnerSystem.Run(`git config --global advice.detachedHead false`);
|
||||
RemoteClientLogger.log(`Cloning the repository being built:`);
|
||||
|
|
@ -78,6 +91,7 @@ export class RemoteClient {
|
|||
assert(fs.existsSync(`.git`), 'git folder exists');
|
||||
RemoteClientLogger.log(`${CloudRunner.buildParameters.branch}`);
|
||||
await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.branch}`);
|
||||
await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.gitSha}`);
|
||||
assert(fs.existsSync(path.join(`.git`, `lfs`)), 'LFS folder should not exist before caching');
|
||||
RemoteClientLogger.log(`Checked out ${CloudRunner.buildParameters.branch}`);
|
||||
} catch (error) {
|
||||
|
|
@ -86,10 +100,15 @@ export class RemoteClient {
|
|||
}
|
||||
|
||||
static replaceLargePackageReferencesWithSharedReferences() {
|
||||
const manifest = fs.readFileSync(
|
||||
path.join(CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`),
|
||||
'utf8',
|
||||
);
|
||||
if (CloudRunner.buildParameters.cloudRunnerIntegrationTests) {
|
||||
CloudRunnerLogger.log(
|
||||
fs.readFileSync(path.join(CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`), 'utf8'),
|
||||
);
|
||||
CloudRunnerLogger.log(manifest);
|
||||
}
|
||||
if (CloudRunner.buildParameters.useSharedLargePackages) {
|
||||
manifest.replace(/LargePackages/g, '../../LargePackages');
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -102,8 +121,28 @@ export class RemoteClient {
|
|||
assert(fs.existsSync(CloudRunnerFolders.lfsFolderAbsolute));
|
||||
}
|
||||
|
||||
@CliFunction(`remote-cli`, `sets up a repository, usually before a game-ci build`)
|
||||
@CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
|
||||
static async runRemoteClientJob() {
|
||||
RemoteClient.handleRetainedWorkspace();
|
||||
await RemoteClient.bootstrapRepository();
|
||||
await RemoteClient.runCustomHookFiles(`before-build`);
|
||||
}
|
||||
static async runCustomHookFiles(hookLifecycle: string) {
|
||||
RemoteClientLogger.log(`RunCustomHookFiles: ${hookLifecycle}`);
|
||||
const gameCiCustomHooksPath = path.join(CloudRunnerFolders.repoPathAbsolute, `game-ci`, `hooks`);
|
||||
const files = fs.readdirSync(gameCiCustomHooksPath);
|
||||
for (const file of files) {
|
||||
const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`);
|
||||
const fileContentsObject = YAML.parse(fileContents.toString());
|
||||
if (fileContentsObject.hook === hookLifecycle) {
|
||||
RemoteClientLogger.log(`Active Hook File ${file} contents: ${fileContents}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
static handleRetainedWorkspace() {
|
||||
if (!CloudRunner.buildParameters.retainWorkspace || !CloudRunner.lockedWorkspace) {
|
||||
return;
|
||||
}
|
||||
RemoteClientLogger.log(`Retained Workspace: ${CloudRunner.lockedWorkspace}`);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,9 @@ export class CloudRunnerFolders {
|
|||
// Only the following paths that do not start a path.join with another "Full" suffixed property need to start with an absolute /
|
||||
|
||||
public static get uniqueCloudRunnerJobFolderAbsolute(): string {
|
||||
return path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.buildParameters.buildGuid);
|
||||
return CloudRunner.buildParameters.retainWorkspace && CloudRunner.lockedWorkspace
|
||||
? path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.lockedWorkspace)
|
||||
: path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.buildParameters.buildGuid);
|
||||
}
|
||||
|
||||
public static get cacheFolderFull(): string {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,49 @@
|
|||
import CloudRunner from '../cloud-runner';
|
||||
import { BuildParameters, ImageTag } from '../..';
|
||||
import UnityVersioning from '../../unity-versioning';
|
||||
import { Cli } from '../../cli/cli';
|
||||
import CloudRunnerLogger from '../services/cloud-runner-logger';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import CloudRunnerOptions from '../cloud-runner-options';
|
||||
import setups from './cloud-runner-suite.test';
|
||||
|
||||
async function CreateParameters(overrides) {
|
||||
if (overrides) {
|
||||
Cli.options = overrides;
|
||||
}
|
||||
|
||||
return await BuildParameters.create();
|
||||
}
|
||||
|
||||
describe('Cloud Runner Custom Hooks', () => {
|
||||
it('Responds', () => {});
|
||||
setups();
|
||||
if (CloudRunnerOptions.cloudRunnerTests && CloudRunnerOptions.cloudRunnerCluster !== `k8s`) {
|
||||
it('Run one build it should not already be retained, run subsequent build which should use retained workspace', async () => {
|
||||
const overrides = {
|
||||
versioning: 'None',
|
||||
projectPath: 'test-project',
|
||||
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
|
||||
targetPlatform: 'StandaloneLinux64',
|
||||
cacheKey: `test-case-${uuidv4()}`,
|
||||
retainWorkspaces: true,
|
||||
};
|
||||
const buildParameter2 = await CreateParameters(overrides);
|
||||
const baseImage2 = new ImageTag(buildParameter2);
|
||||
const results2 = await CloudRunner.run(buildParameter2, baseImage2.toString());
|
||||
CloudRunnerLogger.log(`run 2 succeeded`);
|
||||
|
||||
const build2ContainsRetainedWorkspacePhrase = results2.includes(`Retained Workspace:`);
|
||||
const build2ContainsWorkspaceExistsAlreadyPhrase = results2.includes(`Retained Workspace Already Exists!`);
|
||||
const build2ContainsBuildSucceeded = results2.includes('Build succeeded');
|
||||
const build2ContainsPreBuildHookMessage = results2.includes('pre-build test!');
|
||||
const build2ContainsPostBuildHookMessage = results2.includes('post-build test!');
|
||||
|
||||
expect(build2ContainsRetainedWorkspacePhrase).toBeTruthy();
|
||||
expect(build2ContainsWorkspaceExistsAlreadyPhrase).toBeTruthy();
|
||||
expect(build2ContainsBuildSucceeded).toBeTruthy();
|
||||
expect(build2ContainsPreBuildHookMessage).toBeTruthy();
|
||||
expect(build2ContainsPostBuildHookMessage).toBeTruthy();
|
||||
}, 10000000);
|
||||
}
|
||||
});
|
||||
|
|
@ -19,7 +19,7 @@ describe('Cloud Runner Retain Workspace', () => {
|
|||
it('Responds', () => {});
|
||||
setups();
|
||||
if (CloudRunnerOptions.cloudRunnerTests && CloudRunnerOptions.cloudRunnerCluster !== `k8s`) {
|
||||
it('Run one build it should not use cache, run subsequent build which should use cache', async () => {
|
||||
it('Run one build it should not already be retained, run subsequent build which should use retained workspace', async () => {
|
||||
const overrides = {
|
||||
versioning: 'None',
|
||||
projectPath: 'test-project',
|
||||
|
|
@ -51,6 +51,8 @@ describe('Cloud Runner Retain Workspace', () => {
|
|||
|
||||
const build2ContainsCacheKey = results2.includes(buildParameter.cacheKey);
|
||||
const build2ContainsBuildGuid1FromRetainedWorkspace = results2.includes(buildParameter.buildGuid);
|
||||
const build2ContainsRetainedWorkspacePhrase = results2.includes(`Retained Workspace:`);
|
||||
const build2ContainsWorkspaceExistsAlreadyPhrase = results2.includes(`Retained Workspace Already Exists!`);
|
||||
const build2ContainsBuildSucceeded = results2.includes(buildSucceededString);
|
||||
const build2NotContainsNoLibraryMessage = !results2.includes(libraryString);
|
||||
const build2NotContainsZeroLibraryCacheFilesMessage = !results2.includes(
|
||||
|
|
@ -61,6 +63,8 @@ describe('Cloud Runner Retain Workspace', () => {
|
|||
);
|
||||
|
||||
expect(build2ContainsCacheKey).toBeTruthy();
|
||||
expect(build2ContainsRetainedWorkspacePhrase).toBeTruthy();
|
||||
expect(build2ContainsWorkspaceExistsAlreadyPhrase).toBeTruthy();
|
||||
expect(build2ContainsBuildGuid1FromRetainedWorkspace).toBeTruthy();
|
||||
expect(build2ContainsBuildSucceeded).toBeTruthy();
|
||||
expect(build2NotContainsZeroLibraryCacheFilesMessage).toBeTruthy();
|
||||
|
|
|
|||
|
|
@ -4,51 +4,94 @@ import setups from './cloud-runner-suite.test';
|
|||
import CloudRunnerLogger from '../services/cloud-runner-logger';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import CloudRunnerOptions from '../cloud-runner-options';
|
||||
import UnityVersioning from '../../unity-versioning';
|
||||
import BuildParameters from '../../build-parameters';
|
||||
|
||||
async function CreateParameters(overrides) {
|
||||
if (overrides) {
|
||||
Cli.options = overrides;
|
||||
}
|
||||
|
||||
return await BuildParameters.create();
|
||||
}
|
||||
|
||||
describe('Cloud Runner Locking', () => {
|
||||
setups();
|
||||
it('Responds', () => {});
|
||||
if (CloudRunnerOptions.cloudRunnerTests) {
|
||||
it(`simple locking flow`, async () => {
|
||||
it(`Simple Locking Flow`, async () => {
|
||||
Cli.options.retainWorkspaces = true;
|
||||
const overrides: any = {
|
||||
versioning: 'None',
|
||||
projectPath: 'test-project',
|
||||
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
|
||||
targetPlatform: 'StandaloneLinux64',
|
||||
cacheKey: `test-case-${uuidv4()}`,
|
||||
};
|
||||
const buildParameters = await CreateParameters(overrides);
|
||||
|
||||
const newWorkspaceName = `test-workspace-${uuidv4()}`;
|
||||
const runId = uuidv4();
|
||||
await SharedWorkspaceLocking.CreateLockableWorkspace(newWorkspaceName);
|
||||
await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters);
|
||||
const isExpectedUnlockedBeforeLocking =
|
||||
(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName)) === false;
|
||||
(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === false;
|
||||
expect(isExpectedUnlockedBeforeLocking).toBeTruthy();
|
||||
await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId);
|
||||
const isExpectedLockedAfterLocking = (await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName)) === true;
|
||||
await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters);
|
||||
const isExpectedLockedAfterLocking =
|
||||
(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === true;
|
||||
expect(isExpectedLockedAfterLocking).toBeTruthy();
|
||||
const locksBeforeRelease = await SharedWorkspaceLocking.GetAllLocks(newWorkspaceName);
|
||||
const locksBeforeRelease = await SharedWorkspaceLocking.GetAllLocks(newWorkspaceName, buildParameters);
|
||||
CloudRunnerLogger.log(JSON.stringify(locksBeforeRelease, undefined, 4));
|
||||
expect(locksBeforeRelease.length > 1).toBeTruthy();
|
||||
await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId);
|
||||
const locks = await SharedWorkspaceLocking.GetAllLocks(newWorkspaceName);
|
||||
await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters);
|
||||
const locks = await SharedWorkspaceLocking.GetAllLocks(newWorkspaceName, buildParameters);
|
||||
expect(locks.length === 1).toBeTruthy();
|
||||
const isExpectedLockedAfterReleasing =
|
||||
(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName)) === false;
|
||||
(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === false;
|
||||
expect(isExpectedLockedAfterReleasing).toBeTruthy();
|
||||
}, 150000);
|
||||
it('Locking', async () => {
|
||||
it.skip('All Locking Actions', async () => {
|
||||
Cli.options.retainWorkspaces = true;
|
||||
CloudRunnerLogger.log(`GetAllWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetAllWorkspaces())}`);
|
||||
CloudRunnerLogger.log(`GetFreeWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetFreeWorkspaces())}`);
|
||||
const overrides: any = {
|
||||
versioning: 'None',
|
||||
projectPath: 'test-project',
|
||||
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
|
||||
targetPlatform: 'StandaloneLinux64',
|
||||
cacheKey: `test-case-${uuidv4()}`,
|
||||
};
|
||||
const buildParameters = await CreateParameters(overrides);
|
||||
|
||||
CloudRunnerLogger.log(
|
||||
`IsWorkspaceLocked ${JSON.stringify(await SharedWorkspaceLocking.IsWorkspaceLocked('test-workspace'))}`,
|
||||
`GetAllWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetAllWorkspaces(buildParameters))}`,
|
||||
);
|
||||
CloudRunnerLogger.log(`GetFreeWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetFreeWorkspaces())}`);
|
||||
CloudRunnerLogger.log(
|
||||
`LockWorkspace ${JSON.stringify(await SharedWorkspaceLocking.LockWorkspace('test-workspace', uuidv4()))}`,
|
||||
`GetFreeWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetFreeWorkspaces(buildParameters))}`,
|
||||
);
|
||||
CloudRunnerLogger.log(
|
||||
`IsWorkspaceLocked ${JSON.stringify(
|
||||
await SharedWorkspaceLocking.IsWorkspaceLocked(`test-workspace-${uuidv4()}`, buildParameters),
|
||||
)}`,
|
||||
);
|
||||
CloudRunnerLogger.log(
|
||||
`GetFreeWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetFreeWorkspaces(buildParameters))}`,
|
||||
);
|
||||
CloudRunnerLogger.log(
|
||||
`LockWorkspace ${JSON.stringify(
|
||||
await SharedWorkspaceLocking.LockWorkspace(`test-workspace-${uuidv4()}`, uuidv4(), buildParameters),
|
||||
)}`,
|
||||
);
|
||||
CloudRunnerLogger.log(
|
||||
`CreateLockableWorkspace ${JSON.stringify(
|
||||
await SharedWorkspaceLocking.CreateLockableWorkspace('test-workspace-2'),
|
||||
await SharedWorkspaceLocking.CreateWorkspace(`test-workspace-${uuidv4()}`, buildParameters),
|
||||
)}`,
|
||||
);
|
||||
CloudRunnerLogger.log(
|
||||
`GetLockedWorkspace ${JSON.stringify(
|
||||
await SharedWorkspaceLocking.GetLockedWorkspace('test-workspace-2', uuidv4()),
|
||||
await SharedWorkspaceLocking.GetOrCreateLockedWorkspace(
|
||||
`test-workspace-${uuidv4()}`,
|
||||
uuidv4(),
|
||||
buildParameters,
|
||||
),
|
||||
)}`,
|
||||
);
|
||||
}, 3000000);
|
||||
|
|
|
|||
|
|
@ -26,11 +26,15 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
|
|||
|
||||
if (CloudRunnerOptions.retainWorkspaces) {
|
||||
const workspace =
|
||||
(await SharedWorkspaceLocking.GetLockedWorkspace(
|
||||
(await SharedWorkspaceLocking.GetOrCreateLockedWorkspace(
|
||||
`test-workspace-${CloudRunner.buildParameters.buildGuid}`,
|
||||
CloudRunner.buildParameters.buildGuid,
|
||||
CloudRunner.buildParameters,
|
||||
)) || CloudRunner.buildParameters.buildGuid;
|
||||
|
||||
process.env.LOCKED_WORKSPACE = workspace;
|
||||
CloudRunner.lockedWorkspace = workspace;
|
||||
|
||||
CloudRunnerLogger.logLine(`Using workspace ${workspace}`);
|
||||
cloudRunnerStepState.environment = [
|
||||
...cloudRunnerStepState.environment,
|
||||
|
|
@ -38,16 +42,16 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
|
|||
];
|
||||
}
|
||||
|
||||
if (!CloudRunner.buildParameters.isCliMode) core.startGroup('pre build steps');
|
||||
let output = '';
|
||||
if (CloudRunner.buildParameters.preBuildSteps !== '') {
|
||||
if (!CloudRunner.buildParameters.isCliMode) core.startGroup('pre build steps');
|
||||
output += await CustomWorkflow.runCustomJob(
|
||||
CloudRunner.buildParameters.preBuildSteps,
|
||||
cloudRunnerStepState.environment,
|
||||
cloudRunnerStepState.secrets,
|
||||
);
|
||||
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
|
||||
}
|
||||
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
|
||||
CloudRunnerLogger.logWithTime('Configurable pre build step(s) time');
|
||||
|
||||
if (!CloudRunner.buildParameters.isCliMode) core.startGroup('build');
|
||||
|
|
@ -67,22 +71,24 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
|
|||
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
|
||||
CloudRunnerLogger.logWithTime('Build time');
|
||||
|
||||
if (!CloudRunner.buildParameters.isCliMode) core.startGroup('post build steps');
|
||||
if (CloudRunner.buildParameters.postBuildSteps !== '') {
|
||||
if (!CloudRunner.buildParameters.isCliMode) core.startGroup('post build steps');
|
||||
output += await CustomWorkflow.runCustomJob(
|
||||
CloudRunner.buildParameters.postBuildSteps,
|
||||
cloudRunnerStepState.environment,
|
||||
cloudRunnerStepState.secrets,
|
||||
);
|
||||
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
|
||||
}
|
||||
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
|
||||
CloudRunnerLogger.logWithTime('Configurable post build step(s) time');
|
||||
|
||||
if (CloudRunnerOptions.retainWorkspaces) {
|
||||
await SharedWorkspaceLocking.ReleaseWorkspace(
|
||||
`test-workspace-${CloudRunner.buildParameters.buildGuid}`,
|
||||
CloudRunner.buildParameters.buildGuid,
|
||||
CloudRunner.buildParameters,
|
||||
);
|
||||
CloudRunner.lockedWorkspace = undefined;
|
||||
}
|
||||
|
||||
CloudRunnerLogger.log(`Cloud Runner finished running standard build automation`);
|
||||
|
|
@ -113,25 +119,28 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
|
|||
${BuildAutomationWorkflow.setupCommands(builderPath)}
|
||||
${setupHooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
|
||||
${buildHooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
|
||||
${BuildAutomationWorkflow.BuildCommands(builderPath, CloudRunner.buildParameters.buildGuid)}
|
||||
${BuildAutomationWorkflow.BuildCommands(builderPath)}
|
||||
${buildHooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}`;
|
||||
}
|
||||
|
||||
private static setupCommands(builderPath) {
|
||||
const commands = `mkdir -p ${CloudRunnerFolders.ToLinuxFolder(
|
||||
CloudRunnerFolders.builderPathAbsolute,
|
||||
)} && git clone -q -b ${CloudRunner.buildParameters.cloudRunnerBranch} ${CloudRunnerFolders.ToLinuxFolder(
|
||||
CloudRunnerFolders.unityBuilderRepoUrl,
|
||||
)} "${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.builderPathAbsolute)}" && chmod +x ${builderPath}`;
|
||||
|
||||
return `export GIT_DISCOVERY_ACROSS_FILESYSTEM=1
|
||||
echo "game ci cloud runner clone"
|
||||
mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.builderPathAbsolute)}
|
||||
git clone -q -b ${CloudRunner.buildParameters.cloudRunnerBranch} ${CloudRunnerFolders.ToLinuxFolder(
|
||||
CloudRunnerFolders.unityBuilderRepoUrl,
|
||||
)} "${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.builderPathAbsolute)}"
|
||||
chmod +x ${builderPath}
|
||||
if [ -e "${CloudRunnerFolders.ToLinuxFolder(
|
||||
CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute,
|
||||
)}" ]; then echo "Retained Workspace Already Exists!"; else ${commands}; fi
|
||||
echo "game ci cloud runner bootstrap"
|
||||
node ${builderPath} -m remote-cli`;
|
||||
node ${builderPath} -m remote-cli-pre-build`;
|
||||
}
|
||||
|
||||
// ToDo: Replace with a very simple "node ${builderPath} -m build-cli" to run the scripts below without enlarging the request size
|
||||
private static BuildCommands(builderPath, guid) {
|
||||
const linuxCacheFolder = CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderFull);
|
||||
private static BuildCommands(builderPath) {
|
||||
const distFolder = path.join(CloudRunnerFolders.builderPathAbsolute, 'dist');
|
||||
const ubuntuPlatformsFolder = path.join(CloudRunnerFolders.builderPathAbsolute, 'dist', 'platforms', 'ubuntu');
|
||||
|
||||
|
|
@ -147,18 +156,6 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
|
|||
/entrypoint.sh
|
||||
echo "game ci cloud runner push library to cache"
|
||||
chmod +x ${builderPath}
|
||||
# node ${builderPath} -m remote-cli-post
|
||||
node ${builderPath} -m cache-push --cachePushFrom ${CloudRunnerFolders.ToLinuxFolder(
|
||||
CloudRunnerFolders.libraryFolderAbsolute,
|
||||
)} --artifactName lib-${guid} --cachePushTo ${CloudRunnerFolders.ToLinuxFolder(`${linuxCacheFolder}/Library`)}
|
||||
echo "game ci cloud runner push build to cache"
|
||||
node ${builderPath} -m cache-push --cachePushFrom ${CloudRunnerFolders.ToLinuxFolder(
|
||||
CloudRunnerFolders.projectBuildFolderAbsolute,
|
||||
)} --artifactName build-${guid} --cachePushTo ${`${CloudRunnerFolders.ToLinuxFolder(`${linuxCacheFolder}/build`)}`}
|
||||
${BuildAutomationWorkflow.GetCleanupCommand(CloudRunnerFolders.projectPathAbsolute)}`;
|
||||
}
|
||||
|
||||
private static GetCleanupCommand(cleanupPath: string) {
|
||||
return CloudRunnerOptions.retainWorkspaces ? `` : `rm -r ${CloudRunnerFolders.ToLinuxFolder(cleanupPath)}`;
|
||||
node ${builderPath} -m remote-cli-post-build`;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue