cleanup logging for github check updates

pull/524/head
Frostebite 2023-03-18 23:14:26 +00:00
parent 4afa20ec5c
commit e1f3cb3e99
17 changed files with 127 additions and 125 deletions

115
dist/index.js generated vendored
View File

@ -207,6 +207,9 @@ const cli_1 = __nccwpck_require__(55651);
const github_1 = __importDefault(__nccwpck_require__(83654));
const cloud_runner_options_1 = __importDefault(__nccwpck_require__(96552));
class BuildParameters {
static useRetainedWorkspaceMode(buildParameters) {
return buildParameters.retainWorkspaces > 0;
}
static async create() {
const buildFile = this.parseBuildFile(input_1.default.buildName, input_1.default.targetPlatform, input_1.default.androidExportType);
const editorVersion = unity_versioning_1.default.determineUnityVersion(input_1.default.projectPath, input_1.default.unityVersion);
@ -289,14 +292,13 @@ class BuildParameters {
logId: nanoid_1.customAlphabet(cloud_runner_constants_1.default.alphabet, 9)(),
buildGuid: cloud_runner_guid_1.default.generateGuid(input_1.default.runNumber, input_1.default.targetPlatform),
customJobHooks: cloud_runner_options_1.default.customJobHooks(),
readInputOverrideCommand: cloud_runner_options_1.default.readInputOverrideCommand(),
readInputFromOverrideList: cloud_runner_options_1.default.readInputFromOverrideList(),
inputOverrideCommand: cloud_runner_options_1.default.inputOverrideCommand(),
inputOverrides: cloud_runner_options_1.default.inputOverrides(),
kubeStorageClass: cloud_runner_options_1.default.kubeStorageClass,
cacheKey: cloud_runner_options_1.default.cacheKey,
retainWorkspaces: cloud_runner_options_1.default.retainWorkspaces,
useSharedLargePackages: cloud_runner_options_1.default.useSharedLargePackages,
useLz4Compression: cloud_runner_options_1.default.useLz4Compression,
maxRetainedWorkspaces: cloud_runner_options_1.default.maxRetainedWorkspaces,
compressionStrategy: cloud_runner_options_1.default.compressionStrategy,
garbageCollectionMaxAge: cloud_runner_options_1.default.garbageCollectionMaxAge,
githubChecks: cloud_runner_options_1.default.githubChecks,
asyncWorkflow: cloud_runner_options_1.default.asyncCloudRunner,
@ -617,7 +619,7 @@ class Cli {
core.info(`Running POST build tasks`);
await caching_1.Caching.PushToCache(cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(`${cloud_runner_folders_1.CloudRunnerFolders.cacheFolderForCacheKeyFull}/Library`), cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.libraryFolderAbsolute), `lib-${__1.CloudRunner.buildParameters.buildGuid}`);
await caching_1.Caching.PushToCache(cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(`${cloud_runner_folders_1.CloudRunnerFolders.cacheFolderForCacheKeyFull}/build`), cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.projectBuildFolderAbsolute), `build-${__1.CloudRunner.buildParameters.buildGuid}`);
if (!__1.CloudRunner.buildParameters.retainWorkspaces) {
if (!__1.BuildParameters.useRetainedWorkspaceMode(__1.CloudRunner.buildParameters)) {
await cloud_runner_system_1.CloudRunnerSystem.Run(`rm -r ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`);
}
await remote_client_1.RemoteClient.runCustomHookFiles(`after-build`);
@ -814,11 +816,11 @@ class CloudRunnerOptions {
// ### ### ###
// Input override handling
// ### ### ###
static readInputFromOverrideList() {
return CloudRunnerOptions.getInput('readInputFromOverrideList') || '';
static inputOverrides() {
return CloudRunnerOptions.getInput('inputOverrides') || '';
}
static readInputOverrideCommand() {
const value = CloudRunnerOptions.getInput('readInputOverrideCommand');
static inputOverrideCommand() {
const value = CloudRunnerOptions.getInput('inputOverrideCommand');
if (value === 'gcp-secret-manager') {
return 'gcloud secrets versions access 1 --secret="{0}"';
}
@ -860,14 +862,10 @@ class CloudRunnerOptions {
static get cloudRunnerDebug() {
return (CloudRunnerOptions.getInput(`cloudRunnerTests`) === `true` ||
CloudRunnerOptions.getInput(`cloudRunnerDebug`) === `true` ||
CloudRunnerOptions.getInput(`cloudRunnerDebug`) === `true` ||
CloudRunnerOptions.getInput(`cloudRunnerDebugEnv`) === `true` ||
false);
}
static get cloudRunnerDebugTree() {
return CloudRunnerOptions.getInput(`cloudRunnerDebugTree`) === `true` || false;
}
static get cloudRunnerDebugEnv() {
return CloudRunnerOptions.getInput(`cloudRunnerDebugEnv`) === `true` || false;
}
static get skipLFS() {
return CloudRunnerOptions.getInput(`skipLFS`) === `true` || false;
}
@ -883,8 +881,8 @@ class CloudRunnerOptions {
static get useSharedBuilder() {
return (CloudRunnerOptions.getInput(`useSharedBuilder`) || 'false') === 'true';
}
static get useLz4Compression() {
return (CloudRunnerOptions.getInput(`useLz4Compression`) || 'false') === 'true';
static get compressionStrategy() {
return (CloudRunnerOptions.getInput(`compressionStrategy`) || 'false') === 'true';
}
static get useCleanupCron() {
return (CloudRunnerOptions.getInput(`useCleanupCron`) || 'true') === 'true';
@ -893,10 +891,7 @@ class CloudRunnerOptions {
// Retained Workspace
// ### ### ###
static get retainWorkspaces() {
return CloudRunnerOptions.getInput(`retainWorkspaces`) || false;
}
static get maxRetainedWorkspaces() {
return Number(CloudRunnerOptions.getInput(`maxRetainedWorkspaces`)) || 3;
return CloudRunnerOptions.getInput(`retainWorkspaces`) || 0;
}
// ### ### ###
// Garbage Collection
@ -1014,7 +1009,7 @@ class CloudRunner {
// CloudRunnerLogger.log(`Cloud Runner output ${Input.ToEnvVarFormat(element)} = ${buildParameters[element]}`);
core.setOutput(__1.Input.ToEnvVarFormat(element), buildParameters[element]);
}
core.setOutput(__1.Input.ToEnvVarFormat(`buildArtifact`), `build-${CloudRunner.buildParameters.buildGuid}.tar${CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''}`);
core.setOutput(__1.Input.ToEnvVarFormat(`buildArtifact`), `build-${CloudRunner.buildParameters.buildGuid}.tar${CloudRunner.buildParameters.compressionStrategy ? '.lz4' : ''}`);
}
}
static setupSelectedBuildPlatform() {
@ -1045,7 +1040,7 @@ class CloudRunner {
if (!CloudRunner.buildParameters.isCliMode)
core.endGroup();
try {
if (buildParameters.retainWorkspaces) {
if (__1.BuildParameters.useRetainedWorkspaceMode(buildParameters)) {
CloudRunner.lockedWorkspace = shared_workspace_locking_1.default.NewWorkspaceName();
const result = await shared_workspace_locking_1.default.GetOrCreateLockedWorkspace(CloudRunner.lockedWorkspace, CloudRunner.buildParameters.buildGuid, CloudRunner.buildParameters);
if (result) {
@ -1056,8 +1051,8 @@ class CloudRunner {
];
}
else {
cloud_runner_logger_1.default.log(`Max retained workspaces reached ${buildParameters.maxRetainedWorkspaces}`);
buildParameters.retainWorkspaces = false;
cloud_runner_logger_1.default.log(`Max retained workspaces reached ${buildParameters.retainWorkspaces}`);
buildParameters.retainWorkspaces = 0;
CloudRunner.lockedWorkspace = undefined;
}
}
@ -1074,7 +1069,7 @@ class CloudRunner {
if (!CloudRunner.buildParameters.isCliMode)
core.endGroup();
await github_1.default.updateGitHubCheck(CloudRunner.buildParameters.buildGuid, `success`, `success`, `completed`);
if (CloudRunner.buildParameters.retainWorkspaces) {
if (__1.BuildParameters.useRetainedWorkspaceMode(buildParameters)) {
const workspace = CloudRunner.lockedWorkspace || ``;
await shared_workspace_locking_1.default.ReleaseWorkspace(workspace, CloudRunner.buildParameters.buildGuid, CloudRunner.buildParameters);
const isLocked = await shared_workspace_locking_1.default.IsWorkspaceLocked(workspace, CloudRunner.buildParameters);
@ -2927,9 +2922,9 @@ class LocalDockerCloudRunner {
// eslint-disable-next-line no-unused-vars
defaultSecretsArray) {
const { workspace } = __1.Action;
if (fs.existsSync(`${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${cloud_runner_1.default.buildParameters.useLz4Compression ? '.lz4' : ''}`)) {
if (fs.existsSync(`${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${cloud_runner_1.default.buildParameters.compressionStrategy ? '.lz4' : ''}`)) {
await cloud_runner_system_1.CloudRunnerSystem.Run(`ls ${workspace}/cloud-runner-cache/cache/build/`);
await cloud_runner_system_1.CloudRunnerSystem.Run(`rm -r ${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${cloud_runner_1.default.buildParameters.useLz4Compression ? '.lz4' : ''}`);
await cloud_runner_system_1.CloudRunnerSystem.Run(`rm -r ${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${cloud_runner_1.default.buildParameters.compressionStrategy ? '.lz4' : ''}`);
}
}
setupWorkflow(buildGuid, buildParameters,
@ -3030,6 +3025,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
const k8s = __importStar(__nccwpck_require__(89679));
const __1 = __nccwpck_require__(41359);
const core = __importStar(__nccwpck_require__(42186));
const kubernetes_storage_1 = __importDefault(__nccwpck_require__(49043));
const kubernetes_task_runner_1 = __importDefault(__nccwpck_require__(70801));
@ -3105,7 +3101,9 @@ class Kubernetes {
defaultSecretsArray) {
try {
this.buildParameters = buildParameters;
const id = buildParameters.retainWorkspaces ? cloud_runner_1.default.lockedWorkspace : buildParameters.buildGuid;
const id = __1.BuildParameters.useRetainedWorkspaceMode(buildParameters)
? cloud_runner_1.default.lockedWorkspace
: buildParameters.buildGuid;
this.pvcName = `unity-builder-pvc-${id}`;
this.cleanupCronJobName = `unity-builder-cronjob-${id}`;
this.serviceAccountName = `service-account-${buildParameters.buildGuid}`;
@ -3266,7 +3264,7 @@ class Kubernetes {
branchName,
// eslint-disable-next-line no-unused-vars
defaultSecretsArray) {
if (buildParameters.retainWorkspaces) {
if (__1.BuildParameters.useRetainedWorkspaceMode(buildParameters)) {
return;
}
cloud_runner_logger_1.default.log(`deleting PVC`);
@ -4039,10 +4037,10 @@ class Caching {
cacheArtifactName = cacheArtifactName.replace(' ', '');
const startPath = process.cwd();
let compressionSuffix = '';
if (cloud_runner_1.default.buildParameters.useLz4Compression === true) {
if (cloud_runner_1.default.buildParameters.compressionStrategy === true) {
compressionSuffix = `.lz4`;
}
cloud_runner_logger_1.default.log(`Compression: ${cloud_runner_1.default.buildParameters.useLz4Compression} ${compressionSuffix}`);
cloud_runner_logger_1.default.log(`Compression: ${cloud_runner_1.default.buildParameters.compressionStrategy} ${compressionSuffix}`);
try {
if (!(await fileExists(cacheFolder))) {
await cloud_runner_system_1.CloudRunnerSystem.Run(`mkdir -p ${cacheFolder}`);
@ -4080,7 +4078,7 @@ class Caching {
}
cacheArtifactName = cacheArtifactName.replace(' ', '');
let compressionSuffix = '';
if (cloud_runner_1.default.buildParameters.useLz4Compression === true) {
if (cloud_runner_1.default.buildParameters.compressionStrategy === true) {
compressionSuffix = `.lz4`;
}
const startPath = process.cwd();
@ -4179,6 +4177,7 @@ const cloud_runner_system_1 = __nccwpck_require__(99393);
const yaml_1 = __importDefault(__nccwpck_require__(44603));
const github_1 = __importDefault(__nccwpck_require__(83654));
const task_parameter_serializer_1 = __nccwpck_require__(35346);
const build_parameters_1 = __importDefault(__nccwpck_require__(80787));
class RemoteClient {
static async runRemoteClientJob() {
if (!(await RemoteClient.handleRetainedWorkspace())) {
@ -4236,10 +4235,10 @@ class RemoteClient {
}
static async cloneRepoWithoutLFSFiles() {
process.chdir(`${cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
if (cloud_runner_1.default.buildParameters.retainWorkspaces) {
if (build_parameters_1.default.useRetainedWorkspaceMode(cloud_runner_1.default.buildParameters)) {
if (node_fs_1.default.existsSync(node_path_1.default.join(cloud_runner_folders_1.CloudRunnerFolders.repoPathAbsolute, `.git`))) {
process.chdir(cloud_runner_folders_1.CloudRunnerFolders.repoPathAbsolute);
remote_client_logger_1.RemoteClientLogger.log(`${cloud_runner_folders_1.CloudRunnerFolders.repoPathAbsolute} repo exists - skipping clone - retained workspace mode ${cloud_runner_1.default.buildParameters.retainWorkspaces}`);
remote_client_logger_1.RemoteClientLogger.log(`${cloud_runner_folders_1.CloudRunnerFolders.repoPathAbsolute} repo exists - skipping clone - retained workspace mode ${build_parameters_1.default.useRetainedWorkspaceMode(cloud_runner_1.default.buildParameters)}`);
await cloud_runner_system_1.CloudRunnerSystem.Run(`git fetch && git reset --hard ${cloud_runner_1.default.buildParameters.gitSha}`);
return;
}
@ -4291,7 +4290,7 @@ class RemoteClient {
}
}
static async handleRetainedWorkspace() {
if (!cloud_runner_1.default.buildParameters.retainWorkspaces) {
if (!build_parameters_1.default.useRetainedWorkspaceMode(cloud_runner_1.default.buildParameters)) {
return;
}
remote_client_logger_1.RemoteClientLogger.log(`Retained Workspace: ${cloud_runner_1.default.lockedWorkspace !== undefined}`);
@ -4403,7 +4402,7 @@ class CloudRunnerCustomHooks {
cloud_runner_logger_1.default.log(`Applying hooks ${hooks.length}`);
return `echo "---"
echo "start cloud runner init"
${cloud_runner_options_1.default.cloudRunnerDebugEnv ? `printenv` : `#`}
${cloud_runner_options_1.default.cloudRunnerDebug ? `printenv` : `#`}
echo "start of cloud runner job"
${hooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
${commands}
@ -4571,8 +4570,8 @@ class CloudRunnerCustomSteps {
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
aws configure set region $AWS_DEFAULT_REGION --profile default
aws s3 cp /data/cache/$CI_CACHE_KEY/build/build-${cloud_runner_1.default.buildParameters.buildGuid}.tar${cloud_runner_1.default.buildParameters.useLz4Compression ? '.lz4' : ''} s3://${cloud_runner_1.default.buildParameters.awsBaseStackName}/cloud-runner-cache/$CI_CACHE_KEY/build/build-$BUILD_GUID.tar${cloud_runner_1.default.buildParameters.useLz4Compression ? '.lz4' : ''}
rm /data/cache/$CI_CACHE_KEY/build/build-${cloud_runner_1.default.buildParameters.buildGuid}.tar${cloud_runner_1.default.buildParameters.useLz4Compression ? '.lz4' : ''}
aws s3 cp /data/cache/$CI_CACHE_KEY/build/build-${cloud_runner_1.default.buildParameters.buildGuid}.tar${cloud_runner_1.default.buildParameters.compressionStrategy ? '.lz4' : ''} s3://${cloud_runner_1.default.buildParameters.awsBaseStackName}/cloud-runner-cache/$CI_CACHE_KEY/build/build-$BUILD_GUID.tar${cloud_runner_1.default.buildParameters.compressionStrategy ? '.lz4' : ''}
rm /data/cache/$CI_CACHE_KEY/build/build-${cloud_runner_1.default.buildParameters.buildGuid}.tar${cloud_runner_1.default.buildParameters.compressionStrategy ? '.lz4' : ''}
secrets:
- name: awsAccessKeyId
value: ${process.env.AWS_ACCESS_KEY_ID || ``}
@ -4588,7 +4587,7 @@ class CloudRunnerCustomSteps {
aws configure set region $CI_AWS_DEFAULT_REGION --profile default
aws s3 ls ${cloud_runner_1.default.buildParameters.awsBaseStackName}/cloud-runner-cache/ || true
aws s3 ls ${cloud_runner_1.default.buildParameters.awsBaseStackName}/cloud-runner-cache/$CI_CACHE_KEY/build || true
aws s3 cp s3://${cloud_runner_1.default.buildParameters.awsBaseStackName}/cloud-runner-cache/$CI_CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${cloud_runner_1.default.buildParameters.useLz4Compression ? '.lz4' : ''} /data/cache/$CI_CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${cloud_runner_1.default.buildParameters.useLz4Compression ? '.lz4' : ''}
aws s3 cp s3://${cloud_runner_1.default.buildParameters.awsBaseStackName}/cloud-runner-cache/$CI_CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${cloud_runner_1.default.buildParameters.compressionStrategy ? '.lz4' : ''} /data/cache/$CI_CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${cloud_runner_1.default.buildParameters.compressionStrategy ? '.lz4' : ''}
secrets:
- name: CI_AWS_ACCESS_KEY_ID
- name: CI_AWS_SECRET_ACCESS_KEY
@ -4677,8 +4676,8 @@ class CloudRunnerCustomSteps {
hook: after
commands: |
apt-get update > /dev/null
${cloud_runner_options_1.default.cloudRunnerDebugTree ? `apt-get install -y tree > /dev/null` : `#`}
${cloud_runner_options_1.default.cloudRunnerDebugTree ? `tree -L 3 /data/cache` : `#`}
${cloud_runner_options_1.default.cloudRunnerDebug ? `apt-get install -y tree > /dev/null` : `#`}
${cloud_runner_options_1.default.cloudRunnerDebug ? `tree -L 3 /data/cache` : `#`}
secrets:
- name: awsAccessKeyId
value: ${process.env.AWS_ACCESS_KEY_ID || ``}
@ -4796,13 +4795,16 @@ exports.CloudRunnerFolders = void 0;
const node_path_1 = __importDefault(__nccwpck_require__(49411));
const cloud_runner_options_1 = __importDefault(__nccwpck_require__(96552));
const cloud_runner_1 = __importDefault(__nccwpck_require__(79144));
const build_parameters_1 = __importDefault(__nccwpck_require__(80787));
class CloudRunnerFolders {
static ToLinuxFolder(folder) {
return folder.replace(/\\/g, `/`);
}
// Only the following paths that do not start a path.join with another "Full" suffixed property need to start with an absolute /
static get uniqueCloudRunnerJobFolderAbsolute() {
return cloud_runner_1.default.buildParameters && cloud_runner_1.default.buildParameters.retainWorkspaces && cloud_runner_1.default.lockedWorkspace
return cloud_runner_1.default.buildParameters &&
build_parameters_1.default.useRetainedWorkspaceMode(cloud_runner_1.default.buildParameters) &&
cloud_runner_1.default.lockedWorkspace
? node_path_1.default.join(`/`, CloudRunnerFolders.buildVolumeFolder, cloud_runner_1.default.lockedWorkspace)
: node_path_1.default.join(`/`, CloudRunnerFolders.buildVolumeFolder, cloud_runner_1.default.buildParameters.buildGuid);
}
@ -4995,10 +4997,10 @@ class CloudRunnerQueryOverride {
return;
}
static shouldUseOverride(query) {
if (cloud_runner_options_1.default.readInputOverrideCommand() !== '') {
if (cloud_runner_options_1.default.readInputFromOverrideList() !== '') {
const doesInclude = cloud_runner_options_1.default.readInputFromOverrideList().split(',').includes(query) ||
cloud_runner_options_1.default.readInputFromOverrideList().split(',').includes(input_1.default.ToEnvVarFormat(query));
if (cloud_runner_options_1.default.inputOverrideCommand() !== '') {
if (cloud_runner_options_1.default.inputOverrides() !== '') {
const doesInclude = cloud_runner_options_1.default.inputOverrides().split(',').includes(query) ||
cloud_runner_options_1.default.inputOverrides().split(',').includes(input_1.default.ToEnvVarFormat(query));
return doesInclude ? true : false;
}
else {
@ -5010,10 +5012,10 @@ class CloudRunnerQueryOverride {
if (!this.shouldUseOverride(query)) {
throw new Error(`Should not be trying to run override query on ${query}`);
}
return await generic_input_reader_1.GenericInputReader.Run(formatFunction(cloud_runner_options_1.default.readInputOverrideCommand(), [{ key: 0, value: query }]));
return await generic_input_reader_1.GenericInputReader.Run(formatFunction(cloud_runner_options_1.default.inputOverrideCommand(), [{ key: 0, value: query }]));
}
static async PopulateQueryOverrideInput() {
const queries = cloud_runner_options_1.default.readInputFromOverrideList().split(',');
const queries = cloud_runner_options_1.default.inputOverrides().split(',');
CloudRunnerQueryOverride.queryOverrides = {};
for (const element of queries) {
if (CloudRunnerQueryOverride.shouldUseOverride(element)) {
@ -5263,7 +5265,7 @@ exports.SharedWorkspaceLocking = void 0;
const cloud_runner_system_1 = __nccwpck_require__(99393);
const node_fs_1 = __importDefault(__nccwpck_require__(87561));
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
const cloud_runner_options_1 = __importDefault(__nccwpck_require__(96552));
const build_parameters_1 = __importDefault(__nccwpck_require__(80787));
const cloud_runner_1 = __importDefault(__nccwpck_require__(79144));
class SharedWorkspaceLocking {
static get workspaceBucketRoot() {
@ -5309,7 +5311,7 @@ class SharedWorkspaceLocking {
.filter((x) => x.includes(workspace) && x.endsWith(`_lock`));
}
static async GetOrCreateLockedWorkspace(workspace, runId, buildParametersContext) {
if (!cloud_runner_options_1.default.retainWorkspaces) {
if (!build_parameters_1.default.useRetainedWorkspaceMode(buildParametersContext)) {
return false;
}
if (await SharedWorkspaceLocking.DoesCacheKeyTopLevelExist(buildParametersContext)) {
@ -5366,8 +5368,7 @@ class SharedWorkspaceLocking {
static async IsWorkspaceBelowMax(workspace, buildParametersContext) {
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
if (workspace === ``) {
return (workspaces.length < buildParametersContext.maxRetainedWorkspaces ||
buildParametersContext.maxRetainedWorkspaces === 0);
return (workspaces.length < buildParametersContext.retainWorkspaces || buildParametersContext.retainWorkspaces === 0);
}
const ordered = [];
for (const ws of workspaces) {
@ -5379,8 +5380,8 @@ class SharedWorkspaceLocking {
ordered.sort((x) => x.timestamp);
const matches = ordered.filter((x) => x.name.includes(workspace));
const isWorkspaceBelowMax = matches.length > 0 &&
(ordered.indexOf(matches[0]) < buildParametersContext.maxRetainedWorkspaces ||
buildParametersContext.maxRetainedWorkspaces === 0);
(ordered.indexOf(matches[0]) < buildParametersContext.retainWorkspaces ||
buildParametersContext.retainWorkspaces === 0);
return isWorkspaceBelowMax;
}
static async GetWorkspaceTimestamp(workspace, buildParametersContext) {
@ -5417,7 +5418,7 @@ class SharedWorkspaceLocking {
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
cloud_runner_logger_1.default.log(`All workspaces ${workspaces}`);
if (!(await SharedWorkspaceLocking.IsWorkspaceBelowMax(workspace, buildParametersContext))) {
cloud_runner_logger_1.default.log(`Workspace is above max ${workspaces} ${buildParametersContext.maxRetainedWorkspaces}`);
cloud_runner_logger_1.default.log(`Workspace is above max ${workspaces} ${buildParametersContext.retainWorkspaces}`);
await SharedWorkspaceLocking.CleanupWorkspace(workspace, buildParametersContext);
return false;
}
@ -5788,7 +5789,7 @@ rm ${cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolut
node ${builderPath} -m remote-cli-post-build`;
}
static get TreeCommand() {
return cloud_runner_options_1.default.cloudRunnerDebugTree
return cloud_runner_options_1.default.cloudRunnerDebug
? `tree -L 2 ${cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute} && tree -L 2 ${cloud_runner_folders_1.CloudRunnerFolders.cacheFolderForCacheKeyFull} && du -h -s /${cloud_runner_folders_1.CloudRunnerFolders.buildVolumeFolder}/ && du -h -s ${cloud_runner_folders_1.CloudRunnerFolders.cacheFolderForAllFull}`
: `#`;
}

2
dist/index.js.map generated vendored

File diff suppressed because one or more lines are too long

View File

@ -53,8 +53,8 @@ class BuildParameters {
public kubeStorageClass!: string;
public chownFilesTo!: string;
public customJobHooks!: string;
public readInputFromOverrideList!: string;
public readInputOverrideCommand!: string;
public inputOverrides!: string;
public inputOverrideCommand!: string;
public cacheKey!: string;
public postBuildSteps!: string;
@ -70,10 +70,9 @@ class BuildParameters {
public cloudRunnerDebug!: boolean | undefined;
public cloudRunnerBuilderPlatform!: string | undefined;
public isCliMode!: boolean;
public retainWorkspaces!: boolean;
public maxRetainedWorkspaces!: number;
public retainWorkspaces!: number;
public useSharedLargePackages!: boolean;
public useLz4Compression!: boolean;
public compressionStrategy!: boolean;
public garbageCollectionMaxAge!: number;
public githubChecks!: boolean;
public asyncWorkflow!: boolean;
@ -83,6 +82,9 @@ class BuildParameters {
public skipCache!: boolean;
public cacheUnityInstallationOnMac!: boolean;
public unityHubVersionOnMac!: string;
public static useRetainedWorkspaceMode(buildParameters: BuildParameters) {
return buildParameters.retainWorkspaces > 0;
}
static async create(): Promise<BuildParameters> {
const buildFile = this.parseBuildFile(Input.buildName, Input.targetPlatform, Input.androidExportType);
@ -170,14 +172,13 @@ class BuildParameters {
logId: customAlphabet(CloudRunnerConstants.alphabet, 9)(),
buildGuid: CloudRunnerBuildGuid.generateGuid(Input.runNumber, Input.targetPlatform),
customJobHooks: CloudRunnerOptions.customJobHooks(),
readInputOverrideCommand: CloudRunnerOptions.readInputOverrideCommand(),
readInputFromOverrideList: CloudRunnerOptions.readInputFromOverrideList(),
inputOverrideCommand: CloudRunnerOptions.inputOverrideCommand(),
inputOverrides: CloudRunnerOptions.inputOverrides(),
kubeStorageClass: CloudRunnerOptions.kubeStorageClass,
cacheKey: CloudRunnerOptions.cacheKey,
retainWorkspaces: CloudRunnerOptions.retainWorkspaces,
useSharedLargePackages: CloudRunnerOptions.useSharedLargePackages,
useLz4Compression: CloudRunnerOptions.useLz4Compression,
maxRetainedWorkspaces: CloudRunnerOptions.maxRetainedWorkspaces,
compressionStrategy: CloudRunnerOptions.compressionStrategy,
garbageCollectionMaxAge: CloudRunnerOptions.garbageCollectionMaxAge,
githubChecks: CloudRunnerOptions.githubChecks,
asyncWorkflow: CloudRunnerOptions.asyncCloudRunner,

View File

@ -191,7 +191,7 @@ export class Cli {
`build-${CloudRunner.buildParameters.buildGuid}`,
);
if (!CloudRunner.buildParameters.retainWorkspaces) {
if (!BuildParameters.useRetainedWorkspaceMode(CloudRunner.buildParameters)) {
await CloudRunnerSystem.Run(
`rm -r ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
);

View File

@ -181,12 +181,12 @@ class CloudRunnerOptions {
// Input override handling
// ### ### ###
static readInputFromOverrideList() {
return CloudRunnerOptions.getInput('readInputFromOverrideList') || '';
static inputOverrides() {
return CloudRunnerOptions.getInput('inputOverrides') || '';
}
static readInputOverrideCommand() {
const value = CloudRunnerOptions.getInput('readInputOverrideCommand');
static inputOverrideCommand() {
const value = CloudRunnerOptions.getInput('inputOverrideCommand');
if (value === 'gcp-secret-manager') {
return 'gcloud secrets versions access 1 --secret="{0}"';
@ -241,15 +241,11 @@ class CloudRunnerOptions {
return (
CloudRunnerOptions.getInput(`cloudRunnerTests`) === `true` ||
CloudRunnerOptions.getInput(`cloudRunnerDebug`) === `true` ||
CloudRunnerOptions.getInput(`cloudRunnerDebug`) === `true` ||
CloudRunnerOptions.getInput(`cloudRunnerDebugEnv`) === `true` ||
false
);
}
static get cloudRunnerDebugTree(): boolean {
return CloudRunnerOptions.getInput(`cloudRunnerDebugTree`) === `true` || false;
}
static get cloudRunnerDebugEnv(): boolean {
return CloudRunnerOptions.getInput(`cloudRunnerDebugEnv`) === `true` || false;
}
static get skipLFS(): boolean {
return CloudRunnerOptions.getInput(`skipLFS`) === `true` || false;
}
@ -269,8 +265,8 @@ class CloudRunnerOptions {
return (CloudRunnerOptions.getInput(`useSharedBuilder`) || 'false') === 'true';
}
public static get useLz4Compression(): boolean {
return (CloudRunnerOptions.getInput(`useLz4Compression`) || 'false') === 'true';
public static get compressionStrategy(): boolean {
return (CloudRunnerOptions.getInput(`compressionStrategy`) || 'false') === 'true';
}
public static get useCleanupCron(): boolean {
@ -281,12 +277,8 @@ class CloudRunnerOptions {
// Retained Workspace
// ### ### ###
public static get retainWorkspaces(): boolean {
return CloudRunnerOptions.getInput(`retainWorkspaces`) || false;
}
static get maxRetainedWorkspaces(): number {
return Number(CloudRunnerOptions.getInput(`maxRetainedWorkspaces`)) || 3;
public static get retainWorkspaces(): number {
return CloudRunnerOptions.getInput(`retainWorkspaces`) || 0;
}
// ### ### ###

View File

@ -53,7 +53,7 @@ class CloudRunner {
core.setOutput(
Input.ToEnvVarFormat(`buildArtifact`),
`build-${CloudRunner.buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
CloudRunner.buildParameters.compressionStrategy ? '.lz4' : ''
}`,
);
}
@ -91,7 +91,7 @@ class CloudRunner {
);
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
try {
if (buildParameters.retainWorkspaces) {
if (BuildParameters.useRetainedWorkspaceMode(buildParameters)) {
CloudRunner.lockedWorkspace = SharedWorkspaceLocking.NewWorkspaceName();
const result = await SharedWorkspaceLocking.GetOrCreateLockedWorkspace(
@ -107,8 +107,8 @@ class CloudRunner {
{ name: `CI_LOCKED_WORKSPACE`, value: CloudRunner.lockedWorkspace },
];
} else {
CloudRunnerLogger.log(`Max retained workspaces reached ${buildParameters.maxRetainedWorkspaces}`);
buildParameters.retainWorkspaces = false;
CloudRunnerLogger.log(`Max retained workspaces reached ${buildParameters.retainWorkspaces}`);
buildParameters.retainWorkspaces = 0;
CloudRunner.lockedWorkspace = undefined;
}
}
@ -131,7 +131,7 @@ class CloudRunner {
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
await GitHub.updateGitHubCheck(CloudRunner.buildParameters.buildGuid, `success`, `success`, `completed`);
if (CloudRunner.buildParameters.retainWorkspaces) {
if (BuildParameters.useRetainedWorkspaceMode(buildParameters)) {
const workspace = CloudRunner.lockedWorkspace || ``;
await SharedWorkspaceLocking.ReleaseWorkspace(
workspace,

View File

@ -51,14 +51,14 @@ class LocalDockerCloudRunner implements ProviderInterface {
if (
fs.existsSync(
`${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
CloudRunner.buildParameters.compressionStrategy ? '.lz4' : ''
}`,
)
) {
await CloudRunnerSystem.Run(`ls ${workspace}/cloud-runner-cache/cache/build/`);
await CloudRunnerSystem.Run(
`rm -r ${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
CloudRunner.buildParameters.compressionStrategy ? '.lz4' : ''
}`,
);
}

View File

@ -95,7 +95,9 @@ class Kubernetes implements ProviderInterface {
) {
try {
this.buildParameters = buildParameters;
const id = buildParameters.retainWorkspaces ? CloudRunner.lockedWorkspace : buildParameters.buildGuid;
const id = BuildParameters.useRetainedWorkspaceMode(buildParameters)
? CloudRunner.lockedWorkspace
: buildParameters.buildGuid;
this.pvcName = `unity-builder-pvc-${id}`;
this.cleanupCronJobName = `unity-builder-cronjob-${id}`;
this.serviceAccountName = `service-account-${buildParameters.buildGuid}`;
@ -332,7 +334,7 @@ class Kubernetes implements ProviderInterface {
// eslint-disable-next-line no-unused-vars
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
) {
if (buildParameters.retainWorkspaces) {
if (BuildParameters.useRetainedWorkspaceMode(buildParameters)) {
return;
}
CloudRunnerLogger.log(`deleting PVC`);

View File

@ -48,10 +48,10 @@ export class Caching {
cacheArtifactName = cacheArtifactName.replace(' ', '');
const startPath = process.cwd();
let compressionSuffix = '';
if (CloudRunner.buildParameters.useLz4Compression === true) {
if (CloudRunner.buildParameters.compressionStrategy === true) {
compressionSuffix = `.lz4`;
}
CloudRunnerLogger.log(`Compression: ${CloudRunner.buildParameters.useLz4Compression} ${compressionSuffix}`);
CloudRunnerLogger.log(`Compression: ${CloudRunner.buildParameters.compressionStrategy} ${compressionSuffix}`);
try {
if (!(await fileExists(cacheFolder))) {
await CloudRunnerSystem.Run(`mkdir -p ${cacheFolder}`);
@ -106,7 +106,7 @@ export class Caching {
}
cacheArtifactName = cacheArtifactName.replace(' ', '');
let compressionSuffix = '';
if (CloudRunner.buildParameters.useLz4Compression === true) {
if (CloudRunner.buildParameters.compressionStrategy === true) {
compressionSuffix = `.lz4`;
}
const startPath = process.cwd();

View File

@ -12,6 +12,7 @@ import { CloudRunnerSystem } from '../services/cloud-runner-system';
import YAML from 'yaml';
import GitHub from '../../github';
import { TaskParameterSerializer } from '../services/task-parameter-serializer';
import BuildParameters from '../../build-parameters';
export class RemoteClient {
@CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
@ -89,11 +90,15 @@ export class RemoteClient {
private static async cloneRepoWithoutLFSFiles() {
process.chdir(`${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
if (CloudRunner.buildParameters.retainWorkspaces) {
if (BuildParameters.useRetainedWorkspaceMode(CloudRunner.buildParameters)) {
if (fs.existsSync(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`))) {
process.chdir(CloudRunnerFolders.repoPathAbsolute);
RemoteClientLogger.log(
`${CloudRunnerFolders.repoPathAbsolute} repo exists - skipping clone - retained workspace mode ${CloudRunner.buildParameters.retainWorkspaces}`,
`${
CloudRunnerFolders.repoPathAbsolute
} repo exists - skipping clone - retained workspace mode ${BuildParameters.useRetainedWorkspaceMode(
CloudRunner.buildParameters,
)}`,
);
await CloudRunnerSystem.Run(`git fetch && git reset --hard ${CloudRunner.buildParameters.gitSha}`);
@ -152,7 +157,7 @@ export class RemoteClient {
}
}
static async handleRetainedWorkspace() {
if (!CloudRunner.buildParameters.retainWorkspaces) {
if (!BuildParameters.useRetainedWorkspaceMode(CloudRunner.buildParameters)) {
return;
}
RemoteClientLogger.log(`Retained Workspace: ${CloudRunner.lockedWorkspace !== undefined}`);

View File

@ -16,7 +16,7 @@ export class CloudRunnerCustomHooks {
return `echo "---"
echo "start cloud runner init"
${CloudRunnerOptions.cloudRunnerDebugEnv ? `printenv` : `#`}
${CloudRunnerOptions.cloudRunnerDebug ? `printenv` : `#`}
echo "start of cloud runner job"
${hooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
${commands}

View File

@ -46,14 +46,14 @@ export class CloudRunnerCustomSteps {
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
aws configure set region $AWS_DEFAULT_REGION --profile default
aws s3 cp /data/cache/$CI_CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
CloudRunner.buildParameters.compressionStrategy ? '.lz4' : ''
} s3://${
CloudRunner.buildParameters.awsBaseStackName
}/cloud-runner-cache/$CI_CACHE_KEY/build/build-$BUILD_GUID.tar${
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
CloudRunner.buildParameters.compressionStrategy ? '.lz4' : ''
}
rm /data/cache/$CI_CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
CloudRunner.buildParameters.compressionStrategy ? '.lz4' : ''
}
secrets:
- name: awsAccessKeyId
@ -73,9 +73,9 @@ export class CloudRunnerCustomSteps {
aws s3 cp s3://${
CloudRunner.buildParameters.awsBaseStackName
}/cloud-runner-cache/$CI_CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
CloudRunner.buildParameters.compressionStrategy ? '.lz4' : ''
} /data/cache/$CI_CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
CloudRunner.buildParameters.compressionStrategy ? '.lz4' : ''
}
secrets:
- name: CI_AWS_ACCESS_KEY_ID
@ -169,8 +169,8 @@ export class CloudRunnerCustomSteps {
hook: after
commands: |
apt-get update > /dev/null
${CloudRunnerOptions.cloudRunnerDebugTree ? `apt-get install -y tree > /dev/null` : `#`}
${CloudRunnerOptions.cloudRunnerDebugTree ? `tree -L 3 /data/cache` : `#`}
${CloudRunnerOptions.cloudRunnerDebug ? `apt-get install -y tree > /dev/null` : `#`}
${CloudRunnerOptions.cloudRunnerDebug ? `tree -L 3 /data/cache` : `#`}
secrets:
- name: awsAccessKeyId
value: ${process.env.AWS_ACCESS_KEY_ID || ``}

View File

@ -1,6 +1,7 @@
import path from 'node:path';
import CloudRunnerOptions from '../cloud-runner-options';
import CloudRunner from './../cloud-runner';
import BuildParameters from '../../build-parameters';
export class CloudRunnerFolders {
public static readonly repositoryFolder = 'repo';
@ -12,7 +13,9 @@ export class CloudRunnerFolders {
// Only the following paths that do not start a path.join with another "Full" suffixed property need to start with an absolute /
public static get uniqueCloudRunnerJobFolderAbsolute(): string {
return CloudRunner.buildParameters && CloudRunner.buildParameters.retainWorkspaces && CloudRunner.lockedWorkspace
return CloudRunner.buildParameters &&
BuildParameters.useRetainedWorkspaceMode(CloudRunner.buildParameters) &&
CloudRunner.lockedWorkspace
? path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.lockedWorkspace)
: path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.buildParameters.buildGuid);
}

View File

@ -31,11 +31,11 @@ class CloudRunnerQueryOverride {
}
private static shouldUseOverride(query: string) {
if (CloudRunnerOptions.readInputOverrideCommand() !== '') {
if (CloudRunnerOptions.readInputFromOverrideList() !== '') {
if (CloudRunnerOptions.inputOverrideCommand() !== '') {
if (CloudRunnerOptions.inputOverrides() !== '') {
const doesInclude =
CloudRunnerOptions.readInputFromOverrideList().split(',').includes(query) ||
CloudRunnerOptions.readInputFromOverrideList().split(',').includes(Input.ToEnvVarFormat(query));
CloudRunnerOptions.inputOverrides().split(',').includes(query) ||
CloudRunnerOptions.inputOverrides().split(',').includes(Input.ToEnvVarFormat(query));
return doesInclude ? true : false;
} else {
@ -50,12 +50,12 @@ class CloudRunnerQueryOverride {
}
return await GenericInputReader.Run(
formatFunction(CloudRunnerOptions.readInputOverrideCommand(), [{ key: 0, value: query }]),
formatFunction(CloudRunnerOptions.inputOverrideCommand(), [{ key: 0, value: query }]),
);
}
public static async PopulateQueryOverrideInput() {
const queries = CloudRunnerOptions.readInputFromOverrideList().split(',');
const queries = CloudRunnerOptions.inputOverrides().split(',');
CloudRunnerQueryOverride.queryOverrides = {};
for (const element of queries) {
if (CloudRunnerQueryOverride.shouldUseOverride(element)) {

View File

@ -1,7 +1,6 @@
import { CloudRunnerSystem } from './cloud-runner-system';
import fs from 'node:fs';
import CloudRunnerLogger from './cloud-runner-logger';
import CloudRunnerOptions from '../cloud-runner-options';
import BuildParameters from '../../build-parameters';
import CloudRunner from '../cloud-runner';
export class SharedWorkspaceLocking {
@ -68,7 +67,7 @@ export class SharedWorkspaceLocking {
runId: string,
buildParametersContext: BuildParameters,
) {
if (!CloudRunnerOptions.retainWorkspaces) {
if (!BuildParameters.useRetainedWorkspaceMode(buildParametersContext)) {
return false;
}
@ -153,8 +152,7 @@ export class SharedWorkspaceLocking {
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
if (workspace === ``) {
return (
workspaces.length < buildParametersContext.maxRetainedWorkspaces ||
buildParametersContext.maxRetainedWorkspaces === 0
workspaces.length < buildParametersContext.retainWorkspaces || buildParametersContext.retainWorkspaces === 0
);
}
const ordered: any[] = [];
@ -168,8 +166,8 @@ export class SharedWorkspaceLocking {
const matches = ordered.filter((x) => x.name.includes(workspace));
const isWorkspaceBelowMax =
matches.length > 0 &&
(ordered.indexOf(matches[0]) < buildParametersContext.maxRetainedWorkspaces ||
buildParametersContext.maxRetainedWorkspaces === 0);
(ordered.indexOf(matches[0]) < buildParametersContext.retainWorkspaces ||
buildParametersContext.retainWorkspaces === 0);
return isWorkspaceBelowMax;
}
@ -230,7 +228,7 @@ export class SharedWorkspaceLocking {
CloudRunnerLogger.log(`All workspaces ${workspaces}`);
if (!(await SharedWorkspaceLocking.IsWorkspaceBelowMax(workspace, buildParametersContext))) {
CloudRunnerLogger.log(`Workspace is above max ${workspaces} ${buildParametersContext.maxRetainedWorkspaces}`);
CloudRunnerLogger.log(`Workspace is above max ${workspaces} ${buildParametersContext.retainWorkspaces}`);
await SharedWorkspaceLocking.CleanupWorkspace(workspace, buildParametersContext);
return false;

View File

@ -23,7 +23,7 @@ describe('Cloud Runner Retain Workspace', () => {
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
retainWorkspaces: true,
retainWorkspaces: 1,
};
const buildParameter = await CreateParameters(overrides);
expect(buildParameter.projectPath).toEqual(overrides.projectPath);

View File

@ -120,7 +120,7 @@ rm ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}/setEnv.sh`;
}
private static get TreeCommand(): string {
return CloudRunnerOptions.cloudRunnerDebugTree
return CloudRunnerOptions.cloudRunnerDebug
? `tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute} && tree -L 2 ${CloudRunnerFolders.cacheFolderForCacheKeyFull} && du -h -s /${CloudRunnerFolders.buildVolumeFolder}/ && du -h -s ${CloudRunnerFolders.cacheFolderForAllFull}`
: `#`;
}