actions
/
unity-builder
mirror of https://github.com/game-ci/unity-builder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

track cloud runner storage consumption AND enable constant garbage collect flag

pull/437/head
Frostebite 2022-10-20 17:04:10 +01:00
parent 8954e68489
commit de49da5005
10 changed files with 85 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
dist/index.js vendored
View File

@ -300,6 +300,8 @@ class BuildParameters {
useSharedLargePackages: cloud_runner_options_1.default.useSharedLargePackages,
useLz4Compression: cloud_runner_options_1.default.useLz4Compression,
maxRetainedWorkspaces: cloud_runner_options_1.default.maxRetainedWorkspaces,
constantGarbageCollection: cloud_runner_options_1.default.constantGarbageCollection,
garbageCollectionMaxAge: cloud_runner_options_1.default.garbageCollectionMaxAge,
};
});
}
@ -614,12 +616,18 @@ class Cli {
static PostCLIBuild() {
return __awaiter(this, void 0, void 0, function* () {
core.info(`Running POST build tasks`);
yield caching_1.Caching.PushToCache(cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(`${cloud_runner_folders_1.CloudRunnerFolders.cacheFolderFull}/Library`), cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.libraryFolderAbsolute), `lib-${__1.CloudRunner.buildParameters.buildGuid}`);
yield caching_1.Caching.PushToCache(cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(`${cloud_runner_folders_1.CloudRunnerFolders.cacheFolderFull}/build`), cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.projectBuildFolderAbsolute), `build-${__1.CloudRunner.buildParameters.buildGuid}`);
yield caching_1.Caching.PushToCache(cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(`${cloud_runner_folders_1.CloudRunnerFolders.cacheFolderForCacheKeyFull}/Library`), cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.libraryFolderAbsolute), `lib-${__1.CloudRunner.buildParameters.buildGuid}`);
yield caching_1.Caching.PushToCache(cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(`${cloud_runner_folders_1.CloudRunnerFolders.cacheFolderForCacheKeyFull}/build`), cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.projectBuildFolderAbsolute), `build-${__1.CloudRunner.buildParameters.buildGuid}`);
if (!__1.CloudRunner.buildParameters.retainWorkspace) {
yield cloud_runner_system_1.CloudRunnerSystem.Run(`rm -r ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`);
}
yield remote_client_1.RemoteClient.runCustomHookFiles(`after-build`);
const parameters = yield __1.BuildParameters.create();
__1.CloudRunner.setup(parameters);
if (parameters.constantGarbageCollection) {
yield cloud_runner_system_1.CloudRunnerSystem.Run(`find ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.buildVolumeFolder)} -name '*.*' -mmin +${parameters.garbageCollectionMaxAge * 60} -delete`);
yield cloud_runner_system_1.CloudRunnerSystem.Run(`find ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.cacheFolderForAllFull)} -name '*.*' -mmin +${parameters.garbageCollectionMaxAge * 60} -delete`);
}
return new Promise((result) => result(``));
});
}
@ -865,6 +873,15 @@ class CloudRunnerOptions {
static get maxRetainedWorkspaces() {
return Number(CloudRunnerOptions.getInput(`maxRetainedWorkspaces`)) || 3;
}
// ### ### ###
// Garbage Collection
// ### ### ###
static get constantGarbageCollection() {
return CloudRunnerOptions.getInput(`constantGarbageCollection`) || true;
}
static get garbageCollectionMaxAge() {
return Number(CloudRunnerOptions.getInput(`garbageCollectionMaxAge`)) || 24;
}
}
exports["default"] = CloudRunnerOptions;
@ -1033,6 +1050,9 @@ class CloudRunner {
yield shared_workspace_locking_1.default.ReleaseWorkspace(CloudRunner.lockedWorkspace || ``, CloudRunner.buildParameters.buildGuid, CloudRunner.buildParameters);
CloudRunner.lockedWorkspace = undefined;
}
if (buildParameters.constantGarbageCollection) {
CloudRunner.Provider.garbageCollect(``, true, buildParameters.garbageCollectionMaxAge, true, true);
}
return output;
}
catch (error) {
@ -2758,6 +2778,7 @@ const docker_1 = __importDefault(__nccwpck_require__(16934));
const __1 = __nccwpck_require__(41359);
const fs_1 = __nccwpck_require__(57147);
const cloud_runner_1 = __importDefault(__nccwpck_require__(79144));
const cloud_runner_system_1 = __nccwpck_require__(99393);
class LocalDockerCloudRunner {
listResources() {
throw new Error('Method not implemented.');
@ -2845,6 +2866,8 @@ cp -a ${sharedFolder}. /github/workspace/cloud-runner-cache/
if (cloud_runner_1.default.buildParameters.cloudRunnerDebug) {
cloud_runner_logger_1.default.log(`Running local-docker: \n ${fileContents}`);
}
yield cloud_runner_system_1.CloudRunnerSystem.Run(`ls /github/workspace/`);
yield cloud_runner_system_1.CloudRunnerSystem.Run(`ls /github/workspace/cloud-runner-cache`);
yield docker_1.default.run(image, Object.assign({ workspace, actionFolder }, this.buildParameters), false, `chmod +x /github/workspace/${entrypointFilePath} && /github/workspace/${entrypointFilePath}`, content, {
listeners: {
stdout: (data) => {
@ -4028,9 +4051,8 @@ class RemoteClient {
static bootstrapRepository() {
return __awaiter(this, void 0, void 0, function* () {
try {
yield cloud_runner_system_1.CloudRunnerSystem.Run(`mkdir -p ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`);
yield cloud_runner_system_1.CloudRunnerSystem.Run(`mkdir -p ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.repoPathAbsolute)}`);
yield cloud_runner_system_1.CloudRunnerSystem.Run(`mkdir -p ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.cacheFolderFull)}`);
yield cloud_runner_system_1.CloudRunnerSystem.Run(`mkdir -p ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.cacheFolderForCacheKeyFull)}`);
process.chdir(cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.repoPathAbsolute));
yield RemoteClient.cloneRepoWithoutLFSFiles();
RemoteClient.replaceLargePackageReferencesWithSharedReferences();
@ -4584,7 +4606,10 @@ class CloudRunnerFolders {
? path_1.default.join(`/`, CloudRunnerFolders.buildVolumeFolder, cloud_runner_1.default.lockedWorkspace)
: path_1.default.join(`/`, CloudRunnerFolders.buildVolumeFolder, cloud_runner_1.default.buildParameters.buildGuid);
}
static get cacheFolderFull() {
static get cacheFolderForAllFull() {
return path_1.default.join('/', CloudRunnerFolders.buildVolumeFolder, CloudRunnerFolders.cacheFolder);
}
static get cacheFolderForCacheKeyFull() {
return path_1.default.join('/', CloudRunnerFolders.buildVolumeFolder, CloudRunnerFolders.cacheFolder, cloud_runner_1.default.buildParameters.cacheKey);
}
static get builderPathAbsolute() {
@ -4609,10 +4634,10 @@ class CloudRunnerFolders {
return process.env.PURGE_REMOTE_BUILDER_CACHE !== undefined;
}
static get lfsCacheFolderFull() {
return path_1.default.join(CloudRunnerFolders.cacheFolderFull, `lfs`);
return path_1.default.join(CloudRunnerFolders.cacheFolderForCacheKeyFull, `lfs`);
}
static get libraryCacheFolderFull() {
return path_1.default.join(CloudRunnerFolders.cacheFolderFull, `Library`);
return path_1.default.join(CloudRunnerFolders.cacheFolderForCacheKeyFull, `Library`);
}
static get unityBuilderRepoUrl() {
return `https://${cloud_runner_1.default.buildParameters.gitPrivateToken}@github.com/game-ci/unity-builder.git`;
@ -5607,7 +5632,7 @@ class BuildAutomationWorkflow {
}
static get TreeCommand() {
return cloud_runner_options_1.default.cloudRunnerDebugTree
? `tree -L 2 ${cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute} && tree -L 2 ${cloud_runner_folders_1.CloudRunnerFolders.cacheFolderFull}`
? `tree -L 2 ${cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute} && tree -L 2 ${cloud_runner_folders_1.CloudRunnerFolders.cacheFolderForCacheKeyFull} && du -h -s ${cloud_runner_folders_1.CloudRunnerFolders.buildVolumeFolder} && du -h -s ${cloud_runner_folders_1.CloudRunnerFolders.cacheFolderForCacheKeyFull}`
: `#`;
}
}

2
dist/index.js.map vendored
View File

File diff suppressed because one or more lines are too long

4
src/model/build-parameters.ts
View File

@ -67,6 +67,8 @@ class BuildParameters {
public maxRetainedWorkspaces!: number;
public useSharedLargePackages!: boolean;
public useLz4Compression!: boolean;
public garbageCollectionMaxAge!: number;
public constantGarbageCollection!: boolean;
static async create(): Promise<BuildParameters> {
const buildFile = this.parseBuildFile(Input.buildName, Input.targetPlatform, Input.androidAppBundle);
@ -145,6 +147,8 @@ class BuildParameters {
useSharedLargePackages: CloudRunnerOptions.useSharedLargePackages,
useLz4Compression: CloudRunnerOptions.useLz4Compression,
maxRetainedWorkspaces: CloudRunnerOptions.maxRetainedWorkspaces,
constantGarbageCollection: CloudRunnerOptions.constantGarbageCollection,
garbageCollectionMaxAge: CloudRunnerOptions.garbageCollectionMaxAge,
};
}

19
src/model/cli/cli.ts
View File

@ -168,13 +168,13 @@ export class Cli {
core.info(`Running POST build tasks`);
await Caching.PushToCache(
CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderFull}/Library`),
CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderForCacheKeyFull}/Library`),
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute),
`lib-${CloudRunner.buildParameters.buildGuid}`,
);
await Caching.PushToCache(
CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderFull}/build`),
CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderForCacheKeyFull}/build`),
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.projectBuildFolderAbsolute),
`build-${CloudRunner.buildParameters.buildGuid}`,
);
@ -187,6 +187,21 @@ export class Cli {
await RemoteClient.runCustomHookFiles(`after-build`);
const parameters = await BuildParameters.create();
CloudRunner.setup(parameters);
if (parameters.constantGarbageCollection) {
await CloudRunnerSystem.Run(
`find ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.buildVolumeFolder)} -name '*.*' -mmin +${
parameters.garbageCollectionMaxAge * 60
} -delete`,
);
await CloudRunnerSystem.Run(
`find ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderForAllFull)} -name '*.*' -mmin +${
parameters.garbageCollectionMaxAge * 60
} -delete`,
);
}
return new Promise((result) => result(``));
}
}

12
src/model/cloud-runner/cloud-runner-options.ts
View File

@ -244,6 +244,18 @@ class CloudRunnerOptions {
static get maxRetainedWorkspaces(): number {
return Number(CloudRunnerOptions.getInput(`maxRetainedWorkspaces`)) || 3;
}
// ### ### ###
// Garbage Collection
// ### ### ###
static get constantGarbageCollection(): boolean {
return CloudRunnerOptions.getInput(`constantGarbageCollection`) || true;
}
static get garbageCollectionMaxAge(): number {
return Number(CloudRunnerOptions.getInput(`garbageCollectionMaxAge`)) || 24;
}
}
export default CloudRunnerOptions;

4
src/model/cloud-runner/cloud-runner.ts
View File

@ -118,6 +118,10 @@ class CloudRunner {
CloudRunner.lockedWorkspace = undefined;
}
if (buildParameters.constantGarbageCollection) {
CloudRunner.Provider.garbageCollect(``, true, buildParameters.garbageCollectionMaxAge, true, true);
}
return output;
} catch (error) {
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();

4
src/model/cloud-runner/providers/docker/index.ts
View File

@ -9,6 +9,7 @@ import { writeFileSync } from 'fs';
import CloudRunner from '../../cloud-runner';
import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow';
import { CloudRunnerSystem } from '../../services/cloud-runner-system';
class LocalDockerCloudRunner implements ProviderInterface {
public buildParameters: BuildParameters | undefined;
@ -117,7 +118,8 @@ cp -a ${sharedFolder}. /github/workspace/cloud-runner-cache/
if (CloudRunner.buildParameters.cloudRunnerDebug) {
CloudRunnerLogger.log(`Running local-docker: \n ${fileContents}`);
}
await CloudRunnerSystem.Run(`ls /github/workspace/`);
await CloudRunnerSystem.Run(`ls /github/workspace/cloud-runner-cache`);
await Docker.run(
image,
{ workspace, actionFolder, ...this.buildParameters },

7
src/model/cloud-runner/remote-client/index.ts
View File

@ -14,11 +14,10 @@ import YAML from 'yaml';
export class RemoteClient {
public static async bootstrapRepository() {
try {
await CloudRunnerSystem.Run(
`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
);
await CloudRunnerSystem.Run(`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute)}`);
await CloudRunnerSystem.Run(`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderFull)}`);
await CloudRunnerSystem.Run(
`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderForCacheKeyFull)}`,
);
process.chdir(CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute));
await RemoteClient.cloneRepoWithoutLFSFiles();
RemoteClient.replaceLargePackageReferencesWithSharedReferences();

10
src/model/cloud-runner/services/cloud-runner-folders.ts
View File

@ -16,7 +16,11 @@ export class CloudRunnerFolders {
: path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.buildParameters.buildGuid);
}
public static get cacheFolderFull(): string {
public static get cacheFolderForAllFull(): string {
return path.join('/', CloudRunnerFolders.buildVolumeFolder, CloudRunnerFolders.cacheFolder);
}
public static get cacheFolderForCacheKeyFull(): string {
return path.join(
'/',
CloudRunnerFolders.buildVolumeFolder,
@ -54,11 +58,11 @@ export class CloudRunnerFolders {
}
public static get lfsCacheFolderFull() {
return path.join(CloudRunnerFolders.cacheFolderFull, `lfs`);
return path.join(CloudRunnerFolders.cacheFolderForCacheKeyFull, `lfs`);
}
public static get libraryCacheFolderFull() {
return path.join(CloudRunnerFolders.cacheFolderFull, `Library`);
return path.join(CloudRunnerFolders.cacheFolderForCacheKeyFull, `Library`);
}
public static get unityBuilderRepoUrl(): string {

2
src/model/cloud-runner/workflows/build-automation-workflow.ts
View File

@ -131,7 +131,7 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
private static get TreeCommand(): string {
return CloudRunnerOptions.cloudRunnerDebugTree
? `tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute} && tree -L 2 ${CloudRunnerFolders.cacheFolderFull}`
? `tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute} && tree -L 2 ${CloudRunnerFolders.cacheFolderForCacheKeyFull} && du -h -s ${CloudRunnerFolders.buildVolumeFolder} && du -h -s ${CloudRunnerFolders.cacheFolderForCacheKeyFull}`
: `#`;
}
}