track cloud runner storage consumption AND enable constant garbage collect flag

dist/index.js

@@ -300,6 +300,8 @@ class BuildParameters {
                 useSharedLargePackages: cloud_runner_options_1.default.useSharedLargePackages,
                 useLz4Compression: cloud_runner_options_1.default.useLz4Compression,
                 maxRetainedWorkspaces: cloud_runner_options_1.default.maxRetainedWorkspaces,
+                constantGarbageCollection: cloud_runner_options_1.default.constantGarbageCollection,
+                garbageCollectionMaxAge: cloud_runner_options_1.default.garbageCollectionMaxAge,
             };
         });
     }
@@ -614,12 +616,18 @@ class Cli {
     static PostCLIBuild() {
         return __awaiter(this, void 0, void 0, function* () {
             core.info(`Running POST build tasks`);
-            yield caching_1.Caching.PushToCache(cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(`${cloud_runner_folders_1.CloudRunnerFolders.cacheFolderFull}/Library`), cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.libraryFolderAbsolute), `lib-${__1.CloudRunner.buildParameters.buildGuid}`);
+            yield caching_1.Caching.PushToCache(cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(`${cloud_runner_folders_1.CloudRunnerFolders.cacheFolderForCacheKeyFull}/Library`), cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.libraryFolderAbsolute), `lib-${__1.CloudRunner.buildParameters.buildGuid}`);
-            yield caching_1.Caching.PushToCache(cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(`${cloud_runner_folders_1.CloudRunnerFolders.cacheFolderFull}/build`), cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.projectBuildFolderAbsolute), `build-${__1.CloudRunner.buildParameters.buildGuid}`);
+            yield caching_1.Caching.PushToCache(cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(`${cloud_runner_folders_1.CloudRunnerFolders.cacheFolderForCacheKeyFull}/build`), cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.projectBuildFolderAbsolute), `build-${__1.CloudRunner.buildParameters.buildGuid}`);
             if (!__1.CloudRunner.buildParameters.retainWorkspace) {
                 yield cloud_runner_system_1.CloudRunnerSystem.Run(`rm -r ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`);
             }
             yield remote_client_1.RemoteClient.runCustomHookFiles(`after-build`);
+            const parameters = yield __1.BuildParameters.create();
+            __1.CloudRunner.setup(parameters);
+            if (parameters.constantGarbageCollection) {
+                yield cloud_runner_system_1.CloudRunnerSystem.Run(`find ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.buildVolumeFolder)} -name '*.*' -mmin +${parameters.garbageCollectionMaxAge * 60} -delete`);
+                yield cloud_runner_system_1.CloudRunnerSystem.Run(`find ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.cacheFolderForAllFull)} -name '*.*' -mmin +${parameters.garbageCollectionMaxAge * 60} -delete`);
+            }
             return new Promise((result) => result(``));
         });
     }
@@ -865,6 +873,15 @@ class CloudRunnerOptions {
     static get maxRetainedWorkspaces() {
         return Number(CloudRunnerOptions.getInput(`maxRetainedWorkspaces`)) || 3;
     }
+    // ### ### ###
+    // Garbage Collection
+    // ### ### ###
+    static get constantGarbageCollection() {
+        return CloudRunnerOptions.getInput(`constantGarbageCollection`) || true;
+    }
+    static get garbageCollectionMaxAge() {
+        return Number(CloudRunnerOptions.getInput(`garbageCollectionMaxAge`)) || 24;
+    }
 }
 exports["default"] = CloudRunnerOptions;
 
@@ -1033,6 +1050,9 @@ class CloudRunner {
                     yield shared_workspace_locking_1.default.ReleaseWorkspace(CloudRunner.lockedWorkspace || ``, CloudRunner.buildParameters.buildGuid, CloudRunner.buildParameters);
                     CloudRunner.lockedWorkspace = undefined;
                 }
+                if (buildParameters.constantGarbageCollection) {
+                    CloudRunner.Provider.garbageCollect(``, true, buildParameters.garbageCollectionMaxAge, true, true);
+                }
                 return output;
             }
             catch (error) {
@@ -2758,6 +2778,7 @@ const docker_1 = __importDefault(__nccwpck_require__(16934));
 const __1 = __nccwpck_require__(41359);
 const fs_1 = __nccwpck_require__(57147);
 const cloud_runner_1 = __importDefault(__nccwpck_require__(79144));
+const cloud_runner_system_1 = __nccwpck_require__(99393);
 class LocalDockerCloudRunner {
     listResources() {
         throw new Error('Method not implemented.');
@@ -2845,6 +2866,8 @@ cp -a ${sharedFolder}. /github/workspace/cloud-runner-cache/
             if (cloud_runner_1.default.buildParameters.cloudRunnerDebug) {
                 cloud_runner_logger_1.default.log(`Running local-docker: \n ${fileContents}`);
             }
+            yield cloud_runner_system_1.CloudRunnerSystem.Run(`ls /github/workspace/`);
+            yield cloud_runner_system_1.CloudRunnerSystem.Run(`ls /github/workspace/cloud-runner-cache`);
             yield docker_1.default.run(image, Object.assign({ workspace, actionFolder }, this.buildParameters), false, `chmod +x /github/workspace/${entrypointFilePath} && /github/workspace/${entrypointFilePath}`, content, {
                 listeners: {
                     stdout: (data) => {
@@ -4028,9 +4051,8 @@ class RemoteClient {
     static bootstrapRepository() {
         return __awaiter(this, void 0, void 0, function* () {
             try {
-                yield cloud_runner_system_1.CloudRunnerSystem.Run(`mkdir -p ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`);
                 yield cloud_runner_system_1.CloudRunnerSystem.Run(`mkdir -p ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.repoPathAbsolute)}`);
-                yield cloud_runner_system_1.CloudRunnerSystem.Run(`mkdir -p ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.cacheFolderFull)}`);
+                yield cloud_runner_system_1.CloudRunnerSystem.Run(`mkdir -p ${cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.cacheFolderForCacheKeyFull)}`);
                 process.chdir(cloud_runner_folders_1.CloudRunnerFolders.ToLinuxFolder(cloud_runner_folders_1.CloudRunnerFolders.repoPathAbsolute));
                 yield RemoteClient.cloneRepoWithoutLFSFiles();
                 RemoteClient.replaceLargePackageReferencesWithSharedReferences();
@@ -4584,7 +4606,10 @@ class CloudRunnerFolders {
             ? path_1.default.join(`/`, CloudRunnerFolders.buildVolumeFolder, cloud_runner_1.default.lockedWorkspace)
             : path_1.default.join(`/`, CloudRunnerFolders.buildVolumeFolder, cloud_runner_1.default.buildParameters.buildGuid);
     }
-    static get cacheFolderFull() {
+    static get cacheFolderForAllFull() {
+        return path_1.default.join('/', CloudRunnerFolders.buildVolumeFolder, CloudRunnerFolders.cacheFolder);
+    }
+    static get cacheFolderForCacheKeyFull() {
         return path_1.default.join('/', CloudRunnerFolders.buildVolumeFolder, CloudRunnerFolders.cacheFolder, cloud_runner_1.default.buildParameters.cacheKey);
     }
     static get builderPathAbsolute() {
@@ -4609,10 +4634,10 @@ class CloudRunnerFolders {
         return process.env.PURGE_REMOTE_BUILDER_CACHE !== undefined;
     }
     static get lfsCacheFolderFull() {
-        return path_1.default.join(CloudRunnerFolders.cacheFolderFull, `lfs`);
+        return path_1.default.join(CloudRunnerFolders.cacheFolderForCacheKeyFull, `lfs`);
     }
     static get libraryCacheFolderFull() {
-        return path_1.default.join(CloudRunnerFolders.cacheFolderFull, `Library`);
+        return path_1.default.join(CloudRunnerFolders.cacheFolderForCacheKeyFull, `Library`);
     }
     static get unityBuilderRepoUrl() {
         return `https://${cloud_runner_1.default.buildParameters.gitPrivateToken}@github.com/game-ci/unity-builder.git`;
@@ -5607,7 +5632,7 @@ class BuildAutomationWorkflow {
     }
     static get TreeCommand() {
         return cloud_runner_options_1.default.cloudRunnerDebugTree
-            ? `tree -L 2 ${cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute} && tree -L 2 ${cloud_runner_folders_1.CloudRunnerFolders.cacheFolderFull}`
+            ? `tree -L 2 ${cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute} && tree -L 2 ${cloud_runner_folders_1.CloudRunnerFolders.cacheFolderForCacheKeyFull} && du -h -s ${cloud_runner_folders_1.CloudRunnerFolders.buildVolumeFolder} && du -h -s ${cloud_runner_folders_1.CloudRunnerFolders.cacheFolderForCacheKeyFull}`
             : `#`;
     }
 }

src/model/build-parameters.ts

@@ -67,6 +67,8 @@ class BuildParameters {
   public maxRetainedWorkspaces!: number;
   public useSharedLargePackages!: boolean;
   public useLz4Compression!: boolean;
+  public garbageCollectionMaxAge!: number;
+  public constantGarbageCollection!: boolean;
 
   static async create(): Promise<BuildParameters> {
     const buildFile = this.parseBuildFile(Input.buildName, Input.targetPlatform, Input.androidAppBundle);
@@ -145,6 +147,8 @@ class BuildParameters {
       useSharedLargePackages: CloudRunnerOptions.useSharedLargePackages,
       useLz4Compression: CloudRunnerOptions.useLz4Compression,
       maxRetainedWorkspaces: CloudRunnerOptions.maxRetainedWorkspaces,
+      constantGarbageCollection: CloudRunnerOptions.constantGarbageCollection,
+      garbageCollectionMaxAge: CloudRunnerOptions.garbageCollectionMaxAge,
     };
   }
 

src/model/cli/cli.ts

@@ -168,13 +168,13 @@ export class Cli {
     core.info(`Running POST build tasks`);
 
     await Caching.PushToCache(
-      CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderFull}/Library`),
+      CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderForCacheKeyFull}/Library`),
       CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute),
       `lib-${CloudRunner.buildParameters.buildGuid}`,
     );
 
     await Caching.PushToCache(
-      CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderFull}/build`),
+      CloudRunnerFolders.ToLinuxFolder(`${CloudRunnerFolders.cacheFolderForCacheKeyFull}/build`),
       CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.projectBuildFolderAbsolute),
       `build-${CloudRunner.buildParameters.buildGuid}`,
     );
@@ -187,6 +187,21 @@ export class Cli {
 
     await RemoteClient.runCustomHookFiles(`after-build`);
 
+    const parameters = await BuildParameters.create();
+    CloudRunner.setup(parameters);
+    if (parameters.constantGarbageCollection) {
+      await CloudRunnerSystem.Run(
+        `find ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.buildVolumeFolder)} -name '*.*' -mmin +${
+          parameters.garbageCollectionMaxAge * 60
+        } -delete`,
+      );
+      await CloudRunnerSystem.Run(
+        `find ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderForAllFull)} -name '*.*' -mmin +${
+          parameters.garbageCollectionMaxAge * 60
+        } -delete`,
+      );
+    }
+
     return new Promise((result) => result(``));
   }
 }

src/model/cloud-runner/cloud-runner-options.ts

@@ -244,6 +244,18 @@ class CloudRunnerOptions {
   static get maxRetainedWorkspaces(): number {
     return Number(CloudRunnerOptions.getInput(`maxRetainedWorkspaces`)) || 3;
   }
+
+  // ### ### ###
+  // Garbage Collection
+  // ### ### ###
+
+  static get constantGarbageCollection(): boolean {
+    return CloudRunnerOptions.getInput(`constantGarbageCollection`) || true;
+  }
+
+  static get garbageCollectionMaxAge(): number {
+    return Number(CloudRunnerOptions.getInput(`garbageCollectionMaxAge`)) || 24;
+  }
 }
 
 export default CloudRunnerOptions;

src/model/cloud-runner/cloud-runner.ts

@@ -118,6 +118,10 @@ class CloudRunner {
         CloudRunner.lockedWorkspace = undefined;
       }
 
+      if (buildParameters.constantGarbageCollection) {
+        CloudRunner.Provider.garbageCollect(``, true, buildParameters.garbageCollectionMaxAge, true, true);
+      }
+
       return output;
     } catch (error) {
       if (!CloudRunner.buildParameters.isCliMode) core.endGroup();

src/model/cloud-runner/providers/docker/index.ts

@@ -9,6 +9,7 @@ import { writeFileSync } from 'fs';
 import CloudRunner from '../../cloud-runner';
 import { ProviderResource } from '../provider-resource';
 import { ProviderWorkflow } from '../provider-workflow';
+import { CloudRunnerSystem } from '../../services/cloud-runner-system';
 
 class LocalDockerCloudRunner implements ProviderInterface {
   public buildParameters: BuildParameters | undefined;
@@ -117,7 +118,8 @@ cp -a ${sharedFolder}. /github/workspace/cloud-runner-cache/
     if (CloudRunner.buildParameters.cloudRunnerDebug) {
       CloudRunnerLogger.log(`Running local-docker: \n ${fileContents}`);
     }
-
+    await CloudRunnerSystem.Run(`ls /github/workspace/`);
+    await CloudRunnerSystem.Run(`ls /github/workspace/cloud-runner-cache`);
     await Docker.run(
       image,
       { workspace, actionFolder, ...this.buildParameters },

src/model/cloud-runner/remote-client/index.ts

@@ -14,11 +14,10 @@ import YAML from 'yaml';
 export class RemoteClient {
   public static async bootstrapRepository() {
     try {
-      await CloudRunnerSystem.Run(
-        `mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
-      );
       await CloudRunnerSystem.Run(`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute)}`);
-      await CloudRunnerSystem.Run(`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderFull)}`);
+      await CloudRunnerSystem.Run(
+        `mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderForCacheKeyFull)}`,
+      );
       process.chdir(CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute));
       await RemoteClient.cloneRepoWithoutLFSFiles();
       RemoteClient.replaceLargePackageReferencesWithSharedReferences();

src/model/cloud-runner/services/cloud-runner-folders.ts

@@ -16,7 +16,11 @@ export class CloudRunnerFolders {
       : path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.buildParameters.buildGuid);
   }
 
-  public static get cacheFolderFull(): string {
+  public static get cacheFolderForAllFull(): string {
+    return path.join('/', CloudRunnerFolders.buildVolumeFolder, CloudRunnerFolders.cacheFolder);
+  }
+
+  public static get cacheFolderForCacheKeyFull(): string {
     return path.join(
       '/',
       CloudRunnerFolders.buildVolumeFolder,
@@ -54,11 +58,11 @@ export class CloudRunnerFolders {
   }
 
   public static get lfsCacheFolderFull() {
-    return path.join(CloudRunnerFolders.cacheFolderFull, `lfs`);
+    return path.join(CloudRunnerFolders.cacheFolderForCacheKeyFull, `lfs`);
   }
 
   public static get libraryCacheFolderFull() {
-    return path.join(CloudRunnerFolders.cacheFolderFull, `Library`);
+    return path.join(CloudRunnerFolders.cacheFolderForCacheKeyFull, `Library`);
   }
 
   public static get unityBuilderRepoUrl(): string {

src/model/cloud-runner/workflows/build-automation-workflow.ts

@@ -131,7 +131,7 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
 
   private static get TreeCommand(): string {
     return CloudRunnerOptions.cloudRunnerDebugTree
-      ? `tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute} && tree -L 2 ${CloudRunnerFolders.cacheFolderFull}`
+      ? `tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute} && tree -L 2 ${CloudRunnerFolders.cacheFolderForCacheKeyFull} && du -h -s ${CloudRunnerFolders.buildVolumeFolder} && du -h -s ${CloudRunnerFolders.cacheFolderForCacheKeyFull}`
       : `#`;
   }
 }