Better locking

dist/index.js

@@ -618,42 +618,80 @@ Object.defineProperty(exports, "__esModule", ({ value: true }));
 exports.SharedWorkspaceLocking = void 0;
 const cloud_runner_system_1 = __nccwpck_require__(99393);
 const fs = __importStar(__nccwpck_require__(57147));
-const cloud_runner_1 = __importDefault(__nccwpck_require__(79144));
 const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
 const cloud_runner_options_1 = __importDefault(__nccwpck_require__(96552));
 class SharedWorkspaceLocking {
-    static GetLockedWorkspace() {
+    static GetLockedWorkspace(workspaceIfCreated, runId) {
         return __awaiter(this, void 0, void 0, function* () {
             if (!cloud_runner_options_1.default.retainWorkspaces) {
                 return;
             }
-            const workspaces = SharedWorkspaceLocking.GetFreeWorkspaces();
+            const workspaces = yield SharedWorkspaceLocking.GetFreeWorkspaces();
             for (const element of workspaces) {
-                if (yield SharedWorkspaceLocking.LockWorkspace(element)) {
+                if (yield SharedWorkspaceLocking.LockWorkspace(element, runId)) {
                     return element;
                 }
             }
-            return yield SharedWorkspaceLocking.CreateLockableWorkspace(cloud_runner_1.default.buildParameters.buildGuid);
+            return yield SharedWorkspaceLocking.CreateLockableWorkspace(workspaceIfCreated);
         });
     }
     static GetFreeWorkspaces() {
-        return [];
+        return __awaiter(this, void 0, void 0, function* () {
+            const result = [];
+            const workspaces = yield SharedWorkspaceLocking.GetAllWorkspaces();
+            for (const element of workspaces) {
+                if (!(yield SharedWorkspaceLocking.IsWorkspaceLocked(element))) {
+                    result.push(element);
+                }
+            }
+            return result;
+        });
     }
     static GetAllWorkspaces() {
-        return [];
-    }
-    static LockWorkspace(workspace) {
         return __awaiter(this, void 0, void 0, function* () {
-            const file = `${Date.now()}_${cloud_runner_1.default.buildParameters.buildGuid}_lock`;
+            return (yield SharedWorkspaceLocking.ReadLines(`aws s3 ls s3://game-ci-test-storage/locks/`)).map((x) => x.replace(`/`, ``));
+        });
+    }
+    static GetAllLocks(workspace) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return (yield SharedWorkspaceLocking.ReadLines(`aws s3 ls s3://game-ci-test-storage/locks/${workspace}/`)).map((x) => x.replace(`/`, ``));
+        });
+    }
+    static ReadLines(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const result = yield cloud_runner_system_1.CloudRunnerSystem.Run(command, false, true);
+            return result
+                .split(`\n`)
+                .map((x) => x.replace(`\r`, ``))
+                .filter((x) => x !== ``)
+                .map((x) => {
+                const lineValues = x.split(` `);
+                return lineValues[lineValues.length - 1];
+            });
+        });
+    }
+    static LockWorkspace(workspace, runId) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const file = `${Date.now()}_${runId}_lock`;
             fs.writeFileSync(file, '');
-            yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 cp ./${file} s3://game-ci-test-storage/locks/${workspace}/${file}`);
+            yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 cp ./${file} s3://game-ci-test-storage/locks/${workspace}/${file}`, false, true);
             fs.rmSync(file);
             return SharedWorkspaceLocking.HasWorkspaceLock(workspace);
         });
     }
+    static ReleaseWorkspace(workspace, runId) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const file = (yield SharedWorkspaceLocking.GetAllLocks(workspace)).filter((x) => x.includes(`_${runId}_lock`));
+            cloud_runner_logger_1.default.log(`${JSON.stringify(yield SharedWorkspaceLocking.GetAllLocks(workspace))}`);
+            cloud_runner_logger_1.default.log(`Deleting file ${file}`);
+            cloud_runner_logger_1.default.log(`aws s3 rm s3://game-ci-test-storage/locks/${workspace}/${file}`);
+            yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 rm s3://game-ci-test-storage/locks/${workspace}/${file}`, false, true);
+            return !SharedWorkspaceLocking.HasWorkspaceLock(workspace);
+        });
+    }
     static HasWorkspaceLock(workspace) {
         return __awaiter(this, void 0, void 0, function* () {
-            cloud_runner_logger_1.default.log((yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 ls s3://game-ci-test-storage/locks/${workspace}/`))
+            cloud_runner_logger_1.default.log((yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 ls s3://game-ci-test-storage/locks/${workspace}/`, false, true))
                 .split('\n')
                 .map((x) => {
                 return x.split(' ');
@@ -662,14 +700,20 @@ class SharedWorkspaceLocking {
             return true;
         });
     }
-    // eslint-disable-next-line no-unused-vars
+    static IsWorkspaceLocked(workspace) {
-    static IsWorkspaceLocked(workspace) { }
+        return __awaiter(this, void 0, void 0, function* () {
+            const files = yield SharedWorkspaceLocking.ReadLines(`aws s3 ls s3://game-ci-test-storage/locks/${workspace}/`);
+            // 1 Because we expect 1 workspace file to exist in every workspace folder
+            return files.length > 1;
+        });
+    }
     static CreateLockableWorkspace(workspace) {
         return __awaiter(this, void 0, void 0, function* () {
-            const file = `${Date.now()}_${cloud_runner_1.default.buildParameters.buildGuid}_workspace`;
+            const file = `${Date.now()}_workspace`;
             fs.writeFileSync(file, '');
-            yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 cp ./${file} s3://game-ci-test-storage/locks/${workspace}/${file}`);
+            yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 cp ./${file} s3://game-ci-test-storage/locks/${workspace}/${file}`, false, true);
-            return cloud_runner_1.default.buildParameters.buildGuid;
+            fs.rmSync(file);
+            return workspace;
         });
     }
     // eslint-disable-next-line no-unused-vars
@@ -2978,6 +3022,7 @@ class Kubernetes {
                 cloud_runner_logger_1.default.log('Abandoning cleanup, build error:');
                 throw error;
             }
+            cloud_runner_logger_1.default.log('cleaning up finished');
             try {
                 yield async_wait_until_1.default(() => __awaiter(this, void 0, void 0, function* () {
                     var _b;
@@ -3005,14 +3050,18 @@ class Kubernetes {
         return __awaiter(this, void 0, void 0, function* () {
             cloud_runner_logger_1.default.log(`deleting PVC`);
             try {
-                yield this.kubeClient.deleteNamespacedPersistentVolumeClaim(this.pvcName, this.namespace);
+                const promise = this.kubeClient.deleteNamespacedPersistentVolumeClaim(this.pvcName, this.namespace);
-            }
+                // eslint-disable-next-line github/no-then
-            catch (error) {
+                promise.catch((error) => {
-                if (error.response.body.reason === `not found`) {
+                    if (error.response.body.reason === `not found`) {
-                    return;
+                        return;
-                }
+                    }
-                cloud_runner_logger_1.default.log(`Cleanup failed ${JSON.stringify(error, undefined, 4)}`);
+                    cloud_runner_logger_1.default.log(`Cleanup failed ${JSON.stringify(error, undefined, 4)}`);
+                });
+                yield promise;
+                // eslint-disable-next-line no-empty
             }
+            catch (_a) { }
         });
     }
     static findPodFromJob(kubeClient, jobName, namespace) {
@@ -4965,7 +5014,8 @@ class BuildAutomationWorkflow {
                 cloud_runner_logger_1.default.logLine(` `);
                 cloud_runner_logger_1.default.logLine('Starting build automation job');
                 // eslint-disable-next-line no-unused-vars
-                const workspace = (yield shared_workspace_locking_1.default.GetLockedWorkspace()) || cloud_runner_1.default.buildParameters.buildGuid;
+                const workspace = (yield shared_workspace_locking_1.default.GetLockedWorkspace(`test-workspace`, cloud_runner_1.default.buildParameters.buildGuid)) ||
+                    cloud_runner_1.default.buildParameters.buildGuid;
                 output += yield cloud_runner_1.default.Provider.runTask(cloud_runner_1.default.buildParameters.buildGuid, baseImage.toString(), BuildAutomationWorkflow.BuildWorkflow, `/${cloud_runner_folders_1.CloudRunnerFolders.buildVolumeFolder}`, `/${cloud_runner_folders_1.CloudRunnerFolders.buildVolumeFolder}/`, cloudRunnerStepState.environment, cloudRunnerStepState.secrets);
                 if (!cloud_runner_1.default.buildParameters.isCliMode)
                     core.endGroup();

src/model/cli/shared-workspace-locking.ts

@@ -1,41 +1,84 @@
 import { CloudRunnerSystem } from '../cloud-runner/services/cloud-runner-system';
 import * as fs from 'fs';
-import CloudRunner from '../cloud-runner/cloud-runner';
 import CloudRunnerLogger from '../cloud-runner/services/cloud-runner-logger';
 import CloudRunnerOptions from '../cloud-runner/cloud-runner-options';
 export class SharedWorkspaceLocking {
-  public static async GetLockedWorkspace() {
+  public static async GetLockedWorkspace(workspaceIfCreated: string, runId: string) {
     if (!CloudRunnerOptions.retainWorkspaces) {
       return;
     }
 
-    const workspaces = SharedWorkspaceLocking.GetFreeWorkspaces();
+    const workspaces = await SharedWorkspaceLocking.GetFreeWorkspaces();
     for (const element of workspaces) {
-      if (await SharedWorkspaceLocking.LockWorkspace(element)) {
+      if (await SharedWorkspaceLocking.LockWorkspace(element, runId)) {
         return element;
       }
     }
 
-    return await SharedWorkspaceLocking.CreateLockableWorkspace(CloudRunner.buildParameters.buildGuid);
+    return await SharedWorkspaceLocking.CreateLockableWorkspace(workspaceIfCreated);
   }
 
-  public static GetFreeWorkspaces(): string[] {
+  public static async GetFreeWorkspaces(): Promise<string[]> {
-    return [];
+    const result: string[] = [];
+    const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces();
+    for (const element of workspaces) {
+      if (!(await SharedWorkspaceLocking.IsWorkspaceLocked(element))) {
+        result.push(element);
+      }
+    }
+
+    return result;
   }
-  public static GetAllWorkspaces(): string[] {
+  public static async GetAllWorkspaces(): Promise<string[]> {
-    return [];
+    return (await SharedWorkspaceLocking.ReadLines(`aws s3 ls s3://game-ci-test-storage/locks/`)).map((x) =>
+      x.replace(`/`, ``),
+    );
   }
-  public static async LockWorkspace(workspace: string): Promise<boolean> {
+  public static async GetAllLocks(workspace: string): Promise<string[]> {
-    const file = `${Date.now()}_${CloudRunner.buildParameters.buildGuid}_lock`;
+    return (await SharedWorkspaceLocking.ReadLines(`aws s3 ls s3://game-ci-test-storage/locks/${workspace}/`)).map(
+      (x) => x.replace(`/`, ``),
+    );
+  }
+
+  private static async ReadLines(command: string): Promise<string[]> {
+    const result = await CloudRunnerSystem.Run(command, false, true);
+
+    return result
+      .split(`\n`)
+      .map((x) => x.replace(`\r`, ``))
+      .filter((x) => x !== ``)
+      .map((x) => {
+        const lineValues = x.split(` `);
+
+        return lineValues[lineValues.length - 1];
+      });
+  }
+
+  public static async LockWorkspace(workspace: string, runId: string): Promise<boolean> {
+    const file = `${Date.now()}_${runId}_lock`;
     fs.writeFileSync(file, '');
-    await CloudRunnerSystem.Run(`aws s3 cp ./${file} s3://game-ci-test-storage/locks/${workspace}/${file}`);
+    await CloudRunnerSystem.Run(
+      `aws s3 cp ./${file} s3://game-ci-test-storage/locks/${workspace}/${file}`,
+      false,
+      true,
+    );
     fs.rmSync(file);
 
     return SharedWorkspaceLocking.HasWorkspaceLock(workspace);
   }
+
+  public static async ReleaseWorkspace(workspace: string, runId: string): Promise<boolean> {
+    const file = (await SharedWorkspaceLocking.GetAllLocks(workspace)).filter((x) => x.includes(`_${runId}_lock`));
+    CloudRunnerLogger.log(`${JSON.stringify(await SharedWorkspaceLocking.GetAllLocks(workspace))}`);
+    CloudRunnerLogger.log(`Deleting file ${file}`);
+    CloudRunnerLogger.log(`aws s3 rm s3://game-ci-test-storage/locks/${workspace}/${file}`);
+    await CloudRunnerSystem.Run(`aws s3 rm s3://game-ci-test-storage/locks/${workspace}/${file}`, false, true);
+
+    return !SharedWorkspaceLocking.HasWorkspaceLock(workspace);
+  }
   public static async HasWorkspaceLock(workspace: string): Promise<boolean> {
     CloudRunnerLogger.log(
-      (await CloudRunnerSystem.Run(`aws s3 ls s3://game-ci-test-storage/locks/${workspace}/`))
+      (await CloudRunnerSystem.Run(`aws s3 ls s3://game-ci-test-storage/locks/${workspace}/`, false, true))
         .split('\n')
         .map((x) => {
           return x.split(' ');
@@ -45,15 +88,25 @@ export class SharedWorkspaceLocking {
 
     return true;
   }
-  // eslint-disable-next-line no-unused-vars
+
-  public static IsWorkspaceLocked(workspace: string) {}
+  public static async IsWorkspaceLocked(workspace: string): Promise<boolean> {
+    const files = await SharedWorkspaceLocking.ReadLines(`aws s3 ls s3://game-ci-test-storage/locks/${workspace}/`);
+
+    // 1 Because we expect 1 workspace file to exist in every workspace folder
+    return files.length > 1;
+  }
 
   public static async CreateLockableWorkspace(workspace: string) {
-    const file = `${Date.now()}_${CloudRunner.buildParameters.buildGuid}_workspace`;
+    const file = `${Date.now()}_workspace`;
     fs.writeFileSync(file, '');
-    await CloudRunnerSystem.Run(`aws s3 cp ./${file} s3://game-ci-test-storage/locks/${workspace}/${file}`);
+    await CloudRunnerSystem.Run(
+      `aws s3 cp ./${file} s3://game-ci-test-storage/locks/${workspace}/${file}`,
+      false,
+      true,
+    );
+    fs.rmSync(file);
 
-    return CloudRunner.buildParameters.buildGuid;
+    return workspace;
   }
   // eslint-disable-next-line no-unused-vars
   public static ReleaseLock(workspace: string) {}

src/model/cloud-runner/providers/k8s/index.ts

@@ -174,6 +174,7 @@ class Kubernetes implements ProviderInterface {
       CloudRunnerLogger.log('Abandoning cleanup, build error:');
       throw error;
     }
+    CloudRunnerLogger.log('cleaning up finished');
     try {
       await waitUntil(
         async () => {
@@ -204,13 +205,17 @@ class Kubernetes implements ProviderInterface {
     CloudRunnerLogger.log(`deleting PVC`);
 
     try {
-      await this.kubeClient.deleteNamespacedPersistentVolumeClaim(this.pvcName, this.namespace);
+      const promise = this.kubeClient.deleteNamespacedPersistentVolumeClaim(this.pvcName, this.namespace);
-    } catch (error: any) {
+      // eslint-disable-next-line github/no-then
-      if (error.response.body.reason === `not found`) {
+      promise.catch((error: any) => {
-        return;
+        if (error.response.body.reason === `not found`) {
-      }
+          return;
-      CloudRunnerLogger.log(`Cleanup failed ${JSON.stringify(error, undefined, 4)}`);
+        }
-    }
+        CloudRunnerLogger.log(`Cleanup failed ${JSON.stringify(error, undefined, 4)}`);
+      });
+      await promise;
+      // eslint-disable-next-line no-empty
+    } catch {}
   }
 
   static async findPodFromJob(kubeClient: CoreV1Api, jobName: string, namespace: string) {

src/model/cloud-runner/tests/shared-workspace-locking.test.ts

@@ -1,11 +1,51 @@
 import SharedWorkspaceLocking from '../../cli/shared-workspace-locking';
 import { Cli } from '../../cli/cli';
 import setups from './cloud-runner-suite.test';
+import CloudRunnerLogger from '../services/cloud-runner-logger';
+import { v4 as uuidv4 } from 'uuid';
 
 describe('Cloud Runner Locking', () => {
   setups();
+  it(`simple locking flow`, async () => {
+    Cli.options.retainWorkspaces = true;
+    const newWorkspaceName = `test-workspace-${uuidv4()}`;
+    const runId = uuidv4();
+    await SharedWorkspaceLocking.CreateLockableWorkspace(newWorkspaceName);
+    const isExpectedUnlockedBeforeLocking =
+      (await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName)) === false;
+    expect(isExpectedUnlockedBeforeLocking).toBeTruthy();
+    await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId);
+    const isExpectedLockedAfterLocking = (await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName)) === true;
+    expect(isExpectedLockedAfterLocking).toBeTruthy();
+    const locksBeforeRelease = await SharedWorkspaceLocking.GetAllLocks(newWorkspaceName);
+    CloudRunnerLogger.log(JSON.stringify(locksBeforeRelease, undefined, 4));
+    expect(locksBeforeRelease.length > 1).toBeTruthy();
+    await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId);
+    const locks = await SharedWorkspaceLocking.GetAllLocks(newWorkspaceName);
+    expect(locks.length === 1).toBeTruthy();
+    const isExpectedLockedAfterReleasing = (await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName)) === false;
+    expect(isExpectedLockedAfterReleasing).toBeTruthy();
+  }, 150000);
   it('Locking', async () => {
     Cli.options.retainWorkspaces = true;
-    await SharedWorkspaceLocking.IsWorkspaceLocked('test-workspace');
+    CloudRunnerLogger.log(`GetAllWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetAllWorkspaces())}`);
-  });
+    CloudRunnerLogger.log(`GetFreeWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetFreeWorkspaces())}`);
+    CloudRunnerLogger.log(
+      `IsWorkspaceLocked ${JSON.stringify(await SharedWorkspaceLocking.IsWorkspaceLocked('test-workspace'))}`,
+    );
+    CloudRunnerLogger.log(`GetFreeWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetFreeWorkspaces())}`);
+    CloudRunnerLogger.log(
+      `LockWorkspace ${JSON.stringify(await SharedWorkspaceLocking.LockWorkspace('test-workspace', uuidv4()))}`,
+    );
+    CloudRunnerLogger.log(
+      `CreateLockableWorkspace ${JSON.stringify(
+        await SharedWorkspaceLocking.CreateLockableWorkspace('test-workspace-2'),
+      )}`,
+    );
+    CloudRunnerLogger.log(
+      `GetLockedWorkspace ${JSON.stringify(
+        await SharedWorkspaceLocking.GetLockedWorkspace('test-workspace-2', uuidv4()),
+      )}`,
+    );
+  }, 150000);
 });

src/model/cloud-runner/workflows/build-automation-workflow.ts

@@ -42,7 +42,9 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
       CloudRunnerLogger.logLine('Starting build automation job');
 
       // eslint-disable-next-line no-unused-vars
-      const workspace = (await SharedWorkspaceLocking.GetLockedWorkspace()) || CloudRunner.buildParameters.buildGuid;
+      const workspace =
+        (await SharedWorkspaceLocking.GetLockedWorkspace(`test-workspace`, CloudRunner.buildParameters.buildGuid)) ||
+        CloudRunner.buildParameters.buildGuid;
 
       output += await CloudRunner.Provider.runTask(
         CloudRunner.buildParameters.buildGuid,