actions
/
unity-builder
mirror of https://github.com/game-ci/unity-builder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Better locking

pull/461/head
Frostebite 2022-09-29 21:30:34 +01:00
parent 026f8f20a4
commit 3a5400080b
7 changed files with 204 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
1664483151394_727700cd-2e8a-4836-8191-a7e15cbd29ea_lock 100644
View File

100
dist/index.js vendored
View File

@ -618,42 +618,80 @@ Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.SharedWorkspaceLocking = void 0;
const cloud_runner_system_1 = __nccwpck_require__(99393);
const fs = __importStar(__nccwpck_require__(57147));
const cloud_runner_1 = __importDefault(__nccwpck_require__(79144));
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
const cloud_runner_options_1 = __importDefault(__nccwpck_require__(96552));
class SharedWorkspaceLocking {
static GetLockedWorkspace() {
static GetLockedWorkspace(workspaceIfCreated, runId) {
return __awaiter(this, void 0, void 0, function* () {
if (!cloud_runner_options_1.default.retainWorkspaces) {
return;
}
const workspaces = SharedWorkspaceLocking.GetFreeWorkspaces();
const workspaces = yield SharedWorkspaceLocking.GetFreeWorkspaces();
for (const element of workspaces) {
if (yield SharedWorkspaceLocking.LockWorkspace(element)) {
if (yield SharedWorkspaceLocking.LockWorkspace(element, runId)) {
return element;
}
}
return yield SharedWorkspaceLocking.CreateLockableWorkspace(cloud_runner_1.default.buildParameters.buildGuid);
return yield SharedWorkspaceLocking.CreateLockableWorkspace(workspaceIfCreated);
});
}
static GetFreeWorkspaces() {
return [];
return __awaiter(this, void 0, void 0, function* () {
const result = [];
const workspaces = yield SharedWorkspaceLocking.GetAllWorkspaces();
for (const element of workspaces) {
if (!(yield SharedWorkspaceLocking.IsWorkspaceLocked(element))) {
result.push(element);
}
}
return result;
});
}
static GetAllWorkspaces() {
return [];
}
static LockWorkspace(workspace) {
return __awaiter(this, void 0, void 0, function* () {
const file = `${Date.now()}_${cloud_runner_1.default.buildParameters.buildGuid}_lock`;
return (yield SharedWorkspaceLocking.ReadLines(`aws s3 ls s3://game-ci-test-storage/locks/`)).map((x) => x.replace(`/`, ``));
});
}
static GetAllLocks(workspace) {
return __awaiter(this, void 0, void 0, function* () {
return (yield SharedWorkspaceLocking.ReadLines(`aws s3 ls s3://game-ci-test-storage/locks/${workspace}/`)).map((x) => x.replace(`/`, ``));
});
}
static ReadLines(command) {
return __awaiter(this, void 0, void 0, function* () {
const result = yield cloud_runner_system_1.CloudRunnerSystem.Run(command, false, true);
return result
.split(`\n`)
.map((x) => x.replace(`\r`, ``))
.filter((x) => x !== ``)
.map((x) => {
const lineValues = x.split(` `);
return lineValues[lineValues.length - 1];
});
});
}
static LockWorkspace(workspace, runId) {
return __awaiter(this, void 0, void 0, function* () {
const file = `${Date.now()}_${runId}_lock`;
fs.writeFileSync(file, '');
yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 cp ./${file} s3://game-ci-test-storage/locks/${workspace}/${file}`);
yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 cp ./${file} s3://game-ci-test-storage/locks/${workspace}/${file}`, false, true);
fs.rmSync(file);
return SharedWorkspaceLocking.HasWorkspaceLock(workspace);
});
}
static ReleaseWorkspace(workspace, runId) {
return __awaiter(this, void 0, void 0, function* () {
const file = (yield SharedWorkspaceLocking.GetAllLocks(workspace)).filter((x) => x.includes(`_${runId}_lock`));
cloud_runner_logger_1.default.log(`${JSON.stringify(yield SharedWorkspaceLocking.GetAllLocks(workspace))}`);
cloud_runner_logger_1.default.log(`Deleting file ${file}`);
cloud_runner_logger_1.default.log(`aws s3 rm s3://game-ci-test-storage/locks/${workspace}/${file}`);
yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 rm s3://game-ci-test-storage/locks/${workspace}/${file}`, false, true);
return !SharedWorkspaceLocking.HasWorkspaceLock(workspace);
});
}
static HasWorkspaceLock(workspace) {
return __awaiter(this, void 0, void 0, function* () {
cloud_runner_logger_1.default.log((yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 ls s3://game-ci-test-storage/locks/${workspace}/`))
cloud_runner_logger_1.default.log((yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 ls s3://game-ci-test-storage/locks/${workspace}/`, false, true))
.split('\n')
.map((x) => {
return x.split(' ');
@ -662,14 +700,20 @@ class SharedWorkspaceLocking {
return true;
});
}
// eslint-disable-next-line no-unused-vars
static IsWorkspaceLocked(workspace) { }
static IsWorkspaceLocked(workspace) {
return __awaiter(this, void 0, void 0, function* () {
const files = yield SharedWorkspaceLocking.ReadLines(`aws s3 ls s3://game-ci-test-storage/locks/${workspace}/`);
// 1 Because we expect 1 workspace file to exist in every workspace folder
return files.length > 1;
});
}
static CreateLockableWorkspace(workspace) {
return __awaiter(this, void 0, void 0, function* () {
const file = `${Date.now()}_${cloud_runner_1.default.buildParameters.buildGuid}_workspace`;
const file = `${Date.now()}_workspace`;
fs.writeFileSync(file, '');
yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 cp ./${file} s3://game-ci-test-storage/locks/${workspace}/${file}`);
return cloud_runner_1.default.buildParameters.buildGuid;
yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 cp ./${file} s3://game-ci-test-storage/locks/${workspace}/${file}`, false, true);
fs.rmSync(file);
return workspace;
});
}
// eslint-disable-next-line no-unused-vars
@ -2978,6 +3022,7 @@ class Kubernetes {
cloud_runner_logger_1.default.log('Abandoning cleanup, build error:');
throw error;
}
cloud_runner_logger_1.default.log('cleaning up finished');
try {
yield async_wait_until_1.default(() => __awaiter(this, void 0, void 0, function* () {
var _b;
@ -3005,14 +3050,18 @@ class Kubernetes {
return __awaiter(this, void 0, void 0, function* () {
cloud_runner_logger_1.default.log(`deleting PVC`);
try {
yield this.kubeClient.deleteNamespacedPersistentVolumeClaim(this.pvcName, this.namespace);
}
catch (error) {
if (error.response.body.reason === `not found`) {
return;
}
cloud_runner_logger_1.default.log(`Cleanup failed ${JSON.stringify(error, undefined, 4)}`);
const promise = this.kubeClient.deleteNamespacedPersistentVolumeClaim(this.pvcName, this.namespace);
// eslint-disable-next-line github/no-then
promise.catch((error) => {
if (error.response.body.reason === `not found`) {
return;
}
cloud_runner_logger_1.default.log(`Cleanup failed ${JSON.stringify(error, undefined, 4)}`);
});
yield promise;
// eslint-disable-next-line no-empty
}
catch (_a) { }
});
}
static findPodFromJob(kubeClient, jobName, namespace) {
@ -4965,7 +5014,8 @@ class BuildAutomationWorkflow {
cloud_runner_logger_1.default.logLine(` `);
cloud_runner_logger_1.default.logLine('Starting build automation job');
// eslint-disable-next-line no-unused-vars
const workspace = (yield shared_workspace_locking_1.default.GetLockedWorkspace()) || cloud_runner_1.default.buildParameters.buildGuid;
const workspace = (yield shared_workspace_locking_1.default.GetLockedWorkspace(`test-workspace`, cloud_runner_1.default.buildParameters.buildGuid)) ||
cloud_runner_1.default.buildParameters.buildGuid;
output += yield cloud_runner_1.default.Provider.runTask(cloud_runner_1.default.buildParameters.buildGuid, baseImage.toString(), BuildAutomationWorkflow.BuildWorkflow, `/${cloud_runner_folders_1.CloudRunnerFolders.buildVolumeFolder}`, `/${cloud_runner_folders_1.CloudRunnerFolders.buildVolumeFolder}/`, cloudRunnerStepState.environment, cloudRunnerStepState.secrets);
if (!cloud_runner_1.default.buildParameters.isCliMode)
core.endGroup();

2
dist/index.js.map vendored
View File

File diff suppressed because one or more lines are too long

89
src/model/cli/shared-workspace-locking.ts
View File

@ -1,41 +1,84 @@
import { CloudRunnerSystem } from '../cloud-runner/services/cloud-runner-system';
import * as fs from 'fs';
import CloudRunner from '../cloud-runner/cloud-runner';
import CloudRunnerLogger from '../cloud-runner/services/cloud-runner-logger';
import CloudRunnerOptions from '../cloud-runner/cloud-runner-options';
export class SharedWorkspaceLocking {
public static async GetLockedWorkspace() {
public static async GetLockedWorkspace(workspaceIfCreated: string, runId: string) {
if (!CloudRunnerOptions.retainWorkspaces) {
return;
}
const workspaces = SharedWorkspaceLocking.GetFreeWorkspaces();
const workspaces = await SharedWorkspaceLocking.GetFreeWorkspaces();
for (const element of workspaces) {
if (await SharedWorkspaceLocking.LockWorkspace(element)) {
if (await SharedWorkspaceLocking.LockWorkspace(element, runId)) {
return element;
}
}
return await SharedWorkspaceLocking.CreateLockableWorkspace(CloudRunner.buildParameters.buildGuid);
return await SharedWorkspaceLocking.CreateLockableWorkspace(workspaceIfCreated);
}
public static GetFreeWorkspaces(): string[] {
return [];
public static async GetFreeWorkspaces(): Promise<string[]> {
const result: string[] = [];
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces();
for (const element of workspaces) {
if (!(await SharedWorkspaceLocking.IsWorkspaceLocked(element))) {
result.push(element);
}
}
return result;
}
public static GetAllWorkspaces(): string[] {
return [];
public static async GetAllWorkspaces(): Promise<string[]> {
return (await SharedWorkspaceLocking.ReadLines(`aws s3 ls s3://game-ci-test-storage/locks/`)).map((x) =>
x.replace(`/`, ``),
);
}
public static async LockWorkspace(workspace: string): Promise<boolean> {
const file = `${Date.now()}_${CloudRunner.buildParameters.buildGuid}_lock`;
public static async GetAllLocks(workspace: string): Promise<string[]> {
return (await SharedWorkspaceLocking.ReadLines(`aws s3 ls s3://game-ci-test-storage/locks/${workspace}/`)).map(
(x) => x.replace(`/`, ``),
);
}
private static async ReadLines(command: string): Promise<string[]> {
const result = await CloudRunnerSystem.Run(command, false, true);
return result
.split(`\n`)
.map((x) => x.replace(`\r`, ``))
.filter((x) => x !== ``)
.map((x) => {
const lineValues = x.split(` `);
return lineValues[lineValues.length - 1];
});
}
public static async LockWorkspace(workspace: string, runId: string): Promise<boolean> {
const file = `${Date.now()}_${runId}_lock`;
fs.writeFileSync(file, '');
await CloudRunnerSystem.Run(`aws s3 cp ./${file} s3://game-ci-test-storage/locks/${workspace}/${file}`);
await CloudRunnerSystem.Run(
`aws s3 cp ./${file} s3://game-ci-test-storage/locks/${workspace}/${file}`,
false,
true,
);
fs.rmSync(file);
return SharedWorkspaceLocking.HasWorkspaceLock(workspace);
}
public static async ReleaseWorkspace(workspace: string, runId: string): Promise<boolean> {
const file = (await SharedWorkspaceLocking.GetAllLocks(workspace)).filter((x) => x.includes(`_${runId}_lock`));
CloudRunnerLogger.log(`${JSON.stringify(await SharedWorkspaceLocking.GetAllLocks(workspace))}`);
CloudRunnerLogger.log(`Deleting file ${file}`);
CloudRunnerLogger.log(`aws s3 rm s3://game-ci-test-storage/locks/${workspace}/${file}`);
await CloudRunnerSystem.Run(`aws s3 rm s3://game-ci-test-storage/locks/${workspace}/${file}`, false, true);
return !SharedWorkspaceLocking.HasWorkspaceLock(workspace);
}
public static async HasWorkspaceLock(workspace: string): Promise<boolean> {
CloudRunnerLogger.log(
(await CloudRunnerSystem.Run(`aws s3 ls s3://game-ci-test-storage/locks/${workspace}/`))
(await CloudRunnerSystem.Run(`aws s3 ls s3://game-ci-test-storage/locks/${workspace}/`, false, true))
.split('\n')
.map((x) => {
return x.split(' ');
@ -45,15 +88,25 @@ export class SharedWorkspaceLocking {
return true;
}
// eslint-disable-next-line no-unused-vars
public static IsWorkspaceLocked(workspace: string) {}
public static async IsWorkspaceLocked(workspace: string): Promise<boolean> {
const files = await SharedWorkspaceLocking.ReadLines(`aws s3 ls s3://game-ci-test-storage/locks/${workspace}/`);
// 1 Because we expect 1 workspace file to exist in every workspace folder
return files.length > 1;
}
public static async CreateLockableWorkspace(workspace: string) {
const file = `${Date.now()}_${CloudRunner.buildParameters.buildGuid}_workspace`;
const file = `${Date.now()}_workspace`;
fs.writeFileSync(file, '');
await CloudRunnerSystem.Run(`aws s3 cp ./${file} s3://game-ci-test-storage/locks/${workspace}/${file}`);
await CloudRunnerSystem.Run(
`aws s3 cp ./${file} s3://game-ci-test-storage/locks/${workspace}/${file}`,
false,
true,
);
fs.rmSync(file);
return CloudRunner.buildParameters.buildGuid;
return workspace;
}
// eslint-disable-next-line no-unused-vars
public static ReleaseLock(workspace: string) {}

19
src/model/cloud-runner/providers/k8s/index.ts
View File

@ -174,6 +174,7 @@ class Kubernetes implements ProviderInterface {
CloudRunnerLogger.log('Abandoning cleanup, build error:');
throw error;
}
CloudRunnerLogger.log('cleaning up finished');
try {
await waitUntil(
async () => {
@ -204,13 +205,17 @@ class Kubernetes implements ProviderInterface {
CloudRunnerLogger.log(`deleting PVC`);
try {
await this.kubeClient.deleteNamespacedPersistentVolumeClaim(this.pvcName, this.namespace);
} catch (error: any) {
if (error.response.body.reason === `not found`) {
return;
}
CloudRunnerLogger.log(`Cleanup failed ${JSON.stringify(error, undefined, 4)}`);
}
const promise = this.kubeClient.deleteNamespacedPersistentVolumeClaim(this.pvcName, this.namespace);
// eslint-disable-next-line github/no-then
promise.catch((error: any) => {
if (error.response.body.reason === `not found`) {
return;
}
CloudRunnerLogger.log(`Cleanup failed ${JSON.stringify(error, undefined, 4)}`);
});
await promise;
// eslint-disable-next-line no-empty
} catch {}
}
static async findPodFromJob(kubeClient: CoreV1Api, jobName: string, namespace: string) {

44
src/model/cloud-runner/tests/shared-workspace-locking.test.ts
View File

@ -1,11 +1,51 @@
import SharedWorkspaceLocking from '../../cli/shared-workspace-locking';
import { Cli } from '../../cli/cli';
import setups from './cloud-runner-suite.test';
import CloudRunnerLogger from '../services/cloud-runner-logger';
import { v4 as uuidv4 } from 'uuid';
describe('Cloud Runner Locking', () => {
setups();
it(`simple locking flow`, async () => {
Cli.options.retainWorkspaces = true;
const newWorkspaceName = `test-workspace-${uuidv4()}`;
const runId = uuidv4();
await SharedWorkspaceLocking.CreateLockableWorkspace(newWorkspaceName);
const isExpectedUnlockedBeforeLocking =
(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName)) === false;
expect(isExpectedUnlockedBeforeLocking).toBeTruthy();
await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId);
const isExpectedLockedAfterLocking = (await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName)) === true;
expect(isExpectedLockedAfterLocking).toBeTruthy();
const locksBeforeRelease = await SharedWorkspaceLocking.GetAllLocks(newWorkspaceName);
CloudRunnerLogger.log(JSON.stringify(locksBeforeRelease, undefined, 4));
expect(locksBeforeRelease.length > 1).toBeTruthy();
await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId);
const locks = await SharedWorkspaceLocking.GetAllLocks(newWorkspaceName);
expect(locks.length === 1).toBeTruthy();
const isExpectedLockedAfterReleasing = (await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName)) === false;
expect(isExpectedLockedAfterReleasing).toBeTruthy();
}, 150000);
it('Locking', async () => {
Cli.options.retainWorkspaces = true;
await SharedWorkspaceLocking.IsWorkspaceLocked('test-workspace');
});
CloudRunnerLogger.log(`GetAllWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetAllWorkspaces())}`);
CloudRunnerLogger.log(`GetFreeWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetFreeWorkspaces())}`);
CloudRunnerLogger.log(
`IsWorkspaceLocked ${JSON.stringify(await SharedWorkspaceLocking.IsWorkspaceLocked('test-workspace'))}`,
);
CloudRunnerLogger.log(`GetFreeWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetFreeWorkspaces())}`);
CloudRunnerLogger.log(
`LockWorkspace ${JSON.stringify(await SharedWorkspaceLocking.LockWorkspace('test-workspace', uuidv4()))}`,
);
CloudRunnerLogger.log(
`CreateLockableWorkspace ${JSON.stringify(
await SharedWorkspaceLocking.CreateLockableWorkspace('test-workspace-2'),
)}`,
);
CloudRunnerLogger.log(
`GetLockedWorkspace ${JSON.stringify(
await SharedWorkspaceLocking.GetLockedWorkspace('test-workspace-2', uuidv4()),
)}`,
);
}, 150000);
});

4
src/model/cloud-runner/workflows/build-automation-workflow.ts
View File

@ -42,7 +42,9 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
CloudRunnerLogger.logLine('Starting build automation job');
// eslint-disable-next-line no-unused-vars
const workspace = (await SharedWorkspaceLocking.GetLockedWorkspace()) || CloudRunner.buildParameters.buildGuid;
const workspace =
(await SharedWorkspaceLocking.GetLockedWorkspace(`test-workspace`, CloudRunner.buildParameters.buildGuid)) ||
CloudRunner.buildParameters.buildGuid;
output += await CloudRunner.Provider.runTask(
CloudRunner.buildParameters.buildGuid,