Ignore garbage creating lock actions in test for now

pull/461/head
Frostebite 2022-10-05 02:20:32 +01:00
parent a3c0d81ac8
commit 893469e946
5 changed files with 137 additions and 140 deletions

90
dist/index.js vendored
View File

@ -648,7 +648,20 @@ const fs = __importStar(__nccwpck_require__(57147));
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855)); const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
const cloud_runner_options_1 = __importDefault(__nccwpck_require__(96552)); const cloud_runner_options_1 = __importDefault(__nccwpck_require__(96552));
class SharedWorkspaceLocking { class SharedWorkspaceLocking {
static GetLockedWorkspace(workspaceIfCreated, runId) { static GetAllWorkspaces() {
return __awaiter(this, void 0, void 0, function* () {
return (yield SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}`)).map((x) => x.replace(`/`, ``));
});
}
static GetAllLocks(workspace) {
return __awaiter(this, void 0, void 0, function* () {
if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace))) {
throw new Error("Workspace doesn't exist, can't call get all locks");
}
return (yield SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${workspace}/`)).map((x) => x.replace(`/`, ``));
});
}
static GetOrCreateLockedWorkspace(workspaceIfCreated, runId) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
if (!cloud_runner_options_1.default.retainWorkspaces) { if (!cloud_runner_options_1.default.retainWorkspaces) {
return; return;
@ -659,7 +672,7 @@ class SharedWorkspaceLocking {
return element; return element;
} }
} }
return yield SharedWorkspaceLocking.CreateLockableWorkspace(workspaceIfCreated); return yield SharedWorkspaceLocking.CreateWorkspace(workspaceIfCreated);
}); });
} }
static DoesWorkspaceExist(workspace) { static DoesWorkspaceExist(workspace) {
@ -667,9 +680,12 @@ class SharedWorkspaceLocking {
return (yield SharedWorkspaceLocking.GetAllWorkspaces()).includes(workspace); return (yield SharedWorkspaceLocking.GetAllWorkspaces()).includes(workspace);
}); });
} }
static CleanupWorkspace(workspace) { static HasWorkspaceLock(workspace, runId) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${workspace} --recursive`, false, true); if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace))) {
return false;
}
return (yield SharedWorkspaceLocking.GetAllLocks(workspace)).filter((x) => x.includes(runId)).length > 0;
}); });
} }
static GetFreeWorkspaces() { static GetFreeWorkspaces() {
@ -684,30 +700,23 @@ class SharedWorkspaceLocking {
return result; return result;
}); });
} }
static GetAllWorkspaces() { static IsWorkspaceLocked(workspace) {
return __awaiter(this, void 0, void 0, function* () {
return (yield SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}`)).map((x) => x.replace(`/`, ``));
});
}
static GetAllLocks(workspace) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace))) { if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace))) {
throw new Error("Workspace doesn't exist, can't call get all locks"); return false;
} }
return (yield SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${workspace}/`)).map((x) => x.replace(`/`, ``)); const files = yield SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${workspace}/`);
// 1 Because we expect 1 workspace file to exist in every workspace folder
return files.length > 1;
}); });
} }
static ReadLines(command) { static CreateWorkspace(workspace) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
const result = yield cloud_runner_system_1.CloudRunnerSystem.Run(command, false, true); const file = `${Date.now()}_workspace`;
return result fs.writeFileSync(file, '');
.split(`\n`) yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${workspace}/${file}`, false, true);
.map((x) => x.replace(`\r`, ``)) fs.rmSync(file);
.filter((x) => x !== ``) return workspace;
.map((x) => {
const lineValues = x.split(` `);
return lineValues[lineValues.length - 1];
});
}); });
} }
static LockWorkspace(workspace, runId) { static LockWorkspace(workspace, runId) {
@ -732,35 +741,24 @@ class SharedWorkspaceLocking {
return !SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId); return !SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId);
}); });
} }
static HasWorkspaceLock(workspace, runId) { static CleanupWorkspace(workspace) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace))) { yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${workspace} --recursive`, false, true);
return false;
}
return (yield SharedWorkspaceLocking.GetAllLocks(workspace)).filter((x) => x.includes(runId)).length > 0;
}); });
} }
static IsWorkspaceLocked(workspace) { static ReadLines(command) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace))) { const result = yield cloud_runner_system_1.CloudRunnerSystem.Run(command, false, true);
return false; return result
} .split(`\n`)
const files = yield SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${workspace}/`); .map((x) => x.replace(`\r`, ``))
// 1 Because we expect 1 workspace file to exist in every workspace folder .filter((x) => x !== ``)
return files.length > 1; .map((x) => {
const lineValues = x.split(` `);
return lineValues[lineValues.length - 1];
});
}); });
} }
static CreateLockableWorkspace(workspace) {
return __awaiter(this, void 0, void 0, function* () {
const file = `${Date.now()}_workspace`;
fs.writeFileSync(file, '');
yield cloud_runner_system_1.CloudRunnerSystem.Run(`aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${workspace}/${file}`, false, true);
fs.rmSync(file);
return workspace;
});
}
// eslint-disable-next-line no-unused-vars
static ReleaseLock(workspace) { }
} }
exports.SharedWorkspaceLocking = SharedWorkspaceLocking; exports.SharedWorkspaceLocking = SharedWorkspaceLocking;
SharedWorkspaceLocking.workspaceRoot = `s3://game-ci-test-storage/locks/`; SharedWorkspaceLocking.workspaceRoot = `s3://game-ci-test-storage/locks/`;
@ -5057,7 +5055,7 @@ class BuildAutomationWorkflow {
try { try {
cloud_runner_logger_1.default.log(`Cloud Runner is running standard build automation`); cloud_runner_logger_1.default.log(`Cloud Runner is running standard build automation`);
if (cloud_runner_options_1.default.retainWorkspaces) { if (cloud_runner_options_1.default.retainWorkspaces) {
const workspace = (yield shared_workspace_locking_1.default.GetLockedWorkspace(`test-workspace-${cloud_runner_1.default.buildParameters.buildGuid}`, cloud_runner_1.default.buildParameters.buildGuid)) || cloud_runner_1.default.buildParameters.buildGuid; const workspace = (yield shared_workspace_locking_1.default.GetOrCreateLockedWorkspace(`test-workspace-${cloud_runner_1.default.buildParameters.buildGuid}`, cloud_runner_1.default.buildParameters.buildGuid)) || cloud_runner_1.default.buildParameters.buildGuid;
process.env.LOCKED_WORKSPACE = workspace; process.env.LOCKED_WORKSPACE = workspace;
cloud_runner_logger_1.default.logLine(`Using workspace ${workspace}`); cloud_runner_logger_1.default.logLine(`Using workspace ${workspace}`);
cloudRunnerStepState.environment = [ cloudRunnerStepState.environment = [

2
dist/index.js.map vendored

File diff suppressed because one or more lines are too long

View File

@ -4,44 +4,6 @@ import CloudRunnerLogger from '../cloud-runner/services/cloud-runner-logger';
import CloudRunnerOptions from '../cloud-runner/cloud-runner-options'; import CloudRunnerOptions from '../cloud-runner/cloud-runner-options';
export class SharedWorkspaceLocking { export class SharedWorkspaceLocking {
private static readonly workspaceRoot = `s3://game-ci-test-storage/locks/`; private static readonly workspaceRoot = `s3://game-ci-test-storage/locks/`;
public static async GetLockedWorkspace(workspaceIfCreated: string, runId: string) {
if (!CloudRunnerOptions.retainWorkspaces) {
return;
}
const workspaces = await SharedWorkspaceLocking.GetFreeWorkspaces();
for (const element of workspaces) {
if (await SharedWorkspaceLocking.LockWorkspace(element, runId)) {
return element;
}
}
return await SharedWorkspaceLocking.CreateLockableWorkspace(workspaceIfCreated);
}
public static async DoesWorkspaceExist(workspace: string) {
return (await SharedWorkspaceLocking.GetAllWorkspaces()).includes(workspace);
}
public static async CleanupWorkspace(workspace: string) {
await CloudRunnerSystem.Run(
`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${workspace} --recursive`,
false,
true,
);
}
public static async GetFreeWorkspaces(): Promise<string[]> {
const result: string[] = [];
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces();
for (const element of workspaces) {
if (!(await SharedWorkspaceLocking.IsWorkspaceLocked(element))) {
result.push(element);
}
}
return result;
}
public static async GetAllWorkspaces(): Promise<string[]> { public static async GetAllWorkspaces(): Promise<string[]> {
return (await SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}`)).map((x) => return (await SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}`)).map((x) =>
x.replace(`/`, ``), x.replace(`/`, ``),
@ -56,19 +18,67 @@ export class SharedWorkspaceLocking {
await SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${workspace}/`) await SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${workspace}/`)
).map((x) => x.replace(`/`, ``)); ).map((x) => x.replace(`/`, ``));
} }
public static async GetOrCreateLockedWorkspace(workspaceIfCreated: string, runId: string) {
if (!CloudRunnerOptions.retainWorkspaces) {
return;
}
private static async ReadLines(command: string): Promise<string[]> { const workspaces = await SharedWorkspaceLocking.GetFreeWorkspaces();
const result = await CloudRunnerSystem.Run(command, false, true); for (const element of workspaces) {
if (await SharedWorkspaceLocking.LockWorkspace(element, runId)) {
return element;
}
}
return result return await SharedWorkspaceLocking.CreateWorkspace(workspaceIfCreated);
.split(`\n`) }
.map((x) => x.replace(`\r`, ``))
.filter((x) => x !== ``)
.map((x) => {
const lineValues = x.split(` `);
return lineValues[lineValues.length - 1]; public static async DoesWorkspaceExist(workspace: string) {
}); return (await SharedWorkspaceLocking.GetAllWorkspaces()).includes(workspace);
}
public static async HasWorkspaceLock(workspace: string, runId: string): Promise<boolean> {
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace))) {
return false;
}
return (await SharedWorkspaceLocking.GetAllLocks(workspace)).filter((x) => x.includes(runId)).length > 0;
}
public static async GetFreeWorkspaces(): Promise<string[]> {
const result: string[] = [];
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces();
for (const element of workspaces) {
if (!(await SharedWorkspaceLocking.IsWorkspaceLocked(element))) {
result.push(element);
}
}
return result;
}
public static async IsWorkspaceLocked(workspace: string): Promise<boolean> {
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace))) {
return false;
}
const files = await SharedWorkspaceLocking.ReadLines(
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${workspace}/`,
);
// 1 Because we expect 1 workspace file to exist in every workspace folder
return files.length > 1;
}
public static async CreateWorkspace(workspace: string) {
const file = `${Date.now()}_workspace`;
fs.writeFileSync(file, '');
await CloudRunnerSystem.Run(
`aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${workspace}/${file}`,
false,
true,
);
fs.rmSync(file);
return workspace;
} }
public static async LockWorkspace(workspace: string, runId: string): Promise<boolean> { public static async LockWorkspace(workspace: string, runId: string): Promise<boolean> {
@ -96,40 +106,28 @@ export class SharedWorkspaceLocking {
return !SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId); return !SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId);
} }
public static async HasWorkspaceLock(workspace: string, runId: string): Promise<boolean> {
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace))) {
return false;
}
return (await SharedWorkspaceLocking.GetAllLocks(workspace)).filter((x) => x.includes(runId)).length > 0; public static async CleanupWorkspace(workspace: string) {
}
public static async IsWorkspaceLocked(workspace: string): Promise<boolean> {
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace))) {
return false;
}
const files = await SharedWorkspaceLocking.ReadLines(
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${workspace}/`,
);
// 1 Because we expect 1 workspace file to exist in every workspace folder
return files.length > 1;
}
public static async CreateLockableWorkspace(workspace: string) {
const file = `${Date.now()}_workspace`;
fs.writeFileSync(file, '');
await CloudRunnerSystem.Run( await CloudRunnerSystem.Run(
`aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${workspace}/${file}`, `aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${workspace} --recursive`,
false, false,
true, true,
); );
fs.rmSync(file);
return workspace;
} }
// eslint-disable-next-line no-unused-vars
public static ReleaseLock(workspace: string) {} private static async ReadLines(command: string): Promise<string[]> {
const result = await CloudRunnerSystem.Run(command, false, true);
return result
.split(`\n`)
.map((x) => x.replace(`\r`, ``))
.filter((x) => x !== ``)
.map((x) => {
const lineValues = x.split(` `);
return lineValues[lineValues.length - 1];
});
}
} }
export default SharedWorkspaceLocking; export default SharedWorkspaceLocking;

View File

@ -13,7 +13,7 @@ describe('Cloud Runner Locking', () => {
Cli.options.retainWorkspaces = true; Cli.options.retainWorkspaces = true;
const newWorkspaceName = `test-workspace-${uuidv4()}`; const newWorkspaceName = `test-workspace-${uuidv4()}`;
const runId = uuidv4(); const runId = uuidv4();
await SharedWorkspaceLocking.CreateLockableWorkspace(newWorkspaceName); await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName);
const isExpectedUnlockedBeforeLocking = const isExpectedUnlockedBeforeLocking =
(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName)) === false; (await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName)) === false;
expect(isExpectedUnlockedBeforeLocking).toBeTruthy(); expect(isExpectedUnlockedBeforeLocking).toBeTruthy();
@ -40,21 +40,22 @@ describe('Cloud Runner Locking', () => {
)}`, )}`,
); );
CloudRunnerLogger.log(`GetFreeWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetFreeWorkspaces())}`); CloudRunnerLogger.log(`GetFreeWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetFreeWorkspaces())}`);
CloudRunnerLogger.log(
`LockWorkspace ${JSON.stringify( // CloudRunnerLogger.log(
await SharedWorkspaceLocking.LockWorkspace(`test-workspace-${uuidv4()}`, uuidv4()), // `LockWorkspace ${JSON.stringify(
)}`, // await SharedWorkspaceLocking.LockWorkspace(`test-workspace-${uuidv4()}`, uuidv4()),
); // )}`,
CloudRunnerLogger.log( // );
`CreateLockableWorkspace ${JSON.stringify( // CloudRunnerLogger.log(
await SharedWorkspaceLocking.CreateLockableWorkspace(`test-workspace-${uuidv4()}`), // `CreateLockableWorkspace ${JSON.stringify(
)}`, // await SharedWorkspaceLocking.CreateWorkspace(`test-workspace-${uuidv4()}`),
); // )}`,
CloudRunnerLogger.log( // );
`GetLockedWorkspace ${JSON.stringify( // CloudRunnerLogger.log(
await SharedWorkspaceLocking.GetLockedWorkspace(`test-workspace-${uuidv4()}`, uuidv4()), // `GetLockedWorkspace ${JSON.stringify(
)}`, // await SharedWorkspaceLocking.GetOrCreateLockedWorkspace(`test-workspace-${uuidv4()}`, uuidv4()),
); // )}`,
// );
}, 3000000); }, 3000000);
} }
}); });

View File

@ -26,7 +26,7 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
if (CloudRunnerOptions.retainWorkspaces) { if (CloudRunnerOptions.retainWorkspaces) {
const workspace = const workspace =
(await SharedWorkspaceLocking.GetLockedWorkspace( (await SharedWorkspaceLocking.GetOrCreateLockedWorkspace(
`test-workspace-${CloudRunner.buildParameters.buildGuid}`, `test-workspace-${CloudRunner.buildParameters.buildGuid}`,
CloudRunner.buildParameters.buildGuid, CloudRunner.buildParameters.buildGuid,
)) || CloudRunner.buildParameters.buildGuid; )) || CloudRunner.buildParameters.buildGuid;