actions
/
unity-builder
mirror of https://github.com/game-ci/unity-builder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Max workspaces and strong consistency locks

pull/437/head
Frostebite 2022-10-18 17:58:17 +01:00
parent 1de2bd651b
commit 641958ea44
6 changed files with 158 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
dist/index.js vendored
View File

@ -299,6 +299,7 @@ class BuildParameters {
retainWorkspace: cloud_runner_options_1.default.retainWorkspaces,
useSharedLargePackages: cloud_runner_options_1.default.useSharedLargePackages,
useLz4Compression: cloud_runner_options_1.default.useLz4Compression,
maxRetainedWorkspaces: cloud_runner_options_1.default.maxRetainedWorkspaces,
};
});
}
@ -803,8 +804,8 @@ class CloudRunnerOptions {
static get retainWorkspaces() {
return CloudRunnerOptions.getInput(`retainWorkspaces`) || false;
}
static get retainWorkspacesMax() {
return Number(CloudRunnerOptions.getInput(`retainWorkspacesMax`)) || 5;
static get maxRetainedWorkspaces() {
return Number(CloudRunnerOptions.getInput(`maxRetainedWorkspaces`)) || 3;
}
}
exports["default"] = CloudRunnerOptions;
@ -942,8 +943,9 @@ class CloudRunner {
CloudRunner.setup(buildParameters);
try {
if (cloud_runner_options_1.default.retainWorkspaces) {
const workspace = (yield shared_workspace_locking_1.default.GetOrCreateLockedWorkspace(`test-workspace-${CloudRunner.buildParameters.buildGuid}`, CloudRunner.buildParameters.buildGuid, CloudRunner.buildParameters)) || CloudRunner.buildParameters.buildGuid;
process.env.LOCKED_WORKSPACE = workspace;
const workspace = `test-workspace-${CloudRunner.buildParameters.buildGuid}`;
const result = (yield shared_workspace_locking_1.default.GetOrCreateLockedWorkspace(workspace, CloudRunner.buildParameters.buildGuid, CloudRunner.buildParameters)) || CloudRunner.buildParameters.buildGuid;
if (result) {
CloudRunner.lockedWorkspace = workspace;
cloud_runner_logger_1.default.logLine(`Using workspace ${workspace}`);
CloudRunner.cloudRunnerEnvironmentVariables = [
@ -951,6 +953,7 @@ class CloudRunner {
{ name: `LOCKED_WORKSPACE`, value: workspace },
];
}
}
if (!CloudRunner.buildParameters.isCliMode)
core.startGroup('Setup shared cloud runner resources');
yield CloudRunner.Provider.setup(CloudRunner.buildParameters.buildGuid, CloudRunner.buildParameters, CloudRunner.buildParameters.branch, CloudRunner.defaultSecrets);
@ -5127,7 +5130,7 @@ class SharedWorkspaceLocking {
return (yield SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/`)).map((x) => x.replace(`/`, ``));
});
}
static GetOrCreateLockedWorkspace(workspaceIfCreated, runId, buildParametersContext) {
static GetOrCreateLockedWorkspace(workspace, runId, buildParametersContext) {
return __awaiter(this, void 0, void 0, function* () {
if (!cloud_runner_options_1.default.retainWorkspaces) {
return;
@ -5138,13 +5141,13 @@ class SharedWorkspaceLocking {
for (const element of workspaces) {
if (yield SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext)) {
cloud_runner_logger_1.default.log(`run agent ${runId} locked workspace: ${element}`);
return element;
return true;
}
}
}
const workspace = yield SharedWorkspaceLocking.CreateWorkspace(workspaceIfCreated, buildParametersContext, runId);
cloud_runner_logger_1.default.log(`run agent ${runId} didn't find a free workspace so created: ${workspace}`);
return workspace;
const createResult = yield SharedWorkspaceLocking.CreateWorkspace(workspace, buildParametersContext, runId);
cloud_runner_logger_1.default.log(`run agent ${runId} didn't find a free workspace so created: ${workspace} createWorkspaceSuccess: ${createResult}`);
return createResult;
});
}
static DoesWorkspaceExist(workspace, buildParametersContext) {
@ -5157,10 +5160,18 @@ class SharedWorkspaceLocking {
if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
return false;
}
cloud_runner_logger_1.default.log(`Checking has workspace ${workspace} was locked`);
const locks = yield SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext);
const includesRunLock = locks.filter((x) => x.includes(runId)).length > 0;
cloud_runner_logger_1.default.log(`Locks ${locks}, includes ${includesRunLock}`);
const locks = (yield SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext))
.filter((x) => x.includes(`_lock`))
.map((x) => {
return {
name: x,
timestamp: Number(x.split(`_`)[0]),
};
})
.sort((x) => x.timestamp);
const lockMatches = locks.filter((x) => x.name.includes(runId));
const includesRunLock = lockMatches.length > 0 && locks.indexOf(lockMatches[0]) === 0;
cloud_runner_logger_1.default.log(`Checking has workspace lock, workspace: ${workspace} \n success: ${includesRunLock} \n locks: ${JSON.stringify(locks, undefined, 4)}`);
return includesRunLock;
});
}
@ -5169,13 +5180,41 @@ class SharedWorkspaceLocking {
const result = [];
const workspaces = yield SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
for (const element of workspaces) {
if (!(yield SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParametersContext))) {
if (!(yield SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParametersContext)) &&
(yield SharedWorkspaceLocking.IsWorkspaceBelowMax(element, buildParametersContext))) {
result.push(element);
}
}
return result;
});
}
static IsWorkspaceBelowMax(workspace, buildParametersContext) {
return __awaiter(this, void 0, void 0, function* () {
const workspaces = yield SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
const ordered = [];
for (const ws of workspaces) {
ordered.push({
name: ws,
timestamp: yield SharedWorkspaceLocking.GetWorkspaceTimestamp(ws, buildParametersContext),
});
}
ordered.sort((x) => x.timestamp);
const matches = ordered.filter((x) => x.name.includes(workspace));
const isWorkspaceBelowMax = matches.length > 0 && ordered.indexOf(matches[0]) < buildParametersContext.maxRetainedWorkspaces;
return isWorkspaceBelowMax;
});
}
static GetWorkspaceTimestamp(workspace, buildParametersContext) {
return __awaiter(this, void 0, void 0, function* () {
if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
throw new Error("Workspace doesn't exist, can't call get all locks");
}
return (yield SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/`))
.map((x) => x.replace(`/`, ``))
.filter((x) => x.includes(`_workspace`))
.map((x) => Number(x))[0];
});
}
static IsWorkspaceLocked(workspace, buildParametersContext) {
return __awaiter(this, void 0, void 0, function* () {
if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
@ -5202,7 +5241,11 @@ class SharedWorkspaceLocking {
fs.rmSync(file);
const workspaces = yield SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`);
cloud_runner_logger_1.default.log(`All workspaces ${workspaces}`);
return workspace;
if (yield SharedWorkspaceLocking.IsWorkspaceBelowMax(workspace, buildParametersContext)) {
yield SharedWorkspaceLocking.CleanupWorkspace(workspace, buildParametersContext);
return false;
}
return true;
});
}
static LockWorkspace(workspace, runId, buildParametersContext) {

2
dist/index.js.map vendored
View File

File diff suppressed because one or more lines are too long

2
src/model/build-parameters.ts
View File

@ -64,6 +64,7 @@ class BuildParameters {
public cloudRunnerBuilderPlatform!: string | undefined;
public isCliMode!: boolean;
public retainWorkspace!: boolean;
public maxRetainedWorkspaces!: number;
public useSharedLargePackages!: boolean;
public useLz4Compression!: boolean;
@ -143,6 +144,7 @@ class BuildParameters {
retainWorkspace: CloudRunnerOptions.retainWorkspaces,
useSharedLargePackages: CloudRunnerOptions.useSharedLargePackages,
useLz4Compression: CloudRunnerOptions.useLz4Compression,
maxRetainedWorkspaces: CloudRunnerOptions.maxRetainedWorkspaces,
};
}

4
src/model/cloud-runner/cloud-runner-options.ts
View File

@ -241,8 +241,8 @@ class CloudRunnerOptions {
return CloudRunnerOptions.getInput(`retainWorkspaces`) || false;
}
static get retainWorkspacesMax(): number {
return Number(CloudRunnerOptions.getInput(`retainWorkspacesMax`)) || 5;
static get maxRetainedWorkspaces(): number {
return Number(CloudRunnerOptions.getInput(`maxRetainedWorkspaces`)) || 3;
}
}

8
src/model/cloud-runner/cloud-runner.ts
View File

@ -67,14 +67,15 @@ class CloudRunner {
CloudRunner.setup(buildParameters);
try {
if (CloudRunnerOptions.retainWorkspaces) {
const workspace =
const workspace = `test-workspace-${CloudRunner.buildParameters.buildGuid}`;
const result =
(await SharedWorkspaceLocking.GetOrCreateLockedWorkspace(
`test-workspace-${CloudRunner.buildParameters.buildGuid}`,
workspace,
CloudRunner.buildParameters.buildGuid,
CloudRunner.buildParameters,
)) || CloudRunner.buildParameters.buildGuid;
process.env.LOCKED_WORKSPACE = workspace;
if (result) {
CloudRunner.lockedWorkspace = workspace;
CloudRunnerLogger.logLine(`Using workspace ${workspace}`);
@ -83,6 +84,7 @@ class CloudRunner {
{ name: `LOCKED_WORKSPACE`, value: workspace },
];
}
}
if (!CloudRunner.buildParameters.isCliMode) core.startGroup('Setup shared cloud runner resources');
await CloudRunner.Provider.setup(
CloudRunner.buildParameters.buildGuid,

90
src/model/cloud-runner/services/shared-workspace-locking.ts
View File

@ -31,7 +31,7 @@ export class SharedWorkspaceLocking {
).map((x) => x.replace(`/`, ``));
}
public static async GetOrCreateLockedWorkspace(
workspaceIfCreated: string,
workspace: string,
runId: string,
buildParametersContext: BuildParameters,
) {
@ -47,15 +47,17 @@ export class SharedWorkspaceLocking {
if (await SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext)) {
CloudRunnerLogger.log(`run agent ${runId} locked workspace: ${element}`);
return element;
return true;
}
}
}
const workspace = await SharedWorkspaceLocking.CreateWorkspace(workspaceIfCreated, buildParametersContext, runId);
CloudRunnerLogger.log(`run agent ${runId} didn't find a free workspace so created: ${workspace}`);
const createResult = await SharedWorkspaceLocking.CreateWorkspace(workspace, buildParametersContext, runId);
CloudRunnerLogger.log(
`run agent ${runId} didn't find a free workspace so created: ${workspace} createWorkspaceSuccess: ${createResult}`,
);
return workspace;
return createResult;
}
public static async DoesWorkspaceExist(workspace: string, buildParametersContext: BuildParameters) {
@ -69,10 +71,24 @@ export class SharedWorkspaceLocking {
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
return false;
}
CloudRunnerLogger.log(`Checking has workspace ${workspace} was locked`);
const locks = await SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext);
const includesRunLock = locks.filter((x) => x.includes(runId)).length > 0;
CloudRunnerLogger.log(`Locks ${locks}, includes ${includesRunLock}`);
const locks = (await SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext))
.filter((x) => x.includes(`_lock`))
.map((x) => {
return {
name: x,
timestamp: Number(x.split(`_`)[0]),
};
})
.sort((x) => x.timestamp);
const lockMatches = locks.filter((x) => x.name.includes(runId));
const includesRunLock = lockMatches.length > 0 && locks.indexOf(lockMatches[0]) === 0;
CloudRunnerLogger.log(
`Checking has workspace lock, workspace: ${workspace} \n success: ${includesRunLock} \n locks: ${JSON.stringify(
locks,
undefined,
4,
)}`,
);
return includesRunLock;
}
@ -81,7 +97,10 @@ export class SharedWorkspaceLocking {
const result: string[] = [];
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
for (const element of workspaces) {
if (!(await SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParametersContext))) {
if (
!(await SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParametersContext)) &&
(await SharedWorkspaceLocking.IsWorkspaceBelowMax(element, buildParametersContext))
) {
result.push(element);
}
}
@ -89,6 +108,44 @@ export class SharedWorkspaceLocking {
return result;
}
public static async IsWorkspaceBelowMax(
workspace: string,
buildParametersContext: BuildParameters,
): Promise<boolean> {
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
const ordered: any[] = [];
for (const ws of workspaces) {
ordered.push({
name: ws,
timestamp: await SharedWorkspaceLocking.GetWorkspaceTimestamp(ws, buildParametersContext),
});
}
ordered.sort((x) => x.timestamp);
const matches = ordered.filter((x) => x.name.includes(workspace));
const isWorkspaceBelowMax =
matches.length > 0 && ordered.indexOf(matches[0]) < buildParametersContext.maxRetainedWorkspaces;
return isWorkspaceBelowMax;
}
public static async GetWorkspaceTimestamp(
workspace: string,
buildParametersContext: BuildParameters,
): Promise<Number> {
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
throw new Error("Workspace doesn't exist, can't call get all locks");
}
return (
await SharedWorkspaceLocking.ReadLines(
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/`,
)
)
.map((x) => x.replace(`/`, ``))
.filter((x) => x.includes(`_workspace`))
.map((x) => Number(x))[0];
}
public static async IsWorkspaceLocked(workspace: string, buildParametersContext: BuildParameters): Promise<boolean> {
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
return false;
@ -110,7 +167,11 @@ export class SharedWorkspaceLocking {
return workspaceFileDoesNotExists || lockFilesExist;
}
public static async CreateWorkspace(workspace: string, buildParametersContext: BuildParameters, lockId: string = ``) {
public static async CreateWorkspace(
workspace: string,
buildParametersContext: BuildParameters,
lockId: string = ``,
): Promise<boolean> {
if (lockId !== ``) {
await SharedWorkspaceLocking.LockWorkspace(workspace, lockId, buildParametersContext);
}
@ -129,8 +190,13 @@ export class SharedWorkspaceLocking {
);
CloudRunnerLogger.log(`All workspaces ${workspaces}`);
if (await SharedWorkspaceLocking.IsWorkspaceBelowMax(workspace, buildParametersContext)) {
await SharedWorkspaceLocking.CleanupWorkspace(workspace, buildParametersContext);
return workspace;
return false;
}
return true;
}
public static async LockWorkspace(