actions
/
unity-builder
mirror of https://github.com/game-ci/unity-builder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Max workspaces and strong consistency locks

pull/437/head
Frostebite 2022-10-18 17:58:17 +01:00
parent 1de2bd651b
commit 641958ea44
6 changed files with 158 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
dist/index.js vendored
View File

@ -299,6 +299,7 @@ class BuildParameters {
retainWorkspace: cloud_runner_options_1.default.retainWorkspaces, retainWorkspace: cloud_runner_options_1.default.retainWorkspaces,
useSharedLargePackages: cloud_runner_options_1.default.useSharedLargePackages, useSharedLargePackages: cloud_runner_options_1.default.useSharedLargePackages,
useLz4Compression: cloud_runner_options_1.default.useLz4Compression, useLz4Compression: cloud_runner_options_1.default.useLz4Compression,
maxRetainedWorkspaces: cloud_runner_options_1.default.maxRetainedWorkspaces,
}; };
}); });
} }
@ -803,8 +804,8 @@ class CloudRunnerOptions {
static get retainWorkspaces() { static get retainWorkspaces() {
return CloudRunnerOptions.getInput(`retainWorkspaces`) || false; return CloudRunnerOptions.getInput(`retainWorkspaces`) || false;
} }
static get retainWorkspacesMax() { static get maxRetainedWorkspaces() {
return Number(CloudRunnerOptions.getInput(`retainWorkspacesMax`)) || 5; return Number(CloudRunnerOptions.getInput(`maxRetainedWorkspaces`)) || 3;
} }
} }
exports["default"] = CloudRunnerOptions; exports["default"] = CloudRunnerOptions;
@ -942,14 +943,16 @@ class CloudRunner {
CloudRunner.setup(buildParameters); CloudRunner.setup(buildParameters);
try { try {
if (cloud_runner_options_1.default.retainWorkspaces) { if (cloud_runner_options_1.default.retainWorkspaces) {
const workspace = (yield shared_workspace_locking_1.default.GetOrCreateLockedWorkspace(`test-workspace-${CloudRunner.buildParameters.buildGuid}`, CloudRunner.buildParameters.buildGuid, CloudRunner.buildParameters)) || CloudRunner.buildParameters.buildGuid; const workspace = `test-workspace-${CloudRunner.buildParameters.buildGuid}`;
process.env.LOCKED_WORKSPACE = workspace; const result = (yield shared_workspace_locking_1.default.GetOrCreateLockedWorkspace(workspace, CloudRunner.buildParameters.buildGuid, CloudRunner.buildParameters)) || CloudRunner.buildParameters.buildGuid;
CloudRunner.lockedWorkspace = workspace; if (result) {
cloud_runner_logger_1.default.logLine(`Using workspace ${workspace}`); CloudRunner.lockedWorkspace = workspace;
CloudRunner.cloudRunnerEnvironmentVariables = [ cloud_runner_logger_1.default.logLine(`Using workspace ${workspace}`);
...CloudRunner.cloudRunnerEnvironmentVariables, CloudRunner.cloudRunnerEnvironmentVariables = [
{ name: `LOCKED_WORKSPACE`, value: workspace }, ...CloudRunner.cloudRunnerEnvironmentVariables,
]; { name: `LOCKED_WORKSPACE`, value: workspace },
];
}
} }
if (!CloudRunner.buildParameters.isCliMode) if (!CloudRunner.buildParameters.isCliMode)
core.startGroup('Setup shared cloud runner resources'); core.startGroup('Setup shared cloud runner resources');
@ -5127,7 +5130,7 @@ class SharedWorkspaceLocking {
return (yield SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/`)).map((x) => x.replace(`/`, ``)); return (yield SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/`)).map((x) => x.replace(`/`, ``));
}); });
} }
static GetOrCreateLockedWorkspace(workspaceIfCreated, runId, buildParametersContext) { static GetOrCreateLockedWorkspace(workspace, runId, buildParametersContext) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
if (!cloud_runner_options_1.default.retainWorkspaces) { if (!cloud_runner_options_1.default.retainWorkspaces) {
return; return;
@ -5138,13 +5141,13 @@ class SharedWorkspaceLocking {
for (const element of workspaces) { for (const element of workspaces) {
if (yield SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext)) { if (yield SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext)) {
cloud_runner_logger_1.default.log(`run agent ${runId} locked workspace: ${element}`); cloud_runner_logger_1.default.log(`run agent ${runId} locked workspace: ${element}`);
return element; return true;
} }
} }
} }
const workspace = yield SharedWorkspaceLocking.CreateWorkspace(workspaceIfCreated, buildParametersContext, runId); const createResult = yield SharedWorkspaceLocking.CreateWorkspace(workspace, buildParametersContext, runId);
cloud_runner_logger_1.default.log(`run agent ${runId} didn't find a free workspace so created: ${workspace}`); cloud_runner_logger_1.default.log(`run agent ${runId} didn't find a free workspace so created: ${workspace} createWorkspaceSuccess: ${createResult}`);
return workspace; return createResult;
}); });
} }
static DoesWorkspaceExist(workspace, buildParametersContext) { static DoesWorkspaceExist(workspace, buildParametersContext) {
@ -5157,10 +5160,18 @@ class SharedWorkspaceLocking {
if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) { if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
return false; return false;
} }
cloud_runner_logger_1.default.log(`Checking has workspace ${workspace} was locked`); const locks = (yield SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext))
const locks = yield SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext); .filter((x) => x.includes(`_lock`))
const includesRunLock = locks.filter((x) => x.includes(runId)).length > 0; .map((x) => {
cloud_runner_logger_1.default.log(`Locks ${locks}, includes ${includesRunLock}`); return {
name: x,
timestamp: Number(x.split(`_`)[0]),
};
})
.sort((x) => x.timestamp);
const lockMatches = locks.filter((x) => x.name.includes(runId));
const includesRunLock = lockMatches.length > 0 && locks.indexOf(lockMatches[0]) === 0;
cloud_runner_logger_1.default.log(`Checking has workspace lock, workspace: ${workspace} \n success: ${includesRunLock} \n locks: ${JSON.stringify(locks, undefined, 4)}`);
return includesRunLock; return includesRunLock;
}); });
} }
@ -5169,13 +5180,41 @@ class SharedWorkspaceLocking {
const result = []; const result = [];
const workspaces = yield SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext); const workspaces = yield SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
for (const element of workspaces) { for (const element of workspaces) {
if (!(yield SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParametersContext))) { if (!(yield SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParametersContext)) &&
(yield SharedWorkspaceLocking.IsWorkspaceBelowMax(element, buildParametersContext))) {
result.push(element); result.push(element);
} }
} }
return result; return result;
}); });
} }
static IsWorkspaceBelowMax(workspace, buildParametersContext) {
return __awaiter(this, void 0, void 0, function* () {
const workspaces = yield SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
const ordered = [];
for (const ws of workspaces) {
ordered.push({
name: ws,
timestamp: yield SharedWorkspaceLocking.GetWorkspaceTimestamp(ws, buildParametersContext),
});
}
ordered.sort((x) => x.timestamp);
const matches = ordered.filter((x) => x.name.includes(workspace));
const isWorkspaceBelowMax = matches.length > 0 && ordered.indexOf(matches[0]) < buildParametersContext.maxRetainedWorkspaces;
return isWorkspaceBelowMax;
});
}
static GetWorkspaceTimestamp(workspace, buildParametersContext) {
return __awaiter(this, void 0, void 0, function* () {
if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
throw new Error("Workspace doesn't exist, can't call get all locks");
}
return (yield SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/`))
.map((x) => x.replace(`/`, ``))
.filter((x) => x.includes(`_workspace`))
.map((x) => Number(x))[0];
});
}
static IsWorkspaceLocked(workspace, buildParametersContext) { static IsWorkspaceLocked(workspace, buildParametersContext) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) { if (!(yield SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
@ -5202,7 +5241,11 @@ class SharedWorkspaceLocking {
fs.rmSync(file); fs.rmSync(file);
const workspaces = yield SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`); const workspaces = yield SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`);
cloud_runner_logger_1.default.log(`All workspaces ${workspaces}`); cloud_runner_logger_1.default.log(`All workspaces ${workspaces}`);
return workspace; if (yield SharedWorkspaceLocking.IsWorkspaceBelowMax(workspace, buildParametersContext)) {
yield SharedWorkspaceLocking.CleanupWorkspace(workspace, buildParametersContext);
return false;
}
return true;
}); });
} }
static LockWorkspace(workspace, runId, buildParametersContext) { static LockWorkspace(workspace, runId, buildParametersContext) {

2
dist/index.js.map vendored
View File

File diff suppressed because one or more lines are too long

2
src/model/build-parameters.ts
View File

@ -64,6 +64,7 @@ class BuildParameters {
public cloudRunnerBuilderPlatform!: string | undefined; public cloudRunnerBuilderPlatform!: string | undefined;
public isCliMode!: boolean; public isCliMode!: boolean;
public retainWorkspace!: boolean; public retainWorkspace!: boolean;
public maxRetainedWorkspaces!: number;
public useSharedLargePackages!: boolean; public useSharedLargePackages!: boolean;
public useLz4Compression!: boolean; public useLz4Compression!: boolean;
@ -143,6 +144,7 @@ class BuildParameters {
retainWorkspace: CloudRunnerOptions.retainWorkspaces, retainWorkspace: CloudRunnerOptions.retainWorkspaces,
useSharedLargePackages: CloudRunnerOptions.useSharedLargePackages, useSharedLargePackages: CloudRunnerOptions.useSharedLargePackages,
useLz4Compression: CloudRunnerOptions.useLz4Compression, useLz4Compression: CloudRunnerOptions.useLz4Compression,
maxRetainedWorkspaces: CloudRunnerOptions.maxRetainedWorkspaces,
}; };
} }

4
src/model/cloud-runner/cloud-runner-options.ts
View File

@ -241,8 +241,8 @@ class CloudRunnerOptions {
return CloudRunnerOptions.getInput(`retainWorkspaces`) || false; return CloudRunnerOptions.getInput(`retainWorkspaces`) || false;
} }
static get retainWorkspacesMax(): number { static get maxRetainedWorkspaces(): number {
return Number(CloudRunnerOptions.getInput(`retainWorkspacesMax`)) || 5; return Number(CloudRunnerOptions.getInput(`maxRetainedWorkspaces`)) || 3;
} }
} }

20
src/model/cloud-runner/cloud-runner.ts
View File

@ -67,21 +67,23 @@ class CloudRunner {
CloudRunner.setup(buildParameters); CloudRunner.setup(buildParameters);
try { try {
if (CloudRunnerOptions.retainWorkspaces) { if (CloudRunnerOptions.retainWorkspaces) {
const workspace = const workspace = `test-workspace-${CloudRunner.buildParameters.buildGuid}`;
const result =
(await SharedWorkspaceLocking.GetOrCreateLockedWorkspace( (await SharedWorkspaceLocking.GetOrCreateLockedWorkspace(
`test-workspace-${CloudRunner.buildParameters.buildGuid}`, workspace,
CloudRunner.buildParameters.buildGuid, CloudRunner.buildParameters.buildGuid,
CloudRunner.buildParameters, CloudRunner.buildParameters,
)) || CloudRunner.buildParameters.buildGuid; )) || CloudRunner.buildParameters.buildGuid;
process.env.LOCKED_WORKSPACE = workspace; if (result) {
CloudRunner.lockedWorkspace = workspace; CloudRunner.lockedWorkspace = workspace;
CloudRunnerLogger.logLine(`Using workspace ${workspace}`); CloudRunnerLogger.logLine(`Using workspace ${workspace}`);
CloudRunner.cloudRunnerEnvironmentVariables = [ CloudRunner.cloudRunnerEnvironmentVariables = [
...CloudRunner.cloudRunnerEnvironmentVariables, ...CloudRunner.cloudRunnerEnvironmentVariables,
{ name: `LOCKED_WORKSPACE`, value: workspace }, { name: `LOCKED_WORKSPACE`, value: workspace },
]; ];
}
} }
if (!CloudRunner.buildParameters.isCliMode) core.startGroup('Setup shared cloud runner resources'); if (!CloudRunner.buildParameters.isCliMode) core.startGroup('Setup shared cloud runner resources');
await CloudRunner.Provider.setup( await CloudRunner.Provider.setup(

90
src/model/cloud-runner/services/shared-workspace-locking.ts
View File

@ -31,7 +31,7 @@ export class SharedWorkspaceLocking {
).map((x) => x.replace(`/`, ``)); ).map((x) => x.replace(`/`, ``));
} }
public static async GetOrCreateLockedWorkspace( public static async GetOrCreateLockedWorkspace(
workspaceIfCreated: string, workspace: string,
runId: string, runId: string,
buildParametersContext: BuildParameters, buildParametersContext: BuildParameters,
) { ) {
@ -47,15 +47,17 @@ export class SharedWorkspaceLocking {
if (await SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext)) { if (await SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext)) {
CloudRunnerLogger.log(`run agent ${runId} locked workspace: ${element}`); CloudRunnerLogger.log(`run agent ${runId} locked workspace: ${element}`);
return element; return true;
} }
} }
} }
const workspace = await SharedWorkspaceLocking.CreateWorkspace(workspaceIfCreated, buildParametersContext, runId); const createResult = await SharedWorkspaceLocking.CreateWorkspace(workspace, buildParametersContext, runId);
CloudRunnerLogger.log(`run agent ${runId} didn't find a free workspace so created: ${workspace}`); CloudRunnerLogger.log(
`run agent ${runId} didn't find a free workspace so created: ${workspace} createWorkspaceSuccess: ${createResult}`,
);
return workspace; return createResult;
} }
public static async DoesWorkspaceExist(workspace: string, buildParametersContext: BuildParameters) { public static async DoesWorkspaceExist(workspace: string, buildParametersContext: BuildParameters) {
@ -69,10 +71,24 @@ export class SharedWorkspaceLocking {
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) { if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
return false; return false;
} }
CloudRunnerLogger.log(`Checking has workspace ${workspace} was locked`); const locks = (await SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext))
const locks = await SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext); .filter((x) => x.includes(`_lock`))
const includesRunLock = locks.filter((x) => x.includes(runId)).length > 0; .map((x) => {
CloudRunnerLogger.log(`Locks ${locks}, includes ${includesRunLock}`); return {
name: x,
timestamp: Number(x.split(`_`)[0]),
};
})
.sort((x) => x.timestamp);
const lockMatches = locks.filter((x) => x.name.includes(runId));
const includesRunLock = lockMatches.length > 0 && locks.indexOf(lockMatches[0]) === 0;
CloudRunnerLogger.log(
`Checking has workspace lock, workspace: ${workspace} \n success: ${includesRunLock} \n locks: ${JSON.stringify(
locks,
undefined,
4,
)}`,
);
return includesRunLock; return includesRunLock;
} }
@ -81,7 +97,10 @@ export class SharedWorkspaceLocking {
const result: string[] = []; const result: string[] = [];
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext); const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
for (const element of workspaces) { for (const element of workspaces) {
if (!(await SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParametersContext))) { if (
!(await SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParametersContext)) &&
(await SharedWorkspaceLocking.IsWorkspaceBelowMax(element, buildParametersContext))
) {
result.push(element); result.push(element);
} }
} }
@ -89,6 +108,44 @@ export class SharedWorkspaceLocking {
return result; return result;
} }
public static async IsWorkspaceBelowMax(
workspace: string,
buildParametersContext: BuildParameters,
): Promise<boolean> {
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
const ordered: any[] = [];
for (const ws of workspaces) {
ordered.push({
name: ws,
timestamp: await SharedWorkspaceLocking.GetWorkspaceTimestamp(ws, buildParametersContext),
});
}
ordered.sort((x) => x.timestamp);
const matches = ordered.filter((x) => x.name.includes(workspace));
const isWorkspaceBelowMax =
matches.length > 0 && ordered.indexOf(matches[0]) < buildParametersContext.maxRetainedWorkspaces;
return isWorkspaceBelowMax;
}
public static async GetWorkspaceTimestamp(
workspace: string,
buildParametersContext: BuildParameters,
): Promise<Number> {
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
throw new Error("Workspace doesn't exist, can't call get all locks");
}
return (
await SharedWorkspaceLocking.ReadLines(
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/`,
)
)
.map((x) => x.replace(`/`, ``))
.filter((x) => x.includes(`_workspace`))
.map((x) => Number(x))[0];
}
public static async IsWorkspaceLocked(workspace: string, buildParametersContext: BuildParameters): Promise<boolean> { public static async IsWorkspaceLocked(workspace: string, buildParametersContext: BuildParameters): Promise<boolean> {
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) { if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
return false; return false;
@ -110,7 +167,11 @@ export class SharedWorkspaceLocking {
return workspaceFileDoesNotExists || lockFilesExist; return workspaceFileDoesNotExists || lockFilesExist;
} }
public static async CreateWorkspace(workspace: string, buildParametersContext: BuildParameters, lockId: string = ``) { public static async CreateWorkspace(
workspace: string,
buildParametersContext: BuildParameters,
lockId: string = ``,
): Promise<boolean> {
if (lockId !== ``) { if (lockId !== ``) {
await SharedWorkspaceLocking.LockWorkspace(workspace, lockId, buildParametersContext); await SharedWorkspaceLocking.LockWorkspace(workspace, lockId, buildParametersContext);
} }
@ -129,8 +190,13 @@ export class SharedWorkspaceLocking {
); );
CloudRunnerLogger.log(`All workspaces ${workspaces}`); CloudRunnerLogger.log(`All workspaces ${workspaces}`);
if (await SharedWorkspaceLocking.IsWorkspaceBelowMax(workspace, buildParametersContext)) {
await SharedWorkspaceLocking.CleanupWorkspace(workspace, buildParametersContext);
return workspace; return false;
}
return true;
} }
public static async LockWorkspace( public static async LockWorkspace(