actions
/
unity-builder
mirror of https://github.com/game-ci/unity-builder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Max workspaces and strong consistency locks

pull/437/head
Frostebite 2022-10-18 22:26:19 +01:00
parent ff30712eb9
commit f05bb6f142
7 changed files with 28 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
dist/index.js vendored
View File

@ -946,14 +946,15 @@ class CloudRunner {
const workspace = `test-workspace-${CloudRunner.buildParameters.buildGuid}`; const workspace = `test-workspace-${CloudRunner.buildParameters.buildGuid}`;
const result = (yield shared_workspace_locking_1.default.GetOrCreateLockedWorkspace(workspace, CloudRunner.buildParameters.buildGuid, CloudRunner.buildParameters)) || CloudRunner.buildParameters.buildGuid; const result = (yield shared_workspace_locking_1.default.GetOrCreateLockedWorkspace(workspace, CloudRunner.buildParameters.buildGuid, CloudRunner.buildParameters)) || CloudRunner.buildParameters.buildGuid;
if (result) { if (result) {
cloud_runner_logger_1.default.logLine(`Using retained workspace ${workspace}`);
CloudRunner.lockedWorkspace = workspace; CloudRunner.lockedWorkspace = workspace;
cloud_runner_logger_1.default.logLine(`Using workspace ${workspace}`);
CloudRunner.cloudRunnerEnvironmentVariables = [ CloudRunner.cloudRunnerEnvironmentVariables = [
...CloudRunner.cloudRunnerEnvironmentVariables, ...CloudRunner.cloudRunnerEnvironmentVariables,
{ name: `LOCKED_WORKSPACE`, value: workspace }, { name: `LOCKED_WORKSPACE`, value: workspace },
]; ];
} }
else { else {
cloud_runner_logger_1.default.log(`Max retained workspaces reached ${buildParameters.maxRetainedWorkspaces}`);
buildParameters.retainWorkspace = false; buildParameters.retainWorkspace = false;
} }
} }
@ -4047,7 +4048,6 @@ const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
const cli_functions_repository_1 = __nccwpck_require__(85301); const cli_functions_repository_1 = __nccwpck_require__(85301);
const cloud_runner_system_1 = __nccwpck_require__(99393); const cloud_runner_system_1 = __nccwpck_require__(99393);
const yaml_1 = __importDefault(__nccwpck_require__(44603)); const yaml_1 = __importDefault(__nccwpck_require__(44603));
const cloud_runner_options_1 = __importDefault(__nccwpck_require__(96552));
class RemoteClient { class RemoteClient {
static bootstrapRepository() { static bootstrapRepository() {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
@ -4088,9 +4088,6 @@ class RemoteClient {
static cloneRepoWithoutLFSFiles() { static cloneRepoWithoutLFSFiles() {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
process.chdir(`${cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`); process.chdir(`${cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
if (cloud_runner_options_1.default.cloudRunnerDebugTree) {
yield cloud_runner_system_1.CloudRunnerSystem.Run(`tree -L 2 ${cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
}
if (cloud_runner_1.default.buildParameters.retainWorkspace && if (cloud_runner_1.default.buildParameters.retainWorkspace &&
fs_1.default.existsSync(path_1.default.join(cloud_runner_folders_1.CloudRunnerFolders.repoPathAbsolute, `.git`))) { fs_1.default.existsSync(path_1.default.join(cloud_runner_folders_1.CloudRunnerFolders.repoPathAbsolute, `.git`))) {
process.chdir(cloud_runner_folders_1.CloudRunnerFolders.repoPathAbsolute); process.chdir(cloud_runner_folders_1.CloudRunnerFolders.repoPathAbsolute);
@ -4144,13 +4141,13 @@ class RemoteClient {
} }
static runRemoteClientJob() { static runRemoteClientJob() {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
yield cloud_runner_system_1.CloudRunnerSystem.Run(`tree -L 2 ${cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`); // await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
RemoteClient.handleRetainedWorkspace(); RemoteClient.handleRetainedWorkspace();
yield cloud_runner_system_1.CloudRunnerSystem.Run(`tree -L 2 ${cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`); // await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
yield RemoteClient.bootstrapRepository(); yield RemoteClient.bootstrapRepository();
yield cloud_runner_system_1.CloudRunnerSystem.Run(`tree -L 2 ${cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`); // await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
yield RemoteClient.runCustomHookFiles(`before-build`); yield RemoteClient.runCustomHookFiles(`before-build`);
yield cloud_runner_system_1.CloudRunnerSystem.Run(`tree -L 2 ${cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`); // await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
}); });
} }
static runCustomHookFiles(hookLifecycle) { static runCustomHookFiles(hookLifecycle) {
@ -5147,8 +5144,9 @@ class SharedWorkspaceLocking {
if (yield SharedWorkspaceLocking.DoesWorkspaceTopLevelExist(buildParametersContext)) { if (yield SharedWorkspaceLocking.DoesWorkspaceTopLevelExist(buildParametersContext)) {
const workspaces = yield SharedWorkspaceLocking.GetFreeWorkspaces(buildParametersContext); const workspaces = yield SharedWorkspaceLocking.GetFreeWorkspaces(buildParametersContext);
for (const element of workspaces) { for (const element of workspaces) {
if (yield SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext)) { const lockResult = yield SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext);
cloud_runner_logger_1.default.log(`run agent ${runId} locked workspace: ${element}`); cloud_runner_logger_1.default.log(`run agent ${runId} try lock workspace: ${element} result: ${lockResult}`);
if (lockResult) {
return true; return true;
} }
} }
@ -5169,7 +5167,6 @@ class SharedWorkspaceLocking {
return false; return false;
} }
const locks = (yield SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext)) const locks = (yield SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext))
.filter((x) => x.includes(`_lock`))
.map((x) => { .map((x) => {
return { return {
name: x, name: x,

2
dist/index.js.map vendored
View File

File diff suppressed because one or more lines are too long

4
src/model/cloud-runner/cloud-runner.ts
View File

@ -76,14 +76,14 @@ class CloudRunner {
)) || CloudRunner.buildParameters.buildGuid; )) || CloudRunner.buildParameters.buildGuid;
if (result) { if (result) {
CloudRunnerLogger.logLine(`Using retained workspace ${workspace}`);
CloudRunner.lockedWorkspace = workspace; CloudRunner.lockedWorkspace = workspace;
CloudRunnerLogger.logLine(`Using workspace ${workspace}`);
CloudRunner.cloudRunnerEnvironmentVariables = [ CloudRunner.cloudRunnerEnvironmentVariables = [
...CloudRunner.cloudRunnerEnvironmentVariables, ...CloudRunner.cloudRunnerEnvironmentVariables,
{ name: `LOCKED_WORKSPACE`, value: workspace }, { name: `LOCKED_WORKSPACE`, value: workspace },
]; ];
} else { } else {
CloudRunnerLogger.log(`Max retained workspaces reached ${buildParameters.maxRetainedWorkspaces}`);
buildParameters.retainWorkspace = false; buildParameters.retainWorkspace = false;
} }
} }

15
src/model/cloud-runner/remote-client/index.ts
View File

@ -10,7 +10,6 @@ import CloudRunnerLogger from '../services/cloud-runner-logger';
import { CliFunction } from '../../cli/cli-functions-repository'; import { CliFunction } from '../../cli/cli-functions-repository';
import { CloudRunnerSystem } from '../services/cloud-runner-system'; import { CloudRunnerSystem } from '../services/cloud-runner-system';
import YAML from 'yaml'; import YAML from 'yaml';
import CloudRunnerOptions from '../cloud-runner-options';
export class RemoteClient { export class RemoteClient {
public static async bootstrapRepository() { public static async bootstrapRepository() {
@ -64,9 +63,6 @@ export class RemoteClient {
private static async cloneRepoWithoutLFSFiles() { private static async cloneRepoWithoutLFSFiles() {
process.chdir(`${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`); process.chdir(`${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
if (CloudRunnerOptions.cloudRunnerDebugTree) {
await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
}
if ( if (
CloudRunner.buildParameters.retainWorkspace && CloudRunner.buildParameters.retainWorkspace &&
@ -132,13 +128,16 @@ export class RemoteClient {
@CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`) @CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
static async runRemoteClientJob() { static async runRemoteClientJob() {
await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`); // await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
RemoteClient.handleRetainedWorkspace(); RemoteClient.handleRetainedWorkspace();
await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
// await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
await RemoteClient.bootstrapRepository(); await RemoteClient.bootstrapRepository();
await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
// await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
await RemoteClient.runCustomHookFiles(`before-build`); await RemoteClient.runCustomHookFiles(`before-build`);
await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
// await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
} }
static async runCustomHookFiles(hookLifecycle: string) { static async runCustomHookFiles(hookLifecycle: string) {
RemoteClientLogger.log(`RunCustomHookFiles: ${hookLifecycle}`); RemoteClientLogger.log(`RunCustomHookFiles: ${hookLifecycle}`);

6
src/model/cloud-runner/services/shared-workspace-locking.ts
View File

@ -50,9 +50,10 @@ export class SharedWorkspaceLocking {
if (await SharedWorkspaceLocking.DoesWorkspaceTopLevelExist(buildParametersContext)) { if (await SharedWorkspaceLocking.DoesWorkspaceTopLevelExist(buildParametersContext)) {
const workspaces = await SharedWorkspaceLocking.GetFreeWorkspaces(buildParametersContext); const workspaces = await SharedWorkspaceLocking.GetFreeWorkspaces(buildParametersContext);
for (const element of workspaces) { for (const element of workspaces) {
if (await SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext)) { const lockResult = await SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext);
CloudRunnerLogger.log(`run agent ${runId} locked workspace: ${element}`); CloudRunnerLogger.log(`run agent ${runId} try lock workspace: ${element} result: ${lockResult}`);
if (lockResult) {
return true; return true;
} }
} }
@ -78,7 +79,6 @@ export class SharedWorkspaceLocking {
return false; return false;
} }
const locks = (await SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext)) const locks = (await SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext))
.filter((x) => x.includes(`_lock`))
.map((x) => { .map((x) => {
return { return {
name: x, name: x,

9
src/model/cloud-runner/tests/cloud-runner-remote-client-caching.test.ts
View File

@ -8,7 +8,6 @@ import { CloudRunnerSystem } from '../services/cloud-runner-system';
import { Caching } from '../remote-client/caching'; import { Caching } from '../remote-client/caching';
import { v4 as uuidv4 } from 'uuid'; import { v4 as uuidv4 } from 'uuid';
import GitHub from '../../github'; import GitHub from '../../github';
import CloudRunnerOptions from '../cloud-runner-options';
describe('Cloud Runner (Remote Client) Caching', () => { describe('Cloud Runner (Remote Client) Caching', () => {
it('responds', () => {}); it('responds', () => {});
if (process.platform === 'linux') { if (process.platform === 'linux') {
@ -45,10 +44,10 @@ describe('Cloud Runner (Remote Client) Caching', () => {
); );
await CloudRunnerSystem.Run(`du -h ${__dirname}`); await CloudRunnerSystem.Run(`du -h ${__dirname}`);
if (CloudRunnerOptions.cloudRunnerDebugTree) { // if (CloudRunnerOptions.cloudRunnerDebugTree) {
await CloudRunnerSystem.Run(`tree ${testFolder}`); // await CloudRunnerSystem.Run(`tree ${testFolder}`);
await CloudRunnerSystem.Run(`tree ${cacheFolder}`); // await CloudRunnerSystem.Run(`tree ${cacheFolder}`);
} // }
// Compare validity to original hash // Compare validity to original hash
expect(fs.readFileSync(path.resolve(testFolder, 'test.txt'), { encoding: 'utf8' }).toString()).toContain( expect(fs.readFileSync(path.resolve(testFolder, 'test.txt'), { encoding: 'utf8' }).toString()).toContain(

4
src/model/cloud-runner/tests/shared-workspace-locking.test.ts
View File

@ -42,10 +42,10 @@ describe('Cloud Runner Locking', () => {
expect(isExpectedLockedAfterLocking).toBeTruthy(); expect(isExpectedLockedAfterLocking).toBeTruthy();
const locksBeforeRelease = await SharedWorkspaceLocking.GetAllLocks(newWorkspaceName, buildParameters); const locksBeforeRelease = await SharedWorkspaceLocking.GetAllLocks(newWorkspaceName, buildParameters);
CloudRunnerLogger.log(JSON.stringify(locksBeforeRelease, undefined, 4)); CloudRunnerLogger.log(JSON.stringify(locksBeforeRelease, undefined, 4));
expect(locksBeforeRelease.length === 1).toBeTruthy(); expect(locksBeforeRelease.length).toBe(1);
await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters); await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters);
const locks = await SharedWorkspaceLocking.GetAllLocks(newWorkspaceName, buildParameters); const locks = await SharedWorkspaceLocking.GetAllLocks(newWorkspaceName, buildParameters);
expect(locks.length === 1).toBeTruthy(); expect(locks.length).toBe(0);
const isExpectedLockedAfterReleasing = const isExpectedLockedAfterReleasing =
(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === false; (await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === false;
expect(isExpectedLockedAfterReleasing).toBeTruthy(); expect(isExpectedLockedAfterReleasing).toBeTruthy();