actions
/
unity-builder
mirror of https://github.com/game-ci/unity-builder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Max workspaces and strong consistency locks

pull/437/head
Frostebite 2022-10-18 22:26:19 +01:00
parent ff30712eb9
commit f05bb6f142
7 changed files with 28 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
dist/index.js vendored
View File

@ -946,14 +946,15 @@ class CloudRunner {
const workspace = `test-workspace-${CloudRunner.buildParameters.buildGuid}`;
const result = (yield shared_workspace_locking_1.default.GetOrCreateLockedWorkspace(workspace, CloudRunner.buildParameters.buildGuid, CloudRunner.buildParameters)) || CloudRunner.buildParameters.buildGuid;
if (result) {
cloud_runner_logger_1.default.logLine(`Using retained workspace ${workspace}`);
CloudRunner.lockedWorkspace = workspace;
cloud_runner_logger_1.default.logLine(`Using workspace ${workspace}`);
CloudRunner.cloudRunnerEnvironmentVariables = [
...CloudRunner.cloudRunnerEnvironmentVariables,
{ name: `LOCKED_WORKSPACE`, value: workspace },
];
}
else {
cloud_runner_logger_1.default.log(`Max retained workspaces reached ${buildParameters.maxRetainedWorkspaces}`);
buildParameters.retainWorkspace = false;
}
}
@ -4047,7 +4048,6 @@ const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
const cli_functions_repository_1 = __nccwpck_require__(85301);
const cloud_runner_system_1 = __nccwpck_require__(99393);
const yaml_1 = __importDefault(__nccwpck_require__(44603));
const cloud_runner_options_1 = __importDefault(__nccwpck_require__(96552));
class RemoteClient {
static bootstrapRepository() {
return __awaiter(this, void 0, void 0, function* () {
@ -4088,9 +4088,6 @@ class RemoteClient {
static cloneRepoWithoutLFSFiles() {
return __awaiter(this, void 0, void 0, function* () {
process.chdir(`${cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
if (cloud_runner_options_1.default.cloudRunnerDebugTree) {
yield cloud_runner_system_1.CloudRunnerSystem.Run(`tree -L 2 ${cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
}
if (cloud_runner_1.default.buildParameters.retainWorkspace &&
fs_1.default.existsSync(path_1.default.join(cloud_runner_folders_1.CloudRunnerFolders.repoPathAbsolute, `.git`))) {
process.chdir(cloud_runner_folders_1.CloudRunnerFolders.repoPathAbsolute);
@ -4144,13 +4141,13 @@ class RemoteClient {
}
static runRemoteClientJob() {
return __awaiter(this, void 0, void 0, function* () {
yield cloud_runner_system_1.CloudRunnerSystem.Run(`tree -L 2 ${cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
// await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
RemoteClient.handleRetainedWorkspace();
yield cloud_runner_system_1.CloudRunnerSystem.Run(`tree -L 2 ${cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
// await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
yield RemoteClient.bootstrapRepository();
yield cloud_runner_system_1.CloudRunnerSystem.Run(`tree -L 2 ${cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
// await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
yield RemoteClient.runCustomHookFiles(`before-build`);
yield cloud_runner_system_1.CloudRunnerSystem.Run(`tree -L 2 ${cloud_runner_folders_1.CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
// await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
});
}
static runCustomHookFiles(hookLifecycle) {
@ -5147,8 +5144,9 @@ class SharedWorkspaceLocking {
if (yield SharedWorkspaceLocking.DoesWorkspaceTopLevelExist(buildParametersContext)) {
const workspaces = yield SharedWorkspaceLocking.GetFreeWorkspaces(buildParametersContext);
for (const element of workspaces) {
if (yield SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext)) {
cloud_runner_logger_1.default.log(`run agent ${runId} locked workspace: ${element}`);
const lockResult = yield SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext);
cloud_runner_logger_1.default.log(`run agent ${runId} try lock workspace: ${element} result: ${lockResult}`);
if (lockResult) {
return true;
}
}
@ -5169,7 +5167,6 @@ class SharedWorkspaceLocking {
return false;
}
const locks = (yield SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext))
.filter((x) => x.includes(`_lock`))
.map((x) => {
return {
name: x,

2
dist/index.js.map vendored
View File

File diff suppressed because one or more lines are too long

4
src/model/cloud-runner/cloud-runner.ts
View File

@ -76,14 +76,14 @@ class CloudRunner {
)) || CloudRunner.buildParameters.buildGuid;
if (result) {
CloudRunnerLogger.logLine(`Using retained workspace ${workspace}`);
CloudRunner.lockedWorkspace = workspace;
CloudRunnerLogger.logLine(`Using workspace ${workspace}`);
CloudRunner.cloudRunnerEnvironmentVariables = [
...CloudRunner.cloudRunnerEnvironmentVariables,
{ name: `LOCKED_WORKSPACE`, value: workspace },
];
} else {
CloudRunnerLogger.log(`Max retained workspaces reached ${buildParameters.maxRetainedWorkspaces}`);
buildParameters.retainWorkspace = false;
}
}

15
src/model/cloud-runner/remote-client/index.ts
View File

@ -10,7 +10,6 @@ import CloudRunnerLogger from '../services/cloud-runner-logger';
import { CliFunction } from '../../cli/cli-functions-repository';
import { CloudRunnerSystem } from '../services/cloud-runner-system';
import YAML from 'yaml';
import CloudRunnerOptions from '../cloud-runner-options';
export class RemoteClient {
public static async bootstrapRepository() {
@ -64,9 +63,6 @@ export class RemoteClient {
private static async cloneRepoWithoutLFSFiles() {
process.chdir(`${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
if (CloudRunnerOptions.cloudRunnerDebugTree) {
await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
}
if (
CloudRunner.buildParameters.retainWorkspace &&
@ -132,13 +128,16 @@ export class RemoteClient {
@CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
static async runRemoteClientJob() {
await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
// await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
RemoteClient.handleRetainedWorkspace();
await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
// await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
await RemoteClient.bootstrapRepository();
await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
// await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
await RemoteClient.runCustomHookFiles(`before-build`);
await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
// await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
}
static async runCustomHookFiles(hookLifecycle: string) {
RemoteClientLogger.log(`RunCustomHookFiles: ${hookLifecycle}`);

6
src/model/cloud-runner/services/shared-workspace-locking.ts
View File

@ -50,9 +50,10 @@ export class SharedWorkspaceLocking {
if (await SharedWorkspaceLocking.DoesWorkspaceTopLevelExist(buildParametersContext)) {
const workspaces = await SharedWorkspaceLocking.GetFreeWorkspaces(buildParametersContext);
for (const element of workspaces) {
if (await SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext)) {
CloudRunnerLogger.log(`run agent ${runId} locked workspace: ${element}`);
const lockResult = await SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext);
CloudRunnerLogger.log(`run agent ${runId} try lock workspace: ${element} result: ${lockResult}`);
if (lockResult) {
return true;
}
}
@ -78,7 +79,6 @@ export class SharedWorkspaceLocking {
return false;
}
const locks = (await SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext))
.filter((x) => x.includes(`_lock`))
.map((x) => {
return {
name: x,

9
src/model/cloud-runner/tests/cloud-runner-remote-client-caching.test.ts
View File

@ -8,7 +8,6 @@ import { CloudRunnerSystem } from '../services/cloud-runner-system';
import { Caching } from '../remote-client/caching';
import { v4 as uuidv4 } from 'uuid';
import GitHub from '../../github';
import CloudRunnerOptions from '../cloud-runner-options';
describe('Cloud Runner (Remote Client) Caching', () => {
it('responds', () => {});
if (process.platform === 'linux') {
@ -45,10 +44,10 @@ describe('Cloud Runner (Remote Client) Caching', () => {
);
await CloudRunnerSystem.Run(`du -h ${__dirname}`);
if (CloudRunnerOptions.cloudRunnerDebugTree) {
await CloudRunnerSystem.Run(`tree ${testFolder}`);
await CloudRunnerSystem.Run(`tree ${cacheFolder}`);
}
// if (CloudRunnerOptions.cloudRunnerDebugTree) {
// await CloudRunnerSystem.Run(`tree ${testFolder}`);
// await CloudRunnerSystem.Run(`tree ${cacheFolder}`);
// }
// Compare validity to original hash
expect(fs.readFileSync(path.resolve(testFolder, 'test.txt'), { encoding: 'utf8' }).toString()).toContain(

4
src/model/cloud-runner/tests/shared-workspace-locking.test.ts
View File

@ -42,10 +42,10 @@ describe('Cloud Runner Locking', () => {
expect(isExpectedLockedAfterLocking).toBeTruthy();
const locksBeforeRelease = await SharedWorkspaceLocking.GetAllLocks(newWorkspaceName, buildParameters);
CloudRunnerLogger.log(JSON.stringify(locksBeforeRelease, undefined, 4));
expect(locksBeforeRelease.length === 1).toBeTruthy();
expect(locksBeforeRelease.length).toBe(1);
await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters);
const locks = await SharedWorkspaceLocking.GetAllLocks(newWorkspaceName, buildParameters);
expect(locks.length === 1).toBeTruthy();
expect(locks.length).toBe(0);
const isExpectedLockedAfterReleasing =
(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === false;
expect(isExpectedLockedAfterReleasing).toBeTruthy();