actions
/
unity-builder
mirror of https://github.com/game-ci/unity-builder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

parameterize s3

pull/479/head
Frostebite 2023-01-19 01:33:35 +00:00
parent b73c33b7f8
commit 16bd7bfef4
8 changed files with 49 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/cloud-runner-pipeline.yml vendored
View File

@ -20,7 +20,7 @@ env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: eu-west-2
AWS_BASE_STACK_NAME: game-ci-github-pipelines
AWS_BASE_STACK_NAME: game-ci-team-pipelines
CLOUD_RUNNER_BRANCH: ${{ github.ref }}
CLOUD_RUNNER_DEBUG: true
CLOUD_RUNNER_DEBUG_TREE: true

33
dist/index.js vendored
View File

@ -2618,7 +2618,7 @@ const task_service_1 = __nccwpck_require__(67205);
const cloud_runner_options_1 = __importDefault(__nccwpck_require__(96552));
class AWSBuildEnvironment {
constructor(buildParameters) {
this.baseStackName = buildParameters.awsBaseStackName;
AWSBuildEnvironment.baseStackName = buildParameters.awsBaseStackName;
}
listResources() {
return __awaiter(this, void 0, void 0, function* () {
@ -2686,8 +2686,8 @@ class AWSBuildEnvironment {
cloud_runner_logger_1.default.log(`AWS Region: ${CF.config.region}`);
const entrypoint = ['/bin/sh'];
const startTimeMs = Date.now();
yield new aws_base_stack_1.AWSBaseStack(this.baseStackName).setupBaseStack(CF);
const taskDef = yield new aws_job_stack_1.AWSJobStack(this.baseStackName).setupCloudFormations(CF, buildGuid, image, entrypoint, commands, mountdir, workingdir, secrets);
yield new aws_base_stack_1.AWSBaseStack(AWSBuildEnvironment.baseStackName).setupBaseStack(CF);
const taskDef = yield new aws_job_stack_1.AWSJobStack(AWSBuildEnvironment.baseStackName).setupCloudFormations(CF, buildGuid, image, entrypoint, commands, mountdir, workingdir, secrets);
let postRunTaskTimeMs;
try {
const postSetupStacksTimeMs = Date.now();
@ -2839,6 +2839,7 @@ const input_1 = __importDefault(__nccwpck_require__(91933));
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
const base_stack_formation_1 = __nccwpck_require__(29643);
const aws_task_runner_1 = __importDefault(__nccwpck_require__(15518));
const __1 = __importDefault(__nccwpck_require__(90480));
class TaskService {
static watch() {
return __awaiter(this, void 0, void 0, function* () {
@ -2971,7 +2972,7 @@ class TaskService {
process.env.AWS_REGION = input_1.default.region;
const s3 = new aws_sdk_1.default.S3();
const listRequest = {
Bucket: `game-ci-test-storage`,
Bucket: __1.default.baseStackName,
};
const results = yield s3.listObjects(listRequest).promise();
return results.Contents || [];
@ -4736,6 +4737,7 @@ const fs = __importStar(__nccwpck_require__(57147));
const input_1 = __importDefault(__nccwpck_require__(91933));
const cloud_runner_options_1 = __importDefault(__nccwpck_require__(96552));
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
const aws_1 = __importDefault(__nccwpck_require__(90480));
class CloudRunnerCustomSteps {
static GetCustomStepsFromFiles(hookLifecycle) {
const results = [];
@ -4766,7 +4768,7 @@ class CloudRunnerCustomSteps {
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
aws configure set region $AWS_DEFAULT_REGION --profile default
aws s3 cp /data/cache/$CACHE_KEY/build/build-${cloud_runner_1.default.buildParameters.buildGuid}.tar${cloud_runner_1.default.buildParameters.useLz4Compression ? '.lz4' : ''} s3://game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID.tar${cloud_runner_1.default.buildParameters.useLz4Compression ? '.lz4' : ''}
aws s3 cp /data/cache/$CACHE_KEY/build/build-${cloud_runner_1.default.buildParameters.buildGuid}.tar${cloud_runner_1.default.buildParameters.useLz4Compression ? '.lz4' : ''} s3://${aws_1.default.baseStackName}/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID.tar${cloud_runner_1.default.buildParameters.useLz4Compression ? '.lz4' : ''}
rm /data/cache/$CACHE_KEY/build/build-${cloud_runner_1.default.buildParameters.buildGuid}.tar${cloud_runner_1.default.buildParameters.useLz4Compression ? '.lz4' : ''}
secrets:
- name: awsAccessKeyId
@ -4781,9 +4783,9 @@ class CloudRunnerCustomSteps {
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
aws configure set region $AWS_DEFAULT_REGION --profile default
aws s3 ls game-ci-test-storage/cloud-runner-cache/ || true
aws s3 ls game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/build || true
aws s3 cp s3://game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${cloud_runner_1.default.buildParameters.useLz4Compression ? '.lz4' : ''} /data/cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${cloud_runner_1.default.buildParameters.useLz4Compression ? '.lz4' : ''}
aws s3 ls ${aws_1.default.baseStackName}/cloud-runner-cache/ || true
aws s3 ls ${aws_1.default.baseStackName}/cloud-runner-cache/$CACHE_KEY/build || true
aws s3 cp s3://${aws_1.default.baseStackName}/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${cloud_runner_1.default.buildParameters.useLz4Compression ? '.lz4' : ''} /data/cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${cloud_runner_1.default.buildParameters.useLz4Compression ? '.lz4' : ''}
secrets:
- name: awsAccessKeyId
- name: awsSecretAccessKey
@ -4832,9 +4834,9 @@ class CloudRunnerCustomSteps {
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
aws configure set region $AWS_DEFAULT_REGION --profile default
aws s3 cp --recursive /data/cache/$CACHE_KEY/lfs s3://game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/lfs
aws s3 cp --recursive /data/cache/$CACHE_KEY/lfs s3://${aws_1.default.baseStackName}/cloud-runner-cache/$CACHE_KEY/lfs
rm -r /data/cache/$CACHE_KEY/lfs
aws s3 cp --recursive /data/cache/$CACHE_KEY/Library s3://game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/Library
aws s3 cp --recursive /data/cache/$CACHE_KEY/Library s3://${aws_1.default.baseStackName}/cloud-runner-cache/$CACHE_KEY/Library
rm -r /data/cache/$CACHE_KEY/Library
secrets:
- name: awsAccessKeyId
@ -4850,13 +4852,13 @@ class CloudRunnerCustomSteps {
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
aws configure set region $AWS_DEFAULT_REGION --profile default
aws s3 ls game-ci-test-storage/cloud-runner-cache/ || true
aws s3 ls game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/ || true
BUCKET1="game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/Library/"
aws s3 ls ${aws_1.default.baseStackName}/cloud-runner-cache/ || true
aws s3 ls ${aws_1.default.baseStackName}/cloud-runner-cache/$CACHE_KEY/ || true
BUCKET1="${aws_1.default.baseStackName}/cloud-runner-cache/$CACHE_KEY/Library/"
aws s3 ls $BUCKET1 || true
OBJECT1="$(aws s3 ls $BUCKET1 | sort | tail -n 1 | awk '{print $4}' || '')"
aws s3 cp s3://$BUCKET1$OBJECT1 /data/cache/$CACHE_KEY/Library/ || true
BUCKET2="game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/lfs/"
BUCKET2="${aws_1.default.baseStackName}/cloud-runner-cache/$CACHE_KEY/lfs/"
aws s3 ls $BUCKET2 || true
OBJECT2="$(aws s3 ls $BUCKET2 | sort | tail -n 1 | awk '{print $4}' || '')"
aws s3 cp s3://$BUCKET2$OBJECT2 /data/cache/$CACHE_KEY/lfs/ || true
@ -5511,6 +5513,7 @@ const fs = __importStar(__nccwpck_require__(57147));
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
const cloud_runner_options_1 = __importDefault(__nccwpck_require__(96552));
const cloud_runner_1 = __importDefault(__nccwpck_require__(79144));
const aws_1 = __importDefault(__nccwpck_require__(90480));
class SharedWorkspaceLocking {
static GetAllWorkspaces(buildParametersContext) {
return __awaiter(this, void 0, void 0, function* () {
@ -5707,7 +5710,7 @@ class SharedWorkspaceLocking {
}
}
exports.SharedWorkspaceLocking = SharedWorkspaceLocking;
SharedWorkspaceLocking.workspaceBucketRoot = `s3://game-ci-test-storage/`;
SharedWorkspaceLocking.workspaceBucketRoot = `s3://${aws_1.default.baseStackName}/`;
SharedWorkspaceLocking.workspaceRoot = `${SharedWorkspaceLocking.workspaceBucketRoot}locks/`;
exports["default"] = SharedWorkspaceLocking;

2
dist/index.js.map vendored
View File

File diff suppressed because one or more lines are too long

8
src/model/cloud-runner/providers/aws/index.ts
View File

@ -16,10 +16,10 @@ import { TaskService } from './services/task-service';
import CloudRunnerOptions from '../../cloud-runner-options';
class AWSBuildEnvironment implements ProviderInterface {
private baseStackName: string;
public static baseStackName: string;
constructor(buildParameters: BuildParameters) {
this.baseStackName = buildParameters.awsBaseStackName;
AWSBuildEnvironment.baseStackName = buildParameters.awsBaseStackName;
}
async listResources(): Promise<ProviderResource[]> {
await TaskService.getCloudFormationJobStacks();
@ -95,8 +95,8 @@ class AWSBuildEnvironment implements ProviderInterface {
const entrypoint = ['/bin/sh'];
const startTimeMs = Date.now();
await new AwsBaseStack(this.baseStackName).setupBaseStack(CF);
const taskDef = await new AwsJobStack(this.baseStackName).setupCloudFormations(
await new AwsBaseStack(AWSBuildEnvironment.baseStackName).setupBaseStack(CF);
const taskDef = await new AwsJobStack(AWSBuildEnvironment.baseStackName).setupCloudFormations(
CF,
buildGuid,
image,

3
src/model/cloud-runner/providers/aws/services/task-service.ts
View File

@ -4,6 +4,7 @@ import CloudRunnerLogger from '../../../services/cloud-runner-logger';
import { BaseStackFormation } from '../cloud-formations/base-stack-formation';
import AwsTaskRunner from '../aws-task-runner';
import { ListObjectsRequest } from 'aws-sdk/clients/s3';
import AWSBuildEnvironment from '..';
export class TaskService {
static async watch() {
@ -158,7 +159,7 @@ export class TaskService {
process.env.AWS_REGION = Input.region;
const s3 = new AWS.S3();
const listRequest: ListObjectsRequest = {
Bucket: `game-ci-test-storage`,
Bucket: AWSBuildEnvironment.baseStackName,
};
const results = await s3.listObjects(listRequest).promise();

31
src/model/cloud-runner/services/cloud-runner-custom-steps.ts
View File

@ -9,6 +9,7 @@ import Input from '../../input';
import CloudRunnerOptions from '../cloud-runner-options';
import CloudRunnerLogger from './cloud-runner-logger';
import { CustomStep } from './custom-step';
import AWSBuildEnvironment from '../providers/aws';
export class CloudRunnerCustomSteps {
static GetCustomStepsFromFiles(hookLifecycle: string): CustomStep[] {
@ -45,7 +46,7 @@ export class CloudRunnerCustomSteps {
aws configure set region $AWS_DEFAULT_REGION --profile default
aws s3 cp /data/cache/$CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
} s3://game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID.tar${
} s3://${AWSBuildEnvironment.baseStackName}/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID.tar${
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
}
rm /data/cache/$CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${
@ -64,11 +65,13 @@ export class CloudRunnerCustomSteps {
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
aws configure set region $AWS_DEFAULT_REGION --profile default
aws s3 ls game-ci-test-storage/cloud-runner-cache/ || true
aws s3 ls game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/build || true
aws s3 cp s3://game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
} /data/cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
aws s3 ls ${AWSBuildEnvironment.baseStackName}/cloud-runner-cache/ || true
aws s3 ls ${AWSBuildEnvironment.baseStackName}/cloud-runner-cache/$CACHE_KEY/build || true
aws s3 cp s3://${
AWSBuildEnvironment.baseStackName
}/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
} /data/cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
}
secrets:
@ -119,9 +122,13 @@ export class CloudRunnerCustomSteps {
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
aws configure set region $AWS_DEFAULT_REGION --profile default
aws s3 cp --recursive /data/cache/$CACHE_KEY/lfs s3://game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/lfs
aws s3 cp --recursive /data/cache/$CACHE_KEY/lfs s3://${
AWSBuildEnvironment.baseStackName
}/cloud-runner-cache/$CACHE_KEY/lfs
rm -r /data/cache/$CACHE_KEY/lfs
aws s3 cp --recursive /data/cache/$CACHE_KEY/Library s3://game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/Library
aws s3 cp --recursive /data/cache/$CACHE_KEY/Library s3://${
AWSBuildEnvironment.baseStackName
}/cloud-runner-cache/$CACHE_KEY/Library
rm -r /data/cache/$CACHE_KEY/Library
secrets:
- name: awsAccessKeyId
@ -137,13 +144,13 @@ export class CloudRunnerCustomSteps {
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
aws configure set region $AWS_DEFAULT_REGION --profile default
aws s3 ls game-ci-test-storage/cloud-runner-cache/ || true
aws s3 ls game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/ || true
BUCKET1="game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/Library/"
aws s3 ls ${AWSBuildEnvironment.baseStackName}/cloud-runner-cache/ || true
aws s3 ls ${AWSBuildEnvironment.baseStackName}/cloud-runner-cache/$CACHE_KEY/ || true
BUCKET1="${AWSBuildEnvironment.baseStackName}/cloud-runner-cache/$CACHE_KEY/Library/"
aws s3 ls $BUCKET1 || true
OBJECT1="$(aws s3 ls $BUCKET1 | sort | tail -n 1 | awk '{print $4}' || '')"
aws s3 cp s3://$BUCKET1$OBJECT1 /data/cache/$CACHE_KEY/Library/ || true
BUCKET2="game-ci-test-storage/cloud-runner-cache/$CACHE_KEY/lfs/"
BUCKET2="${AWSBuildEnvironment.baseStackName}/cloud-runner-cache/$CACHE_KEY/lfs/"
aws s3 ls $BUCKET2 || true
OBJECT2="$(aws s3 ls $BUCKET2 | sort | tail -n 1 | awk '{print $4}' || '')"
aws s3 cp s3://$BUCKET2$OBJECT2 /data/cache/$CACHE_KEY/lfs/ || true

3
src/model/cloud-runner/services/shared-workspace-locking.ts
View File

@ -4,8 +4,9 @@ import CloudRunnerLogger from './cloud-runner-logger';
import CloudRunnerOptions from '../cloud-runner-options';
import BuildParameters from '../../build-parameters';
import CloudRunner from '../cloud-runner';
import AWSBuildEnvironment from '../providers/aws';
export class SharedWorkspaceLocking {
private static readonly workspaceBucketRoot = `s3://game-ci-test-storage/`;
private static readonly workspaceBucketRoot = `s3://${AWSBuildEnvironment.baseStackName}/`;
private static readonly workspaceRoot = `${SharedWorkspaceLocking.workspaceBucketRoot}locks/`;
public static async GetAllWorkspaces(buildParametersContext: BuildParameters): Promise<string[]> {
if (!(await SharedWorkspaceLocking.DoesWorkspaceTopLevelExist(buildParametersContext))) {

3
src/model/cloud-runner/tests/cloud-runner-s3-prebuilt-steps.test.ts
View File

@ -7,6 +7,7 @@ import { v4 as uuidv4 } from 'uuid';
import CloudRunnerOptions from '../cloud-runner-options';
import setups from './cloud-runner-suite.test';
import { CloudRunnerSystem } from '../services/cloud-runner-system';
import AWSBuildEnvironment from '../providers/aws';
async function CreateParameters(overrides) {
if (overrides) {
@ -38,7 +39,7 @@ describe('Cloud Runner pre-built S3 steps', () => {
expect(build2ContainsBuildSucceeded).toBeTruthy();
const results = await CloudRunnerSystem.RunAndReadLines(
`aws s3 ls s3://game-ci-test-storage/cloud-runner-cache/${buildParameter2.cacheKey}/`,
`aws s3 ls s3://${AWSBuildEnvironment.baseStackName}/cloud-runner-cache/${buildParameter2.cacheKey}/`,
);
CloudRunnerLogger.log(results.join(`,`));
}, 1_000_000_000);