actions
/
unity-builder
mirror of https://github.com/game-ci/unity-builder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

experimental pre-post hooks

pull/310/head
Frostebite 2022-01-01 17:44:41 +00:00
parent cc92a8c57a
commit dc48c34ef3
6 changed files with 127 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
.github/workflows/cloud-runner-aws-pipeline.yml vendored
View File

@ -45,6 +45,12 @@ jobs:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: eu-west-2
## we will auth to google as well for log aggregation
- uses: google-github-actions/setup-gcloud@master
with:
version: '288.0.0'
service_account_email: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_EMAIL }}
service_account_key: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
- uses: actions/setup-node@v2
with:
node-version: 12.x
@ -69,25 +75,6 @@ jobs:
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: eu-west-2
DEBUG: true
EXPERIMENTAL_HOOK: |
- name: FUSE filesystem to s3 for global scale caching
image: amazon/aws-cli
commands: |
sudo amazon-linux-extras install epel
sudo yum install s3fs-fuse
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
aws configure set region $AWS_DEFAULT_REGION --profile default
echo ACCESS_KEY_ID:SECRET_ACCESS_KEY > ${HOME}/.passwd-s3fs
chmod 600 ${HOME}/.passwd-s3fs
s3fs game-ci-test-storage /data/cache -o passwd_file=${HOME}/.passwd-s3fs
secrets:
- name: awsAccessKeyId
value: ${{ secrets.AWS_ACCESS_KEY_ID }}
- name: awsSecretAccessKey
value: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
- name: awsDefaultRegion
value: eu-west-2
with:
cloudRunnerCluster: aws
versioning: None

20
.github/workflows/cloud-runner-k8s-pipeline.yml vendored
View File

@ -63,6 +63,26 @@ jobs:
GITHUUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
KUBE_CONFIG: ${{ steps.read-base64.outputs.base64 }}
CLOUD_RUNNER_BRANCH: remote-builder/unified-providers
# Use FUSE to mount a storage bucket to a cache folder
EXPERIMENTAL_HOOK: |
- name: FUSE filesystem to s3 for global scale caching
hook: before
commands: |
sudo amazon-linux-extras install epel
sudo yum install s3fs-fuse
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
aws configure set region $AWS_DEFAULT_REGION --profile default
echo ACCESS_KEY_ID:SECRET_ACCESS_KEY > ${HOME}/.passwd-s3fs
chmod 600 ${HOME}/.passwd-s3fs
s3fs game-ci-test-storage /data/cache -o passwd_file=${HOME}/.passwd-s3fs
secrets:
- name: awsAccessKeyId
value: ${{ secrets.AWS_ACCESS_KEY_ID }}
- name: awsSecretAccessKey
value: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
- name: awsDefaultRegion
value: eu-west-2
###########################
# Cloud Runner Build Test #

51
dist/index.js vendored
View File

@ -2492,26 +2492,65 @@ exports.default = KubernetesTaskRunner;
/***/ }),
/***/ 71899:
/***/ ((__unused_webpack_module, exports, __webpack_require__) => {
/***/ (function(__unused_webpack_module, exports, __webpack_require__) {
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.CloudRunnerBuildCommandProcessor = void 0;
exports.Hook = exports.CloudRunnerBuildCommandProcessor = void 0;
const __1 = __webpack_require__(41359);
const yaml_1 = __importDefault(__webpack_require__(13552));
const cloud_runner_logger_1 = __importDefault(__webpack_require__(22855));
class CloudRunnerBuildCommandProcessor {
static ProcessCommands(commands, buildParameters) {
let hooks = CloudRunnerBuildCommandProcessor.getHooks();
if (__1.Input.cloudRunnerTests) {
cloud_runner_logger_1.default.log(JSON.stringify(hooks, undefined, 4));
}
else {
hooks = [];
}
return `echo "---"
echo "start cloud runner init"
${__1.Input.cloudRunnerTests ? '' : '#'} printenv
echo "start cloud runner job"
${hooks
.filter((x) => x.hook !== undefined && x.hook.length > 0 && x.hook.includes(`before`))
.map((x) => x.commands)
.join(`\n`) || ' '}
${commands}
${hooks
.filter((x) => x.hook !== undefined && x.hook.length > 0 && x.hook.includes(`before`))
.map((x) => x.commands)
.join(`\n`) || ' '}
echo "end of cloud runner job
---${buildParameters.logId}"
`;
}
static getHooks() {
const experimentHooks = process.env.EXPERIMENTAL_HOOKS;
let output = new Hook[0]();
if (experimentHooks && experimentHooks !== '') {
try {
output = yaml_1.default.parse(experimentHooks);
}
catch (error) {
throw error;
}
}
return output;
}
}
exports.CloudRunnerBuildCommandProcessor = CloudRunnerBuildCommandProcessor;
class Hook {
constructor() {
this.secrets = [];
}
}
exports.Hook = Hook;
/***/ }),
@ -2625,6 +2664,7 @@ exports.TaskParameterSerializer = void 0;
const __1 = __webpack_require__(41359);
const image_environment_factory_1 = __importDefault(__webpack_require__(25145));
const cloud_runner_state_1 = __webpack_require__(70912);
const cloud_runner_build_command_process_1 = __webpack_require__(71899);
class TaskParameterSerializer {
static readBuildEnvironmentVariables() {
TaskParameterSerializer.setupDefaultSecrets();
@ -2648,6 +2688,13 @@ class TaskParameterSerializer {
let array = new Array();
array = TaskParameterSerializer.readBuildParameters(array);
array = TaskParameterSerializer.readInput(array);
const configurableHooks = cloud_runner_build_command_process_1.CloudRunnerBuildCommandProcessor.getHooks()
.map((x) => x.secrets)
// eslint-disable-next-line unicorn/no-array-reduce
.reduce((x, y) =>
// eslint-disable-next-line unicorn/prefer-spread
x.concat(y));
array.push(configurableHooks);
array = array.filter((x) => x.value !== undefined && x.name !== '0' && x.value !== '' && x.name !== 'prototype' && x.name !== 'length');
array = array.map((x) => {
x.name = __1.Input.ToEnvVarFormat(x.name);

2
dist/index.js.map vendored
View File

File diff suppressed because one or more lines are too long

42
src/model/cloud-runner/services/cloud-runner-build-command-process.ts
View File

@ -1,14 +1,56 @@
import { BuildParameters, Input } from '../..';
import YAML from 'yaml';
import CloudRunnerSecret from './cloud-runner-secret';
import CloudRunnerLogger from './cloud-runner-logger';
export class CloudRunnerBuildCommandProcessor {
public static ProcessCommands(commands: string, buildParameters: BuildParameters): string {
let hooks = CloudRunnerBuildCommandProcessor.getHooks();
if (Input.cloudRunnerTests) {
CloudRunnerLogger.log(JSON.stringify(hooks, undefined, 4));
} else {
hooks = [];
}
return `echo "---"
echo "start cloud runner init"
${Input.cloudRunnerTests ? '' : '#'} printenv
echo "start cloud runner job"
${
hooks
.filter((x) => x.hook !== undefined && x.hook.length > 0 && x.hook.includes(`before`))
.map((x) => x.commands)
.join(`\n`) || ' '
}
${commands}
${
hooks
.filter((x) => x.hook !== undefined && x.hook.length > 0 && x.hook.includes(`before`))
.map((x) => x.commands)
.join(`\n`) || ' '
}
echo "end of cloud runner job
---${buildParameters.logId}"
`;
}
public static getHooks(): Hook[] {
const experimentHooks = process.env.EXPERIMENTAL_HOOKS;
let output = new Hook[0]();
if (experimentHooks && experimentHooks !== '') {
try {
output = YAML.parse(experimentHooks);
} catch (error) {
throw error;
}
}
return output;
}
}
export class Hook {
public commands;
public secrets: CloudRunnerSecret[] = [];
public name;
public hook;
}

9
src/model/cloud-runner/services/task-parameter-serializer.ts
View File

@ -2,6 +2,7 @@ import { Input } from '../..';
import ImageEnvironmentFactory from '../../image-environment-factory';
import CloudRunnerEnvironmentVariable from './cloud-runner-environment-variable';
import { CloudRunnerState } from '../state/cloud-runner-state';
import { CloudRunnerBuildCommandProcessor } from './cloud-runner-build-command-process';
export class TaskParameterSerializer {
public static readBuildEnvironmentVariables(): CloudRunnerEnvironmentVariable[] {
@ -26,6 +27,14 @@ export class TaskParameterSerializer {
let array = new Array();
array = TaskParameterSerializer.readBuildParameters(array);
array = TaskParameterSerializer.readInput(array);
const configurableHooks = CloudRunnerBuildCommandProcessor.getHooks()
.map((x) => x.secrets)
// eslint-disable-next-line unicorn/no-array-reduce
.reduce((x, y) =>
// eslint-disable-next-line unicorn/prefer-spread
x.concat(y),
);
array.push(configurableHooks);
array = array.filter(
(x) => x.value !== undefined && x.name !== '0' && x.value !== '' && x.name !== 'prototype' && x.name !== 'length',