experimental pre-post hooks

.github/workflows/cloud-runner-aws-pipeline.yml

@@ -45,6 +45,12 @@ jobs:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: eu-west-2
+      ## we will auth to google as well for log aggregation
+      - uses: google-github-actions/setup-gcloud@master
+        with:
+          version: '288.0.0'
+          service_account_email: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_EMAIL }}
+          service_account_key: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
       - uses: actions/setup-node@v2
         with:
           node-version: 12.x
@@ -69,25 +75,6 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_DEFAULT_REGION: eu-west-2
           DEBUG: true
-          EXPERIMENTAL_HOOK: |
-            - name: FUSE filesystem to s3 for global scale caching
-              image: amazon/aws-cli
-              commands: |
-                sudo amazon-linux-extras install epel
-                sudo yum install s3fs-fuse
-                aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
-                aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
-                aws configure set region $AWS_DEFAULT_REGION --profile default
-                echo ACCESS_KEY_ID:SECRET_ACCESS_KEY > ${HOME}/.passwd-s3fs
-                chmod 600 ${HOME}/.passwd-s3fs
-                s3fs game-ci-test-storage /data/cache -o passwd_file=${HOME}/.passwd-s3fs
-              secrets:
-              - name: awsAccessKeyId
-                value: ${{ secrets.AWS_ACCESS_KEY_ID }}
-              - name: awsSecretAccessKey
-                value: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-              - name: awsDefaultRegion
-                value: eu-west-2
         with:
           cloudRunnerCluster: aws
           versioning: None

.github/workflows/cloud-runner-k8s-pipeline.yml

@@ -63,6 +63,26 @@ jobs:
           GITHUUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           KUBE_CONFIG: ${{ steps.read-base64.outputs.base64 }}
           CLOUD_RUNNER_BRANCH: remote-builder/unified-providers
+          # Use FUSE to mount a storage bucket to a cache folder
+          EXPERIMENTAL_HOOK: |
+            - name: FUSE filesystem to s3 for global scale caching
+              hook: before
+              commands: |
+                sudo amazon-linux-extras install epel
+                sudo yum install s3fs-fuse
+                aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
+                aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
+                aws configure set region $AWS_DEFAULT_REGION --profile default
+                echo ACCESS_KEY_ID:SECRET_ACCESS_KEY > ${HOME}/.passwd-s3fs
+                chmod 600 ${HOME}/.passwd-s3fs
+                s3fs game-ci-test-storage /data/cache -o passwd_file=${HOME}/.passwd-s3fs
+              secrets:
+              - name: awsAccessKeyId
+                value: ${{ secrets.AWS_ACCESS_KEY_ID }}
+              - name: awsSecretAccessKey
+                value: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+              - name: awsDefaultRegion
+                value: eu-west-2
 
       ###########################
       # Cloud Runner Build Test #

dist/index.js

@@ -2492,26 +2492,65 @@ exports.default = KubernetesTaskRunner;
 /***/ }),
 
 /***/ 71899:
-/***/ ((__unused_webpack_module, exports, __webpack_require__) => {
+/***/ (function(__unused_webpack_module, exports, __webpack_require__) {
 
 "use strict";
 
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", ({ value: true }));
-exports.CloudRunnerBuildCommandProcessor = void 0;
+exports.Hook = exports.CloudRunnerBuildCommandProcessor = void 0;
 const __1 = __webpack_require__(41359);
+const yaml_1 = __importDefault(__webpack_require__(13552));
+const cloud_runner_logger_1 = __importDefault(__webpack_require__(22855));
 class CloudRunnerBuildCommandProcessor {
     static ProcessCommands(commands, buildParameters) {
+        let hooks = CloudRunnerBuildCommandProcessor.getHooks();
+        if (__1.Input.cloudRunnerTests) {
+            cloud_runner_logger_1.default.log(JSON.stringify(hooks, undefined, 4));
+        }
+        else {
+            hooks = [];
+        }
         return `echo "---"
       echo "start cloud runner init"
       ${__1.Input.cloudRunnerTests ? '' : '#'} printenv
       echo "start cloud runner job"
+      ${hooks
+            .filter((x) => x.hook !== undefined && x.hook.length > 0 && x.hook.includes(`before`))
+            .map((x) => x.commands)
+            .join(`\n`) || ' '}
       ${commands}
+      ${hooks
+            .filter((x) => x.hook !== undefined && x.hook.length > 0 && x.hook.includes(`before`))
+            .map((x) => x.commands)
+            .join(`\n`) || ' '}
       echo "end of cloud runner job
       ---${buildParameters.logId}"
     `;
     }
+    static getHooks() {
+        const experimentHooks = process.env.EXPERIMENTAL_HOOKS;
+        let output = new Hook[0]();
+        if (experimentHooks && experimentHooks !== '') {
+            try {
+                output = yaml_1.default.parse(experimentHooks);
+            }
+            catch (error) {
+                throw error;
+            }
+        }
+        return output;
+    }
 }
 exports.CloudRunnerBuildCommandProcessor = CloudRunnerBuildCommandProcessor;
+class Hook {
+    constructor() {
+        this.secrets = [];
+    }
+}
+exports.Hook = Hook;
 
 
 /***/ }),
@@ -2625,6 +2664,7 @@ exports.TaskParameterSerializer = void 0;
 const __1 = __webpack_require__(41359);
 const image_environment_factory_1 = __importDefault(__webpack_require__(25145));
 const cloud_runner_state_1 = __webpack_require__(70912);
+const cloud_runner_build_command_process_1 = __webpack_require__(71899);
 class TaskParameterSerializer {
     static readBuildEnvironmentVariables() {
         TaskParameterSerializer.setupDefaultSecrets();
@@ -2648,6 +2688,13 @@ class TaskParameterSerializer {
         let array = new Array();
         array = TaskParameterSerializer.readBuildParameters(array);
         array = TaskParameterSerializer.readInput(array);
+        const configurableHooks = cloud_runner_build_command_process_1.CloudRunnerBuildCommandProcessor.getHooks()
+            .map((x) => x.secrets)
+            // eslint-disable-next-line unicorn/no-array-reduce
+            .reduce((x, y) => 
+        // eslint-disable-next-line unicorn/prefer-spread
+        x.concat(y));
+        array.push(configurableHooks);
         array = array.filter((x) => x.value !== undefined && x.name !== '0' && x.value !== '' && x.name !== 'prototype' && x.name !== 'length');
         array = array.map((x) => {
             x.name = __1.Input.ToEnvVarFormat(x.name);

src/model/cloud-runner/services/cloud-runner-build-command-process.ts

@@ -1,14 +1,56 @@
 import { BuildParameters, Input } from '../..';
+import YAML from 'yaml';
+import CloudRunnerSecret from './cloud-runner-secret';
+import CloudRunnerLogger from './cloud-runner-logger';
 
 export class CloudRunnerBuildCommandProcessor {
   public static ProcessCommands(commands: string, buildParameters: BuildParameters): string {
+    let hooks = CloudRunnerBuildCommandProcessor.getHooks();
+
+    if (Input.cloudRunnerTests) {
+      CloudRunnerLogger.log(JSON.stringify(hooks, undefined, 4));
+    } else {
+      hooks = [];
+    }
+
     return `echo "---"
       echo "start cloud runner init"
       ${Input.cloudRunnerTests ? '' : '#'} printenv
       echo "start cloud runner job"
+      ${
+        hooks
+          .filter((x) => x.hook !== undefined && x.hook.length > 0 && x.hook.includes(`before`))
+          .map((x) => x.commands)
+          .join(`\n`) || ' '
+      }
       ${commands}
+      ${
+        hooks
+          .filter((x) => x.hook !== undefined && x.hook.length > 0 && x.hook.includes(`before`))
+          .map((x) => x.commands)
+          .join(`\n`) || ' '
+      }
       echo "end of cloud runner job
       ---${buildParameters.logId}"
     `;
   }
+
+  public static getHooks(): Hook[] {
+    const experimentHooks = process.env.EXPERIMENTAL_HOOKS;
+    let output = new Hook[0]();
+    if (experimentHooks && experimentHooks !== '') {
+      try {
+        output = YAML.parse(experimentHooks);
+      } catch (error) {
+        throw error;
+      }
+    }
+    return output;
+  }
+}
+export class Hook {
+  public commands;
+  public secrets: CloudRunnerSecret[] = [];
+  public name;
+  public hook;
 }

src/model/cloud-runner/services/task-parameter-serializer.ts

@@ -2,6 +2,7 @@ import { Input } from '../..';
 import ImageEnvironmentFactory from '../../image-environment-factory';
 import CloudRunnerEnvironmentVariable from './cloud-runner-environment-variable';
 import { CloudRunnerState } from '../state/cloud-runner-state';
+import { CloudRunnerBuildCommandProcessor } from './cloud-runner-build-command-process';
 
 export class TaskParameterSerializer {
   public static readBuildEnvironmentVariables(): CloudRunnerEnvironmentVariable[] {
@@ -26,6 +27,14 @@ export class TaskParameterSerializer {
     let array = new Array();
     array = TaskParameterSerializer.readBuildParameters(array);
     array = TaskParameterSerializer.readInput(array);
+    const configurableHooks = CloudRunnerBuildCommandProcessor.getHooks()
+      .map((x) => x.secrets)
+      // eslint-disable-next-line unicorn/no-array-reduce
+      .reduce((x, y) =>
+        // eslint-disable-next-line unicorn/prefer-spread
+        x.concat(y),
+      );
+    array.push(configurableHooks);
 
     array = array.filter(
       (x) => x.value !== undefined && x.name !== '0' && x.value !== '' && x.name !== 'prototype' && x.name !== 'length',