import * as k8s from '@kubernetes/client-node'; import { BuildParameters } from '..'; import * as core from '@actions/core'; import { KubeConfig, Log } from '@kubernetes/client-node'; import { Writable } from 'stream'; import { RemoteBuilderProviderInterface } from './remote-builder-provider-interface'; import RemoteBuilderSecret from './remote-builder-secret'; import { waitUntil } from 'async-wait-until'; import KubernetesStorage from './kubernetes-storage'; import RemoteBuilderEnvironmentVariable from './remote-builder-environment-variable'; const base64 = require('base-64'); class Kubernetes implements RemoteBuilderProviderInterface { private kubeConfig: KubeConfig; private kubeClient: k8s.CoreV1Api; private kubeClientBatch: k8s.BatchV1Api; private buildId: string = ''; private buildCorrelationId: string = ''; private buildParameters: BuildParameters; private baseImage: any; private pvcName: string = ''; private secretName: string = ''; private jobName: string = ''; private namespace: string; private podName: string = ''; private containerName: string = ''; constructor(buildParameters: BuildParameters, baseImage) { const kc = new k8s.KubeConfig(); kc.loadFromDefault(); const k8sApi = kc.makeApiClient(k8s.CoreV1Api); const k8sBatchApi = kc.makeApiClient(k8s.BatchV1Api); core.info('Loaded default Kubernetes configuration for this environment'); this.kubeConfig = kc; this.kubeClient = k8sApi; this.kubeClientBatch = k8sBatchApi; this.buildCorrelationId = Kubernetes.uuidv4(); this.namespace = 'default'; this.buildParameters = buildParameters; this.baseImage = baseImage; } async runBuildTask( buildId: string, stackName: string, image: string, commands: string[], mountdir: string, workingdir: string, environment: RemoteBuilderEnvironmentVariable[], secrets: RemoteBuilderSecret[], ): Promise { try { this.setUniqueBuildId(); const defaultSecretsArray: RemoteBuilderSecret[] = [ { ParameterKey: 'GithubToken', EnvironmentVariable: 'GITHUB_TOKEN', ParameterValue: this.buildParameters.githubToken, }, { ParameterKey: 'UNITY_LICENSE', EnvironmentVariable: 'UNITY_LICENSE', ParameterValue: process.env.UNITY_LICENSE || '', }, { ParameterKey: 'ANDROID_KEYSTORE_BASE64', EnvironmentVariable: 'ANDROID_KEYSTORE_BASE64', ParameterValue: this.buildParameters.androidKeystoreBase64, }, { ParameterKey: 'ANDROID_KEYSTORE_PASS', EnvironmentVariable: 'ANDROID_KEYSTORE_PASS', ParameterValue: this.buildParameters.androidKeystorePass, }, { ParameterKey: 'ANDROID_KEYALIAS_PASS', EnvironmentVariable: 'ANDROID_KEYALIAS_PASS', ParameterValue: this.buildParameters.androidKeyaliasPass, }, ]; defaultSecretsArray.push(...secrets); // setup await this.createSecret(defaultSecretsArray); await KubernetesStorage.createPersistentVolumeClaim( this.buildParameters, this.pvcName, this.kubeClient, this.namespace, ); //run const jobSpec = this.getJobSpec(commands, image); core.info('Creating build job'); await this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec); core.info('Job created'); await KubernetesStorage.watchUntilPVCNotPending(this.kubeClient, this.pvcName, this.namespace); core.info('PVC Bound'); this.setPodNameAndContainerName(await this.findPod()); core.info('Watching pod until running'); await this.watchUntilPodRunning(); core.info('Pod running, streaming logs'); await this.streamLogs(); await this.cleanup(); } catch (error) { core.info('Running job failed'); await this.cleanup(); throw error; } } setUniqueBuildId() { const buildId = Kubernetes.uuidv4(); const pvcName = `unity-builder-pvc-${buildId}`; const secretName = `build-credentials-${buildId}`; const jobName = `unity-builder-job-${buildId}`; this.buildId = buildId; this.pvcName = pvcName; this.secretName = secretName; this.jobName = jobName; } async runFullBuildFlow() { core.info('Running Remote Builder on Kubernetes'); try { await this.runCloneJob(); await this.runBuildJob(); } catch (error) { core.error(error); core.error(JSON.stringify(error.response, undefined, 4)); throw error; } core.setOutput('volume', this.pvcName); } async createSecret(secrets: RemoteBuilderSecret[]) { const secret = new k8s.V1Secret(); secret.apiVersion = 'v1'; secret.kind = 'Secret'; secret.type = 'Opaque'; secret.metadata = { name: this.secretName, }; secret.data = {}; for (const buildSecret of secrets) { secret.data[buildSecret.EnvironmentVariable] = base64.encode(buildSecret.ParameterValue); secret.data[`${buildSecret.EnvironmentVariable}_NAME`] = base64.encode(buildSecret.ParameterKey); } try { await this.kubeClient.createNamespacedSecret(this.namespace, secret); } catch (error) { throw error; } } getJobSpec(command: string[], image: string) { const job = new k8s.V1Job(); job.apiVersion = 'batch/v1'; job.kind = 'Job'; job.metadata = { name: this.jobName, labels: { app: 'unity-builder', }, }; job.spec = { template: { spec: { volumes: [ { name: 'data', persistentVolumeClaim: { claimName: this.pvcName, }, }, { name: 'credentials', secret: { secretName: this.secretName, }, }, ], containers: [ { name: 'main', image, command, resources: { requests: { memory: this.buildParameters.remoteBuildMemory, cpu: this.buildParameters.remoteBuildCpu, }, }, env: [ { name: 'GITHUB_SHA', value: this.buildId, }, { name: 'GITHUB_WORKSPACE', value: '/data/repo', }, { name: 'PROJECT_PATH', value: this.buildParameters.projectPath, }, { name: 'BUILD_PATH', value: this.buildParameters.buildPath, }, { name: 'BUILD_FILE', value: this.buildParameters.buildFile, }, { name: 'BUILD_NAME', value: this.buildParameters.buildName, }, { name: 'BUILD_METHOD', value: this.buildParameters.buildMethod, }, { name: 'CUSTOM_PARAMETERS', value: this.buildParameters.customParameters, }, { name: 'CHOWN_FILES_TO', value: this.buildParameters.chownFilesTo, }, { name: 'BUILD_TARGET', value: this.buildParameters.platform, }, { name: 'ANDROID_VERSION_CODE', value: this.buildParameters.androidVersionCode.toString(), }, { name: 'ANDROID_KEYSTORE_NAME', value: this.buildParameters.androidKeystoreName, }, { name: 'ANDROID_KEYALIAS_NAME', value: this.buildParameters.androidKeyaliasName, }, ], volumeMounts: [ { name: 'data', mountPath: '/data', }, { name: 'credentials', mountPath: '/credentials', readOnly: true, }, ], lifecycle: { preStop: { exec: { command: [ 'bin/bash', '-c', `cd /data/builder/action/steps; chmod +x /return_license.sh; /return_license.sh;`, ], }, }, }, }, ], restartPolicy: 'Never', }, }, }; job.spec.backoffLimit = 1; return job; } async runJobInKubernetesPod(command: string[], image: string) { try { this.setUniqueBuildId(); const defaultSecretsArray: RemoteBuilderSecret[] = [ { ParameterKey: 'GithubToken', EnvironmentVariable: 'GITHUB_TOKEN', ParameterValue: this.buildParameters.githubToken, }, { ParameterKey: 'UNITY_LICENSE', EnvironmentVariable: 'UNITY_LICENSE', ParameterValue: process.env.UNITY_LICENSE || '', }, { ParameterKey: 'ANDROID_KEYSTORE_BASE64', EnvironmentVariable: 'ANDROID_KEYSTORE_BASE64', ParameterValue: this.buildParameters.androidKeystoreBase64, }, { ParameterKey: 'ANDROID_KEYSTORE_PASS', EnvironmentVariable: 'ANDROID_KEYSTORE_PASS', ParameterValue: this.buildParameters.androidKeystorePass, }, { ParameterKey: 'ANDROID_KEYALIAS_PASS', EnvironmentVariable: 'ANDROID_KEYALIAS_PASS', ParameterValue: this.buildParameters.androidKeyaliasPass, }, ]; // setup await this.createSecret(defaultSecretsArray); await KubernetesStorage.createPersistentVolumeClaim( this.buildParameters, this.pvcName, this.kubeClient, this.namespace, ); //run const jobSpec = this.getJobSpec(command, image); core.info('Creating build job'); await this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec); core.info('Job created'); await KubernetesStorage.watchUntilPVCNotPending(this.kubeClient, this.pvcName, this.namespace); core.info('PVC Bound'); this.setPodNameAndContainerName(await this.findPod()); core.info('Watching pod until running'); await this.watchUntilPodRunning(); core.info('Pod running, streaming logs'); await this.streamLogs(); await this.cleanup(); } catch (error) { core.info('Running job failed'); await this.cleanup(); throw error; } } async findPod() { const pod = (await this.kubeClient.listNamespacedPod(this.namespace)).body.items.find( (x) => x.metadata?.labels?.['job-name'] === this.jobName, ); if (pod === undefined) { throw new Error("pod with job-name label doesn't exist"); } return pod; } async runCloneJob() { await this.runBuildTask( this.buildCorrelationId, '', 'alpine/git', [ '/bin/ash', '-c', `apk update; apk add unzip; apk add git-lfs; apk add jq; ls /credentials/ export GITHUB_TOKEN=$(cat /credentials/GITHUB_TOKEN); cd /data; git clone https://github.com/${process.env.GITHUB_REPOSITORY}.git repo; git clone https://github.com/webbertakken/unity-builder.git builder; cd repo; git checkout $GITHUB_SHA; ls echo "end"`, ], '', '', [], [], ); } async runBuildJob() { await this.runBuildTask( this.buildCorrelationId, '', this.baseImage.toString(), [ 'bin/bash', '-c', `ls for f in ./credentials/*; do export $(basename $f)="$(cat $f)"; done ls /data ls /data/builder ls /data/builder/dist cp -r /data/builder/dist/default-build-script /UnityBuilderAction cp -r /data/builder/dist/entrypoint.sh /entrypoint.sh cp -r /data/builder/dist/steps /steps chmod -R +x /entrypoint.sh chmod -R +x /steps /entrypoint.sh `, ], '', '', [], [], ); } async watchUntilPodRunning() { let success: boolean = false; core.info(`Watching ${this.podName} ${this.namespace}`); await waitUntil( async () => { const phase = (await this.kubeClient.readNamespacedPodStatus(this.podName, this.namespace))?.body.status?.phase; success = phase === 'Running'; if (success || phase !== 'Pending') return true; return false; }, { timeout: 500000, intervalBetweenAttempts: 15000, }, ); return success; } setPodNameAndContainerName(pod: k8s.V1Pod) { this.podName = pod.metadata?.name || ''; this.containerName = pod.status?.containerStatuses?.[0].name || ''; } async streamLogs() { core.info(`Streaming logs from pod: ${this.podName} container: ${this.containerName} namespace: ${this.namespace}`); const stream = new Writable(); stream._write = (chunk, encoding, next) => { core.info(chunk.toString()); next(); }; const logOptions = { follow: true, pretty: true, previous: true, }; try { const resultError = await new Promise((resolve) => new Log(this.kubeConfig).log(this.namespace, this.podName, this.containerName, stream, resolve, logOptions), ); if (resultError) { throw resultError; } } catch (error) { throw error; } core.info('end of log stream'); } async cleanup() { core.info('cleaning up'); try { await this.kubeClientBatch.deleteNamespacedJob(this.jobName, this.namespace); await this.kubeClient.deleteNamespacedPersistentVolumeClaim(this.pvcName, this.namespace); await this.kubeClient.deleteNamespacedSecret(this.secretName, this.namespace); } catch (error) { core.info('Failed to cleanup, error:'); core.error(JSON.stringify(error, undefined, 4)); core.info('Abandoning cleanup, build error:'); } } static uuidv4() { return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => { const r = Math.trunc(Math.random() * 16); const v = c === 'x' ? r : (r & 0x3) | 0x8; return v.toString(16); }); } } export default Kubernetes;