actions
/
unity-builder
mirror of https://github.com/game-ci/unity-builder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

podname logs for log service

pull/531/head
Frostebite 2023-08-14 18:59:05 +01:00
parent 14819a6338
commit 2019e730cb
7 changed files with 20 additions and 427 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

185
dist/index.js generated vendored
View File

@ -3262,7 +3262,6 @@ const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(42864));
const cloud_runner_1 = __importDefault(__nccwpck_require__(79144));
const remote_client_logger_1 = __nccwpck_require__(59412);
const kubernetes_role_1 = __nccwpck_require__(88231);
const kubernetes_log_service_1 = __importDefault(__nccwpck_require__(62552));
const cloud_runner_system_1 = __nccwpck_require__(4197);
class Kubernetes {
// eslint-disable-next-line no-unused-vars
@ -3370,8 +3369,6 @@ class Kubernetes {
await kubernetes_secret_1.default.createSecret(secrets, this.secretName, this.namespace, this.kubeClient);
let output = '';
try {
this.ip =
(await kubernetes_log_service_1.default.createLogDeployment(this.namespace, this.kubeClientApps, this.kubeClient)) || ``;
cloud_runner_logger_1.default.log('Job does not exist');
await this.createJob(commands, image, mountdir, workingdir, environment, secrets);
cloud_runner_logger_1.default.log('Watching pod until running');
@ -3428,7 +3425,6 @@ class Kubernetes {
cloud_runner_logger_1.default.log(`Build job created`);
await new Promise((promise) => setTimeout(promise, 5000));
cloud_runner_logger_1.default.log('Job created');
await kubernetes_log_service_1.default.cleanupLogDeployment(this.namespace, this.kubeClientApps, this.kubeClient);
return result.body.metadata?.name;
}
catch (error) {
@ -3447,7 +3443,6 @@ class Kubernetes {
await this.kubeClientBatch.deleteNamespacedJob(this.jobName, this.namespace);
await this.kubeClient.deleteNamespacedPod(this.podName, this.namespace);
await kubernetes_role_1.KubernetesRole.deleteRole(this.serviceAccountName, this.namespace, this.rbacAuthorizationV1Api);
await kubernetes_log_service_1.default.cleanupLogDeployment(this.namespace, this.kubeClientApps, this.kubeClient);
}
catch (error) {
cloud_runner_logger_1.default.log(`Failed to cleanup`);
@ -3612,172 +3607,6 @@ class KubernetesJobSpecFactory {
exports["default"] = KubernetesJobSpecFactory;
/***/ }),
/***/ 62552:
/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) {
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
const k8s = __importStar(__nccwpck_require__(89679));
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(42864));
const cloud_runner_system_1 = __nccwpck_require__(4197);
class KubernetesLogService {
static async cleanupLogDeployment(namespace, kubeClientApps, kubeClient) {
if (!process.env.LOG_SERVICE_IP) {
return Promise.resolve();
}
await kubeClient.deleteNamespacedService('http-fileserver', namespace);
await kubeClientApps.deleteNamespacedDeployment('http-fileserver', namespace);
}
static async createLogService(serviceAccountName, namespace, kubeClient) {
const serviceAccount = new k8s.V1ServiceAccount();
serviceAccount.apiVersion = 'v1';
serviceAccount.kind = 'ServiceAccount';
serviceAccount.metadata = {
name: serviceAccountName,
};
serviceAccount.automountServiceAccountToken = true;
return kubeClient.createNamespacedServiceAccount(namespace, serviceAccount);
}
static async createLogDeployment(namespace, kubeClient, kubeClientCore) {
if (!process.env.LOG_SERVICE_IP) {
return `0.0.0.0`;
}
// create a deployment with above json
const deployment = new k8s.V1Deployment();
deployment.apiVersion = 'apps/v1';
deployment.kind = 'Deployment';
deployment.metadata = {
name: 'http-fileserver',
labels: {
service: 'http-fileserver',
},
};
deployment.spec = {
selector: {
matchLabels: {
service: 'http-fileserver',
},
},
replicas: 1,
strategy: {},
template: {
metadata: {
labels: {
service: 'http-fileserver',
},
},
spec: {
containers: [
{
image: 'node:18',
imagePullPolicy: 'Always',
name: 'http-fileserver',
ports: [{ containerPort: 80 }],
command: [
'bash',
'-c',
'while true; do sleep 30; npm i files-upload-server -g; files-upload-server "downloads"; done;',
],
resources: {
requests: {
memory: '750M',
cpu: '0.25',
},
},
},
],
restartPolicy: 'Always',
},
},
};
await kubeClient.createNamespacedDeployment(namespace, deployment);
await this.createLogServiceExpose(namespace, kubeClientCore);
// wait in loop until serivce ip address is exposed
for (let index = 0; index < 10; index++) {
// wait for service to share ip address
await new Promise((resolve) => setTimeout(resolve, 10000));
// get cluster ip address of service
const service = await kubeClientCore.readNamespacedService('http-fileserver', namespace);
// get podname of deployment
const podname = await cloud_runner_system_1.CloudRunnerSystem.Run(`kubectl get pods -n ${namespace} -l service=http-fileserver -o jsonpath='{.items[0].metadata.name}'`, false, true);
// if status of pod is not running, then continue
const podStatus = await cloud_runner_system_1.CloudRunnerSystem.Run(`kubectl get pods -n ${namespace} ${podname} -o jsonpath='{.status.phase}'`, false, true);
if (podStatus !== 'Running') {
cloud_runner_logger_1.default.log(`Pod status: ${podStatus}`);
continue;
}
const logs = await cloud_runner_system_1.CloudRunnerSystem.Run(`kubectl logs ${podname} -f --timestamps -p`, false, true);
cloud_runner_logger_1.default.log(`Logs: ${logs}`);
// get cluster ip
const ip = service.body?.spec?.clusterIP;
if (ip && ip.length > 0) {
// log service json
cloud_runner_logger_1.default.log(`Service: ${JSON.stringify(service.body, undefined, 4)}`);
cloud_runner_logger_1.default.log(`Service IP: ${ip}`);
return ip;
}
}
}
// create kubernetes service to expose deployment
static async createLogServiceExpose(namespace, kubeClient) {
if (!process.env.LOG_SERVICE_IP) {
return;
}
// create a service with above json
const service = new k8s.V1Service();
service.apiVersion = 'v1';
service.kind = 'Service';
service.metadata = {
name: 'http-fileserver',
labels: {
service: 'http-fileserver',
},
};
service.spec = {
ports: [
{
name: '80-80',
port: 80,
protocol: 'TCP',
targetPort: 80,
},
],
selector: {
service: 'http-fileserver',
},
type: 'LoadBalancer',
};
await kubeClient.createNamespacedService(namespace, service);
}
}
exports["default"] = KubernetesLogService;
/***/ }),
/***/ 90740:
@ -4605,7 +4434,7 @@ class RemoteClient {
}
await RemoteClient.runCustomHookFiles(`after-build`);
// WIP - need to give the pod permissions to create config map
await remote_client_logger_1.RemoteClientLogger.printCollectedLogs();
await remote_client_logger_1.RemoteClientLogger.handleLogManagementPostJob();
return new Promise((result) => result(``));
}
static async runCustomHookFiles(hookLifecycle) {
@ -4756,7 +4585,6 @@ const node_fs_1 = __importDefault(__nccwpck_require__(87561));
const node_path_1 = __importDefault(__nccwpck_require__(49411));
const cloud_runner_1 = __importDefault(__nccwpck_require__(79144));
const cloud_runner_options_1 = __importDefault(__nccwpck_require__(66965));
const k8s_1 = __importDefault(__nccwpck_require__(21613));
class RemoteClientLogger {
static get LogFilePath() {
return node_path_1.default.join(`/home`, `job-log.txt`);
@ -4780,15 +4608,20 @@ class RemoteClientLogger {
node_fs_1.default.appendFileSync(RemoteClientLogger.LogFilePath, `${message}\n`);
}
}
static async printCollectedLogs() {
static async handleLogManagementPostJob() {
if (cloud_runner_options_1.default.providerStrategy !== 'k8s') {
return;
}
cloud_runner_logger_1.default.log(`Collected Logs`);
cloud_runner_logger_1.default.log(process.env[`LOG_SERVICE_IP`] || ``);
const hashedLogs = node_fs_1.default.readFileSync(RemoteClientLogger.LogFilePath).toString();
cloud_runner_logger_1.default.log(hashedLogs);
const logs = node_fs_1.default.readFileSync(RemoteClientLogger.LogFilePath).toString();
cloud_runner_logger_1.default.log(logs);
await k8s_1.default.Instance.PushLogUpdate(logs);
// loop for 5 mins logging the logs every minute
for (let index = 0; index < 5; index++) {
await new Promise((resolve) => setTimeout(resolve, 60000));
cloud_runner_logger_1.default.log(logs);
}
}
}
exports.RemoteClientLogger = RemoteClientLogger;

2
dist/index.js.map generated vendored
View File

File diff suppressed because one or more lines are too long

5
src/model/cloud-runner/providers/k8s/index.ts
View File

@ -16,7 +16,6 @@ import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow';
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
import { KubernetesRole } from './kubernetes-role';
import KubernetesLogService from './kubernetes-log-service';
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
class Kubernetes implements ProviderInterface {
@ -159,8 +158,6 @@ class Kubernetes implements ProviderInterface {
await KubernetesSecret.createSecret(secrets, this.secretName, this.namespace, this.kubeClient);
let output = '';
try {
this.ip =
(await KubernetesLogService.createLogDeployment(this.namespace, this.kubeClientApps, this.kubeClient)) || ``;
CloudRunnerLogger.log('Job does not exist');
await this.createJob(commands, image, mountdir, workingdir, environment, secrets);
CloudRunnerLogger.log('Watching pod until running');
@ -271,7 +268,6 @@ class Kubernetes implements ProviderInterface {
CloudRunnerLogger.log(`Build job created`);
await new Promise((promise) => setTimeout(promise, 5000));
CloudRunnerLogger.log('Job created');
await KubernetesLogService.cleanupLogDeployment(this.namespace, this.kubeClientApps, this.kubeClient);
return result.body.metadata?.name;
} catch (error) {
@ -292,7 +288,6 @@ class Kubernetes implements ProviderInterface {
await this.kubeClientBatch.deleteNamespacedJob(this.jobName, this.namespace);
await this.kubeClient.deleteNamespacedPod(this.podName, this.namespace);
await KubernetesRole.deleteRole(this.serviceAccountName, this.namespace, this.rbacAuthorizationV1Api);
await KubernetesLogService.cleanupLogDeployment(this.namespace, this.kubeClientApps, this.kubeClient);
} catch (error: any) {
CloudRunnerLogger.log(`Failed to cleanup`);
if (error.response.body.reason !== `NotFound`) {

158
src/model/cloud-runner/providers/k8s/kubernetes-log-service.ts
View File

@ -1,158 +0,0 @@
import { CoreV1Api } from '@kubernetes/client-node';
import * as k8s from '@kubernetes/client-node';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
class KubernetesLogService {
static async cleanupLogDeployment(namespace: string, kubeClientApps: k8s.AppsV1Api, kubeClient: CoreV1Api) {
if (!process.env.LOG_SERVICE_IP) {
return Promise.resolve();
}
await kubeClient.deleteNamespacedService('http-fileserver', namespace);
await kubeClientApps.deleteNamespacedDeployment('http-fileserver', namespace);
}
static async createLogService(serviceAccountName: string, namespace: string, kubeClient: CoreV1Api) {
const serviceAccount = new k8s.V1ServiceAccount();
serviceAccount.apiVersion = 'v1';
serviceAccount.kind = 'ServiceAccount';
serviceAccount.metadata = {
name: serviceAccountName,
};
serviceAccount.automountServiceAccountToken = true;
return kubeClient.createNamespacedServiceAccount(namespace, serviceAccount);
}
static async createLogDeployment(namespace: string, kubeClient: k8s.AppsV1Api, kubeClientCore: CoreV1Api) {
if (!process.env.LOG_SERVICE_IP) {
return `0.0.0.0`;
}
// create a deployment with above json
const deployment = new k8s.V1Deployment();
deployment.apiVersion = 'apps/v1';
deployment.kind = 'Deployment';
deployment.metadata = {
name: 'http-fileserver',
labels: {
service: 'http-fileserver',
},
};
deployment.spec = {
selector: {
matchLabels: {
service: 'http-fileserver',
},
},
replicas: 1,
strategy: {},
template: {
metadata: {
labels: {
service: 'http-fileserver',
},
},
spec: {
containers: [
{
image: 'node:18',
imagePullPolicy: 'Always',
name: 'http-fileserver',
ports: [{ containerPort: 80 }],
command: [
'bash',
'-c',
'while true; do sleep 30; npm i files-upload-server -g; files-upload-server "downloads"; done;',
],
resources: {
requests: {
memory: '750M',
cpu: '0.25',
},
},
},
],
restartPolicy: 'Always',
},
},
};
await kubeClient.createNamespacedDeployment(namespace, deployment);
await this.createLogServiceExpose(namespace, kubeClientCore);
// wait in loop until serivce ip address is exposed
for (let index = 0; index < 10; index++) {
// wait for service to share ip address
await new Promise((resolve) => setTimeout(resolve, 10000));
// get cluster ip address of service
const service = await kubeClientCore.readNamespacedService('http-fileserver', namespace);
// get podname of deployment
const podname = await CloudRunnerSystem.Run(
`kubectl get pods -n ${namespace} -l service=http-fileserver -o jsonpath='{.items[0].metadata.name}'`,
false,
true,
);
// if status of pod is not running, then continue
const podStatus = await CloudRunnerSystem.Run(
`kubectl get pods -n ${namespace} ${podname} -o jsonpath='{.status.phase}'`,
false,
true,
);
if (podStatus !== 'Running') {
CloudRunnerLogger.log(`Pod status: ${podStatus}`);
continue;
}
const logs = await CloudRunnerSystem.Run(`kubectl logs ${podname} -f --timestamps -p`, false, true);
CloudRunnerLogger.log(`Logs: ${logs}`);
// get cluster ip
const ip = service.body?.spec?.clusterIP;
if (ip && ip.length > 0) {
// log service json
CloudRunnerLogger.log(`Service: ${JSON.stringify(service.body, undefined, 4)}`);
CloudRunnerLogger.log(`Service IP: ${ip}`);
return ip;
}
}
}
// create kubernetes service to expose deployment
static async createLogServiceExpose(namespace: string, kubeClient: CoreV1Api) {
if (!process.env.LOG_SERVICE_IP) {
return;
}
// create a service with above json
const service = new k8s.V1Service();
service.apiVersion = 'v1';
service.kind = 'Service';
service.metadata = {
name: 'http-fileserver',
labels: {
service: 'http-fileserver',
},
};
service.spec = {
ports: [
{
name: '80-80',
port: 80,
protocol: 'TCP',
targetPort: 80,
},
],
selector: {
service: 'http-fileserver',
},
type: 'LoadBalancer',
};
await kubeClient.createNamespacedService(namespace, service);
}
}
export default KubernetesLogService;

2
src/model/cloud-runner/remote-client/index.ts
View File

@ -83,7 +83,7 @@ export class RemoteClient {
await RemoteClient.runCustomHookFiles(`after-build`);
// WIP - need to give the pod permissions to create config map
await RemoteClientLogger.printCollectedLogs();
await RemoteClientLogger.handleLogManagementPostJob();
return new Promise((result) => result(``));
}

13
src/model/cloud-runner/remote-client/remote-client-logger.ts
View File

@ -3,7 +3,6 @@ import fs from 'node:fs';
import path from 'node:path';
import CloudRunner from '../cloud-runner';
import CloudRunnerOptions from '../options/cloud-runner-options';
import Kubernetes from '../providers/k8s';
export class RemoteClientLogger {
private static get LogFilePath() {
@ -34,14 +33,20 @@ export class RemoteClientLogger {
}
}
public static async printCollectedLogs() {
public static async handleLogManagementPostJob() {
if (CloudRunnerOptions.providerStrategy !== 'k8s') {
return;
}
CloudRunnerLogger.log(`Collected Logs`);
CloudRunnerLogger.log(process.env[`LOG_SERVICE_IP`] || ``);
const hashedLogs = fs.readFileSync(RemoteClientLogger.LogFilePath).toString();
CloudRunnerLogger.log(hashedLogs);
const logs = fs.readFileSync(RemoteClientLogger.LogFilePath).toString();
CloudRunnerLogger.log(logs);
await Kubernetes.Instance.PushLogUpdate(logs);
// loop for 5 mins logging the logs every minute
for (let index = 0; index < 5; index++) {
await new Promise((resolve) => setTimeout(resolve, 60000));
CloudRunnerLogger.log(logs);
}
}
}

82
src/model/cloud-runner/tests/cloud-runner-kubernetes.test.ts
View File

@ -1,82 +0,0 @@
import BuildParameters from '../../build-parameters';
import { Cli } from '../../cli/cli';
import UnityVersioning from '../../unity-versioning';
import CloudRunner from '../cloud-runner';
import CloudRunnerOptions from '../options/cloud-runner-options';
import KubernetesLogService from '../providers/k8s/kubernetes-log-service';
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
import setups from './cloud-runner-suite.test';
import { v4 as uuidv4 } from 'uuid';
import * as k8s from '@kubernetes/client-node';
import ImageTag from '../../image-tag';
async function CreateParameters(overrides: any) {
if (overrides) {
Cli.options = overrides;
}
return await BuildParameters.create();
}
describe('Cloud Runner Kubernetes', () => {
it('Responds', () => {});
setups();
if (CloudRunnerOptions.cloudRunnerDebug) {
it('Build create log service', async () => {
const overrides = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
customJob: `
- name: 'step 1'
image: 'ubuntu'
commands: 'curl http://$LOG_SERVICE_IP:80''`,
};
if (CloudRunnerOptions.providerStrategy !== `k8s`) {
return;
}
const buildParameter = await CreateParameters(overrides);
expect(buildParameter.projectPath).toEqual(overrides.projectPath);
await CloudRunner.setup(buildParameter);
const kubeConfig = new k8s.KubeConfig();
kubeConfig.loadFromDefault();
const kubeClient = kubeConfig.makeApiClient(k8s.CoreV1Api);
const kubeClientApps = kubeConfig.makeApiClient(k8s.AppsV1Api);
await KubernetesLogService.createLogDeployment('test', kubeClientApps, kubeClient);
CloudRunnerLogger.log(`run 1 succeeded`);
}, 1_000_000_000);
it('curl log service', async () => {
const overrides = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.read('test-project'),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
customJob: `
- name: 'step 1'
image: 'ubuntu'
commands: |
apt-get update
apt-get install curl -y
curl http://$LOG_SERVICE_IP:80`,
};
if (CloudRunnerOptions.providerStrategy !== `k8s`) {
return;
}
const buildParameter = await CreateParameters(overrides);
expect(buildParameter.projectPath).toEqual(overrides.projectPath);
const baseImage = new ImageTag(buildParameter);
const results = await CloudRunner.run(buildParameter, baseImage.toString());
const buildSucceededString = 'Build succeeded';
expect(results).toContain(buildSucceededString);
CloudRunnerLogger.log(`run 1 succeeded`);
}, 1_000_000_000);
}
});