podname logs for log service

dist/index.js

@@ -3262,7 +3262,6 @@ const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(42864));
 const cloud_runner_1 = __importDefault(__nccwpck_require__(79144));
 const remote_client_logger_1 = __nccwpck_require__(59412);
 const kubernetes_role_1 = __nccwpck_require__(88231);
-const kubernetes_log_service_1 = __importDefault(__nccwpck_require__(62552));
 const cloud_runner_system_1 = __nccwpck_require__(4197);
 class Kubernetes {
     // eslint-disable-next-line no-unused-vars
@@ -3370,8 +3369,6 @@ class Kubernetes {
             await kubernetes_secret_1.default.createSecret(secrets, this.secretName, this.namespace, this.kubeClient);
             let output = '';
             try {
-                this.ip =
-                    (await kubernetes_log_service_1.default.createLogDeployment(this.namespace, this.kubeClientApps, this.kubeClient)) || ``;
                 cloud_runner_logger_1.default.log('Job does not exist');
                 await this.createJob(commands, image, mountdir, workingdir, environment, secrets);
                 cloud_runner_logger_1.default.log('Watching pod until running');
@@ -3428,7 +3425,6 @@ class Kubernetes {
                 cloud_runner_logger_1.default.log(`Build job created`);
                 await new Promise((promise) => setTimeout(promise, 5000));
                 cloud_runner_logger_1.default.log('Job created');
-                await kubernetes_log_service_1.default.cleanupLogDeployment(this.namespace, this.kubeClientApps, this.kubeClient);
                 return result.body.metadata?.name;
             }
             catch (error) {
@@ -3447,7 +3443,6 @@ class Kubernetes {
             await this.kubeClientBatch.deleteNamespacedJob(this.jobName, this.namespace);
             await this.kubeClient.deleteNamespacedPod(this.podName, this.namespace);
             await kubernetes_role_1.KubernetesRole.deleteRole(this.serviceAccountName, this.namespace, this.rbacAuthorizationV1Api);
-            await kubernetes_log_service_1.default.cleanupLogDeployment(this.namespace, this.kubeClientApps, this.kubeClient);
         }
         catch (error) {
             cloud_runner_logger_1.default.log(`Failed to cleanup`);
@@ -3612,172 +3607,6 @@ class KubernetesJobSpecFactory {
 exports["default"] = KubernetesJobSpecFactory;
 
 
-/***/ }),
-
-/***/ 62552:
-/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) {
-
-"use strict";
-
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", ({ value: true }));
-const k8s = __importStar(__nccwpck_require__(89679));
-const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(42864));
-const cloud_runner_system_1 = __nccwpck_require__(4197);
-class KubernetesLogService {
-    static async cleanupLogDeployment(namespace, kubeClientApps, kubeClient) {
-        if (!process.env.LOG_SERVICE_IP) {
-            return Promise.resolve();
-        }
-        await kubeClient.deleteNamespacedService('http-fileserver', namespace);
-        await kubeClientApps.deleteNamespacedDeployment('http-fileserver', namespace);
-    }
-    static async createLogService(serviceAccountName, namespace, kubeClient) {
-        const serviceAccount = new k8s.V1ServiceAccount();
-        serviceAccount.apiVersion = 'v1';
-        serviceAccount.kind = 'ServiceAccount';
-        serviceAccount.metadata = {
-            name: serviceAccountName,
-        };
-        serviceAccount.automountServiceAccountToken = true;
-        return kubeClient.createNamespacedServiceAccount(namespace, serviceAccount);
-    }
-    static async createLogDeployment(namespace, kubeClient, kubeClientCore) {
-        if (!process.env.LOG_SERVICE_IP) {
-            return `0.0.0.0`;
-        }
-        // create a deployment with above json
-        const deployment = new k8s.V1Deployment();
-        deployment.apiVersion = 'apps/v1';
-        deployment.kind = 'Deployment';
-        deployment.metadata = {
-            name: 'http-fileserver',
-            labels: {
-                service: 'http-fileserver',
-            },
-        };
-        deployment.spec = {
-            selector: {
-                matchLabels: {
-                    service: 'http-fileserver',
-                },
-            },
-            replicas: 1,
-            strategy: {},
-            template: {
-                metadata: {
-                    labels: {
-                        service: 'http-fileserver',
-                    },
-                },
-                spec: {
-                    containers: [
-                        {
-                            image: 'node:18',
-                            imagePullPolicy: 'Always',
-                            name: 'http-fileserver',
-                            ports: [{ containerPort: 80 }],
-                            command: [
-                                'bash',
-                                '-c',
-                                'while true; do sleep 30; npm i files-upload-server -g; files-upload-server "downloads"; done;',
-                            ],
-                            resources: {
-                                requests: {
-                                    memory: '750M',
-                                    cpu: '0.25',
-                                },
-                            },
-                        },
-                    ],
-                    restartPolicy: 'Always',
-                },
-            },
-        };
-        await kubeClient.createNamespacedDeployment(namespace, deployment);
-        await this.createLogServiceExpose(namespace, kubeClientCore);
-        // wait in loop until serivce ip address is exposed
-        for (let index = 0; index < 10; index++) {
-            // wait for service to share ip address
-            await new Promise((resolve) => setTimeout(resolve, 10000));
-            // get cluster ip address of service
-            const service = await kubeClientCore.readNamespacedService('http-fileserver', namespace);
-            // get podname of deployment
-            const podname = await cloud_runner_system_1.CloudRunnerSystem.Run(`kubectl get pods -n ${namespace} -l service=http-fileserver -o jsonpath='{.items[0].metadata.name}'`, false, true);
-            // if status of pod is not running, then continue
-            const podStatus = await cloud_runner_system_1.CloudRunnerSystem.Run(`kubectl get pods -n ${namespace} ${podname} -o jsonpath='{.status.phase}'`, false, true);
-            if (podStatus !== 'Running') {
-                cloud_runner_logger_1.default.log(`Pod status: ${podStatus}`);
-                continue;
-            }
-            const logs = await cloud_runner_system_1.CloudRunnerSystem.Run(`kubectl logs ${podname} -f --timestamps -p`, false, true);
-            cloud_runner_logger_1.default.log(`Logs: ${logs}`);
-            // get cluster ip
-            const ip = service.body?.spec?.clusterIP;
-            if (ip && ip.length > 0) {
-                // log service json
-                cloud_runner_logger_1.default.log(`Service: ${JSON.stringify(service.body, undefined, 4)}`);
-                cloud_runner_logger_1.default.log(`Service IP: ${ip}`);
-                return ip;
-            }
-        }
-    }
-    // create kubernetes service to expose deployment
-    static async createLogServiceExpose(namespace, kubeClient) {
-        if (!process.env.LOG_SERVICE_IP) {
-            return;
-        }
-        // create a service with above json
-        const service = new k8s.V1Service();
-        service.apiVersion = 'v1';
-        service.kind = 'Service';
-        service.metadata = {
-            name: 'http-fileserver',
-            labels: {
-                service: 'http-fileserver',
-            },
-        };
-        service.spec = {
-            ports: [
-                {
-                    name: '80-80',
-                    port: 80,
-                    protocol: 'TCP',
-                    targetPort: 80,
-                },
-            ],
-            selector: {
-                service: 'http-fileserver',
-            },
-            type: 'LoadBalancer',
-        };
-        await kubeClient.createNamespacedService(namespace, service);
-    }
-}
-exports["default"] = KubernetesLogService;
-
-
 /***/ }),
 
 /***/ 90740:
@@ -4605,7 +4434,7 @@ class RemoteClient {
         }
         await RemoteClient.runCustomHookFiles(`after-build`);
         // WIP - need to give the pod permissions to create config map
-        await remote_client_logger_1.RemoteClientLogger.printCollectedLogs();
+        await remote_client_logger_1.RemoteClientLogger.handleLogManagementPostJob();
         return new Promise((result) => result(``));
     }
     static async runCustomHookFiles(hookLifecycle) {
@@ -4756,7 +4585,6 @@ const node_fs_1 = __importDefault(__nccwpck_require__(87561));
 const node_path_1 = __importDefault(__nccwpck_require__(49411));
 const cloud_runner_1 = __importDefault(__nccwpck_require__(79144));
 const cloud_runner_options_1 = __importDefault(__nccwpck_require__(66965));
-const k8s_1 = __importDefault(__nccwpck_require__(21613));
 class RemoteClientLogger {
     static get LogFilePath() {
         return node_path_1.default.join(`/home`, `job-log.txt`);
@@ -4780,15 +4608,20 @@ class RemoteClientLogger {
             node_fs_1.default.appendFileSync(RemoteClientLogger.LogFilePath, `${message}\n`);
         }
     }
-    static async printCollectedLogs() {
+    static async handleLogManagementPostJob() {
         if (cloud_runner_options_1.default.providerStrategy !== 'k8s') {
             return;
         }
         cloud_runner_logger_1.default.log(`Collected Logs`);
-        cloud_runner_logger_1.default.log(process.env[`LOG_SERVICE_IP`] || ``);
+        const hashedLogs = node_fs_1.default.readFileSync(RemoteClientLogger.LogFilePath).toString();
+        cloud_runner_logger_1.default.log(hashedLogs);
         const logs = node_fs_1.default.readFileSync(RemoteClientLogger.LogFilePath).toString();
         cloud_runner_logger_1.default.log(logs);
-        await k8s_1.default.Instance.PushLogUpdate(logs);
+        // loop for 5 mins logging the logs every minute
+        for (let index = 0; index < 5; index++) {
+            await new Promise((resolve) => setTimeout(resolve, 60000));
+            cloud_runner_logger_1.default.log(logs);
+        }
     }
 }
 exports.RemoteClientLogger = RemoteClientLogger;

src/model/cloud-runner/providers/k8s/index.ts

@@ -16,7 +16,6 @@ import { ProviderResource } from '../provider-resource';
 import { ProviderWorkflow } from '../provider-workflow';
 import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
 import { KubernetesRole } from './kubernetes-role';
-import KubernetesLogService from './kubernetes-log-service';
 import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
 
 class Kubernetes implements ProviderInterface {
@@ -159,8 +158,6 @@ class Kubernetes implements ProviderInterface {
       await KubernetesSecret.createSecret(secrets, this.secretName, this.namespace, this.kubeClient);
       let output = '';
       try {
-        this.ip =
-          (await KubernetesLogService.createLogDeployment(this.namespace, this.kubeClientApps, this.kubeClient)) || ``;
         CloudRunnerLogger.log('Job does not exist');
         await this.createJob(commands, image, mountdir, workingdir, environment, secrets);
         CloudRunnerLogger.log('Watching pod until running');
@@ -271,7 +268,6 @@ class Kubernetes implements ProviderInterface {
         CloudRunnerLogger.log(`Build job created`);
         await new Promise((promise) => setTimeout(promise, 5000));
         CloudRunnerLogger.log('Job created');
-        await KubernetesLogService.cleanupLogDeployment(this.namespace, this.kubeClientApps, this.kubeClient);
 
         return result.body.metadata?.name;
       } catch (error) {
@@ -292,7 +288,6 @@ class Kubernetes implements ProviderInterface {
       await this.kubeClientBatch.deleteNamespacedJob(this.jobName, this.namespace);
       await this.kubeClient.deleteNamespacedPod(this.podName, this.namespace);
       await KubernetesRole.deleteRole(this.serviceAccountName, this.namespace, this.rbacAuthorizationV1Api);
-      await KubernetesLogService.cleanupLogDeployment(this.namespace, this.kubeClientApps, this.kubeClient);
     } catch (error: any) {
       CloudRunnerLogger.log(`Failed to cleanup`);
       if (error.response.body.reason !== `NotFound`) {

src/model/cloud-runner/providers/k8s/kubernetes-log-service.ts

@@ -1,158 +0,0 @@
-import { CoreV1Api } from '@kubernetes/client-node';
-import * as k8s from '@kubernetes/client-node';
-import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
-import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
-class KubernetesLogService {
-  static async cleanupLogDeployment(namespace: string, kubeClientApps: k8s.AppsV1Api, kubeClient: CoreV1Api) {
-    if (!process.env.LOG_SERVICE_IP) {
-      return Promise.resolve();
-    }
-    await kubeClient.deleteNamespacedService('http-fileserver', namespace);
-    await kubeClientApps.deleteNamespacedDeployment('http-fileserver', namespace);
-  }
-
-  static async createLogService(serviceAccountName: string, namespace: string, kubeClient: CoreV1Api) {
-    const serviceAccount = new k8s.V1ServiceAccount();
-    serviceAccount.apiVersion = 'v1';
-    serviceAccount.kind = 'ServiceAccount';
-    serviceAccount.metadata = {
-      name: serviceAccountName,
-    };
-    serviceAccount.automountServiceAccountToken = true;
-
-    return kubeClient.createNamespacedServiceAccount(namespace, serviceAccount);
-  }
-
-  static async createLogDeployment(namespace: string, kubeClient: k8s.AppsV1Api, kubeClientCore: CoreV1Api) {
-    if (!process.env.LOG_SERVICE_IP) {
-      return `0.0.0.0`;
-    }
-
-    // create a deployment with above json
-    const deployment = new k8s.V1Deployment();
-    deployment.apiVersion = 'apps/v1';
-    deployment.kind = 'Deployment';
-    deployment.metadata = {
-      name: 'http-fileserver',
-      labels: {
-        service: 'http-fileserver',
-      },
-    };
-    deployment.spec = {
-      selector: {
-        matchLabels: {
-          service: 'http-fileserver',
-        },
-      },
-      replicas: 1,
-      strategy: {},
-      template: {
-        metadata: {
-          labels: {
-            service: 'http-fileserver',
-          },
-        },
-        spec: {
-          containers: [
-            {
-              image: 'node:18',
-              imagePullPolicy: 'Always',
-              name: 'http-fileserver',
-              ports: [{ containerPort: 80 }],
-              command: [
-                'bash',
-                '-c',
-                'while true; do sleep 30; npm i files-upload-server -g; files-upload-server "downloads"; done;',
-              ],
-              resources: {
-                requests: {
-                  memory: '750M',
-                  cpu: '0.25',
-                },
-              },
-            },
-          ],
-          restartPolicy: 'Always',
-        },
-      },
-    };
-    await kubeClient.createNamespacedDeployment(namespace, deployment);
-    await this.createLogServiceExpose(namespace, kubeClientCore);
-
-    // wait in loop until serivce ip address is exposed
-
-    for (let index = 0; index < 10; index++) {
-      // wait for service to share ip address
-      await new Promise((resolve) => setTimeout(resolve, 10000));
-
-      // get cluster ip address of service
-      const service = await kubeClientCore.readNamespacedService('http-fileserver', namespace);
-
-      // get podname of deployment
-
-      const podname = await CloudRunnerSystem.Run(
-        `kubectl get pods -n ${namespace} -l service=http-fileserver -o jsonpath='{.items[0].metadata.name}'`,
-        false,
-        true,
-      );
-
-      // if status of pod is not running, then continue
-      const podStatus = await CloudRunnerSystem.Run(
-        `kubectl get pods -n ${namespace} ${podname} -o jsonpath='{.status.phase}'`,
-        false,
-        true,
-      );
-      if (podStatus !== 'Running') {
-        CloudRunnerLogger.log(`Pod status: ${podStatus}`);
-        continue;
-      }
-
-      const logs = await CloudRunnerSystem.Run(`kubectl logs ${podname} -f --timestamps -p`, false, true);
-      CloudRunnerLogger.log(`Logs: ${logs}`);
-
-      // get cluster ip
-      const ip = service.body?.spec?.clusterIP;
-      if (ip && ip.length > 0) {
-        // log service json
-        CloudRunnerLogger.log(`Service: ${JSON.stringify(service.body, undefined, 4)}`);
-        CloudRunnerLogger.log(`Service IP: ${ip}`);
-
-        return ip;
-      }
-    }
-  }
-
-  // create kubernetes service to expose deployment
-  static async createLogServiceExpose(namespace: string, kubeClient: CoreV1Api) {
-    if (!process.env.LOG_SERVICE_IP) {
-      return;
-    }
-
-    // create a service with above json
-    const service = new k8s.V1Service();
-    service.apiVersion = 'v1';
-    service.kind = 'Service';
-    service.metadata = {
-      name: 'http-fileserver',
-      labels: {
-        service: 'http-fileserver',
-      },
-    };
-    service.spec = {
-      ports: [
-        {
-          name: '80-80',
-          port: 80,
-          protocol: 'TCP',
-          targetPort: 80,
-        },
-      ],
-      selector: {
-        service: 'http-fileserver',
-      },
-      type: 'LoadBalancer',
-    };
-    await kubeClient.createNamespacedService(namespace, service);
-  }
-}
-export default KubernetesLogService;

src/model/cloud-runner/remote-client/index.ts

@@ -83,7 +83,7 @@ export class RemoteClient {
     await RemoteClient.runCustomHookFiles(`after-build`);
 
     // WIP - need to give the pod permissions to create config map
-    await RemoteClientLogger.printCollectedLogs();
+    await RemoteClientLogger.handleLogManagementPostJob();
 
     return new Promise((result) => result(``));
   }

src/model/cloud-runner/remote-client/remote-client-logger.ts

@@ -3,7 +3,6 @@ import fs from 'node:fs';
 import path from 'node:path';
 import CloudRunner from '../cloud-runner';
 import CloudRunnerOptions from '../options/cloud-runner-options';
-import Kubernetes from '../providers/k8s';
 
 export class RemoteClientLogger {
   private static get LogFilePath() {
@@ -34,14 +33,20 @@ export class RemoteClientLogger {
     }
   }
 
-  public static async printCollectedLogs() {
+  public static async handleLogManagementPostJob() {
     if (CloudRunnerOptions.providerStrategy !== 'k8s') {
       return;
     }
     CloudRunnerLogger.log(`Collected Logs`);
-    CloudRunnerLogger.log(process.env[`LOG_SERVICE_IP`] || ``);
+    const hashedLogs = fs.readFileSync(RemoteClientLogger.LogFilePath).toString();
+    CloudRunnerLogger.log(hashedLogs);
     const logs = fs.readFileSync(RemoteClientLogger.LogFilePath).toString();
     CloudRunnerLogger.log(logs);
-    await Kubernetes.Instance.PushLogUpdate(logs);
+
+    // loop for 5 mins logging the logs every minute
+    for (let index = 0; index < 5; index++) {
+      await new Promise((resolve) => setTimeout(resolve, 60000));
+      CloudRunnerLogger.log(logs);
+    }
   }
 }

src/model/cloud-runner/tests/cloud-runner-kubernetes.test.ts

@@ -1,82 +0,0 @@
-import BuildParameters from '../../build-parameters';
-import { Cli } from '../../cli/cli';
-import UnityVersioning from '../../unity-versioning';
-import CloudRunner from '../cloud-runner';
-import CloudRunnerOptions from '../options/cloud-runner-options';
-import KubernetesLogService from '../providers/k8s/kubernetes-log-service';
-import CloudRunnerLogger from '../services/core/cloud-runner-logger';
-import setups from './cloud-runner-suite.test';
-import { v4 as uuidv4 } from 'uuid';
-import * as k8s from '@kubernetes/client-node';
-import ImageTag from '../../image-tag';
-
-async function CreateParameters(overrides: any) {
-  if (overrides) {
-    Cli.options = overrides;
-  }
-
-  return await BuildParameters.create();
-}
-
-describe('Cloud Runner Kubernetes', () => {
-  it('Responds', () => {});
-  setups();
-  if (CloudRunnerOptions.cloudRunnerDebug) {
-    it('Build create log service', async () => {
-      const overrides = {
-        versioning: 'None',
-        projectPath: 'test-project',
-        unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
-        targetPlatform: 'StandaloneLinux64',
-        cacheKey: `test-case-${uuidv4()}`,
-        customJob: `
-        - name: 'step 1'
-          image: 'ubuntu'
-          commands: 'curl http://$LOG_SERVICE_IP:80''`,
-      };
-      if (CloudRunnerOptions.providerStrategy !== `k8s`) {
-        return;
-      }
-      const buildParameter = await CreateParameters(overrides);
-      expect(buildParameter.projectPath).toEqual(overrides.projectPath);
-
-      await CloudRunner.setup(buildParameter);
-      const kubeConfig = new k8s.KubeConfig();
-      kubeConfig.loadFromDefault();
-      const kubeClient = kubeConfig.makeApiClient(k8s.CoreV1Api);
-      const kubeClientApps = kubeConfig.makeApiClient(k8s.AppsV1Api);
-
-      await KubernetesLogService.createLogDeployment('test', kubeClientApps, kubeClient);
-
-      CloudRunnerLogger.log(`run 1 succeeded`);
-    }, 1_000_000_000);
-    it('curl log service', async () => {
-      const overrides = {
-        versioning: 'None',
-        projectPath: 'test-project',
-        unityVersion: UnityVersioning.read('test-project'),
-        targetPlatform: 'StandaloneLinux64',
-        cacheKey: `test-case-${uuidv4()}`,
-        customJob: `
-        - name: 'step 1'
-          image: 'ubuntu'
-          commands: |
-            apt-get update
-            apt-get install curl -y
-            curl http://$LOG_SERVICE_IP:80`,
-      };
-      if (CloudRunnerOptions.providerStrategy !== `k8s`) {
-        return;
-      }
-      const buildParameter = await CreateParameters(overrides);
-      expect(buildParameter.projectPath).toEqual(overrides.projectPath);
-
-      const baseImage = new ImageTag(buildParameter);
-      const results = await CloudRunner.run(buildParameter, baseImage.toString());
-      const buildSucceededString = 'Build succeeded';
-      expect(results).toContain(buildSucceededString);
-
-      CloudRunnerLogger.log(`run 1 succeeded`);
-    }, 1_000_000_000);
-  }
-});