Improve k8s logging accuracy

dist/index.js

@@ -3304,18 +3304,17 @@ class Kubernetes {
                 this.jobName = `unity-builder-job-${this.buildGuid}`;
                 this.containerName = `main`;
                 yield kubernetes_secret_1.default.createSecret(secrets, this.secretName, this.namespace, this.kubeClient);
-                yield this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets);
-                this.setPodNameAndContainerName(yield Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace));
-                cloud_runner_logger_1.default.log('Watching pod until running');
-                yield kubernetes_task_runner_1.default.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);
                 let output = '';
                 // eslint-disable-next-line no-constant-condition
                 while (true) {
                     try {
                         const existsAlready = yield this.doesJobExist(this.jobName);
-                        if (!existsAlready) {
+                        const existsFailedAlready = yield this.doesFailedJobExist();
+                        if (!existsAlready || existsFailedAlready) {
                             cloud_runner_logger_1.default.log('Job does not exist');
                             yield this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets);
+                            const find = yield Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace);
+                            this.setPodNameAndContainerName(find);
                             cloud_runner_logger_1.default.log('Watching pod until running');
                             yield kubernetes_task_runner_1.default.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);
                         }
@@ -3373,6 +3372,13 @@ class Kubernetes {
             return jobs.body.items.some((x) => { var _a; return ((_a = x.metadata) === null || _a === void 0 ? void 0 : _a.name) === name; });
         });
     }
+    doesFailedJobExist() {
+        var _a;
+        return __awaiter(this, void 0, void 0, function* () {
+            const podStatus = yield this.kubeClient.readNamespacedPodStatus(this.podName, this.namespace);
+            return ((_a = podStatus.body.status) === null || _a === void 0 ? void 0 : _a.phase) === `Failed`;
+        });
+    }
     createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets) {
         var _a;
         return __awaiter(this, void 0, void 0, function* () {
@@ -3384,6 +3390,7 @@ class Kubernetes {
                     cloud_runner_logger_1.default.log(`Build job created`);
                     yield new Promise((promise) => setTimeout(promise, 5000));
                     cloud_runner_logger_1.default.log('Job created');
+                    this.podName;
                     return (_a = result.body.metadata) === null || _a === void 0 ? void 0 : _a.name;
                 }
                 catch (error) {

src/model/cloud-runner/providers/k8s/index.ts

@@ -129,18 +129,17 @@ class Kubernetes implements ProviderInterface {
       this.jobName = `unity-builder-job-${this.buildGuid}`;
       this.containerName = `main`;
       await KubernetesSecret.createSecret(secrets, this.secretName, this.namespace, this.kubeClient);
-      await this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets);
-      this.setPodNameAndContainerName(await Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace));
-      CloudRunnerLogger.log('Watching pod until running');
-      await KubernetesTaskRunner.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);
       let output = '';
       // eslint-disable-next-line no-constant-condition
       while (true) {
         try {
           const existsAlready = await this.doesJobExist(this.jobName);
-          if (!existsAlready) {
+          const existsFailedAlready = await this.doesFailedJobExist();
+          if (!existsAlready || existsFailedAlready) {
             CloudRunnerLogger.log('Job does not exist');
             await this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets);
+            const find = await Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace);
+            this.setPodNameAndContainerName(find);
             CloudRunnerLogger.log('Watching pod until running');
             await KubernetesTaskRunner.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);
           }
@@ -206,6 +205,12 @@ class Kubernetes implements ProviderInterface {
     return jobs.body.items.some((x) => x.metadata?.name === name);
   }
 
+  private async doesFailedJobExist() {
+    const podStatus = await this.kubeClient.readNamespacedPodStatus(this.podName, this.namespace);
+
+    return podStatus.body.status?.phase === `Failed`;
+  }
+
   private async createNamespacedJob(
     commands: string,
     image: string,
@@ -235,6 +240,7 @@ class Kubernetes implements ProviderInterface {
         CloudRunnerLogger.log(`Build job created`);
         await new Promise((promise) => setTimeout(promise, 5000));
         CloudRunnerLogger.log('Job created');
+        this.podName;
 
         return result.body.metadata?.name;
       } catch (error) {