fix: k8s stability

.github/workflows/cloud-runner-dev-pipeline.yml

@@ -100,7 +100,7 @@ jobs:
       # download from cloud storage
       - uses: actions/upload-artifact@v2
         with:
-          name: AWS Build (${{ matrix.targetPlatform }})
+          name: ${{ matrix.cloudRunnerCluster }} Build (${{ matrix.targetPlatform }})
           path: build-${{ steps.unity-build.outputs.BUILD_GUID }}.tar.lz4
           retention-days: 14
       - run: yarn run test-i --detectOpenHandles --forceExit --runInBand

dist/index.js

@@ -3070,8 +3070,17 @@ class Kubernetes {
                         yield kubernetes_task_runner_1.default.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);
                         cloud_runner_logger_1.default.log('Pod running, streaming logs');
                         output = yield kubernetes_task_runner_1.default.runTask(this.kubeConfig, this.kubeClient, this.jobName, this.podName, 'main', this.namespace);
+                        if (!(yield this.kubeClientBatch.listNamespacedJob(this.namespace)).body.items
+                            .map((x) => { var _a; return ((_a = x.metadata) === null || _a === void 0 ? void 0 : _a.name) || undefined; })
+                            .filter((x) => x !== undefined)
+                            .includes(this.jobName)) {
+                            cloud_runner_logger_1.default.log('Job not found, assumed ended!');
                             break;
                         }
+                        else {
+                            cloud_runner_logger_1.default.log('Job still running, recovering stream...');
+                        }
+                    }
                     catch (error) {
                         cloud_runner_logger_1.default.log('error running k8s workflow');
                         cloud_runner_logger_1.default.log(error.message);

src/model/cloud-runner/providers/k8s/index.ts

@@ -151,7 +151,17 @@ class Kubernetes implements ProviderInterface {
             'main',
             this.namespace,
           );
+          if (
+            !(await this.kubeClientBatch.listNamespacedJob(this.namespace)).body.items
+              .map((x) => x.metadata?.name || undefined)
+              .filter((x) => x !== undefined)
+              .includes(this.jobName)
+          ) {
+            CloudRunnerLogger.log('Job not found, assumed ended!');
             break;
+          } else {
+            CloudRunnerLogger.log('Job still running, recovering stream...');
+          }
         } catch (error: any) {
           CloudRunnerLogger.log('error running k8s workflow');
           CloudRunnerLogger.log(error.message);