Delete k8s pod explicitly to free up pvc

dist/index.js

@@ -1943,7 +1943,6 @@ class Kubernetes {
         });
     }
     runTask(buildGuid, image, commands, mountdir, workingdir, environment, secrets) {
-        var _a;
         return __awaiter(this, void 0, void 0, function* () {
             try {
                 // setup
@@ -1954,9 +1953,8 @@ class Kubernetes {
                 yield kubernetes_secret_1.default.createSecret(secrets, this.secretName, this.namespace, this.kubeClient);
                 const jobSpec = kubernetes_job_spec_factory_1.default.getJobSpec(commands, image, mountdir, workingdir, environment, secrets, this.buildGuid, this.buildParameters, this.secretName, this.pvcName, this.jobName, k8s);
                 //run
-                cloud_runner_logger_1.default.log('Creating build job');
+                const jobResult = yield this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec);
-                this.jobName =
+                cloud_runner_logger_1.default.log(`Creating build job ${JSON.stringify(jobResult, undefined, 4)}`);
-                    ((_a = (yield this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec)).body.metadata) === null || _a === void 0 ? void 0 : _a.name) || this.jobName;
                 yield new Promise((promise) => setTimeout(promise, 5000));
                 cloud_runner_logger_1.default.log('Job created');
                 this.setPodNameAndContainerName(yield Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace));
@@ -1998,21 +1996,23 @@ class Kubernetes {
     cleanupTaskResources() {
         return __awaiter(this, void 0, void 0, function* () {
             cloud_runner_logger_1.default.log('cleaning up');
+            try {
+                yield this.kubeClientBatch.deleteNamespacedJob(this.jobName, this.namespace);
+                yield this.kubeClient.deleteNamespacedPod(this.podName, this.namespace);
+                yield this.kubeClient.deleteNamespacedSecret(this.secretName, this.namespace);
+            }
+            catch (error) {
+                cloud_runner_logger_1.default.log('Failed to cleanup, error:');
+                core.error(JSON.stringify(error, undefined, 4));
+                cloud_runner_logger_1.default.log('Abandoning cleanup, build error:');
+                throw error;
+            }
             try {
                 yield async_wait_until_1.default(() => __awaiter(this, void 0, void 0, function* () {
                     var _b;
-                    try {
-                        yield this.kubeClientBatch.deleteNamespacedJob(this.jobName, this.namespace);
-                        yield this.kubeClient.deleteNamespacedSecret(this.secretName, this.namespace);
-                    }
-                    catch (error) {
-                        cloud_runner_logger_1.default.log('Failed to cleanup, error:');
-                        core.error(JSON.stringify(error, undefined, 4));
-                        cloud_runner_logger_1.default.log('Abandoning cleanup, build error:');
-                        throw error;
-                    }
                     const jobBody = (yield this.kubeClientBatch.readNamespacedJob(this.jobName, this.namespace)).body;
-                    return jobBody === null || ((_b = jobBody.status) === null || _b === void 0 ? void 0 : _b.active) === 0;
+                    const podname = (yield this.kubeClient.readNamespacedPod(this.podName, this.namespace)).body;
+                    return (jobBody === null || ((_b = jobBody.status) === null || _b === void 0 ? void 0 : _b.active) === 0) && podname === null;
                 }), {
                     timeout: 500000,
                     intervalBetweenAttempts: 15000,

src/model/cloud-runner/k8s/index.ts

@@ -95,9 +95,8 @@ class Kubernetes implements CloudRunnerProviderInterface {
       );
 
       //run
-      CloudRunnerLogger.log('Creating build job');
+      const jobResult = await this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec);
-      this.jobName =
+      CloudRunnerLogger.log(`Creating build job ${JSON.stringify(jobResult, undefined, 4)}`);
-        (await this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec)).body.metadata?.name || this.jobName;
 
       await new Promise((promise) => setTimeout(promise, 5000));
       CloudRunnerLogger.log('Job created');
@@ -144,20 +143,22 @@ class Kubernetes implements CloudRunnerProviderInterface {
 
   async cleanupTaskResources() {
     CloudRunnerLogger.log('cleaning up');
+    try {
+      await this.kubeClientBatch.deleteNamespacedJob(this.jobName, this.namespace);
+      await this.kubeClient.deleteNamespacedPod(this.podName, this.namespace);
+      await this.kubeClient.deleteNamespacedSecret(this.secretName, this.namespace);
+    } catch (error) {
+      CloudRunnerLogger.log('Failed to cleanup, error:');
+      core.error(JSON.stringify(error, undefined, 4));
+      CloudRunnerLogger.log('Abandoning cleanup, build error:');
+      throw error;
+    }
     try {
       await waitUntil(
         async () => {
-          try {
-            await this.kubeClientBatch.deleteNamespacedJob(this.jobName, this.namespace);
-            await this.kubeClient.deleteNamespacedSecret(this.secretName, this.namespace);
-          } catch (error) {
-            CloudRunnerLogger.log('Failed to cleanup, error:');
-            core.error(JSON.stringify(error, undefined, 4));
-            CloudRunnerLogger.log('Abandoning cleanup, build error:');
-            throw error;
-          }
           const jobBody = (await this.kubeClientBatch.readNamespacedJob(this.jobName, this.namespace)).body;
-          return jobBody === null || jobBody.status?.active === 0;
+          const podname = (await this.kubeClient.readNamespacedPod(this.podName, this.namespace)).body;
+          return (jobBody === null || jobBody.status?.active === 0) && podname === null;
         },
         {
           timeout: 500000,