Fix kubernetes cleanup to retry delete
							parent
							
								
									85068a482d
								
							
						
					
					
						commit
						ad0dfc7b72
					
				|  | @ -1997,17 +1997,21 @@ class Kubernetes { | ||||||
|         return __awaiter(this, void 0, void 0, function* () { |         return __awaiter(this, void 0, void 0, function* () { | ||||||
|             cloud_runner_logger_1.default.log('cleaning up'); |             cloud_runner_logger_1.default.log('cleaning up'); | ||||||
|             try { |             try { | ||||||
|                 yield this.kubeClientBatch.deleteNamespacedJob(this.jobName, this.namespace); |                 yield async_wait_until_1.default(() => __awaiter(this, void 0, void 0, function* () { | ||||||
|                 yield this.kubeClient.deleteNamespacedSecret(this.secretName, this.namespace); |                     var _b; | ||||||
|             } |                     try { | ||||||
|             catch (error) { |                         yield this.kubeClientBatch.deleteNamespacedJob(this.jobName, this.namespace); | ||||||
|                 cloud_runner_logger_1.default.log('Failed to cleanup, error:'); |                         yield this.kubeClient.deleteNamespacedSecret(this.secretName, this.namespace); | ||||||
|                 core.error(JSON.stringify(error, undefined, 4)); |                     } | ||||||
|                 cloud_runner_logger_1.default.log('Abandoning cleanup, build error:'); |                     catch (error) { | ||||||
|                 throw error; |                         cloud_runner_logger_1.default.log('Failed to cleanup, error:'); | ||||||
|             } |                         core.error(JSON.stringify(error, undefined, 4)); | ||||||
|             try { |                         cloud_runner_logger_1.default.log('Abandoning cleanup, build error:'); | ||||||
|                 yield async_wait_until_1.default(() => __awaiter(this, void 0, void 0, function* () { return (yield this.kubeClientBatch.readNamespacedJob(this.jobName, this.namespace)).body === null; }), { |                         throw error; | ||||||
|  |                     } | ||||||
|  |                     const jobBody = (yield this.kubeClientBatch.readNamespacedJob(this.jobName, this.namespace)).body; | ||||||
|  |                     return jobBody === null || ((_b = jobBody.status) === null || _b === void 0 ? void 0 : _b.active) === 0; | ||||||
|  |                 }), { | ||||||
|                     timeout: 500000, |                     timeout: 500000, | ||||||
|                     intervalBetweenAttempts: 15000, |                     intervalBetweenAttempts: 15000, | ||||||
|                 }); |                 }); | ||||||
|  |  | ||||||
										
											
												File diff suppressed because one or more lines are too long
											
										
									
								
							|  | @ -142,18 +142,21 @@ class Kubernetes implements CloudRunnerProviderInterface { | ||||||
| 
 | 
 | ||||||
|   async cleanupTaskResources() { |   async cleanupTaskResources() { | ||||||
|     CloudRunnerLogger.log('cleaning up'); |     CloudRunnerLogger.log('cleaning up'); | ||||||
|     try { |  | ||||||
|       await this.kubeClientBatch.deleteNamespacedJob(this.jobName, this.namespace); |  | ||||||
|       await this.kubeClient.deleteNamespacedSecret(this.secretName, this.namespace); |  | ||||||
|     } catch (error) { |  | ||||||
|       CloudRunnerLogger.log('Failed to cleanup, error:'); |  | ||||||
|       core.error(JSON.stringify(error, undefined, 4)); |  | ||||||
|       CloudRunnerLogger.log('Abandoning cleanup, build error:'); |  | ||||||
|       throw error; |  | ||||||
|     } |  | ||||||
|     try { |     try { | ||||||
|       await waitUntil( |       await waitUntil( | ||||||
|         async () => (await this.kubeClientBatch.readNamespacedJob(this.jobName, this.namespace)).body === null, |         async () => { | ||||||
|  |           try { | ||||||
|  |             await this.kubeClientBatch.deleteNamespacedJob(this.jobName, this.namespace); | ||||||
|  |             await this.kubeClient.deleteNamespacedSecret(this.secretName, this.namespace); | ||||||
|  |           } catch (error) { | ||||||
|  |             CloudRunnerLogger.log('Failed to cleanup, error:'); | ||||||
|  |             core.error(JSON.stringify(error, undefined, 4)); | ||||||
|  |             CloudRunnerLogger.log('Abandoning cleanup, build error:'); | ||||||
|  |             throw error; | ||||||
|  |           } | ||||||
|  |           const jobBody = (await this.kubeClientBatch.readNamespacedJob(this.jobName, this.namespace)).body; | ||||||
|  |           return jobBody === null || jobBody.status?.active === 0; | ||||||
|  |         }, | ||||||
|         { |         { | ||||||
|           timeout: 500000, |           timeout: 500000, | ||||||
|           intervalBetweenAttempts: 15000, |           intervalBetweenAttempts: 15000, | ||||||
|  |  | ||||||
		Loading…
	
		Reference in New Issue