Fix kubernetes cleanup to retry delete
parent
85068a482d
commit
ad0dfc7b72
|
|
@ -1997,17 +1997,21 @@ class Kubernetes {
|
||||||
return __awaiter(this, void 0, void 0, function* () {
|
return __awaiter(this, void 0, void 0, function* () {
|
||||||
cloud_runner_logger_1.default.log('cleaning up');
|
cloud_runner_logger_1.default.log('cleaning up');
|
||||||
try {
|
try {
|
||||||
yield this.kubeClientBatch.deleteNamespacedJob(this.jobName, this.namespace);
|
yield async_wait_until_1.default(() => __awaiter(this, void 0, void 0, function* () {
|
||||||
yield this.kubeClient.deleteNamespacedSecret(this.secretName, this.namespace);
|
var _b;
|
||||||
}
|
try {
|
||||||
catch (error) {
|
yield this.kubeClientBatch.deleteNamespacedJob(this.jobName, this.namespace);
|
||||||
cloud_runner_logger_1.default.log('Failed to cleanup, error:');
|
yield this.kubeClient.deleteNamespacedSecret(this.secretName, this.namespace);
|
||||||
core.error(JSON.stringify(error, undefined, 4));
|
}
|
||||||
cloud_runner_logger_1.default.log('Abandoning cleanup, build error:');
|
catch (error) {
|
||||||
throw error;
|
cloud_runner_logger_1.default.log('Failed to cleanup, error:');
|
||||||
}
|
core.error(JSON.stringify(error, undefined, 4));
|
||||||
try {
|
cloud_runner_logger_1.default.log('Abandoning cleanup, build error:');
|
||||||
yield async_wait_until_1.default(() => __awaiter(this, void 0, void 0, function* () { return (yield this.kubeClientBatch.readNamespacedJob(this.jobName, this.namespace)).body === null; }), {
|
throw error;
|
||||||
|
}
|
||||||
|
const jobBody = (yield this.kubeClientBatch.readNamespacedJob(this.jobName, this.namespace)).body;
|
||||||
|
return jobBody === null || ((_b = jobBody.status) === null || _b === void 0 ? void 0 : _b.active) === 0;
|
||||||
|
}), {
|
||||||
timeout: 500000,
|
timeout: 500000,
|
||||||
intervalBetweenAttempts: 15000,
|
intervalBetweenAttempts: 15000,
|
||||||
});
|
});
|
||||||
|
|
|
||||||
File diff suppressed because one or more lines are too long
|
|
@ -142,18 +142,21 @@ class Kubernetes implements CloudRunnerProviderInterface {
|
||||||
|
|
||||||
async cleanupTaskResources() {
|
async cleanupTaskResources() {
|
||||||
CloudRunnerLogger.log('cleaning up');
|
CloudRunnerLogger.log('cleaning up');
|
||||||
try {
|
|
||||||
await this.kubeClientBatch.deleteNamespacedJob(this.jobName, this.namespace);
|
|
||||||
await this.kubeClient.deleteNamespacedSecret(this.secretName, this.namespace);
|
|
||||||
} catch (error) {
|
|
||||||
CloudRunnerLogger.log('Failed to cleanup, error:');
|
|
||||||
core.error(JSON.stringify(error, undefined, 4));
|
|
||||||
CloudRunnerLogger.log('Abandoning cleanup, build error:');
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
await waitUntil(
|
await waitUntil(
|
||||||
async () => (await this.kubeClientBatch.readNamespacedJob(this.jobName, this.namespace)).body === null,
|
async () => {
|
||||||
|
try {
|
||||||
|
await this.kubeClientBatch.deleteNamespacedJob(this.jobName, this.namespace);
|
||||||
|
await this.kubeClient.deleteNamespacedSecret(this.secretName, this.namespace);
|
||||||
|
} catch (error) {
|
||||||
|
CloudRunnerLogger.log('Failed to cleanup, error:');
|
||||||
|
core.error(JSON.stringify(error, undefined, 4));
|
||||||
|
CloudRunnerLogger.log('Abandoning cleanup, build error:');
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
const jobBody = (await this.kubeClientBatch.readNamespacedJob(this.jobName, this.namespace)).body;
|
||||||
|
return jobBody === null || jobBody.status?.active === 0;
|
||||||
|
},
|
||||||
{
|
{
|
||||||
timeout: 500000,
|
timeout: 500000,
|
||||||
intervalBetweenAttempts: 15000,
|
intervalBetweenAttempts: 15000,
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue