aws/k8s smoke test

pull/496/head
Frostebite 2023-02-16 01:05:28 +00:00
parent 22c14f5b5f
commit 2f91bba5c6
4 changed files with 34 additions and 20 deletions

23
dist/index.js generated vendored
View File

@ -3340,18 +3340,15 @@ class Kubernetes {
} }
if (!existsAlready || ((_b = status.state) === null || _b === void 0 ? void 0 : _b.terminated) !== undefined) { if (!existsAlready || ((_b = status.state) === null || _b === void 0 ? void 0 : _b.terminated) !== undefined) {
cloud_runner_logger_1.default.log('Job does not exist'); cloud_runner_logger_1.default.log('Job does not exist');
yield this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets); yield this.createJob(commands, image, mountdir, workingdir, environment, secrets);
const find = yield Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace);
this.setPodNameAndContainerName(find);
cloud_runner_logger_1.default.log('Watching pod until running'); cloud_runner_logger_1.default.log('Watching pod until running');
yield kubernetes_task_runner_1.default.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace); yield kubernetes_task_runner_1.default.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);
} }
cloud_runner_logger_1.default.log('Pod running, streaming logs'); cloud_runner_logger_1.default.log('Pod running, streaming logs');
output += yield kubernetes_task_runner_1.default.runTask(this.kubeConfig, this.kubeClient, this.jobName, this.podName, 'main', this.namespace);
const running = yield kubernetes_pods_1.default.IsPodRunning(this.podName, this.namespace, this.kubeClient); const running = yield kubernetes_pods_1.default.IsPodRunning(this.podName, this.namespace, this.kubeClient);
output += yield kubernetes_task_runner_1.default.runTask(this.kubeConfig, this.kubeClient, this.jobName, this.podName, 'main', this.namespace, running);
status = yield kubernetes_pods_1.default.GetPodStatus(this.podName, this.namespace, this.kubeClient);
if (!running) { if (!running) {
status = yield kubernetes_pods_1.default.GetPodStatus(this.podName, this.namespace, this.kubeClient);
cloud_runner_logger_1.default.log(`Pod not found, assumed ended!`);
if (!follow_log_stream_service_1.FollowLogStreamService.DidReceiveEndOfTransmission && status === `Succeeded`) { if (!follow_log_stream_service_1.FollowLogStreamService.DidReceiveEndOfTransmission && status === `Succeeded`) {
output += yield kubernetes_task_runner_1.default.runTask(this.kubeConfig, this.kubeClient, this.jobName, this.podName, 'main', this.namespace, true); output += yield kubernetes_task_runner_1.default.runTask(this.kubeConfig, this.kubeClient, this.jobName, this.podName, 'main', this.namespace, true);
break; break;
@ -3360,9 +3357,7 @@ class Kubernetes {
break; break;
} }
} }
else { cloud_runner_logger_1.default.log(`Pod status ${status}, retrying log stream...`);
cloud_runner_logger_1.default.log('Pod still running, recovering stream...');
}
} }
catch (error) { catch (error) {
let errorParsed; let errorParsed;
@ -3401,6 +3396,13 @@ class Kubernetes {
} }
}); });
} }
createJob(commands, image, mountdir, workingdir, environment, secrets) {
return __awaiter(this, void 0, void 0, function* () {
yield this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets);
const find = yield Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace);
this.setPodNameAndContainerName(find);
});
}
doesJobExist(name) { doesJobExist(name) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
const jobs = yield this.kubeClientBatch.listNamespacedJob(this.namespace); const jobs = yield this.kubeClientBatch.listNamespacedJob(this.namespace);
@ -4103,7 +4105,8 @@ class KubernetesTaskRunner {
stream.destroy(); stream.destroy();
} }
cloud_runner_logger_1.default.log('k8s task runner failed'); cloud_runner_logger_1.default.log('k8s task runner failed');
cloud_runner_logger_1.default.log(JSON.stringify(((_a = error === null || error === void 0 ? void 0 : error.response) === null || _a === void 0 ? void 0 : _a.body) || error, undefined, 4)); cloud_runner_logger_1.default.log(JSON.stringify((_a = error === null || error === void 0 ? void 0 : error.response) === null || _a === void 0 ? void 0 : _a.body, undefined, 4));
cloud_runner_logger_1.default.log(JSON.stringify(error, undefined, 4));
} }
cloud_runner_logger_1.default.log('end of log stream'); cloud_runner_logger_1.default.log('end of log stream');
return output; return output;

2
dist/index.js.map generated vendored

File diff suppressed because one or more lines are too long

View File

@ -145,13 +145,12 @@ class Kubernetes implements ProviderInterface {
} }
if (!existsAlready || status.state?.terminated !== undefined) { if (!existsAlready || status.state?.terminated !== undefined) {
CloudRunnerLogger.log('Job does not exist'); CloudRunnerLogger.log('Job does not exist');
await this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets); await this.createJob(commands, image, mountdir, workingdir, environment, secrets);
const find = await Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace);
this.setPodNameAndContainerName(find);
CloudRunnerLogger.log('Watching pod until running'); CloudRunnerLogger.log('Watching pod until running');
await KubernetesTaskRunner.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace); await KubernetesTaskRunner.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);
} }
CloudRunnerLogger.log('Pod running, streaming logs'); CloudRunnerLogger.log('Pod running, streaming logs');
const running = await KubernetesPods.IsPodRunning(this.podName, this.namespace, this.kubeClient);
output += await KubernetesTaskRunner.runTask( output += await KubernetesTaskRunner.runTask(
this.kubeConfig, this.kubeConfig,
this.kubeClient, this.kubeClient,
@ -159,12 +158,11 @@ class Kubernetes implements ProviderInterface {
this.podName, this.podName,
'main', 'main',
this.namespace, this.namespace,
running,
); );
const running = await KubernetesPods.IsPodRunning(this.podName, this.namespace, this.kubeClient); status = await KubernetesPods.GetPodStatus(this.podName, this.namespace, this.kubeClient);
if (!running) { if (!running) {
status = await KubernetesPods.GetPodStatus(this.podName, this.namespace, this.kubeClient);
CloudRunnerLogger.log(`Pod not found, assumed ended!`);
if (!FollowLogStreamService.DidReceiveEndOfTransmission && status === `Succeeded`) { if (!FollowLogStreamService.DidReceiveEndOfTransmission && status === `Succeeded`) {
output += await KubernetesTaskRunner.runTask( output += await KubernetesTaskRunner.runTask(
this.kubeConfig, this.kubeConfig,
@ -181,9 +179,8 @@ class Kubernetes implements ProviderInterface {
if (FollowLogStreamService.DidReceiveEndOfTransmission) { if (FollowLogStreamService.DidReceiveEndOfTransmission) {
break; break;
} }
} else {
CloudRunnerLogger.log('Pod still running, recovering stream...');
} }
CloudRunnerLogger.log(`Pod status ${status}, retrying log stream...`);
} catch (error: any) { } catch (error: any) {
let errorParsed; let errorParsed;
try { try {
@ -223,6 +220,19 @@ class Kubernetes implements ProviderInterface {
} }
} }
private async createJob(
commands: string,
image: string,
mountdir: string,
workingdir: string,
environment: CloudRunnerEnvironmentVariable[],
secrets: CloudRunnerSecret[],
) {
await this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets);
const find = await Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace);
this.setPodNameAndContainerName(find);
}
private async doesJobExist(name) { private async doesJobExist(name) {
const jobs = await this.kubeClientBatch.listNamespacedJob(this.namespace); const jobs = await this.kubeClientBatch.listNamespacedJob(this.namespace);

View File

@ -114,7 +114,8 @@ class KubernetesTaskRunner {
stream.destroy(); stream.destroy();
} }
CloudRunnerLogger.log('k8s task runner failed'); CloudRunnerLogger.log('k8s task runner failed');
CloudRunnerLogger.log(JSON.stringify(error?.response?.body || error, undefined, 4)); CloudRunnerLogger.log(JSON.stringify(error?.response?.body, undefined, 4));
CloudRunnerLogger.log(JSON.stringify(error, undefined, 4));
} }
CloudRunnerLogger.log('end of log stream'); CloudRunnerLogger.log('end of log stream');