events log cleanup

pull/496/head
Frostebite 2023-02-15 01:12:47 +00:00
parent 3a8d624592
commit beff5eb6ff
3 changed files with 60 additions and 24 deletions

67
dist/index.js generated vendored
View File

@ -1728,10 +1728,15 @@ class AWSTaskRunner {
}
cloud_runner_logger_1.default.log(`Streaming...`);
const { output, shouldCleanup } = yield this.streamLogsUntilTaskStops(cluster, taskArn, streamName);
yield new Promise((resolve) => resolve(10000));
const taskData = yield AWSTaskRunner.describeTasks(cluster, taskArn);
const containerState = (_q = taskData.containers) === null || _q === void 0 ? void 0 : _q[0];
const exitCode = containerState === null || containerState === void 0 ? void 0 : containerState.exitCode;
let exitCode;
let containerState;
let taskData;
while (exitCode === undefined) {
yield new Promise((resolve) => resolve(10000));
taskData = yield AWSTaskRunner.describeTasks(cluster, taskArn);
containerState = (_q = taskData.containers) === null || _q === void 0 ? void 0 : _q[0];
exitCode = containerState === null || containerState === void 0 ? void 0 : containerState.exitCode;
}
cloud_runner_logger_1.default.log(`Container State: ${JSON.stringify(containerState, undefined, 4)}`);
if (exitCode === undefined) {
cloud_runner_logger_1.default.logWarning(`Undefined exitcode for container`);
@ -3308,7 +3313,7 @@ class Kubernetes {
});
}
runTaskInWorkflow(buildGuid, image, commands, mountdir, workingdir, environment, secrets) {
var _a, _b;
var _a, _b, _c, _d;
return __awaiter(this, void 0, void 0, function* () {
try {
cloud_runner_logger_1.default.log('Cloud Runner K8s workflow!');
@ -3329,7 +3334,7 @@ class Kubernetes {
cloud_runner_logger_1.default.log(JSON.stringify((_a = status.body.status) === null || _a === void 0 ? void 0 : _a.containerStatuses, undefined, 4));
existsAlready = true;
}
catch (_c) {
catch (_e) {
// empty
}
if (!existsAlready || ((_b = status.state) === null || _b === void 0 ? void 0 : _b.terminated) !== undefined) {
@ -3353,21 +3358,20 @@ class Kubernetes {
yield this.cleanupTaskResources();
}
catch (error) {
// let errorParsed;
// try {
// // errorParsed = JSON.parse(error);
// } catch {
// // errorParsed = error;
// }
// const errorMessage =
// errorParsed.name || errorParsed.reason || errorParsed.response?.body?.reason || errorParsed.message;
const continueStreaming = false;
// errorMessage.includes(`dial timeout, backstop`) ||
// errorMessage.includes(`HttpError`) ||
// errorMessage.includes(`HttpError: HTTP request failed`) ||
// errorMessage.includes(`an error occurred when try to find container`) ||
// errorMessage.includes(`not found`) ||
// errorMessage.includes(`Not Found`);
let errorParsed;
try {
errorParsed = JSON.parse(error);
}
catch (_f) {
errorParsed = error;
}
const errorMessage = errorParsed.name || errorParsed.reason || ((_d = (_c = errorParsed.response) === null || _c === void 0 ? void 0 : _c.body) === null || _d === void 0 ? void 0 : _d.reason) || errorParsed.message;
const continueStreaming = errorMessage.includes(`dial timeout, backstop`) ||
errorMessage.includes(`HttpError`) ||
errorMessage.includes(`HttpError: HTTP request failed`) ||
errorMessage.includes(`an error occurred when try to find container`) ||
errorMessage.includes(`not found`) ||
errorMessage.includes(`Not Found`);
if (continueStreaming) {
cloud_runner_logger_1.default.log('Log Stream Container Not Found');
yield new Promise((resolve) => resolve(5000));
@ -4090,13 +4094,21 @@ class KubernetesTaskRunner {
return __awaiter(this, void 0, void 0, function* () {
let success = false;
cloud_runner_logger_1.default.log(`Watching ${podName} ${namespace}`);
cloud_runner_logger_1.default.log(JSON.stringify((yield kubeClient.listNamespacedEvent(namespace)).body.items
.map((x) => {
return {
message: x.message || ``,
name: x.metadata.name || ``,
reason: x.reason || ``,
};
})
.filter((x) => x.name.includes(podName)), undefined, 4));
yield async_wait_until_1.default(() => __awaiter(this, void 0, void 0, function* () {
var _a, _b, _c, _d, _e, _f;
const status = yield kubeClient.readNamespacedPodStatus(podName, namespace);
const events = yield kubeClient.readNamespacedEvent(podName, namespace);
const phase = (_a = status === null || status === void 0 ? void 0 : status.body.status) === null || _a === void 0 ? void 0 : _a.phase;
success = phase === 'Running';
cloud_runner_logger_1.default.log(`${(_b = status.body.status) === null || _b === void 0 ? void 0 : _b.phase} ${((_d = (_c = status.body.status) === null || _c === void 0 ? void 0 : _c.conditions) === null || _d === void 0 ? void 0 : _d[0].reason) || ''} ${((_f = (_e = status.body.status) === null || _e === void 0 ? void 0 : _e.conditions) === null || _f === void 0 ? void 0 : _f[0].message) || ''} \n ${events}`);
cloud_runner_logger_1.default.log(`${(_b = status.body.status) === null || _b === void 0 ? void 0 : _b.phase} ${((_d = (_c = status.body.status) === null || _c === void 0 ? void 0 : _c.conditions) === null || _d === void 0 ? void 0 : _d[0].reason) || ''} ${((_f = (_e = status.body.status) === null || _e === void 0 ? void 0 : _e.conditions) === null || _f === void 0 ? void 0 : _f[0].message) || ''}`);
if (success || phase !== 'Pending')
return true;
return false;
@ -4104,6 +4116,15 @@ class KubernetesTaskRunner {
timeout: 2000000,
intervalBetweenAttempts: 15000,
});
cloud_runner_logger_1.default.log(JSON.stringify((yield kubeClient.listNamespacedEvent(namespace)).body.items
.map((x) => {
return {
message: x.message || ``,
name: x.metadata.name || ``,
reason: x.reason || ``,
};
})
.filter((x) => x.name.includes(podName)), undefined, 4));
return success;
});
}

2
dist/index.js.map generated vendored

File diff suppressed because one or more lines are too long

View File

@ -120,6 +120,21 @@ class KubernetesTaskRunner {
static async watchUntilPodRunning(kubeClient: CoreV1Api, podName: string, namespace: string) {
let success: boolean = false;
CloudRunnerLogger.log(`Watching ${podName} ${namespace}`);
CloudRunnerLogger.log(
JSON.stringify(
(await kubeClient.listNamespacedEvent(namespace)).body.items
.map((x) => {
return {
message: x.message || ``,
name: x.metadata.name || ``,
reason: x.reason || ``,
};
})
.filter((x) => x.name.includes(podName)),
undefined,
4,
),
);
await waitUntil(
async () => {
const status = await kubeClient.readNamespacedPodStatus(podName, namespace);