events log cleanup
parent
3a8d624592
commit
beff5eb6ff
|
@ -1728,10 +1728,15 @@ class AWSTaskRunner {
|
|||
}
|
||||
cloud_runner_logger_1.default.log(`Streaming...`);
|
||||
const { output, shouldCleanup } = yield this.streamLogsUntilTaskStops(cluster, taskArn, streamName);
|
||||
yield new Promise((resolve) => resolve(10000));
|
||||
const taskData = yield AWSTaskRunner.describeTasks(cluster, taskArn);
|
||||
const containerState = (_q = taskData.containers) === null || _q === void 0 ? void 0 : _q[0];
|
||||
const exitCode = containerState === null || containerState === void 0 ? void 0 : containerState.exitCode;
|
||||
let exitCode;
|
||||
let containerState;
|
||||
let taskData;
|
||||
while (exitCode === undefined) {
|
||||
yield new Promise((resolve) => resolve(10000));
|
||||
taskData = yield AWSTaskRunner.describeTasks(cluster, taskArn);
|
||||
containerState = (_q = taskData.containers) === null || _q === void 0 ? void 0 : _q[0];
|
||||
exitCode = containerState === null || containerState === void 0 ? void 0 : containerState.exitCode;
|
||||
}
|
||||
cloud_runner_logger_1.default.log(`Container State: ${JSON.stringify(containerState, undefined, 4)}`);
|
||||
if (exitCode === undefined) {
|
||||
cloud_runner_logger_1.default.logWarning(`Undefined exitcode for container`);
|
||||
|
@ -3308,7 +3313,7 @@ class Kubernetes {
|
|||
});
|
||||
}
|
||||
runTaskInWorkflow(buildGuid, image, commands, mountdir, workingdir, environment, secrets) {
|
||||
var _a, _b;
|
||||
var _a, _b, _c, _d;
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
try {
|
||||
cloud_runner_logger_1.default.log('Cloud Runner K8s workflow!');
|
||||
|
@ -3329,7 +3334,7 @@ class Kubernetes {
|
|||
cloud_runner_logger_1.default.log(JSON.stringify((_a = status.body.status) === null || _a === void 0 ? void 0 : _a.containerStatuses, undefined, 4));
|
||||
existsAlready = true;
|
||||
}
|
||||
catch (_c) {
|
||||
catch (_e) {
|
||||
// empty
|
||||
}
|
||||
if (!existsAlready || ((_b = status.state) === null || _b === void 0 ? void 0 : _b.terminated) !== undefined) {
|
||||
|
@ -3353,21 +3358,20 @@ class Kubernetes {
|
|||
yield this.cleanupTaskResources();
|
||||
}
|
||||
catch (error) {
|
||||
// let errorParsed;
|
||||
// try {
|
||||
// // errorParsed = JSON.parse(error);
|
||||
// } catch {
|
||||
// // errorParsed = error;
|
||||
// }
|
||||
// const errorMessage =
|
||||
// errorParsed.name || errorParsed.reason || errorParsed.response?.body?.reason || errorParsed.message;
|
||||
const continueStreaming = false;
|
||||
// errorMessage.includes(`dial timeout, backstop`) ||
|
||||
// errorMessage.includes(`HttpError`) ||
|
||||
// errorMessage.includes(`HttpError: HTTP request failed`) ||
|
||||
// errorMessage.includes(`an error occurred when try to find container`) ||
|
||||
// errorMessage.includes(`not found`) ||
|
||||
// errorMessage.includes(`Not Found`);
|
||||
let errorParsed;
|
||||
try {
|
||||
errorParsed = JSON.parse(error);
|
||||
}
|
||||
catch (_f) {
|
||||
errorParsed = error;
|
||||
}
|
||||
const errorMessage = errorParsed.name || errorParsed.reason || ((_d = (_c = errorParsed.response) === null || _c === void 0 ? void 0 : _c.body) === null || _d === void 0 ? void 0 : _d.reason) || errorParsed.message;
|
||||
const continueStreaming = errorMessage.includes(`dial timeout, backstop`) ||
|
||||
errorMessage.includes(`HttpError`) ||
|
||||
errorMessage.includes(`HttpError: HTTP request failed`) ||
|
||||
errorMessage.includes(`an error occurred when try to find container`) ||
|
||||
errorMessage.includes(`not found`) ||
|
||||
errorMessage.includes(`Not Found`);
|
||||
if (continueStreaming) {
|
||||
cloud_runner_logger_1.default.log('Log Stream Container Not Found');
|
||||
yield new Promise((resolve) => resolve(5000));
|
||||
|
@ -4090,13 +4094,21 @@ class KubernetesTaskRunner {
|
|||
return __awaiter(this, void 0, void 0, function* () {
|
||||
let success = false;
|
||||
cloud_runner_logger_1.default.log(`Watching ${podName} ${namespace}`);
|
||||
cloud_runner_logger_1.default.log(JSON.stringify((yield kubeClient.listNamespacedEvent(namespace)).body.items
|
||||
.map((x) => {
|
||||
return {
|
||||
message: x.message || ``,
|
||||
name: x.metadata.name || ``,
|
||||
reason: x.reason || ``,
|
||||
};
|
||||
})
|
||||
.filter((x) => x.name.includes(podName)), undefined, 4));
|
||||
yield async_wait_until_1.default(() => __awaiter(this, void 0, void 0, function* () {
|
||||
var _a, _b, _c, _d, _e, _f;
|
||||
const status = yield kubeClient.readNamespacedPodStatus(podName, namespace);
|
||||
const events = yield kubeClient.readNamespacedEvent(podName, namespace);
|
||||
const phase = (_a = status === null || status === void 0 ? void 0 : status.body.status) === null || _a === void 0 ? void 0 : _a.phase;
|
||||
success = phase === 'Running';
|
||||
cloud_runner_logger_1.default.log(`${(_b = status.body.status) === null || _b === void 0 ? void 0 : _b.phase} ${((_d = (_c = status.body.status) === null || _c === void 0 ? void 0 : _c.conditions) === null || _d === void 0 ? void 0 : _d[0].reason) || ''} ${((_f = (_e = status.body.status) === null || _e === void 0 ? void 0 : _e.conditions) === null || _f === void 0 ? void 0 : _f[0].message) || ''} \n ${events}`);
|
||||
cloud_runner_logger_1.default.log(`${(_b = status.body.status) === null || _b === void 0 ? void 0 : _b.phase} ${((_d = (_c = status.body.status) === null || _c === void 0 ? void 0 : _c.conditions) === null || _d === void 0 ? void 0 : _d[0].reason) || ''} ${((_f = (_e = status.body.status) === null || _e === void 0 ? void 0 : _e.conditions) === null || _f === void 0 ? void 0 : _f[0].message) || ''}`);
|
||||
if (success || phase !== 'Pending')
|
||||
return true;
|
||||
return false;
|
||||
|
@ -4104,6 +4116,15 @@ class KubernetesTaskRunner {
|
|||
timeout: 2000000,
|
||||
intervalBetweenAttempts: 15000,
|
||||
});
|
||||
cloud_runner_logger_1.default.log(JSON.stringify((yield kubeClient.listNamespacedEvent(namespace)).body.items
|
||||
.map((x) => {
|
||||
return {
|
||||
message: x.message || ``,
|
||||
name: x.metadata.name || ``,
|
||||
reason: x.reason || ``,
|
||||
};
|
||||
})
|
||||
.filter((x) => x.name.includes(podName)), undefined, 4));
|
||||
return success;
|
||||
});
|
||||
}
|
||||
|
|
File diff suppressed because one or more lines are too long
|
@ -120,6 +120,21 @@ class KubernetesTaskRunner {
|
|||
static async watchUntilPodRunning(kubeClient: CoreV1Api, podName: string, namespace: string) {
|
||||
let success: boolean = false;
|
||||
CloudRunnerLogger.log(`Watching ${podName} ${namespace}`);
|
||||
CloudRunnerLogger.log(
|
||||
JSON.stringify(
|
||||
(await kubeClient.listNamespacedEvent(namespace)).body.items
|
||||
.map((x) => {
|
||||
return {
|
||||
message: x.message || ``,
|
||||
name: x.metadata.name || ``,
|
||||
reason: x.reason || ``,
|
||||
};
|
||||
})
|
||||
.filter((x) => x.name.includes(podName)),
|
||||
undefined,
|
||||
4,
|
||||
),
|
||||
);
|
||||
await waitUntil(
|
||||
async () => {
|
||||
const status = await kubeClient.readNamespacedPodStatus(podName, namespace);
|
||||
|
|
Loading…
Reference in New Issue