actions
/
unity-builder
mirror of https://github.com/game-ci/unity-builder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

garbage collection cleanup

pull/473/head
Frostebite 2022-11-16 20:41:33 +00:00
parent 371170540e
commit f7098896dc
6 changed files with 141 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

134
dist/index.js vendored
View File

@ -2347,7 +2347,6 @@ const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
const aws_job_stack_1 = __nccwpck_require__(70633);
const aws_base_stack_1 = __nccwpck_require__(26112);
const __1 = __nccwpck_require__(41359);
const tertiary_resources_service_1 = __nccwpck_require__(61627);
const garbage_collection_service_1 = __nccwpck_require__(53859);
const task_service_1 = __nccwpck_require__(67205);
class AWSBuildEnvironment {
@ -2356,10 +2355,9 @@ class AWSBuildEnvironment {
}
listResources() {
return __awaiter(this, void 0, void 0, function* () {
yield task_service_1.TaskService.awsListJobs();
yield tertiary_resources_service_1.TertiaryResourcesService.awsListLogGroups();
yield task_service_1.TaskService.awsListTasks();
yield task_service_1.TaskService.awsListStacks();
yield task_service_1.TaskService.getCloudFormationJobStacks();
yield task_service_1.TaskService.getLogGroups();
yield task_service_1.TaskService.getTasks();
return [];
});
}
@ -2373,7 +2371,7 @@ class AWSBuildEnvironment {
}
listOtherResources() {
return __awaiter(this, void 0, void 0, function* () {
yield tertiary_resources_service_1.TertiaryResourcesService.awsListLogGroups();
yield task_service_1.TaskService.getLogGroups();
return '';
});
}
@ -2487,29 +2485,30 @@ const aws_sdk_1 = __importDefault(__nccwpck_require__(71786));
const input_1 = __importDefault(__nccwpck_require__(91933));
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
const task_service_1 = __nccwpck_require__(67205);
const tertiary_resources_service_1 = __nccwpck_require__(61627);
class GarbageCollectionService {
static isOlderThan1day(date) {
const ageDate = new Date(date.getTime() - Date.now());
return ageDate.getDay() > 0;
}
static cleanup(deleteResources = false, OneDayOlderOnly = false) {
var _a, _b;
return __awaiter(this, void 0, void 0, function* () {
process.env.AWS_REGION = input_1.default.region;
const CF = new aws_sdk_1.default.CloudFormation();
const ecs = new aws_sdk_1.default.ECS();
const cwl = new aws_sdk_1.default.CloudWatchLogs();
const taskDefinitionsInUse = new Array();
yield task_service_1.TaskService.awsListTasks((taskElement, element) => __awaiter(this, void 0, void 0, function* () {
var _a;
const tasks = yield task_service_1.TaskService.getTasks();
for (const task of tasks) {
const { taskElement, element } = task;
taskDefinitionsInUse.push(taskElement.taskDefinitionArn);
if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(taskElement.CreatedAt))) {
cloud_runner_logger_1.default.log(`Stopping task ${(_a = taskElement.containers) === null || _a === void 0 ? void 0 : _a[0].name}`);
yield ecs.stopTask({ task: taskElement.taskArn || '', cluster: element }).promise();
}
}));
yield task_service_1.TaskService.awsListStacks((element) => __awaiter(this, void 0, void 0, function* () {
var _b;
}
const jobStacks = yield task_service_1.TaskService.getCloudFormationJobStacks();
for (const element of jobStacks) {
if ((_b = (yield CF.describeStackResources({ StackName: element.StackName }).promise()).StackResources) === null || _b === void 0 ? void 0 : _b.some((x) => x.ResourceType === 'AWS::ECS::TaskDefinition' && taskDefinitionsInUse.includes(x.PhysicalResourceId))) {
cloud_runner_logger_1.default.log(`Skipping ${element.StackName} - active task was running not deleting`);
return;
@ -2523,14 +2522,19 @@ class GarbageCollectionService {
const deleteStackInput = { StackName: element.StackName };
yield CF.deleteStack(deleteStackInput).promise();
}
}));
yield tertiary_resources_service_1.TertiaryResourcesService.awsListLogGroups((element) => __awaiter(this, void 0, void 0, function* () {
}
const logGroups = yield task_service_1.TaskService.getLogGroups();
for (const element of logGroups) {
if (deleteResources &&
(!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(new Date(element.createdAt)))) {
cloud_runner_logger_1.default.log(`Deleting ${element.logGroupName}`);
yield cwl.deleteLogGroup({ logGroupName: element.logGroupName || '' }).promise();
}
}));
}
const locks = yield task_service_1.TaskService.getLocks();
for (const element of locks) {
cloud_runner_logger_1.default.log(`Lock: ${element.Key}`);
}
});
}
}
@ -2571,40 +2575,43 @@ class TaskService {
return output;
});
}
static awsListStacks(perResultCallback = false) {
static getCloudFormationJobStacks() {
var _a, _b;
return __awaiter(this, void 0, void 0, function* () {
cloud_runner_logger_1.default.log(`List Stacks`);
const result = [];
cloud_runner_logger_1.default.log(``);
cloud_runner_logger_1.default.log(`List Cloud Formation Stacks`);
process.env.AWS_REGION = input_1.default.region;
const CF = new aws_sdk_1.default.CloudFormation();
const stacks = ((_a = (yield CF.listStacks().promise()).StackSummaries) === null || _a === void 0 ? void 0 : _a.filter((_x) => _x.StackStatus !== 'DELETE_COMPLETE')) || [];
cloud_runner_logger_1.default.log(`Stacks ${stacks.length}`);
const stacks = ((_a = (yield CF.listStacks().promise()).StackSummaries) === null || _a === void 0 ? void 0 : _a.filter((_x) => _x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription !== base_stack_formation_1.BaseStackFormation.baseStackDecription)) || [];
cloud_runner_logger_1.default.log(``);
cloud_runner_logger_1.default.log(`Cloud Formation Stacks ${stacks.length}`);
for (const element of stacks) {
const ageDate = new Date(Date.now() - element.CreationTime.getTime());
cloud_runner_logger_1.default.log(`Task Stack ${element.StackName} - Age D${Math.floor(ageDate.getHours() / 24)} H${ageDate.getHours()} M${ageDate.getMinutes()}`);
if (perResultCallback)
yield perResultCallback(element);
result.push(element);
}
const baseStacks = ((_b = (yield CF.listStacks().promise()).StackSummaries) === null || _b === void 0 ? void 0 : _b.filter((_x) => _x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription === base_stack_formation_1.BaseStackFormation.baseStackDecription)) || [];
cloud_runner_logger_1.default.log(``);
cloud_runner_logger_1.default.log(`Base Stacks ${baseStacks.length}`);
for (const element of baseStacks) {
const ageDate = new Date(Date.now() - element.CreationTime.getTime());
cloud_runner_logger_1.default.log(`Task Stack ${element.StackName} - Age D${Math.floor(ageDate.getHours() / 24)} H${ageDate.getHours()} M${ageDate.getMinutes()}`);
if (perResultCallback)
yield perResultCallback(element);
}
if (stacks === undefined) {
return;
result.push(element);
}
cloud_runner_logger_1.default.log(``);
return result;
});
}
static awsListTasks(perResultCallback = false) {
static getTasks() {
return __awaiter(this, void 0, void 0, function* () {
const result = [];
cloud_runner_logger_1.default.log(``);
cloud_runner_logger_1.default.log(`List Tasks`);
process.env.AWS_REGION = input_1.default.region;
const ecs = new aws_sdk_1.default.ECS();
const clusters = (yield ecs.listClusters().promise()).clusterArns || [];
cloud_runner_logger_1.default.log(`Clusters ${clusters.length}`);
cloud_runner_logger_1.default.log(`Task Clusters ${clusters.length}`);
for (const element of clusters) {
const input = {
cluster: element,
@ -2614,6 +2621,7 @@ class TaskService {
const describeInput = { tasks: list, cluster: element };
const describeList = (yield ecs.describeTasks(describeInput).promise()).tasks || [];
if (describeList.length === 0) {
cloud_runner_logger_1.default.log(`No Tasks`);
continue;
}
cloud_runner_logger_1.default.log(`Tasks ${describeList.length}`);
@ -2627,27 +2635,12 @@ class TaskService {
cloud_runner_logger_1.default.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`);
continue;
}
if (perResultCallback)
yield perResultCallback(taskElement, element);
result.push({ taskElement, element });
}
}
}
});
}
static awsListJobs(perResultCallback = false) {
var _a;
return __awaiter(this, void 0, void 0, function* () {
cloud_runner_logger_1.default.log(`List Jobs`);
process.env.AWS_REGION = input_1.default.region;
const CF = new aws_sdk_1.default.CloudFormation();
const stacks = ((_a = (yield CF.listStacks().promise()).StackSummaries) === null || _a === void 0 ? void 0 : _a.filter((_x) => _x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription !== base_stack_formation_1.BaseStackFormation.baseStackDecription)) || [];
cloud_runner_logger_1.default.log(`Stacks ${stacks.length}`);
for (const element of stacks) {
const ageDate = new Date(Date.now() - element.CreationTime.getTime());
cloud_runner_logger_1.default.log(`Task Stack ${element.StackName} - Age D${Math.floor(ageDate.getHours() / 24)} H${ageDate.getHours()} M${ageDate.getMinutes()}`);
if (perResultCallback)
yield perResultCallback(element);
}
cloud_runner_logger_1.default.log(``);
return result;
});
}
static awsDescribeJob(job) {
@ -2673,37 +2666,9 @@ class TaskService {
return message;
});
}
}
exports.TaskService = TaskService;
/***/ }),
/***/ 61627:
/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) {
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.TertiaryResourcesService = void 0;
const aws_sdk_1 = __importDefault(__nccwpck_require__(71786));
const input_1 = __importDefault(__nccwpck_require__(91933));
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
class TertiaryResourcesService {
static awsListLogGroups(perResultCallback = false) {
static getLogGroups() {
return __awaiter(this, void 0, void 0, function* () {
const result = [];
process.env.AWS_REGION = input_1.default.region;
const ecs = new aws_sdk_1.default.CloudWatchLogs();
let logStreamInput = {
@ -2724,13 +2689,24 @@ class TertiaryResourcesService {
}
const ageDate = new Date(Date.now() - element.creationTime);
cloud_runner_logger_1.default.log(`Task Stack ${element.logGroupName} - Age D${Math.floor(ageDate.getHours() / 24)} H${ageDate.getHours()} M${ageDate.getMinutes()}`);
if (perResultCallback)
yield perResultCallback(element, element);
result.push(element);
}
return result;
});
}
static getLocks() {
return __awaiter(this, void 0, void 0, function* () {
process.env.AWS_REGION = input_1.default.region;
const s3 = new aws_sdk_1.default.S3();
const listRequest = {
Bucket: `game-ci-test-storage`,
};
const results = yield s3.listObjects(listRequest).promise();
return results.Contents || [];
});
}
}
exports.TertiaryResourcesService = TertiaryResourcesService;
exports.TaskService = TaskService;
/***/ }),

2
dist/index.js.map vendored
View File

File diff suppressed because one or more lines are too long

10
src/model/cloud-runner/providers/aws/index.ts
View File

@ -9,7 +9,6 @@ import CloudRunnerLogger from '../../services/cloud-runner-logger';
import { AWSJobStack as AwsJobStack } from './aws-job-stack';
import { AWSBaseStack as AwsBaseStack } from './aws-base-stack';
import { Input } from '../../..';
import { TertiaryResourcesService } from './services/tertiary-resources-service';
import { GarbageCollectionService } from './services/garbage-collection-service';
import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow';
@ -22,10 +21,9 @@ class AWSBuildEnvironment implements ProviderInterface {
this.baseStackName = buildParameters.awsBaseStackName;
}
async listResources(): Promise<ProviderResource[]> {
await TaskService.awsListJobs();
await TertiaryResourcesService.awsListLogGroups();
await TaskService.awsListTasks();
await TaskService.awsListStacks();
await TaskService.getCloudFormationJobStacks();
await TaskService.getLogGroups();
await TaskService.getTasks();
return [];
}
@ -37,7 +35,7 @@ class AWSBuildEnvironment implements ProviderInterface {
}
async listOtherResources(): Promise<string> {
await TertiaryResourcesService.awsListLogGroups();
await TaskService.getLogGroups();
return '';
}

22
src/model/cloud-runner/providers/aws/services/garbage-collection-service.ts
View File

@ -2,7 +2,6 @@ import AWS from 'aws-sdk';
import Input from '../../../../input';
import CloudRunnerLogger from '../../../services/cloud-runner-logger';
import { TaskService } from './task-service';
import { TertiaryResourcesService } from './tertiary-resources-service';
export class GarbageCollectionService {
static isOlderThan1day(date: any) {
@ -17,14 +16,17 @@ export class GarbageCollectionService {
const ecs = new AWS.ECS();
const cwl = new AWS.CloudWatchLogs();
const taskDefinitionsInUse = new Array();
await TaskService.awsListTasks(async (taskElement, element) => {
const tasks = await TaskService.getTasks();
for (const task of tasks) {
const { taskElement, element } = task;
taskDefinitionsInUse.push(taskElement.taskDefinitionArn);
if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(taskElement.CreatedAt))) {
CloudRunnerLogger.log(`Stopping task ${taskElement.containers?.[0].name}`);
await ecs.stopTask({ task: taskElement.taskArn || '', cluster: element }).promise();
}
});
await TaskService.awsListStacks(async (element) => {
}
const jobStacks = await TaskService.getCloudFormationJobStacks();
for (const element of jobStacks) {
if (
(await CF.describeStackResources({ StackName: element.StackName }).promise()).StackResources?.some(
(x) => x.ResourceType === 'AWS::ECS::TaskDefinition' && taskDefinitionsInUse.includes(x.PhysicalResourceId),
@ -44,8 +46,9 @@ export class GarbageCollectionService {
const deleteStackInput: AWS.CloudFormation.DeleteStackInput = { StackName: element.StackName };
await CF.deleteStack(deleteStackInput).promise();
}
});
await TertiaryResourcesService.awsListLogGroups(async (element) => {
}
const logGroups = await TaskService.getLogGroups();
for (const element of logGroups) {
if (
deleteResources &&
(!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(new Date(element.createdAt)))
@ -53,6 +56,11 @@ export class GarbageCollectionService {
CloudRunnerLogger.log(`Deleting ${element.logGroupName}`);
await cwl.deleteLogGroup({ logGroupName: element.logGroupName || '' }).promise();
}
});
}
const locks = await TaskService.getLocks();
for (const element of locks) {
CloudRunnerLogger.log(`Lock: ${element.Key}`);
}
}
}

99
src/model/cloud-runner/providers/aws/services/task-service.ts
View File

@ -3,6 +3,7 @@ import Input from '../../../../input';
import CloudRunnerLogger from '../../../services/cloud-runner-logger';
import { BaseStackFormation } from '../cloud-formations/base-stack-formation';
import AwsTaskRunner from '../aws-task-runner';
import { ListObjectsRequest } from 'aws-sdk/clients/s3';
export class TaskService {
static async watch() {
@ -15,16 +16,19 @@ export class TaskService {
return output;
}
public static async awsListStacks(perResultCallback: any = false) {
CloudRunnerLogger.log(`List Stacks`);
public static async getCloudFormationJobStacks() {
const result: any[] = [];
CloudRunnerLogger.log(``);
CloudRunnerLogger.log(`List Cloud Formation Stacks`);
process.env.AWS_REGION = Input.region;
const CF = new AWS.CloudFormation();
const stacks =
(await CF.listStacks().promise()).StackSummaries?.filter(
(_x) => _x.StackStatus !== 'DELETE_COMPLETE', // &&
// _x.TemplateDescription === TaskDefinitionFormation.description.replace('\n', ''),
(_x) =>
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription !== BaseStackFormation.baseStackDecription,
) || [];
CloudRunnerLogger.log(`Stacks ${stacks.length}`);
CloudRunnerLogger.log(``);
CloudRunnerLogger.log(`Cloud Formation Stacks ${stacks.length}`);
for (const element of stacks) {
const ageDate: Date = new Date(Date.now() - element.CreationTime.getTime());
@ -33,13 +37,14 @@ export class TaskService {
ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
);
if (perResultCallback) await perResultCallback(element);
result.push(element);
}
const baseStacks =
(await CF.listStacks().promise()).StackSummaries?.filter(
(_x) =>
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription === BaseStackFormation.baseStackDecription,
) || [];
CloudRunnerLogger.log(``);
CloudRunnerLogger.log(`Base Stacks ${baseStacks.length}`);
for (const element of baseStacks) {
const ageDate: Date = new Date(Date.now() - element.CreationTime.getTime());
@ -49,18 +54,20 @@ export class TaskService {
ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
);
if (perResultCallback) await perResultCallback(element);
}
if (stacks === undefined) {
return;
result.push(element);
}
CloudRunnerLogger.log(``);
return result;
}
public static async awsListTasks(perResultCallback: any = false) {
public static async getTasks() {
const result: any[] = [];
CloudRunnerLogger.log(``);
CloudRunnerLogger.log(`List Tasks`);
process.env.AWS_REGION = Input.region;
const ecs = new AWS.ECS();
const clusters = (await ecs.listClusters().promise()).clusterArns || [];
CloudRunnerLogger.log(`Clusters ${clusters.length}`);
CloudRunnerLogger.log(`Task Clusters ${clusters.length}`);
for (const element of clusters) {
const input: AWS.ECS.ListTasksRequest = {
cluster: element,
@ -71,6 +78,7 @@ export class TaskService {
const describeInput: AWS.ECS.DescribeTasksRequest = { tasks: list, cluster: element };
const describeList = (await ecs.describeTasks(describeInput).promise()).tasks || [];
if (describeList.length === 0) {
CloudRunnerLogger.log(`No Tasks`);
continue;
}
CloudRunnerLogger.log(`Tasks ${describeList.length}`);
@ -84,31 +92,13 @@ export class TaskService {
CloudRunnerLogger.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`);
continue;
}
if (perResultCallback) await perResultCallback(taskElement, element);
result.push({ taskElement, element });
}
}
}
}
public static async awsListJobs(perResultCallback: any = false) {
CloudRunnerLogger.log(`List Jobs`);
process.env.AWS_REGION = Input.region;
const CF = new AWS.CloudFormation();
const stacks =
(await CF.listStacks().promise()).StackSummaries?.filter(
(_x) =>
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription !== BaseStackFormation.baseStackDecription,
) || [];
CloudRunnerLogger.log(`Stacks ${stacks.length}`);
for (const element of stacks) {
const ageDate: Date = new Date(Date.now() - element.CreationTime.getTime());
CloudRunnerLogger.log(``);
CloudRunnerLogger.log(
`Task Stack ${element.StackName} - Age D${Math.floor(
ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
);
if (perResultCallback) await perResultCallback(element);
}
return result;
}
public static async awsDescribeJob(job: string) {
process.env.AWS_REGION = Input.region;
@ -131,4 +121,47 @@ export class TaskService {
return message;
}
public static async getLogGroups() {
const result: any[] = [];
process.env.AWS_REGION = Input.region;
const ecs = new AWS.CloudWatchLogs();
let logStreamInput: AWS.CloudWatchLogs.DescribeLogGroupsRequest = {
/* logGroupNamePrefix: 'game-ci' */
};
let logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise();
const logGroups = logGroupsDescribe.logGroups || [];
while (logGroupsDescribe.nextToken) {
logStreamInput = { /* logGroupNamePrefix: 'game-ci',*/ nextToken: logGroupsDescribe.nextToken };
logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise();
logGroups.push(...(logGroupsDescribe?.logGroups || []));
}
CloudRunnerLogger.log(`Log Groups ${logGroups.length}`);
for (const element of logGroups) {
if (element.creationTime === undefined) {
CloudRunnerLogger.log(`Skipping ${element.logGroupName} no createdAt date`);
continue;
}
const ageDate: Date = new Date(Date.now() - element.creationTime);
CloudRunnerLogger.log(
`Task Stack ${element.logGroupName} - Age D${Math.floor(
ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
);
result.push(element);
}
return result;
}
public static async getLocks() {
process.env.AWS_REGION = Input.region;
const s3 = new AWS.S3();
const listRequest: ListObjectsRequest = {
Bucket: `game-ci-test-storage`,
};
const results = await s3.listObjects(listRequest).promise();
return results.Contents || [];
}
}

36
src/model/cloud-runner/providers/aws/services/tertiary-resources-service.ts
View File

@ -1,36 +0,0 @@
import AWS from 'aws-sdk';
import Input from '../../../../input';
import CloudRunnerLogger from '../../../services/cloud-runner-logger';
export class TertiaryResourcesService {
public static async awsListLogGroups(perResultCallback: any = false) {
process.env.AWS_REGION = Input.region;
const ecs = new AWS.CloudWatchLogs();
let logStreamInput: AWS.CloudWatchLogs.DescribeLogGroupsRequest = {
/* logGroupNamePrefix: 'game-ci' */
};
let logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise();
const logGroups = logGroupsDescribe.logGroups || [];
while (logGroupsDescribe.nextToken) {
logStreamInput = { /* logGroupNamePrefix: 'game-ci',*/ nextToken: logGroupsDescribe.nextToken };
logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise();
logGroups.push(...(logGroupsDescribe?.logGroups || []));
}
CloudRunnerLogger.log(`Log Groups ${logGroups.length}`);
for (const element of logGroups) {
if (element.creationTime === undefined) {
CloudRunnerLogger.log(`Skipping ${element.logGroupName} no createdAt date`);
continue;
}
const ageDate: Date = new Date(Date.now() - element.creationTime);
CloudRunnerLogger.log(
`Task Stack ${element.logGroupName} - Age D${Math.floor(
ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
);
if (perResultCallback) await perResultCallback(element, element);
}
}
}