actions
/
unity-builder
mirror of https://github.com/game-ci/unity-builder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

custom hooks and renaming

pull/419/head
Frostebite 2022-08-02 19:37:26 +01:00
parent 388a42d514
commit dcbdfba6ad
19 changed files with 998 additions and 363 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

614
dist/index.js vendored
View File

@ -1202,7 +1202,7 @@ const zlib = __importStar(__nccwpck_require__(59796));
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
const __1 = __nccwpck_require__(41359);
const cloud_runner_1 = __importDefault(__nccwpck_require__(79144));
const cloud_runner_build_command_process_1 = __nccwpck_require__(71899);
const cloud_runner_custom_hooks_1 = __nccwpck_require__(58873);
const follow_log_stream_service_1 = __nccwpck_require__(64121);
class AWSTaskRunner {
static runTask(taskDef, ECS, CF, environment, buildGuid, commands) {
@ -1223,7 +1223,7 @@ class AWSTaskRunner {
{
name: taskDef.taskDefStackName,
environment,
command: ['-c', cloud_runner_build_command_process_1.CloudRunnerBuildCommandProcessor.ProcessCommands(commands, cloud_runner_1.default.buildParameters)],
command: ['-c', cloud_runner_custom_hooks_1.CloudRunnerCustomHooks.ApplyHooksToCommands(commands, cloud_runner_1.default.buildParameters)],
},
],
},
@ -1971,16 +1971,12 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.AwsCliCommands = void 0;
const aws_sdk_1 = __importDefault(__nccwpck_require__(71786));
const cli_functions_repository_1 = __nccwpck_require__(85301);
const input_1 = __importDefault(__nccwpck_require__(91933));
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
const base_stack_formation_1 = __nccwpck_require__(29643);
const task_service_1 = __nccwpck_require__(67205);
const garbage_collection_service_1 = __nccwpck_require__(53859);
const tertiary_resources_service_1 = __nccwpck_require__(61627);
class AwsCliCommands {
static awsListAll() {
return __awaiter(this, void 0, void 0, function* () {
@ -1991,167 +1987,42 @@ class AwsCliCommands {
}
static garbageCollectAws() {
return __awaiter(this, void 0, void 0, function* () {
yield AwsCliCommands.cleanup(false);
yield garbage_collection_service_1.GarbageCollectionService.cleanup(false);
});
}
static garbageCollectAwsAll() {
return __awaiter(this, void 0, void 0, function* () {
yield AwsCliCommands.cleanup(true);
yield garbage_collection_service_1.GarbageCollectionService.cleanup(true);
});
}
static garbageCollectAwsAllOlderThanOneDay() {
return __awaiter(this, void 0, void 0, function* () {
yield AwsCliCommands.cleanup(true, true);
yield garbage_collection_service_1.GarbageCollectionService.cleanup(true, true);
});
}
static isOlderThan1day(date) {
const ageDate = new Date(date.getTime() - Date.now());
return ageDate.getDay() > 0;
}
static awsListStacks(perResultCallback = false) {
var _a, _b;
return __awaiter(this, void 0, void 0, function* () {
process.env.AWS_REGION = input_1.default.region;
const CF = new aws_sdk_1.default.CloudFormation();
const stacks = ((_a = (yield CF.listStacks().promise()).StackSummaries) === null || _a === void 0 ? void 0 : _a.filter((_x) => _x.StackStatus !== 'DELETE_COMPLETE')) || [];
cloud_runner_logger_1.default.log(`Stacks ${stacks.length}`);
for (const element of stacks) {
const ageDate = new Date(Date.now() - element.CreationTime.getTime());
// if (verbose)
cloud_runner_logger_1.default.log(`Task Stack ${element.StackName} - Age D${Math.floor(ageDate.getHours() / 24)} H${ageDate.getHours()} M${ageDate.getMinutes()}`);
if (perResultCallback)
yield perResultCallback(element);
}
const baseStacks = ((_b = (yield CF.listStacks().promise()).StackSummaries) === null || _b === void 0 ? void 0 : _b.filter((_x) => _x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription === base_stack_formation_1.BaseStackFormation.baseStackDecription)) || [];
cloud_runner_logger_1.default.log(`Base Stacks ${baseStacks.length}`);
for (const element of baseStacks) {
const ageDate = new Date(Date.now() - element.CreationTime.getTime());
// if (verbose)
cloud_runner_logger_1.default.log(`Task Stack ${element.StackName} - Age D${Math.floor(ageDate.getHours() / 24)} H${ageDate.getHours()} M${ageDate.getMinutes()}`);
if (perResultCallback)
yield perResultCallback(element);
}
if (stacks === undefined) {
return;
}
return task_service_1.TaskService.awsListStacks(perResultCallback);
});
}
static awsListTasks(perResultCallback = false) {
return __awaiter(this, void 0, void 0, function* () {
process.env.AWS_REGION = input_1.default.region;
const ecs = new aws_sdk_1.default.ECS();
const clusters = (yield ecs.listClusters().promise()).clusterArns || [];
cloud_runner_logger_1.default.log(`Clusters ${clusters.length}`);
for (const element of clusters) {
const input = {
cluster: element,
};
const list = (yield ecs.listTasks(input).promise()).taskArns || [];
if (list.length > 0) {
const describeInput = { tasks: list, cluster: element };
const describeList = (yield ecs.describeTasks(describeInput).promise()).tasks || [];
if (describeList === []) {
continue;
}
cloud_runner_logger_1.default.log(`Tasks ${describeList.length}`);
for (const taskElement of describeList) {
if (taskElement === undefined) {
continue;
}
taskElement.overrides = {};
taskElement.attachments = [];
if (taskElement.createdAt === undefined) {
cloud_runner_logger_1.default.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`);
continue;
}
if (perResultCallback)
yield perResultCallback(taskElement, element);
}
}
}
return task_service_1.TaskService.awsListJobs(perResultCallback);
});
}
static listTasks(perResultCallback = false) {
return __awaiter(this, void 0, void 0, function* () {
return task_service_1.TaskService.awsListJobs(perResultCallback);
});
}
static awsListLogGroups(perResultCallback = false) {
return __awaiter(this, void 0, void 0, function* () {
process.env.AWS_REGION = input_1.default.region;
const ecs = new aws_sdk_1.default.CloudWatchLogs();
let logStreamInput = {
/* logGroupNamePrefix: 'game-ci' */
};
let logGroupsDescribe = yield ecs.describeLogGroups(logStreamInput).promise();
const logGroups = logGroupsDescribe.logGroups || [];
while (logGroupsDescribe.nextToken) {
logStreamInput = { /* logGroupNamePrefix: 'game-ci',*/ nextToken: logGroupsDescribe.nextToken };
logGroupsDescribe = yield ecs.describeLogGroups(logStreamInput).promise();
logGroups.push(...((logGroupsDescribe === null || logGroupsDescribe === void 0 ? void 0 : logGroupsDescribe.logGroups) || []));
}
cloud_runner_logger_1.default.log(`Log Groups ${logGroups.length}`);
for (const element of logGroups) {
if (element.creationTime === undefined) {
cloud_runner_logger_1.default.log(`Skipping ${element.logGroupName} no createdAt date`);
continue;
}
const ageDate = new Date(Date.now() - element.creationTime);
// if (verbose)
cloud_runner_logger_1.default.log(`Task Stack ${element.logGroupName} - Age D${Math.floor(ageDate.getHours() / 24)} H${ageDate.getHours()} M${ageDate.getMinutes()}`);
if (perResultCallback)
yield perResultCallback(element, element);
}
yield tertiary_resources_service_1.TertiaryResourcesService.AwsListLogGroups(perResultCallback);
});
}
static awsListJobs(perResultCallback = false) {
var _a;
return __awaiter(this, void 0, void 0, function* () {
process.env.AWS_REGION = input_1.default.region;
const CF = new aws_sdk_1.default.CloudFormation();
const stacks = ((_a = (yield CF.listStacks().promise()).StackSummaries) === null || _a === void 0 ? void 0 : _a.filter((_x) => _x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription !== base_stack_formation_1.BaseStackFormation.baseStackDecription)) || [];
cloud_runner_logger_1.default.log(`Stacks ${stacks.length}`);
for (const element of stacks) {
const ageDate = new Date(Date.now() - element.CreationTime.getTime());
// if (verbose)
cloud_runner_logger_1.default.log(`Task Stack ${element.StackName} - Age D${Math.floor(ageDate.getHours() / 24)} H${ageDate.getHours()} M${ageDate.getMinutes()}`);
if (perResultCallback)
yield perResultCallback(element);
}
});
}
static cleanup(deleteResources = false, OneDayOlderOnly = false) {
return __awaiter(this, void 0, void 0, function* () {
process.env.AWS_REGION = input_1.default.region;
const CF = new aws_sdk_1.default.CloudFormation();
const ecs = new aws_sdk_1.default.ECS();
const cwl = new aws_sdk_1.default.CloudWatchLogs();
const taskDefinitionsInUse = new Array();
yield AwsCliCommands.awsListTasks((taskElement, element) => __awaiter(this, void 0, void 0, function* () {
var _a;
taskDefinitionsInUse.push(taskElement.taskDefinitionArn);
if (deleteResources && (!OneDayOlderOnly || AwsCliCommands.isOlderThan1day(taskElement.CreatedAt))) {
cloud_runner_logger_1.default.log(`Stopping task ${(_a = taskElement.containers) === null || _a === void 0 ? void 0 : _a[0].name}`);
yield ecs.stopTask({ task: taskElement.taskArn || '', cluster: element }).promise();
}
}));
yield AwsCliCommands.awsListStacks((element) => __awaiter(this, void 0, void 0, function* () {
var _b;
if ((_b = (yield CF.describeStackResources({ StackName: element.StackName }).promise()).StackResources) === null || _b === void 0 ? void 0 : _b.some((x) => x.ResourceType === 'AWS::ECS::TaskDefinition' && taskDefinitionsInUse.includes(x.PhysicalResourceId))) {
cloud_runner_logger_1.default.log(`Skipping ${element.StackName} - active task was running not deleting`);
return;
}
if (deleteResources && (!OneDayOlderOnly || AwsCliCommands.isOlderThan1day(element.CreationTime))) {
if (element.StackName === 'game-ci' || element.TemplateDescription === 'Game-CI base stack') {
cloud_runner_logger_1.default.log(`Skipping ${element.StackName} ignore list`);
return;
}
cloud_runner_logger_1.default.log(`Deleting ${element.logGroupName}`);
const deleteStackInput = { StackName: element.StackName };
yield CF.deleteStack(deleteStackInput).promise();
}
}));
yield AwsCliCommands.awsListLogGroups((element) => __awaiter(this, void 0, void 0, function* () {
if (deleteResources && (!OneDayOlderOnly || AwsCliCommands.isOlderThan1day(new Date(element.createdAt)))) {
cloud_runner_logger_1.default.log(`Deleting ${element.logGroupName}`);
yield cwl.deleteLogGroup({ logGroupName: element.logGroupName || '' }).promise();
}
}));
return task_service_1.TaskService.awsListJobs(perResultCallback);
});
}
}
@ -2173,6 +2044,9 @@ __decorate([
__decorate([
cli_functions_repository_1.CliFunction(`aws-list-tasks`, `List tasks`)
], AwsCliCommands, "awsListTasks", null);
__decorate([
cli_functions_repository_1.CliFunction(`list-tasks`, `List tasks`)
], AwsCliCommands, "listTasks", null);
__decorate([
cli_functions_repository_1.CliFunction(`aws-list-log-groups`, `List tasks`)
], AwsCliCommands, "awsListLogGroups", null);
@ -2227,10 +2101,45 @@ const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
const aws_job_stack_1 = __nccwpck_require__(70633);
const aws_base_stack_1 = __nccwpck_require__(26112);
const __1 = __nccwpck_require__(41359);
const aws_cli_commands_1 = __nccwpck_require__(34340);
const task_service_1 = __nccwpck_require__(67205);
const tertiary_resources_service_1 = __nccwpck_require__(61627);
class AWSBuildEnvironment {
constructor(buildParameters) {
this.baseStackName = buildParameters.awsBaseStackName;
}
inspect() {
throw new Error('Method not implemented.');
}
watch() {
throw new Error('Method not implemented.');
}
listOtherResources() {
return __awaiter(this, void 0, void 0, function* () {
yield tertiary_resources_service_1.TertiaryResourcesService.AwsListLogGroups();
return '';
});
}
garbageCollect(
// eslint-disable-next-line no-unused-vars
filter,
// eslint-disable-next-line no-unused-vars
previewOnly) {
throw new Error('Method not implemented.');
}
listAll() {
return __awaiter(this, void 0, void 0, function* () {
yield aws_cli_commands_1.AwsCliCommands.awsListAll();
return '';
});
}
listTasks() {
return __awaiter(this, void 0, void 0, function* () {
yield aws_cli_commands_1.AwsCliCommands.awsListJobs();
yield task_service_1.TaskService.awsListTasks();
return '';
});
}
cleanup(
// eslint-disable-next-line no-unused-vars
buildGuid,
@ -2301,6 +2210,277 @@ class AWSBuildEnvironment {
exports["default"] = AWSBuildEnvironment;
/***/ }),
/***/ 53859:
/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) {
"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.GarbageCollectionService = void 0;
const aws_sdk_1 = __importDefault(__nccwpck_require__(71786));
const cli_functions_repository_1 = __nccwpck_require__(85301);
const input_1 = __importDefault(__nccwpck_require__(91933));
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
const aws_cli_commands_1 = __nccwpck_require__(34340);
class GarbageCollectionService {
static garbageCollectAws() {
return __awaiter(this, void 0, void 0, function* () {
yield GarbageCollectionService.cleanup(false);
});
}
static garbageCollectAwsAll() {
return __awaiter(this, void 0, void 0, function* () {
yield GarbageCollectionService.cleanup(true);
});
}
static garbageCollectAwsAllOlderThanOneDay() {
return __awaiter(this, void 0, void 0, function* () {
yield GarbageCollectionService.cleanup(true, true);
});
}
static isOlderThan1day(date) {
const ageDate = new Date(date.getTime() - Date.now());
return ageDate.getDay() > 0;
}
static cleanup(deleteResources = false, OneDayOlderOnly = false) {
return __awaiter(this, void 0, void 0, function* () {
process.env.AWS_REGION = input_1.default.region;
const CF = new aws_sdk_1.default.CloudFormation();
const ecs = new aws_sdk_1.default.ECS();
const cwl = new aws_sdk_1.default.CloudWatchLogs();
const taskDefinitionsInUse = new Array();
yield aws_cli_commands_1.AwsCliCommands.awsListTasks((taskElement, element) => __awaiter(this, void 0, void 0, function* () {
var _a;
taskDefinitionsInUse.push(taskElement.taskDefinitionArn);
if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(taskElement.CreatedAt))) {
cloud_runner_logger_1.default.log(`Stopping task ${(_a = taskElement.containers) === null || _a === void 0 ? void 0 : _a[0].name}`);
yield ecs.stopTask({ task: taskElement.taskArn || '', cluster: element }).promise();
}
}));
yield aws_cli_commands_1.AwsCliCommands.awsListStacks((element) => __awaiter(this, void 0, void 0, function* () {
var _b;
if ((_b = (yield CF.describeStackResources({ StackName: element.StackName }).promise()).StackResources) === null || _b === void 0 ? void 0 : _b.some((x) => x.ResourceType === 'AWS::ECS::TaskDefinition' && taskDefinitionsInUse.includes(x.PhysicalResourceId))) {
cloud_runner_logger_1.default.log(`Skipping ${element.StackName} - active task was running not deleting`);
return;
}
if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(element.CreationTime))) {
if (element.StackName === 'game-ci' || element.TemplateDescription === 'Game-CI base stack') {
cloud_runner_logger_1.default.log(`Skipping ${element.StackName} ignore list`);
return;
}
cloud_runner_logger_1.default.log(`Deleting ${element.logGroupName}`);
const deleteStackInput = { StackName: element.StackName };
yield CF.deleteStack(deleteStackInput).promise();
}
}));
yield aws_cli_commands_1.AwsCliCommands.awsListLogGroups((element) => __awaiter(this, void 0, void 0, function* () {
if (deleteResources &&
(!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(new Date(element.createdAt)))) {
cloud_runner_logger_1.default.log(`Deleting ${element.logGroupName}`);
yield cwl.deleteLogGroup({ logGroupName: element.logGroupName || '' }).promise();
}
}));
});
}
}
__decorate([
cli_functions_repository_1.CliFunction(`aws-garbage-collect-list`, `garbage collect aws resources not in use !WIP!`)
], GarbageCollectionService, "garbageCollectAws", null);
__decorate([
cli_functions_repository_1.CliFunction(`aws-garbage-collect-all`, `garbage collect aws resources regardless of whether they are in use`)
], GarbageCollectionService, "garbageCollectAwsAll", null);
__decorate([
cli_functions_repository_1.CliFunction(`aws-garbage-collect-all-1d-older`, `garbage collect aws resources created more than 1d ago (ignore if they are in use)`)
], GarbageCollectionService, "garbageCollectAwsAllOlderThanOneDay", null);
exports.GarbageCollectionService = GarbageCollectionService;
/***/ }),
/***/ 67205:
/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) {
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.TaskService = void 0;
const aws_sdk_1 = __importDefault(__nccwpck_require__(71786));
const input_1 = __importDefault(__nccwpck_require__(91933));
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
const base_stack_formation_1 = __nccwpck_require__(29643);
class TaskService {
static awsListStacks(perResultCallback = false) {
var _a, _b;
return __awaiter(this, void 0, void 0, function* () {
process.env.AWS_REGION = input_1.default.region;
const CF = new aws_sdk_1.default.CloudFormation();
const stacks = ((_a = (yield CF.listStacks().promise()).StackSummaries) === null || _a === void 0 ? void 0 : _a.filter((_x) => _x.StackStatus !== 'DELETE_COMPLETE')) || [];
cloud_runner_logger_1.default.log(`Stacks ${stacks.length}`);
for (const element of stacks) {
const ageDate = new Date(Date.now() - element.CreationTime.getTime());
// if (verbose)
cloud_runner_logger_1.default.log(`Task Stack ${element.StackName} - Age D${Math.floor(ageDate.getHours() / 24)} H${ageDate.getHours()} M${ageDate.getMinutes()}`);
if (perResultCallback)
yield perResultCallback(element);
}
const baseStacks = ((_b = (yield CF.listStacks().promise()).StackSummaries) === null || _b === void 0 ? void 0 : _b.filter((_x) => _x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription === base_stack_formation_1.BaseStackFormation.baseStackDecription)) || [];
cloud_runner_logger_1.default.log(`Base Stacks ${baseStacks.length}`);
for (const element of baseStacks) {
const ageDate = new Date(Date.now() - element.CreationTime.getTime());
// if (verbose)
cloud_runner_logger_1.default.log(`Task Stack ${element.StackName} - Age D${Math.floor(ageDate.getHours() / 24)} H${ageDate.getHours()} M${ageDate.getMinutes()}`);
if (perResultCallback)
yield perResultCallback(element);
}
if (stacks === undefined) {
return;
}
});
}
static awsListTasks(perResultCallback = false) {
return __awaiter(this, void 0, void 0, function* () {
process.env.AWS_REGION = input_1.default.region;
const ecs = new aws_sdk_1.default.ECS();
const clusters = (yield ecs.listClusters().promise()).clusterArns || [];
cloud_runner_logger_1.default.log(`Clusters ${clusters.length}`);
for (const element of clusters) {
const input = {
cluster: element,
};
const list = (yield ecs.listTasks(input).promise()).taskArns || [];
if (list.length > 0) {
const describeInput = { tasks: list, cluster: element };
const describeList = (yield ecs.describeTasks(describeInput).promise()).tasks || [];
if (describeList === []) {
continue;
}
cloud_runner_logger_1.default.log(`Tasks ${describeList.length}`);
for (const taskElement of describeList) {
if (taskElement === undefined) {
continue;
}
taskElement.overrides = {};
taskElement.attachments = [];
if (taskElement.createdAt === undefined) {
cloud_runner_logger_1.default.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`);
continue;
}
if (perResultCallback)
yield perResultCallback(taskElement, element);
}
}
}
});
}
static awsListJobs(perResultCallback = false) {
var _a;
return __awaiter(this, void 0, void 0, function* () {
process.env.AWS_REGION = input_1.default.region;
const CF = new aws_sdk_1.default.CloudFormation();
const stacks = ((_a = (yield CF.listStacks().promise()).StackSummaries) === null || _a === void 0 ? void 0 : _a.filter((_x) => _x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription !== base_stack_formation_1.BaseStackFormation.baseStackDecription)) || [];
cloud_runner_logger_1.default.log(`Stacks ${stacks.length}`);
for (const element of stacks) {
const ageDate = new Date(Date.now() - element.CreationTime.getTime());
// if (verbose)
cloud_runner_logger_1.default.log(`Task Stack ${element.StackName} - Age D${Math.floor(ageDate.getHours() / 24)} H${ageDate.getHours()} M${ageDate.getMinutes()}`);
if (perResultCallback)
yield perResultCallback(element);
}
});
}
}
exports.TaskService = TaskService;
/***/ }),
/***/ 61627:
/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) {
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.TertiaryResourcesService = void 0;
const aws_sdk_1 = __importDefault(__nccwpck_require__(71786));
const input_1 = __importDefault(__nccwpck_require__(91933));
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
class TertiaryResourcesService {
static AwsListLogGroups(perResultCallback = false) {
return __awaiter(this, void 0, void 0, function* () {
process.env.AWS_REGION = input_1.default.region;
const ecs = new aws_sdk_1.default.CloudWatchLogs();
let logStreamInput = {
/* logGroupNamePrefix: 'game-ci' */
};
let logGroupsDescribe = yield ecs.describeLogGroups(logStreamInput).promise();
const logGroups = logGroupsDescribe.logGroups || [];
while (logGroupsDescribe.nextToken) {
logStreamInput = { /* logGroupNamePrefix: 'game-ci',*/ nextToken: logGroupsDescribe.nextToken };
logGroupsDescribe = yield ecs.describeLogGroups(logStreamInput).promise();
logGroups.push(...((logGroupsDescribe === null || logGroupsDescribe === void 0 ? void 0 : logGroupsDescribe.logGroups) || []));
}
cloud_runner_logger_1.default.log(`Log Groups ${logGroups.length}`);
for (const element of logGroups) {
if (element.creationTime === undefined) {
cloud_runner_logger_1.default.log(`Skipping ${element.logGroupName} no createdAt date`);
continue;
}
const ageDate = new Date(Date.now() - element.creationTime);
// if (verbose)
cloud_runner_logger_1.default.log(`Task Stack ${element.logGroupName} - Age D${Math.floor(ageDate.getHours() / 24)} H${ageDate.getHours()} M${ageDate.getMinutes()}`);
if (perResultCallback)
yield perResultCallback(element, element);
}
});
}
}
exports.TertiaryResourcesService = TertiaryResourcesService;
/***/ }),
/***/ 21613:
@ -2369,6 +2549,28 @@ class Kubernetes {
this.namespace = 'default';
this.buildParameters = buildParameters;
}
inspect() {
throw new Error('Method not implemented.');
}
watch() {
throw new Error('Method not implemented.');
}
listAll() {
throw new Error('Method not implemented.');
}
listTasks() {
throw new Error('Method not implemented.');
}
listOtherResources() {
throw new Error('Method not implemented.');
}
garbageCollect(
// eslint-disable-next-line no-unused-vars
filter,
// eslint-disable-next-line no-unused-vars
previewOnly) {
throw new Error('Method not implemented.');
}
setup(buildGuid, buildParameters,
// eslint-disable-next-line no-unused-vars
branchName,
@ -2510,7 +2712,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
const client_node_1 = __nccwpck_require__(89679);
const cloud_runner_build_command_process_1 = __nccwpck_require__(71899);
const cloud_runner_custom_hooks_1 = __nccwpck_require__(58873);
const cloud_runner_1 = __importDefault(__nccwpck_require__(79144));
class KubernetesJobSpecFactory {
static getJobSpec(command, image, mountdir, workingDirectory, environment, secrets, buildGuid, buildParameters, secretName, pvcName, jobName, k8s) {
@ -2595,7 +2797,7 @@ class KubernetesJobSpecFactory {
name: 'main',
image,
command: ['/bin/sh'],
args: ['-c', cloud_runner_build_command_process_1.CloudRunnerBuildCommandProcessor.ProcessCommands(command, cloud_runner_1.default.buildParameters)],
args: ['-c', cloud_runner_custom_hooks_1.CloudRunnerCustomHooks.ApplyHooksToCommands(command, cloud_runner_1.default.buildParameters)],
workingDir: `${workingDirectory}`,
resources: {
requests: {
@ -3042,6 +3244,28 @@ Object.defineProperty(exports, "__esModule", ({ value: true }));
const cloud_runner_system_1 = __nccwpck_require__(99393);
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
class LocalDockerCloudRunner {
inspect() {
throw new Error('Method not implemented.');
}
watch() {
throw new Error('Method not implemented.');
}
listAll() {
throw new Error('Method not implemented.');
}
listTasks() {
throw new Error('Method not implemented.');
}
listOtherResources() {
throw new Error('Method not implemented.');
}
garbageCollect(
// eslint-disable-next-line no-unused-vars
filter,
// eslint-disable-next-line no-unused-vars
previewOnly) {
throw new Error('Method not implemented.');
}
cleanup(
// eslint-disable-next-line no-unused-vars
buildGuid,
@ -3102,6 +3326,28 @@ Object.defineProperty(exports, "__esModule", ({ value: true }));
const cloud_runner_system_1 = __nccwpck_require__(99393);
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
class LocalCloudRunner {
inspect() {
throw new Error('Method not implemented.');
}
watch() {
throw new Error('Method not implemented.');
}
listAll() {
throw new Error('Method not implemented.');
}
listTasks() {
throw new Error('Method not implemented.');
}
listOtherResources() {
throw new Error('Method not implemented.');
}
garbageCollect(
// eslint-disable-next-line no-unused-vars
filter,
// eslint-disable-next-line no-unused-vars
previewOnly) {
throw new Error('Method not implemented.');
}
cleanup(
// eslint-disable-next-line no-unused-vars
buildGuid,
@ -3162,6 +3408,28 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
Object.defineProperty(exports, "__esModule", ({ value: true }));
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
class TestCloudRunner {
inspect() {
throw new Error('Method not implemented.');
}
watch() {
throw new Error('Method not implemented.');
}
listAll() {
throw new Error('Method not implemented.');
}
listTasks() {
throw new Error('Method not implemented.');
}
listOtherResources() {
throw new Error('Method not implemented.');
}
garbageCollect(
// eslint-disable-next-line no-unused-vars
filter,
// eslint-disable-next-line no-unused-vars
previewOnly) {
throw new Error('Method not implemented.');
}
cleanup(
// eslint-disable-next-line no-unused-vars
buildGuid,
@ -3537,7 +3805,21 @@ exports.RemoteClientLogger = RemoteClientLogger;
/***/ }),
/***/ 71899:
/***/ 91311:
/***/ ((__unused_webpack_module, exports) => {
"use strict";
Object.defineProperty(exports, "__esModule", ({ value: true }));
class CloudRunnerConstants {
}
CloudRunnerConstants.alphabet = '0123456789abcdefghijklmnopqrstuvwxyz';
exports["default"] = CloudRunnerConstants;
/***/ }),
/***/ 58873:
/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) {
"use strict";
@ -3546,12 +3828,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.Hook = exports.CloudRunnerBuildCommandProcessor = void 0;
exports.Hook = exports.CloudRunnerCustomHooks = void 0;
const yaml_1 = __importDefault(__nccwpck_require__(44603));
const cloud_runner_1 = __importDefault(__nccwpck_require__(79144));
class CloudRunnerBuildCommandProcessor {
static ProcessCommands(commands, buildParameters) {
const hooks = CloudRunnerBuildCommandProcessor.getHooks(buildParameters.customJobHooks).filter((x) => x.step.includes(`all`));
class CloudRunnerCustomHooks {
static ApplyHooksToCommands(commands, buildParameters) {
const hooks = CloudRunnerCustomHooks.getHooks(buildParameters.customJobHooks).filter((x) => x.step.includes(`all`));
return `echo "---"
echo "start cloud runner init"
${cloud_runner_1.default.buildParameters.cloudRunnerIntegrationTests ? '' : '#'} printenv
@ -3576,7 +3858,7 @@ class CloudRunnerBuildCommandProcessor {
return output.filter((x) => x.step !== undefined && x.hook !== undefined && x.hook.length > 0);
}
}
exports.CloudRunnerBuildCommandProcessor = CloudRunnerBuildCommandProcessor;
exports.CloudRunnerCustomHooks = CloudRunnerCustomHooks;
class Hook {
constructor() {
this.secrets = new Array();
@ -3585,20 +3867,6 @@ class Hook {
exports.Hook = Hook;
/***/ }),
/***/ 91311:
/***/ ((__unused_webpack_module, exports) => {
"use strict";
Object.defineProperty(exports, "__esModule", ({ value: true }));
class CloudRunnerConstants {
}
CloudRunnerConstants.alphabet = '0123456789abcdefghijklmnopqrstuvwxyz';
exports["default"] = CloudRunnerConstants;
/***/ }),
/***/ 13527:
@ -4102,7 +4370,7 @@ Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.TaskParameterSerializer = void 0;
const __1 = __nccwpck_require__(41359);
const image_environment_factory_1 = __importDefault(__nccwpck_require__(25145));
const cloud_runner_build_command_process_1 = __nccwpck_require__(71899);
const cloud_runner_custom_hooks_1 = __nccwpck_require__(58873);
const cloud_runner_query_override_1 = __importDefault(__nccwpck_require__(31011));
class TaskParameterSerializer {
static readBuildEnvironmentVariables() {
@ -4126,7 +4394,7 @@ class TaskParameterSerializer {
let array = new Array();
array = TaskParameterSerializer.readBuildParameters(array);
array = TaskParameterSerializer.readInput(array);
const configurableHooks = cloud_runner_build_command_process_1.CloudRunnerBuildCommandProcessor.getHooks(__1.CloudRunner.buildParameters.customJobHooks);
const configurableHooks = cloud_runner_custom_hooks_1.CloudRunnerCustomHooks.getHooks(__1.CloudRunner.buildParameters.customJobHooks);
const secrets = configurableHooks.map((x) => x.secrets).filter((x) => x !== undefined && x.length > 0);
if (secrets.length > 0) {
// eslint-disable-next-line unicorn/no-array-reduce
@ -4244,7 +4512,7 @@ const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
const cloud_runner_folders_1 = __nccwpck_require__(13527);
const custom_workflow_1 = __nccwpck_require__(3786);
const core = __importStar(__nccwpck_require__(42186));
const cloud_runner_build_command_process_1 = __nccwpck_require__(71899);
const cloud_runner_custom_hooks_1 = __nccwpck_require__(58873);
const path_1 = __importDefault(__nccwpck_require__(71017));
const cloud_runner_1 = __importDefault(__nccwpck_require__(79144));
class BuildAutomationWorkflow {
@ -4297,8 +4565,8 @@ class BuildAutomationWorkflow {
});
}
static get BuildWorkflow() {
const setupHooks = cloud_runner_build_command_process_1.CloudRunnerBuildCommandProcessor.getHooks(cloud_runner_1.default.buildParameters.customJobHooks).filter((x) => x.step.includes(`setup`));
const buildHooks = cloud_runner_build_command_process_1.CloudRunnerBuildCommandProcessor.getHooks(cloud_runner_1.default.buildParameters.customJobHooks).filter((x) => x.step.includes(`build`));
const setupHooks = cloud_runner_custom_hooks_1.CloudRunnerCustomHooks.getHooks(cloud_runner_1.default.buildParameters.customJobHooks).filter((x) => x.step.includes(`setup`));
const buildHooks = cloud_runner_custom_hooks_1.CloudRunnerCustomHooks.getHooks(cloud_runner_1.default.buildParameters.customJobHooks).filter((x) => x.step.includes(`build`));
const builderPath = path_1.default.join(cloud_runner_folders_1.CloudRunnerFolders.builderPathAbsolute, 'dist', `index.js`).replace(/\\/g, `/`);
return `apt-get update > /dev/null
apt-get install -y tar tree npm git-lfs jq git > /dev/null

2
dist/index.js.map vendored
View File

File diff suppressed because one or more lines are too long

173
src/model/cloud-runner/cloud-runner-options.ts 100644
View File

@ -0,0 +1,173 @@
import { Cli } from '../cli/cli';
import CloudRunnerQueryOverride from './services/cloud-runner-query-override';
import Input from '../input';
const core = require('@actions/core');
class CloudRunnerOptions {
public static getInput(query) {
if (Input.githubInputEnabled) {
const coreInput = core.getInput(query);
if (coreInput && coreInput !== '') {
return coreInput;
}
}
const alternativeQuery = CloudRunnerOptions.ToEnvVarFormat(query);
// Query input sources
if (Cli.query(query, alternativeQuery)) {
return Cli.query(query, alternativeQuery);
}
if (CloudRunnerQueryOverride.query(query, alternativeQuery)) {
return CloudRunnerQueryOverride.query(query, alternativeQuery);
}
if (process.env[query] !== undefined) {
return process.env[query];
}
if (alternativeQuery !== query && process.env[alternativeQuery] !== undefined) {
return process.env[alternativeQuery];
}
return;
}
static get region(): string {
return CloudRunnerOptions.getInput('region') || 'eu-west-2';
}
static get githubRepo() {
return CloudRunnerOptions.getInput('GITHUB_REPOSITORY') || CloudRunnerOptions.getInput('GITHUB_REPO') || undefined;
}
static get branch() {
if (CloudRunnerOptions.getInput(`GITHUB_REF`)) {
return CloudRunnerOptions.getInput(`GITHUB_REF`).replace('refs/', '').replace(`head/`, '').replace(`heads/`, '');
} else if (CloudRunnerOptions.getInput('branch')) {
return CloudRunnerOptions.getInput('branch');
} else {
return '';
}
}
static get cloudRunnerBuilderPlatform() {
const input = CloudRunnerOptions.getInput('cloudRunnerBuilderPlatform');
if (input) {
return input;
}
if (CloudRunnerOptions.cloudRunnerCluster !== 'local') {
return 'linux';
}
return;
}
static get gitSha() {
if (CloudRunnerOptions.getInput(`GITHUB_SHA`)) {
return CloudRunnerOptions.getInput(`GITHUB_SHA`);
} else if (CloudRunnerOptions.getInput(`GitSHA`)) {
return CloudRunnerOptions.getInput(`GitSHA`);
}
}
static get customJob() {
return CloudRunnerOptions.getInput('customJob') || '';
}
static customJobHooks() {
return CloudRunnerOptions.getInput('customJobHooks') || '';
}
static cachePushOverrideCommand() {
return CloudRunnerOptions.getInput('cachePushOverrideCommand') || '';
}
static cachePullOverrideCommand() {
return CloudRunnerOptions.getInput('cachePullOverrideCommand') || '';
}
static readInputFromOverrideList() {
return CloudRunnerOptions.getInput('readInputFromOverrideList') || '';
}
static readInputOverrideCommand() {
return CloudRunnerOptions.getInput('readInputOverrideCommand') || '';
}
static get cloudRunnerBranch() {
return CloudRunnerOptions.getInput('cloudRunnerBranch') || 'cloud-runner-develop';
}
static get postBuildSteps() {
return CloudRunnerOptions.getInput('postBuildSteps') || '';
}
static get preBuildSteps() {
return CloudRunnerOptions.getInput('preBuildSteps') || '';
}
static get awsBaseStackName() {
return CloudRunnerOptions.getInput('awsBaseStackName') || 'game-ci';
}
static get cloudRunnerCluster() {
if (Cli.isCliMode) {
return CloudRunnerOptions.getInput('cloudRunnerCluster') || 'aws';
}
return CloudRunnerOptions.getInput('cloudRunnerCluster') || 'local';
}
static get cloudRunnerCpu() {
return CloudRunnerOptions.getInput('cloudRunnerCpu');
}
static get cloudRunnerMemory() {
return CloudRunnerOptions.getInput('cloudRunnerMemory');
}
static get kubeConfig() {
return CloudRunnerOptions.getInput('kubeConfig') || '';
}
static get kubeVolume() {
return CloudRunnerOptions.getInput('kubeVolume') || '';
}
static get kubeVolumeSize() {
return CloudRunnerOptions.getInput('kubeVolumeSize') || '5Gi';
}
static get kubeStorageClass(): string {
return CloudRunnerOptions.getInput('kubeStorageClass') || '';
}
static get checkDependencyHealthOverride(): string {
return CloudRunnerOptions.getInput('checkDependencyHealthOverride') || '';
}
static get startDependenciesOverride(): string {
return CloudRunnerOptions.getInput('startDependenciesOverride') || '';
}
static get cacheKey(): string {
return CloudRunnerOptions.getInput('cacheKey') || CloudRunnerOptions.branch;
}
static get cloudRunnerTests(): boolean {
return CloudRunnerOptions.getInput(`cloudRunnerTests`) || false;
}
public static ToEnvVarFormat(input: string) {
if (input.toUpperCase() === input) {
return input;
}
return input
.replace(/([A-Z])/g, ' $1')
.trim()
.toUpperCase()
.replace(/ /g, '_');
}
}
export default CloudRunnerOptions;

4
src/model/cloud-runner/providers/aws/aws-task-runner.ts
View File

@ -6,7 +6,7 @@ import * as zlib from 'zlib';
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import { Input } from '../../..';
import CloudRunner from '../../cloud-runner';
import { CloudRunnerBuildCommandProcessor } from '../../services/cloud-runner-build-command-process';
import { CloudRunnerCustomHooks } from '../../services/cloud-runner-custom-hooks';
import { FollowLogStreamService } from '../../services/follow-log-stream-service';
class AWSTaskRunner {
@ -39,7 +39,7 @@ class AWSTaskRunner {
{
name: taskDef.taskDefStackName,
environment,
command: ['-c', CloudRunnerBuildCommandProcessor.ProcessCommands(commands, CloudRunner.buildParameters)],
command: ['-c', CloudRunnerCustomHooks.ApplyHooksToCommands(commands, CloudRunner.buildParameters)],
},
],
},

190
src/model/cloud-runner/providers/aws/commands/aws-cli-commands.ts
View File

@ -1,207 +1,51 @@
import AWS from 'aws-sdk';
import { CliFunction } from '../../../../cli/cli-functions-repository';
import Input from '../../../../input';
import CloudRunnerLogger from '../../../services/cloud-runner-logger';
import { BaseStackFormation } from '../cloud-formations/base-stack-formation';
import { TaskService } from '../services/task-service';
import { GarbageCollectionService } from '../services/garbage-collection-service';
import { TertiaryResourcesService } from '../services/tertiary-resources-service';
export class AwsCliCommands {
@CliFunction(`aws-list-all`, `List all resources`)
static async awsListAll() {
public static async awsListAll() {
await AwsCliCommands.awsListStacks();
await AwsCliCommands.awsListTasks();
await AwsCliCommands.awsListLogGroups();
}
@CliFunction(`aws-garbage-collect-list`, `garbage collect aws resources not in use !WIP!`)
static async garbageCollectAws() {
await AwsCliCommands.cleanup(false);
await GarbageCollectionService.cleanup(false);
}
@CliFunction(`aws-garbage-collect-all`, `garbage collect aws resources regardless of whether they are in use`)
static async garbageCollectAwsAll() {
await AwsCliCommands.cleanup(true);
await GarbageCollectionService.cleanup(true);
}
@CliFunction(
`aws-garbage-collect-all-1d-older`,
`garbage collect aws resources created more than 1d ago (ignore if they are in use)`,
)
static async garbageCollectAwsAllOlderThanOneDay() {
await AwsCliCommands.cleanup(true, true);
await GarbageCollectionService.cleanup(true, true);
}
static isOlderThan1day(date: any) {
const ageDate = new Date(date.getTime() - Date.now());
return ageDate.getDay() > 0;
}
@CliFunction(`aws-list-stacks`, `List stacks`)
static async awsListStacks(perResultCallback: any = false) {
process.env.AWS_REGION = Input.region;
const CF = new AWS.CloudFormation();
const stacks =
(await CF.listStacks().promise()).StackSummaries?.filter(
(_x) => _x.StackStatus !== 'DELETE_COMPLETE', // &&
// _x.TemplateDescription === TaskDefinitionFormation.description.replace('\n', ''),
) || [];
CloudRunnerLogger.log(`Stacks ${stacks.length}`);
for (const element of stacks) {
const ageDate: Date = new Date(Date.now() - element.CreationTime.getTime());
// if (verbose)
CloudRunnerLogger.log(
`Task Stack ${element.StackName} - Age D${Math.floor(
ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
);
if (perResultCallback) await perResultCallback(element);
}
const baseStacks =
(await CF.listStacks().promise()).StackSummaries?.filter(
(_x) =>
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription === BaseStackFormation.baseStackDecription,
) || [];
CloudRunnerLogger.log(`Base Stacks ${baseStacks.length}`);
for (const element of baseStacks) {
const ageDate: Date = new Date(Date.now() - element.CreationTime.getTime());
// if (verbose)
CloudRunnerLogger.log(
`Task Stack ${element.StackName} - Age D${Math.floor(
ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
);
if (perResultCallback) await perResultCallback(element);
}
if (stacks === undefined) {
return;
}
return TaskService.awsListStacks(perResultCallback);
}
@CliFunction(`aws-list-tasks`, `List tasks`)
static async awsListTasks(perResultCallback: any = false) {
process.env.AWS_REGION = Input.region;
const ecs = new AWS.ECS();
const clusters = (await ecs.listClusters().promise()).clusterArns || [];
CloudRunnerLogger.log(`Clusters ${clusters.length}`);
for (const element of clusters) {
const input: AWS.ECS.ListTasksRequest = {
cluster: element,
};
const list = (await ecs.listTasks(input).promise()).taskArns || [];
if (list.length > 0) {
const describeInput: AWS.ECS.DescribeTasksRequest = { tasks: list, cluster: element };
const describeList = (await ecs.describeTasks(describeInput).promise()).tasks || [];
if (describeList === []) {
continue;
}
CloudRunnerLogger.log(`Tasks ${describeList.length}`);
for (const taskElement of describeList) {
if (taskElement === undefined) {
continue;
}
taskElement.overrides = {};
taskElement.attachments = [];
if (taskElement.createdAt === undefined) {
CloudRunnerLogger.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`);
continue;
}
if (perResultCallback) await perResultCallback(taskElement, element);
}
}
}
return TaskService.awsListJobs(perResultCallback);
}
@CliFunction(`list-tasks`, `List tasks`)
static async listTasks(perResultCallback: any = false) {
return TaskService.awsListJobs(perResultCallback);
}
@CliFunction(`aws-list-log-groups`, `List tasks`)
static async awsListLogGroups(perResultCallback: any = false) {
process.env.AWS_REGION = Input.region;
const ecs = new AWS.CloudWatchLogs();
let logStreamInput: AWS.CloudWatchLogs.DescribeLogGroupsRequest = {
/* logGroupNamePrefix: 'game-ci' */
};
let logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise();
const logGroups = logGroupsDescribe.logGroups || [];
while (logGroupsDescribe.nextToken) {
logStreamInput = { /* logGroupNamePrefix: 'game-ci',*/ nextToken: logGroupsDescribe.nextToken };
logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise();
logGroups.push(...(logGroupsDescribe?.logGroups || []));
}
CloudRunnerLogger.log(`Log Groups ${logGroups.length}`);
for (const element of logGroups) {
if (element.creationTime === undefined) {
CloudRunnerLogger.log(`Skipping ${element.logGroupName} no createdAt date`);
continue;
}
const ageDate: Date = new Date(Date.now() - element.creationTime);
// if (verbose)
CloudRunnerLogger.log(
`Task Stack ${element.logGroupName} - Age D${Math.floor(
ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
);
if (perResultCallback) await perResultCallback(element, element);
}
await TertiaryResourcesService.AwsListLogGroups(perResultCallback);
}
@CliFunction(`aws-list-jobs`, `List tasks`)
static async awsListJobs(perResultCallback: any = false) {
process.env.AWS_REGION = Input.region;
const CF = new AWS.CloudFormation();
const stacks =
(await CF.listStacks().promise()).StackSummaries?.filter(
(_x) =>
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription !== BaseStackFormation.baseStackDecription,
) || [];
CloudRunnerLogger.log(`Stacks ${stacks.length}`);
for (const element of stacks) {
const ageDate: Date = new Date(Date.now() - element.CreationTime.getTime());
// if (verbose)
CloudRunnerLogger.log(
`Task Stack ${element.StackName} - Age D${Math.floor(
ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
);
if (perResultCallback) await perResultCallback(element);
}
}
private static async cleanup(deleteResources = false, OneDayOlderOnly: boolean = false) {
process.env.AWS_REGION = Input.region;
const CF = new AWS.CloudFormation();
const ecs = new AWS.ECS();
const cwl = new AWS.CloudWatchLogs();
const taskDefinitionsInUse = new Array();
await AwsCliCommands.awsListTasks(async (taskElement, element) => {
taskDefinitionsInUse.push(taskElement.taskDefinitionArn);
if (deleteResources && (!OneDayOlderOnly || AwsCliCommands.isOlderThan1day(taskElement.CreatedAt))) {
CloudRunnerLogger.log(`Stopping task ${taskElement.containers?.[0].name}`);
await ecs.stopTask({ task: taskElement.taskArn || '', cluster: element }).promise();
}
});
await AwsCliCommands.awsListStacks(async (element) => {
if (
(await CF.describeStackResources({ StackName: element.StackName }).promise()).StackResources?.some(
(x) => x.ResourceType === 'AWS::ECS::TaskDefinition' && taskDefinitionsInUse.includes(x.PhysicalResourceId),
)
) {
CloudRunnerLogger.log(`Skipping ${element.StackName} - active task was running not deleting`);
return;
}
if (deleteResources && (!OneDayOlderOnly || AwsCliCommands.isOlderThan1day(element.CreationTime))) {
if (element.StackName === 'game-ci' || element.TemplateDescription === 'Game-CI base stack') {
CloudRunnerLogger.log(`Skipping ${element.StackName} ignore list`);
return;
}
CloudRunnerLogger.log(`Deleting ${element.logGroupName}`);
const deleteStackInput: AWS.CloudFormation.DeleteStackInput = { StackName: element.StackName };
await CF.deleteStack(deleteStackInput).promise();
}
});
await AwsCliCommands.awsListLogGroups(async (element) => {
if (deleteResources && (!OneDayOlderOnly || AwsCliCommands.isOlderThan1day(new Date(element.createdAt)))) {
CloudRunnerLogger.log(`Deleting ${element.logGroupName}`);
await cwl.deleteLogGroup({ logGroupName: element.logGroupName || '' }).promise();
}
});
public static async awsListJobs(perResultCallback: any = false) {
return TaskService.awsListJobs(perResultCallback);
}
}

37
src/model/cloud-runner/providers/aws/index.ts
View File

@ -9,6 +9,9 @@ import CloudRunnerLogger from '../../services/cloud-runner-logger';
import { AWSJobStack } from './aws-job-stack';
import { AWSBaseStack } from './aws-base-stack';
import { Input } from '../../..';
import { AwsCliCommands } from './commands/aws-cli-commands';
import { TaskService } from './services/task-service';
import { TertiaryResourcesService } from './services/tertiary-resources-service';
class AWSBuildEnvironment implements ProviderInterface {
private baseStackName: string;
@ -16,6 +19,40 @@ class AWSBuildEnvironment implements ProviderInterface {
constructor(buildParameters: BuildParameters) {
this.baseStackName = buildParameters.awsBaseStackName;
}
inspect(): Promise<string> {
throw new Error('Method not implemented.');
}
watch(): Promise<string> {
throw new Error('Method not implemented.');
}
async listOtherResources(): Promise<string> {
await TertiaryResourcesService.AwsListLogGroups();
return '';
}
garbageCollect(
// eslint-disable-next-line no-unused-vars
filter: string,
// eslint-disable-next-line no-unused-vars
previewOnly: boolean,
): Promise<string> {
throw new Error('Method not implemented.');
}
async listAll() {
await AwsCliCommands.awsListAll();
return '';
}
async listTasks() {
await AwsCliCommands.awsListJobs();
await TaskService.awsListTasks();
return '';
}
async cleanup(
// eslint-disable-next-line no-unused-vars
buildGuid: string,

73
src/model/cloud-runner/providers/aws/services/garbage-collection-service.ts 100644
View File

@ -0,0 +1,73 @@
import AWS from 'aws-sdk';
import { CliFunction } from '../../../../cli/cli-functions-repository';
import Input from '../../../../input';
import CloudRunnerLogger from '../../../services/cloud-runner-logger';
import { AwsCliCommands } from '../commands/aws-cli-commands';
export class GarbageCollectionService {
@CliFunction(`aws-garbage-collect-list`, `garbage collect aws resources not in use !WIP!`)
static async garbageCollectAws() {
await GarbageCollectionService.cleanup(false);
}
@CliFunction(`aws-garbage-collect-all`, `garbage collect aws resources regardless of whether they are in use`)
static async garbageCollectAwsAll() {
await GarbageCollectionService.cleanup(true);
}
@CliFunction(
`aws-garbage-collect-all-1d-older`,
`garbage collect aws resources created more than 1d ago (ignore if they are in use)`,
)
static async garbageCollectAwsAllOlderThanOneDay() {
await GarbageCollectionService.cleanup(true, true);
}
static isOlderThan1day(date: any) {
const ageDate = new Date(date.getTime() - Date.now());
return ageDate.getDay() > 0;
}
public static async cleanup(deleteResources = false, OneDayOlderOnly: boolean = false) {
process.env.AWS_REGION = Input.region;
const CF = new AWS.CloudFormation();
const ecs = new AWS.ECS();
const cwl = new AWS.CloudWatchLogs();
const taskDefinitionsInUse = new Array();
await AwsCliCommands.awsListTasks(async (taskElement, element) => {
taskDefinitionsInUse.push(taskElement.taskDefinitionArn);
if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(taskElement.CreatedAt))) {
CloudRunnerLogger.log(`Stopping task ${taskElement.containers?.[0].name}`);
await ecs.stopTask({ task: taskElement.taskArn || '', cluster: element }).promise();
}
});
await AwsCliCommands.awsListStacks(async (element) => {
if (
(await CF.describeStackResources({ StackName: element.StackName }).promise()).StackResources?.some(
(x) => x.ResourceType === 'AWS::ECS::TaskDefinition' && taskDefinitionsInUse.includes(x.PhysicalResourceId),
)
) {
CloudRunnerLogger.log(`Skipping ${element.StackName} - active task was running not deleting`);
return;
}
if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(element.CreationTime))) {
if (element.StackName === 'game-ci' || element.TemplateDescription === 'Game-CI base stack') {
CloudRunnerLogger.log(`Skipping ${element.StackName} ignore list`);
return;
}
CloudRunnerLogger.log(`Deleting ${element.logGroupName}`);
const deleteStackInput: AWS.CloudFormation.DeleteStackInput = { StackName: element.StackName };
await CF.deleteStack(deleteStackInput).promise();
}
});
await AwsCliCommands.awsListLogGroups(async (element) => {
if (
deleteResources &&
(!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(new Date(element.createdAt)))
) {
CloudRunnerLogger.log(`Deleting ${element.logGroupName}`);
await cwl.deleteLogGroup({ logGroupName: element.logGroupName || '' }).promise();
}
});
}
}

102
src/model/cloud-runner/providers/aws/services/task-service.ts 100644
View File

@ -0,0 +1,102 @@
import AWS from 'aws-sdk';
import Input from '../../../../input';
import CloudRunnerLogger from '../../../services/cloud-runner-logger';
import { BaseStackFormation } from '../cloud-formations/base-stack-formation';
export class TaskService {
public static async awsListStacks(perResultCallback: any = false) {
process.env.AWS_REGION = Input.region;
const CF = new AWS.CloudFormation();
const stacks =
(await CF.listStacks().promise()).StackSummaries?.filter(
(_x) => _x.StackStatus !== 'DELETE_COMPLETE', // &&
// _x.TemplateDescription === TaskDefinitionFormation.description.replace('\n', ''),
) || [];
CloudRunnerLogger.log(`Stacks ${stacks.length}`);
for (const element of stacks) {
const ageDate: Date = new Date(Date.now() - element.CreationTime.getTime());
// if (verbose)
CloudRunnerLogger.log(
`Task Stack ${element.StackName} - Age D${Math.floor(
ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
);
if (perResultCallback) await perResultCallback(element);
}
const baseStacks =
(await CF.listStacks().promise()).StackSummaries?.filter(
(_x) =>
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription === BaseStackFormation.baseStackDecription,
) || [];
CloudRunnerLogger.log(`Base Stacks ${baseStacks.length}`);
for (const element of baseStacks) {
const ageDate: Date = new Date(Date.now() - element.CreationTime.getTime());
// if (verbose)
CloudRunnerLogger.log(
`Task Stack ${element.StackName} - Age D${Math.floor(
ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
);
if (perResultCallback) await perResultCallback(element);
}
if (stacks === undefined) {
return;
}
}
public static async awsListTasks(perResultCallback: any = false) {
process.env.AWS_REGION = Input.region;
const ecs = new AWS.ECS();
const clusters = (await ecs.listClusters().promise()).clusterArns || [];
CloudRunnerLogger.log(`Clusters ${clusters.length}`);
for (const element of clusters) {
const input: AWS.ECS.ListTasksRequest = {
cluster: element,
};
const list = (await ecs.listTasks(input).promise()).taskArns || [];
if (list.length > 0) {
const describeInput: AWS.ECS.DescribeTasksRequest = { tasks: list, cluster: element };
const describeList = (await ecs.describeTasks(describeInput).promise()).tasks || [];
if (describeList === []) {
continue;
}
CloudRunnerLogger.log(`Tasks ${describeList.length}`);
for (const taskElement of describeList) {
if (taskElement === undefined) {
continue;
}
taskElement.overrides = {};
taskElement.attachments = [];
if (taskElement.createdAt === undefined) {
CloudRunnerLogger.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`);
continue;
}
if (perResultCallback) await perResultCallback(taskElement, element);
}
}
}
}
public static async awsListJobs(perResultCallback: any = false) {
process.env.AWS_REGION = Input.region;
const CF = new AWS.CloudFormation();
const stacks =
(await CF.listStacks().promise()).StackSummaries?.filter(
(_x) =>
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription !== BaseStackFormation.baseStackDecription,
) || [];
CloudRunnerLogger.log(`Stacks ${stacks.length}`);
for (const element of stacks) {
const ageDate: Date = new Date(Date.now() - element.CreationTime.getTime());
// if (verbose)
CloudRunnerLogger.log(
`Task Stack ${element.StackName} - Age D${Math.floor(
ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
);
if (perResultCallback) await perResultCallback(element);
}
}
}

37
src/model/cloud-runner/providers/aws/services/tertiary-resources-service.ts 100644
View File

@ -0,0 +1,37 @@
import AWS from 'aws-sdk';
import Input from '../../../../input';
import CloudRunnerLogger from '../../../services/cloud-runner-logger';
export class TertiaryResourcesService {
public static async AwsListLogGroups(perResultCallback: any = false) {
process.env.AWS_REGION = Input.region;
const ecs = new AWS.CloudWatchLogs();
let logStreamInput: AWS.CloudWatchLogs.DescribeLogGroupsRequest = {
/* logGroupNamePrefix: 'game-ci' */
};
let logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise();
const logGroups = logGroupsDescribe.logGroups || [];
while (logGroupsDescribe.nextToken) {
logStreamInput = { /* logGroupNamePrefix: 'game-ci',*/ nextToken: logGroupsDescribe.nextToken };
logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise();
logGroups.push(...(logGroupsDescribe?.logGroups || []));
}
CloudRunnerLogger.log(`Log Groups ${logGroups.length}`);
for (const element of logGroups) {
if (element.creationTime === undefined) {
CloudRunnerLogger.log(`Skipping ${element.logGroupName} no createdAt date`);
continue;
}
const ageDate: Date = new Date(Date.now() - element.creationTime);
// if (verbose)
CloudRunnerLogger.log(
`Task Stack ${element.logGroupName} - Age D${Math.floor(
ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
);
if (perResultCallback) await perResultCallback(element, element);
}
}
}

23
src/model/cloud-runner/providers/k8s/index.ts
View File

@ -39,6 +39,29 @@ class Kubernetes implements ProviderInterface {
this.namespace = 'default';
this.buildParameters = buildParameters;
}
inspect(): Promise<string> {
throw new Error('Method not implemented.');
}
watch(): Promise<string> {
throw new Error('Method not implemented.');
}
listAll(): Promise<string> {
throw new Error('Method not implemented.');
}
listTasks(): Promise<string> {
throw new Error('Method not implemented.');
}
listOtherResources(): Promise<string> {
throw new Error('Method not implemented.');
}
garbageCollect(
// eslint-disable-next-line no-unused-vars
filter: string,
// eslint-disable-next-line no-unused-vars
previewOnly: boolean,
): Promise<string> {
throw new Error('Method not implemented.');
}
public async setup(
buildGuid: string,
buildParameters: BuildParameters,

4
src/model/cloud-runner/providers/k8s/kubernetes-job-spec-factory.ts
View File

@ -1,6 +1,6 @@
import { V1EnvVar, V1EnvVarSource, V1SecretKeySelector } from '@kubernetes/client-node';
import BuildParameters from '../../../build-parameters';
import { CloudRunnerBuildCommandProcessor } from '../../services/cloud-runner-build-command-process';
import { CloudRunnerCustomHooks } from '../../services/cloud-runner-custom-hooks';
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
import CloudRunnerSecret from '../../services/cloud-runner-secret';
import CloudRunner from '../../cloud-runner';
@ -103,7 +103,7 @@ class KubernetesJobSpecFactory {
name: 'main',
image,
command: ['/bin/sh'],
args: ['-c', CloudRunnerBuildCommandProcessor.ProcessCommands(command, CloudRunner.buildParameters)],
args: ['-c', CloudRunnerCustomHooks.ApplyHooksToCommands(command, CloudRunner.buildParameters)],
workingDir: `${workingDirectory}`,
resources: {

23
src/model/cloud-runner/providers/local-docker/index.ts
View File

@ -6,6 +6,29 @@ import { ProviderInterface } from '../provider-interface';
import CloudRunnerSecret from '../../services/cloud-runner-secret';
class LocalDockerCloudRunner implements ProviderInterface {
inspect(): Promise<string> {
throw new Error('Method not implemented.');
}
watch(): Promise<string> {
throw new Error('Method not implemented.');
}
listAll(): Promise<string> {
throw new Error('Method not implemented.');
}
listTasks(): Promise<string> {
throw new Error('Method not implemented.');
}
listOtherResources(): Promise<string> {
throw new Error('Method not implemented.');
}
garbageCollect(
// eslint-disable-next-line no-unused-vars
filter: string,
// eslint-disable-next-line no-unused-vars
previewOnly: boolean,
): Promise<string> {
throw new Error('Method not implemented.');
}
cleanup(
// eslint-disable-next-line no-unused-vars
buildGuid: string,

23
src/model/cloud-runner/providers/local/index.ts
View File

@ -6,6 +6,29 @@ import { ProviderInterface } from '../provider-interface';
import CloudRunnerSecret from '../../services/cloud-runner-secret';
class LocalCloudRunner implements ProviderInterface {
inspect(): Promise<string> {
throw new Error('Method not implemented.');
}
watch(): Promise<string> {
throw new Error('Method not implemented.');
}
listAll(): Promise<string> {
throw new Error('Method not implemented.');
}
listTasks(): Promise<string> {
throw new Error('Method not implemented.');
}
listOtherResources(): Promise<string> {
throw new Error('Method not implemented.');
}
garbageCollect(
// eslint-disable-next-line no-unused-vars
filter: string,
// eslint-disable-next-line no-unused-vars
previewOnly: boolean,
): Promise<string> {
throw new Error('Method not implemented.');
}
cleanup(
// eslint-disable-next-line no-unused-vars
buildGuid: string,

11
src/model/cloud-runner/providers/provider-interface.ts
View File

@ -39,4 +39,15 @@ export interface ProviderInterface {
// eslint-disable-next-line no-unused-vars
secrets: CloudRunnerSecret[],
): Promise<string>;
listAll(): Promise<string>;
listTasks(): Promise<string>;
listOtherResources(): Promise<string>;
garbageCollect(
// eslint-disable-next-line no-unused-vars
filter: string,
// eslint-disable-next-line no-unused-vars
previewOnly: boolean,
): Promise<string>;
inspect(): Promise<string>;
watch(): Promise<string>;
}

23
src/model/cloud-runner/providers/test/index.ts
View File

@ -5,6 +5,29 @@ import { ProviderInterface } from '../provider-interface';
import CloudRunnerSecret from '../../services/cloud-runner-secret';
class TestCloudRunner implements ProviderInterface {
inspect(): Promise<string> {
throw new Error('Method not implemented.');
}
watch(): Promise<string> {
throw new Error('Method not implemented.');
}
listAll(): Promise<string> {
throw new Error('Method not implemented.');
}
listTasks(): Promise<string> {
throw new Error('Method not implemented.');
}
listOtherResources(): Promise<string> {
throw new Error('Method not implemented.');
}
garbageCollect(
// eslint-disable-next-line no-unused-vars
filter: string,
// eslint-disable-next-line no-unused-vars
previewOnly: boolean,
): Promise<string> {
throw new Error('Method not implemented.');
}
cleanup(
// eslint-disable-next-line no-unused-vars
buildGuid: string,

8
src/model/cloud-runner/services/cloud-runner-build-command-process.ts → src/model/cloud-runner/services/cloud-runner-custom-hooks.ts
View File

@ -3,11 +3,9 @@ import YAML from 'yaml';
import CloudRunnerSecret from './cloud-runner-secret';
import CloudRunner from '../cloud-runner';
export class CloudRunnerBuildCommandProcessor {
public static ProcessCommands(commands: string, buildParameters: BuildParameters): string {
const hooks = CloudRunnerBuildCommandProcessor.getHooks(buildParameters.customJobHooks).filter((x) =>
x.step.includes(`all`),
);
export class CloudRunnerCustomHooks {
public static ApplyHooksToCommands(commands: string, buildParameters: BuildParameters): string {
const hooks = CloudRunnerCustomHooks.getHooks(buildParameters.customJobHooks).filter((x) => x.step.includes(`all`));
return `echo "---"
echo "start cloud runner init"

0
src/model/cloud-runner/services/cloud-runner-options-reader.ts 100644
View File

4
src/model/cloud-runner/services/task-parameter-serializer.ts
View File

@ -1,7 +1,7 @@
import { CloudRunner, Input } from '../..';
import ImageEnvironmentFactory from '../../image-environment-factory';
import CloudRunnerEnvironmentVariable from './cloud-runner-environment-variable';
import { CloudRunnerBuildCommandProcessor } from './cloud-runner-build-command-process';
import { CloudRunnerCustomHooks } from './cloud-runner-custom-hooks';
import CloudRunnerSecret from './cloud-runner-secret';
import CloudRunnerQueryOverride from './cloud-runner-query-override';
@ -27,7 +27,7 @@ export class TaskParameterSerializer {
let array = new Array();
array = TaskParameterSerializer.readBuildParameters(array);
array = TaskParameterSerializer.readInput(array);
const configurableHooks = CloudRunnerBuildCommandProcessor.getHooks(CloudRunner.buildParameters.customJobHooks);
const configurableHooks = CloudRunnerCustomHooks.getHooks(CloudRunner.buildParameters.customJobHooks);
const secrets = configurableHooks.map((x) => x.secrets).filter((x) => x !== undefined && x.length > 0);
if (secrets.length > 0) {
// eslint-disable-next-line unicorn/no-array-reduce

10
src/model/cloud-runner/workflows/build-automation-workflow.ts
View File

@ -4,7 +4,7 @@ import { CloudRunnerStepState } from '../cloud-runner-step-state';
import { CustomWorkflow } from './custom-workflow';
import { WorkflowInterface } from './workflow-interface';
import * as core from '@actions/core';
import { CloudRunnerBuildCommandProcessor } from '../services/cloud-runner-build-command-process';
import { CloudRunnerCustomHooks } from '../services/cloud-runner-custom-hooks';
import path from 'path';
import CloudRunner from '../cloud-runner';
@ -62,11 +62,11 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
}
private static get BuildWorkflow() {
const setupHooks = CloudRunnerBuildCommandProcessor.getHooks(CloudRunner.buildParameters.customJobHooks).filter(
(x) => x.step.includes(`setup`),
const setupHooks = CloudRunnerCustomHooks.getHooks(CloudRunner.buildParameters.customJobHooks).filter((x) =>
x.step.includes(`setup`),
);
const buildHooks = CloudRunnerBuildCommandProcessor.getHooks(CloudRunner.buildParameters.customJobHooks).filter(
(x) => x.step.includes(`build`),
const buildHooks = CloudRunnerCustomHooks.getHooks(CloudRunner.buildParameters.customJobHooks).filter((x) =>
x.step.includes(`build`),
);
const builderPath = path.join(CloudRunnerFolders.builderPathAbsolute, 'dist', `index.js`).replace(/\\/g, `/`);