actions
/
unity-builder
mirror of https://github.com/game-ci/unity-builder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix "large package handling" to redirect to shared folder (#473)

pull/477/head
Frostebite 2022-11-17 18:59:33 +00:00 committed by GitHub
parent f300dd27bb
commit d6ac850da1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 167 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

154
dist/index.js generated vendored
View File

@ -744,7 +744,7 @@ class CloudRunnerOptions {
return; return;
} }
static get cloudRunnerBranch() { static get cloudRunnerBranch() {
return CloudRunnerOptions.getInput('cloudRunnerBranch') || 'cloud-runner-develop'; return CloudRunnerOptions.getInput('cloudRunnerBranch') || 'main';
} }
static get cloudRunnerCluster() { static get cloudRunnerCluster() {
if (cli_1.Cli.isCliMode) { if (cli_1.Cli.isCliMode) {
@ -840,17 +840,16 @@ class CloudRunnerOptions {
return CloudRunnerOptions.getInput(`cloudRunnerDebugEnv`) || false; return CloudRunnerOptions.getInput(`cloudRunnerDebugEnv`) || false;
} }
static get watchCloudRunnerToEnd() { static get watchCloudRunnerToEnd() {
const input = CloudRunnerOptions.getInput(`watchToEnd`); return (CloudRunnerOptions.getInput(`watchToEnd`) || true) !== 'false';
return input !== 'false';
} }
static get useSharedLargePackages() { static get useSharedLargePackages() {
return CloudRunnerOptions.getInput(`useSharedLargePackages`) || false; return (CloudRunnerOptions.getInput(`useSharedLargePackages`) || 'false') !== 'false';
} }
static get useSharedBuilder() { static get useSharedBuilder() {
return CloudRunnerOptions.getInput(`useSharedBuilder`) || true; return (CloudRunnerOptions.getInput(`useSharedBuilder`) || true) !== 'false';
} }
static get useLz4Compression() { static get useLz4Compression() {
return CloudRunnerOptions.getInput(`useLz4Compression`) || true; return (CloudRunnerOptions.getInput(`useLz4Compression`) || true) !== false;
} }
// ### ### ### // ### ### ###
// Retained Workspace // Retained Workspace
@ -2348,7 +2347,6 @@ const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
const aws_job_stack_1 = __nccwpck_require__(70633); const aws_job_stack_1 = __nccwpck_require__(70633);
const aws_base_stack_1 = __nccwpck_require__(26112); const aws_base_stack_1 = __nccwpck_require__(26112);
const __1 = __nccwpck_require__(41359); const __1 = __nccwpck_require__(41359);
const tertiary_resources_service_1 = __nccwpck_require__(61627);
const garbage_collection_service_1 = __nccwpck_require__(53859); const garbage_collection_service_1 = __nccwpck_require__(53859);
const task_service_1 = __nccwpck_require__(67205); const task_service_1 = __nccwpck_require__(67205);
class AWSBuildEnvironment { class AWSBuildEnvironment {
@ -2357,10 +2355,9 @@ class AWSBuildEnvironment {
} }
listResources() { listResources() {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
yield task_service_1.TaskService.awsListJobs(); yield task_service_1.TaskService.getCloudFormationJobStacks();
yield tertiary_resources_service_1.TertiaryResourcesService.awsListLogGroups(); yield task_service_1.TaskService.getLogGroups();
yield task_service_1.TaskService.awsListTasks(); yield task_service_1.TaskService.getTasks();
yield task_service_1.TaskService.awsListStacks();
return []; return [];
}); });
} }
@ -2374,7 +2371,7 @@ class AWSBuildEnvironment {
} }
listOtherResources() { listOtherResources() {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
yield tertiary_resources_service_1.TertiaryResourcesService.awsListLogGroups(); yield task_service_1.TaskService.getLogGroups();
return ''; return '';
}); });
} }
@ -2488,29 +2485,30 @@ const aws_sdk_1 = __importDefault(__nccwpck_require__(71786));
const input_1 = __importDefault(__nccwpck_require__(91933)); const input_1 = __importDefault(__nccwpck_require__(91933));
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855)); const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
const task_service_1 = __nccwpck_require__(67205); const task_service_1 = __nccwpck_require__(67205);
const tertiary_resources_service_1 = __nccwpck_require__(61627);
class GarbageCollectionService { class GarbageCollectionService {
static isOlderThan1day(date) { static isOlderThan1day(date) {
const ageDate = new Date(date.getTime() - Date.now()); const ageDate = new Date(date.getTime() - Date.now());
return ageDate.getDay() > 0; return ageDate.getDay() > 0;
} }
static cleanup(deleteResources = false, OneDayOlderOnly = false) { static cleanup(deleteResources = false, OneDayOlderOnly = false) {
var _a, _b;
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
process.env.AWS_REGION = input_1.default.region; process.env.AWS_REGION = input_1.default.region;
const CF = new aws_sdk_1.default.CloudFormation(); const CF = new aws_sdk_1.default.CloudFormation();
const ecs = new aws_sdk_1.default.ECS(); const ecs = new aws_sdk_1.default.ECS();
const cwl = new aws_sdk_1.default.CloudWatchLogs(); const cwl = new aws_sdk_1.default.CloudWatchLogs();
const taskDefinitionsInUse = new Array(); const taskDefinitionsInUse = new Array();
yield task_service_1.TaskService.awsListTasks((taskElement, element) => __awaiter(this, void 0, void 0, function* () { const tasks = yield task_service_1.TaskService.getTasks();
var _a; for (const task of tasks) {
const { taskElement, element } = task;
taskDefinitionsInUse.push(taskElement.taskDefinitionArn); taskDefinitionsInUse.push(taskElement.taskDefinitionArn);
if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(taskElement.CreatedAt))) { if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(taskElement.CreatedAt))) {
cloud_runner_logger_1.default.log(`Stopping task ${(_a = taskElement.containers) === null || _a === void 0 ? void 0 : _a[0].name}`); cloud_runner_logger_1.default.log(`Stopping task ${(_a = taskElement.containers) === null || _a === void 0 ? void 0 : _a[0].name}`);
yield ecs.stopTask({ task: taskElement.taskArn || '', cluster: element }).promise(); yield ecs.stopTask({ task: taskElement.taskArn || '', cluster: element }).promise();
} }
})); }
yield task_service_1.TaskService.awsListStacks((element) => __awaiter(this, void 0, void 0, function* () { const jobStacks = yield task_service_1.TaskService.getCloudFormationJobStacks();
var _b; for (const element of jobStacks) {
if ((_b = (yield CF.describeStackResources({ StackName: element.StackName }).promise()).StackResources) === null || _b === void 0 ? void 0 : _b.some((x) => x.ResourceType === 'AWS::ECS::TaskDefinition' && taskDefinitionsInUse.includes(x.PhysicalResourceId))) { if ((_b = (yield CF.describeStackResources({ StackName: element.StackName }).promise()).StackResources) === null || _b === void 0 ? void 0 : _b.some((x) => x.ResourceType === 'AWS::ECS::TaskDefinition' && taskDefinitionsInUse.includes(x.PhysicalResourceId))) {
cloud_runner_logger_1.default.log(`Skipping ${element.StackName} - active task was running not deleting`); cloud_runner_logger_1.default.log(`Skipping ${element.StackName} - active task was running not deleting`);
return; return;
@ -2524,14 +2522,19 @@ class GarbageCollectionService {
const deleteStackInput = { StackName: element.StackName }; const deleteStackInput = { StackName: element.StackName };
yield CF.deleteStack(deleteStackInput).promise(); yield CF.deleteStack(deleteStackInput).promise();
} }
})); }
yield tertiary_resources_service_1.TertiaryResourcesService.awsListLogGroups((element) => __awaiter(this, void 0, void 0, function* () { const logGroups = yield task_service_1.TaskService.getLogGroups();
for (const element of logGroups) {
if (deleteResources && if (deleteResources &&
(!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(new Date(element.createdAt)))) { (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(new Date(element.createdAt)))) {
cloud_runner_logger_1.default.log(`Deleting ${element.logGroupName}`); cloud_runner_logger_1.default.log(`Deleting ${element.logGroupName}`);
yield cwl.deleteLogGroup({ logGroupName: element.logGroupName || '' }).promise(); yield cwl.deleteLogGroup({ logGroupName: element.logGroupName || '' }).promise();
} }
})); }
const locks = yield task_service_1.TaskService.getLocks();
for (const element of locks) {
cloud_runner_logger_1.default.log(`Lock: ${element.Key}`);
}
}); });
} }
} }
@ -2572,40 +2575,43 @@ class TaskService {
return output; return output;
}); });
} }
static awsListStacks(perResultCallback = false) { static getCloudFormationJobStacks() {
var _a, _b; var _a, _b;
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
cloud_runner_logger_1.default.log(`List Stacks`); const result = [];
cloud_runner_logger_1.default.log(``);
cloud_runner_logger_1.default.log(`List Cloud Formation Stacks`);
process.env.AWS_REGION = input_1.default.region; process.env.AWS_REGION = input_1.default.region;
const CF = new aws_sdk_1.default.CloudFormation(); const CF = new aws_sdk_1.default.CloudFormation();
const stacks = ((_a = (yield CF.listStacks().promise()).StackSummaries) === null || _a === void 0 ? void 0 : _a.filter((_x) => _x.StackStatus !== 'DELETE_COMPLETE')) || []; const stacks = ((_a = (yield CF.listStacks().promise()).StackSummaries) === null || _a === void 0 ? void 0 : _a.filter((_x) => _x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription !== base_stack_formation_1.BaseStackFormation.baseStackDecription)) || [];
cloud_runner_logger_1.default.log(`Stacks ${stacks.length}`); cloud_runner_logger_1.default.log(``);
cloud_runner_logger_1.default.log(`Cloud Formation Stacks ${stacks.length}`);
for (const element of stacks) { for (const element of stacks) {
const ageDate = new Date(Date.now() - element.CreationTime.getTime()); const ageDate = new Date(Date.now() - element.CreationTime.getTime());
cloud_runner_logger_1.default.log(`Task Stack ${element.StackName} - Age D${Math.floor(ageDate.getHours() / 24)} H${ageDate.getHours()} M${ageDate.getMinutes()}`); cloud_runner_logger_1.default.log(`Task Stack ${element.StackName} - Age D${Math.floor(ageDate.getHours() / 24)} H${ageDate.getHours()} M${ageDate.getMinutes()}`);
if (perResultCallback) result.push(element);
yield perResultCallback(element);
} }
const baseStacks = ((_b = (yield CF.listStacks().promise()).StackSummaries) === null || _b === void 0 ? void 0 : _b.filter((_x) => _x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription === base_stack_formation_1.BaseStackFormation.baseStackDecription)) || []; const baseStacks = ((_b = (yield CF.listStacks().promise()).StackSummaries) === null || _b === void 0 ? void 0 : _b.filter((_x) => _x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription === base_stack_formation_1.BaseStackFormation.baseStackDecription)) || [];
cloud_runner_logger_1.default.log(``);
cloud_runner_logger_1.default.log(`Base Stacks ${baseStacks.length}`); cloud_runner_logger_1.default.log(`Base Stacks ${baseStacks.length}`);
for (const element of baseStacks) { for (const element of baseStacks) {
const ageDate = new Date(Date.now() - element.CreationTime.getTime()); const ageDate = new Date(Date.now() - element.CreationTime.getTime());
cloud_runner_logger_1.default.log(`Task Stack ${element.StackName} - Age D${Math.floor(ageDate.getHours() / 24)} H${ageDate.getHours()} M${ageDate.getMinutes()}`); cloud_runner_logger_1.default.log(`Task Stack ${element.StackName} - Age D${Math.floor(ageDate.getHours() / 24)} H${ageDate.getHours()} M${ageDate.getMinutes()}`);
if (perResultCallback) result.push(element);
yield perResultCallback(element);
}
if (stacks === undefined) {
return;
} }
cloud_runner_logger_1.default.log(``);
return result;
}); });
} }
static awsListTasks(perResultCallback = false) { static getTasks() {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
const result = [];
cloud_runner_logger_1.default.log(``);
cloud_runner_logger_1.default.log(`List Tasks`); cloud_runner_logger_1.default.log(`List Tasks`);
process.env.AWS_REGION = input_1.default.region; process.env.AWS_REGION = input_1.default.region;
const ecs = new aws_sdk_1.default.ECS(); const ecs = new aws_sdk_1.default.ECS();
const clusters = (yield ecs.listClusters().promise()).clusterArns || []; const clusters = (yield ecs.listClusters().promise()).clusterArns || [];
cloud_runner_logger_1.default.log(`Clusters ${clusters.length}`); cloud_runner_logger_1.default.log(`Task Clusters ${clusters.length}`);
for (const element of clusters) { for (const element of clusters) {
const input = { const input = {
cluster: element, cluster: element,
@ -2615,6 +2621,7 @@ class TaskService {
const describeInput = { tasks: list, cluster: element }; const describeInput = { tasks: list, cluster: element };
const describeList = (yield ecs.describeTasks(describeInput).promise()).tasks || []; const describeList = (yield ecs.describeTasks(describeInput).promise()).tasks || [];
if (describeList.length === 0) { if (describeList.length === 0) {
cloud_runner_logger_1.default.log(`No Tasks`);
continue; continue;
} }
cloud_runner_logger_1.default.log(`Tasks ${describeList.length}`); cloud_runner_logger_1.default.log(`Tasks ${describeList.length}`);
@ -2628,27 +2635,12 @@ class TaskService {
cloud_runner_logger_1.default.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`); cloud_runner_logger_1.default.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`);
continue; continue;
} }
if (perResultCallback) result.push({ taskElement, element });
yield perResultCallback(taskElement, element);
} }
} }
} }
}); cloud_runner_logger_1.default.log(``);
} return result;
static awsListJobs(perResultCallback = false) {
var _a;
return __awaiter(this, void 0, void 0, function* () {
cloud_runner_logger_1.default.log(`List Jobs`);
process.env.AWS_REGION = input_1.default.region;
const CF = new aws_sdk_1.default.CloudFormation();
const stacks = ((_a = (yield CF.listStacks().promise()).StackSummaries) === null || _a === void 0 ? void 0 : _a.filter((_x) => _x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription !== base_stack_formation_1.BaseStackFormation.baseStackDecription)) || [];
cloud_runner_logger_1.default.log(`Stacks ${stacks.length}`);
for (const element of stacks) {
const ageDate = new Date(Date.now() - element.CreationTime.getTime());
cloud_runner_logger_1.default.log(`Task Stack ${element.StackName} - Age D${Math.floor(ageDate.getHours() / 24)} H${ageDate.getHours()} M${ageDate.getMinutes()}`);
if (perResultCallback)
yield perResultCallback(element);
}
}); });
} }
static awsDescribeJob(job) { static awsDescribeJob(job) {
@ -2674,37 +2666,9 @@ class TaskService {
return message; return message;
}); });
} }
} static getLogGroups() {
exports.TaskService = TaskService;
/***/ }),
/***/ 61627:
/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) {
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.TertiaryResourcesService = void 0;
const aws_sdk_1 = __importDefault(__nccwpck_require__(71786));
const input_1 = __importDefault(__nccwpck_require__(91933));
const cloud_runner_logger_1 = __importDefault(__nccwpck_require__(22855));
class TertiaryResourcesService {
static awsListLogGroups(perResultCallback = false) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
const result = [];
process.env.AWS_REGION = input_1.default.region; process.env.AWS_REGION = input_1.default.region;
const ecs = new aws_sdk_1.default.CloudWatchLogs(); const ecs = new aws_sdk_1.default.CloudWatchLogs();
let logStreamInput = { let logStreamInput = {
@ -2725,13 +2689,24 @@ class TertiaryResourcesService {
} }
const ageDate = new Date(Date.now() - element.creationTime); const ageDate = new Date(Date.now() - element.creationTime);
cloud_runner_logger_1.default.log(`Task Stack ${element.logGroupName} - Age D${Math.floor(ageDate.getHours() / 24)} H${ageDate.getHours()} M${ageDate.getMinutes()}`); cloud_runner_logger_1.default.log(`Task Stack ${element.logGroupName} - Age D${Math.floor(ageDate.getHours() / 24)} H${ageDate.getHours()} M${ageDate.getMinutes()}`);
if (perResultCallback) result.push(element);
yield perResultCallback(element, element);
} }
return result;
});
}
static getLocks() {
return __awaiter(this, void 0, void 0, function* () {
process.env.AWS_REGION = input_1.default.region;
const s3 = new aws_sdk_1.default.S3();
const listRequest = {
Bucket: `game-ci-test-storage`,
};
const results = yield s3.listObjects(listRequest).promise();
return results.Contents || [];
}); });
} }
} }
exports.TertiaryResourcesService = TertiaryResourcesService; exports.TaskService = TaskService;
/***/ }), /***/ }),
@ -4195,12 +4170,15 @@ class RemoteClient {
}); });
} }
static replaceLargePackageReferencesWithSharedReferences() { static replaceLargePackageReferencesWithSharedReferences() {
const manifest = fs_1.default.readFileSync(path_1.default.join(cloud_runner_folders_1.CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`), 'utf8'); if (cloud_runner_1.default.buildParameters.useSharedLargePackages) {
const filePath = path_1.default.join(cloud_runner_folders_1.CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`);
let manifest = fs_1.default.readFileSync(filePath, 'utf8');
manifest = manifest.replace(/LargeContent/g, '../../LargeContent');
fs_1.default.writeFileSync(filePath, manifest);
if (cloud_runner_1.default.buildParameters.cloudRunnerDebug) { if (cloud_runner_1.default.buildParameters.cloudRunnerDebug) {
cloud_runner_logger_1.default.log(`Package Manifest`);
cloud_runner_logger_1.default.log(manifest); cloud_runner_logger_1.default.log(manifest);
} }
if (cloud_runner_1.default.buildParameters.useSharedLargePackages) {
manifest.replace(/LargePackages/g, '../../LargePackages');
} }
} }
static pullLatestLFS() { static pullLatestLFS() {

2
dist/index.js.map generated vendored
View File

File diff suppressed because one or more lines are too long

12
src/model/cloud-runner/cloud-runner-options.ts
View File

@ -98,7 +98,7 @@ class CloudRunnerOptions {
} }
static get cloudRunnerBranch() { static get cloudRunnerBranch() {
return CloudRunnerOptions.getInput('cloudRunnerBranch') || 'cloud-runner-develop'; return CloudRunnerOptions.getInput('cloudRunnerBranch') || 'main';
} }
static get cloudRunnerCluster() { static get cloudRunnerCluster() {
@ -220,21 +220,19 @@ class CloudRunnerOptions {
} }
static get watchCloudRunnerToEnd(): boolean { static get watchCloudRunnerToEnd(): boolean {
const input = CloudRunnerOptions.getInput(`watchToEnd`); return (CloudRunnerOptions.getInput(`watchToEnd`) || true) !== 'false';
return input !== 'false';
} }
public static get useSharedLargePackages(): boolean { public static get useSharedLargePackages(): boolean {
return CloudRunnerOptions.getInput(`useSharedLargePackages`) || false; return (CloudRunnerOptions.getInput(`useSharedLargePackages`) || 'false') !== 'false';
} }
public static get useSharedBuilder(): boolean { public static get useSharedBuilder(): boolean {
return CloudRunnerOptions.getInput(`useSharedBuilder`) || true; return (CloudRunnerOptions.getInput(`useSharedBuilder`) || true) !== 'false';
} }
public static get useLz4Compression(): boolean { public static get useLz4Compression(): boolean {
return CloudRunnerOptions.getInput(`useLz4Compression`) || true; return (CloudRunnerOptions.getInput(`useLz4Compression`) || true) !== false;
} }
// ### ### ### // ### ### ###

10
src/model/cloud-runner/providers/aws/index.ts
View File

@ -9,7 +9,6 @@ import CloudRunnerLogger from '../../services/cloud-runner-logger';
import { AWSJobStack as AwsJobStack } from './aws-job-stack'; import { AWSJobStack as AwsJobStack } from './aws-job-stack';
import { AWSBaseStack as AwsBaseStack } from './aws-base-stack'; import { AWSBaseStack as AwsBaseStack } from './aws-base-stack';
import { Input } from '../../..'; import { Input } from '../../..';
import { TertiaryResourcesService } from './services/tertiary-resources-service';
import { GarbageCollectionService } from './services/garbage-collection-service'; import { GarbageCollectionService } from './services/garbage-collection-service';
import { ProviderResource } from '../provider-resource'; import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow'; import { ProviderWorkflow } from '../provider-workflow';
@ -22,10 +21,9 @@ class AWSBuildEnvironment implements ProviderInterface {
this.baseStackName = buildParameters.awsBaseStackName; this.baseStackName = buildParameters.awsBaseStackName;
} }
async listResources(): Promise<ProviderResource[]> { async listResources(): Promise<ProviderResource[]> {
await TaskService.awsListJobs(); await TaskService.getCloudFormationJobStacks();
await TertiaryResourcesService.awsListLogGroups(); await TaskService.getLogGroups();
await TaskService.awsListTasks(); await TaskService.getTasks();
await TaskService.awsListStacks();
return []; return [];
} }
@ -37,7 +35,7 @@ class AWSBuildEnvironment implements ProviderInterface {
} }
async listOtherResources(): Promise<string> { async listOtherResources(): Promise<string> {
await TertiaryResourcesService.awsListLogGroups(); await TaskService.getLogGroups();
return ''; return '';
} }

22
src/model/cloud-runner/providers/aws/services/garbage-collection-service.ts
View File

@ -2,7 +2,6 @@ import AWS from 'aws-sdk';
import Input from '../../../../input'; import Input from '../../../../input';
import CloudRunnerLogger from '../../../services/cloud-runner-logger'; import CloudRunnerLogger from '../../../services/cloud-runner-logger';
import { TaskService } from './task-service'; import { TaskService } from './task-service';
import { TertiaryResourcesService } from './tertiary-resources-service';
export class GarbageCollectionService { export class GarbageCollectionService {
static isOlderThan1day(date: any) { static isOlderThan1day(date: any) {
@ -17,14 +16,17 @@ export class GarbageCollectionService {
const ecs = new AWS.ECS(); const ecs = new AWS.ECS();
const cwl = new AWS.CloudWatchLogs(); const cwl = new AWS.CloudWatchLogs();
const taskDefinitionsInUse = new Array(); const taskDefinitionsInUse = new Array();
await TaskService.awsListTasks(async (taskElement, element) => { const tasks = await TaskService.getTasks();
for (const task of tasks) {
const { taskElement, element } = task;
taskDefinitionsInUse.push(taskElement.taskDefinitionArn); taskDefinitionsInUse.push(taskElement.taskDefinitionArn);
if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(taskElement.CreatedAt))) { if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(taskElement.CreatedAt))) {
CloudRunnerLogger.log(`Stopping task ${taskElement.containers?.[0].name}`); CloudRunnerLogger.log(`Stopping task ${taskElement.containers?.[0].name}`);
await ecs.stopTask({ task: taskElement.taskArn || '', cluster: element }).promise(); await ecs.stopTask({ task: taskElement.taskArn || '', cluster: element }).promise();
} }
}); }
await TaskService.awsListStacks(async (element) => { const jobStacks = await TaskService.getCloudFormationJobStacks();
for (const element of jobStacks) {
if ( if (
(await CF.describeStackResources({ StackName: element.StackName }).promise()).StackResources?.some( (await CF.describeStackResources({ StackName: element.StackName }).promise()).StackResources?.some(
(x) => x.ResourceType === 'AWS::ECS::TaskDefinition' && taskDefinitionsInUse.includes(x.PhysicalResourceId), (x) => x.ResourceType === 'AWS::ECS::TaskDefinition' && taskDefinitionsInUse.includes(x.PhysicalResourceId),
@ -44,8 +46,9 @@ export class GarbageCollectionService {
const deleteStackInput: AWS.CloudFormation.DeleteStackInput = { StackName: element.StackName }; const deleteStackInput: AWS.CloudFormation.DeleteStackInput = { StackName: element.StackName };
await CF.deleteStack(deleteStackInput).promise(); await CF.deleteStack(deleteStackInput).promise();
} }
}); }
await TertiaryResourcesService.awsListLogGroups(async (element) => { const logGroups = await TaskService.getLogGroups();
for (const element of logGroups) {
if ( if (
deleteResources && deleteResources &&
(!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(new Date(element.createdAt))) (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(new Date(element.createdAt)))
@ -53,6 +56,11 @@ export class GarbageCollectionService {
CloudRunnerLogger.log(`Deleting ${element.logGroupName}`); CloudRunnerLogger.log(`Deleting ${element.logGroupName}`);
await cwl.deleteLogGroup({ logGroupName: element.logGroupName || '' }).promise(); await cwl.deleteLogGroup({ logGroupName: element.logGroupName || '' }).promise();
} }
}); }
const locks = await TaskService.getLocks();
for (const element of locks) {
CloudRunnerLogger.log(`Lock: ${element.Key}`);
}
} }
} }

99
src/model/cloud-runner/providers/aws/services/task-service.ts
View File

@ -3,6 +3,7 @@ import Input from '../../../../input';
import CloudRunnerLogger from '../../../services/cloud-runner-logger'; import CloudRunnerLogger from '../../../services/cloud-runner-logger';
import { BaseStackFormation } from '../cloud-formations/base-stack-formation'; import { BaseStackFormation } from '../cloud-formations/base-stack-formation';
import AwsTaskRunner from '../aws-task-runner'; import AwsTaskRunner from '../aws-task-runner';
import { ListObjectsRequest } from 'aws-sdk/clients/s3';
export class TaskService { export class TaskService {
static async watch() { static async watch() {
@ -15,16 +16,19 @@ export class TaskService {
return output; return output;
} }
public static async awsListStacks(perResultCallback: any = false) { public static async getCloudFormationJobStacks() {
CloudRunnerLogger.log(`List Stacks`); const result: any[] = [];
CloudRunnerLogger.log(``);
CloudRunnerLogger.log(`List Cloud Formation Stacks`);
process.env.AWS_REGION = Input.region; process.env.AWS_REGION = Input.region;
const CF = new AWS.CloudFormation(); const CF = new AWS.CloudFormation();
const stacks = const stacks =
(await CF.listStacks().promise()).StackSummaries?.filter( (await CF.listStacks().promise()).StackSummaries?.filter(
(_x) => _x.StackStatus !== 'DELETE_COMPLETE', // && (_x) =>
// _x.TemplateDescription === TaskDefinitionFormation.description.replace('\n', ''), _x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription !== BaseStackFormation.baseStackDecription,
) || []; ) || [];
CloudRunnerLogger.log(`Stacks ${stacks.length}`); CloudRunnerLogger.log(``);
CloudRunnerLogger.log(`Cloud Formation Stacks ${stacks.length}`);
for (const element of stacks) { for (const element of stacks) {
const ageDate: Date = new Date(Date.now() - element.CreationTime.getTime()); const ageDate: Date = new Date(Date.now() - element.CreationTime.getTime());
@ -33,13 +37,14 @@ export class TaskService {
ageDate.getHours() / 24, ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`, )} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
); );
if (perResultCallback) await perResultCallback(element); result.push(element);
} }
const baseStacks = const baseStacks =
(await CF.listStacks().promise()).StackSummaries?.filter( (await CF.listStacks().promise()).StackSummaries?.filter(
(_x) => (_x) =>
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription === BaseStackFormation.baseStackDecription, _x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription === BaseStackFormation.baseStackDecription,
) || []; ) || [];
CloudRunnerLogger.log(``);
CloudRunnerLogger.log(`Base Stacks ${baseStacks.length}`); CloudRunnerLogger.log(`Base Stacks ${baseStacks.length}`);
for (const element of baseStacks) { for (const element of baseStacks) {
const ageDate: Date = new Date(Date.now() - element.CreationTime.getTime()); const ageDate: Date = new Date(Date.now() - element.CreationTime.getTime());
@ -49,18 +54,20 @@ export class TaskService {
ageDate.getHours() / 24, ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`, )} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
); );
if (perResultCallback) await perResultCallback(element); result.push(element);
} }
if (stacks === undefined) { CloudRunnerLogger.log(``);
return;
return result;
} }
} public static async getTasks() {
public static async awsListTasks(perResultCallback: any = false) { const result: any[] = [];
CloudRunnerLogger.log(``);
CloudRunnerLogger.log(`List Tasks`); CloudRunnerLogger.log(`List Tasks`);
process.env.AWS_REGION = Input.region; process.env.AWS_REGION = Input.region;
const ecs = new AWS.ECS(); const ecs = new AWS.ECS();
const clusters = (await ecs.listClusters().promise()).clusterArns || []; const clusters = (await ecs.listClusters().promise()).clusterArns || [];
CloudRunnerLogger.log(`Clusters ${clusters.length}`); CloudRunnerLogger.log(`Task Clusters ${clusters.length}`);
for (const element of clusters) { for (const element of clusters) {
const input: AWS.ECS.ListTasksRequest = { const input: AWS.ECS.ListTasksRequest = {
cluster: element, cluster: element,
@ -71,6 +78,7 @@ export class TaskService {
const describeInput: AWS.ECS.DescribeTasksRequest = { tasks: list, cluster: element }; const describeInput: AWS.ECS.DescribeTasksRequest = { tasks: list, cluster: element };
const describeList = (await ecs.describeTasks(describeInput).promise()).tasks || []; const describeList = (await ecs.describeTasks(describeInput).promise()).tasks || [];
if (describeList.length === 0) { if (describeList.length === 0) {
CloudRunnerLogger.log(`No Tasks`);
continue; continue;
} }
CloudRunnerLogger.log(`Tasks ${describeList.length}`); CloudRunnerLogger.log(`Tasks ${describeList.length}`);
@ -84,31 +92,13 @@ export class TaskService {
CloudRunnerLogger.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`); CloudRunnerLogger.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`);
continue; continue;
} }
if (perResultCallback) await perResultCallback(taskElement, element); result.push({ taskElement, element });
} }
} }
} }
} CloudRunnerLogger.log(``);
public static async awsListJobs(perResultCallback: any = false) {
CloudRunnerLogger.log(`List Jobs`);
process.env.AWS_REGION = Input.region;
const CF = new AWS.CloudFormation();
const stacks =
(await CF.listStacks().promise()).StackSummaries?.filter(
(_x) =>
_x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription !== BaseStackFormation.baseStackDecription,
) || [];
CloudRunnerLogger.log(`Stacks ${stacks.length}`);
for (const element of stacks) {
const ageDate: Date = new Date(Date.now() - element.CreationTime.getTime());
CloudRunnerLogger.log( return result;
`Task Stack ${element.StackName} - Age D${Math.floor(
ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
);
if (perResultCallback) await perResultCallback(element);
}
} }
public static async awsDescribeJob(job: string) { public static async awsDescribeJob(job: string) {
process.env.AWS_REGION = Input.region; process.env.AWS_REGION = Input.region;
@ -131,4 +121,47 @@ export class TaskService {
return message; return message;
} }
public static async getLogGroups() {
const result: any[] = [];
process.env.AWS_REGION = Input.region;
const ecs = new AWS.CloudWatchLogs();
let logStreamInput: AWS.CloudWatchLogs.DescribeLogGroupsRequest = {
/* logGroupNamePrefix: 'game-ci' */
};
let logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise();
const logGroups = logGroupsDescribe.logGroups || [];
while (logGroupsDescribe.nextToken) {
logStreamInput = { /* logGroupNamePrefix: 'game-ci',*/ nextToken: logGroupsDescribe.nextToken };
logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise();
logGroups.push(...(logGroupsDescribe?.logGroups || []));
}
CloudRunnerLogger.log(`Log Groups ${logGroups.length}`);
for (const element of logGroups) {
if (element.creationTime === undefined) {
CloudRunnerLogger.log(`Skipping ${element.logGroupName} no createdAt date`);
continue;
}
const ageDate: Date = new Date(Date.now() - element.creationTime);
CloudRunnerLogger.log(
`Task Stack ${element.logGroupName} - Age D${Math.floor(
ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
);
result.push(element);
}
return result;
}
public static async getLocks() {
process.env.AWS_REGION = Input.region;
const s3 = new AWS.S3();
const listRequest: ListObjectsRequest = {
Bucket: `game-ci-test-storage`,
};
const results = await s3.listObjects(listRequest).promise();
return results.Contents || [];
}
} }

36
src/model/cloud-runner/providers/aws/services/tertiary-resources-service.ts
View File

@ -1,36 +0,0 @@
import AWS from 'aws-sdk';
import Input from '../../../../input';
import CloudRunnerLogger from '../../../services/cloud-runner-logger';
export class TertiaryResourcesService {
public static async awsListLogGroups(perResultCallback: any = false) {
process.env.AWS_REGION = Input.region;
const ecs = new AWS.CloudWatchLogs();
let logStreamInput: AWS.CloudWatchLogs.DescribeLogGroupsRequest = {
/* logGroupNamePrefix: 'game-ci' */
};
let logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise();
const logGroups = logGroupsDescribe.logGroups || [];
while (logGroupsDescribe.nextToken) {
logStreamInput = { /* logGroupNamePrefix: 'game-ci',*/ nextToken: logGroupsDescribe.nextToken };
logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise();
logGroups.push(...(logGroupsDescribe?.logGroups || []));
}
CloudRunnerLogger.log(`Log Groups ${logGroups.length}`);
for (const element of logGroups) {
if (element.creationTime === undefined) {
CloudRunnerLogger.log(`Skipping ${element.logGroupName} no createdAt date`);
continue;
}
const ageDate: Date = new Date(Date.now() - element.creationTime);
CloudRunnerLogger.log(
`Task Stack ${element.logGroupName} - Age D${Math.floor(
ageDate.getHours() / 24,
)} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
);
if (perResultCallback) await perResultCallback(element, element);
}
}
}

12
src/model/cloud-runner/remote-client/index.ts
View File

@ -105,15 +105,15 @@ export class RemoteClient {
} }
static replaceLargePackageReferencesWithSharedReferences() { static replaceLargePackageReferencesWithSharedReferences() {
const manifest = fs.readFileSync( if (CloudRunner.buildParameters.useSharedLargePackages) {
path.join(CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`), const filePath = path.join(CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`);
'utf8', let manifest = fs.readFileSync(filePath, 'utf8');
); manifest = manifest.replace(/LargeContent/g, '../../LargeContent');
fs.writeFileSync(filePath, manifest);
if (CloudRunner.buildParameters.cloudRunnerDebug) { if (CloudRunner.buildParameters.cloudRunnerDebug) {
CloudRunnerLogger.log(`Package Manifest`);
CloudRunnerLogger.log(manifest); CloudRunnerLogger.log(manifest);
} }
if (CloudRunner.buildParameters.useSharedLargePackages) {
manifest.replace(/LargePackages/g, '../../LargePackages');
} }
} }