final refactoring and logging improvements
parent
0bc0d480de
commit
ec48ac196a
|
|
@ -1165,7 +1165,7 @@ exports.default = Project;
|
|||
|
||||
/***/ }),
|
||||
|
||||
/***/ 1019:
|
||||
/***/ 50148:
|
||||
/***/ (function(__unused_webpack_module, exports, __webpack_require__) {
|
||||
|
||||
"use strict";
|
||||
|
|
@ -1206,17 +1206,21 @@ const SDK = __importStar(__webpack_require__(71786));
|
|||
const nanoid_1 = __webpack_require__(39140);
|
||||
const fs = __importStar(__webpack_require__(35747));
|
||||
const core = __importStar(__webpack_require__(42186));
|
||||
const zlib = __importStar(__webpack_require__(78761));
|
||||
const remote_builder_alphabet_1 = __importDefault(__webpack_require__(41322));
|
||||
class AWS {
|
||||
const aws_build_runner_1 = __importDefault(__webpack_require__(11201));
|
||||
class AWSBuildEnvironment {
|
||||
static run(buildId, stackName, image, commands, mountdir, workingdir, environment, secrets) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
const ECS = new SDK.ECS();
|
||||
const CF = new SDK.CloudFormation();
|
||||
const entrypoint = ['/bin/sh'];
|
||||
const taskDef = yield this.setupCloudFormations(CF, buildId, stackName, image, entrypoint, commands, mountdir, workingdir, secrets);
|
||||
yield this.runTask(taskDef, ECS, CF, environment, buildId);
|
||||
try {
|
||||
yield aws_build_runner_1.default.runTask(taskDef, ECS, CF, environment, buildId);
|
||||
}
|
||||
finally {
|
||||
yield this.cleanupResources(CF, taskDef);
|
||||
}
|
||||
});
|
||||
}
|
||||
static setupCloudFormations(CF, buildUid, stackName, image, entrypoint, commands, mountdir, workingdir, secrets) {
|
||||
|
|
@ -1227,7 +1231,8 @@ class AWS {
|
|||
`;
|
||||
const taskDefStackName = `${stackName}-${buildUid}`;
|
||||
let taskDefCloudFormation = this.readTaskCloudFormationTemplate();
|
||||
core.info(JSON.stringify(secrets, undefined, 4));
|
||||
// Debug secrets
|
||||
// core.info(JSON.stringify(secrets, undefined, 4));
|
||||
for (const secret of secrets) {
|
||||
const insertionStringParameters = 'p1 - input';
|
||||
const insertionStringSecrets = 'p2 - secret';
|
||||
|
|
@ -1268,7 +1273,6 @@ class AWS {
|
|||
taskDefCloudFormation.slice(indexp3),
|
||||
].join('');
|
||||
}
|
||||
core.info(taskDefCloudFormation);
|
||||
const mappedSecrets = secrets.map((x) => {
|
||||
return { ParameterKey: x.ParameterKey.replace(/[^\dA-Za-z]/g, ''), ParameterValue: x.ParameterValue };
|
||||
});
|
||||
|
|
@ -1341,6 +1345,7 @@ class AWS {
|
|||
core.error(error);
|
||||
const events = (yield CF.describeStackEvents({ StackName: taskDefStackName }).promise()).StackEvents;
|
||||
const resources = (yield CF.describeStackResources({ StackName: taskDefStackName }).promise()).StackResources;
|
||||
core.info(taskDefCloudFormation);
|
||||
core.info(JSON.stringify(events, undefined, 4));
|
||||
core.info(JSON.stringify(resources, undefined, 4));
|
||||
throw error;
|
||||
|
|
@ -1365,6 +1370,68 @@ class AWS {
|
|||
static readTaskCloudFormationTemplate() {
|
||||
return fs.readFileSync(`${__dirname}/cloud-formations/task-def-formation.yml`, 'utf8');
|
||||
}
|
||||
static cleanupResources(CF, taskDef) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
yield CF.deleteStack({
|
||||
StackName: taskDef.taskDefStackName,
|
||||
}).promise();
|
||||
yield CF.deleteStack({
|
||||
StackName: taskDef.taskDefStackNameTTL,
|
||||
}).promise();
|
||||
yield CF.waitFor('stackDeleteComplete', {
|
||||
StackName: taskDef.taskDefStackName,
|
||||
}).promise();
|
||||
// Currently too slow and causes too much waiting
|
||||
yield CF.waitFor('stackDeleteComplete', {
|
||||
StackName: taskDef.taskDefStackNameTTL,
|
||||
}).promise();
|
||||
core.info('Cleanup complete');
|
||||
});
|
||||
}
|
||||
}
|
||||
exports.default = AWSBuildEnvironment;
|
||||
|
||||
|
||||
/***/ }),
|
||||
|
||||
/***/ 11201:
|
||||
/***/ (function(__unused_webpack_module, exports, __webpack_require__) {
|
||||
|
||||
"use strict";
|
||||
|
||||
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
|
||||
}) : (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
o[k2] = m[k];
|
||||
}));
|
||||
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
||||
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
||||
}) : function(o, v) {
|
||||
o["default"] = v;
|
||||
});
|
||||
var __importStar = (this && this.__importStar) || function (mod) {
|
||||
if (mod && mod.__esModule) return mod;
|
||||
var result = {};
|
||||
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
||||
__setModuleDefault(result, mod);
|
||||
return result;
|
||||
};
|
||||
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
||||
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
||||
return new (P || (P = Promise))(function (resolve, reject) {
|
||||
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
||||
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
||||
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
||||
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
||||
});
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", ({ value: true }));
|
||||
const AWS = __importStar(__webpack_require__(71786));
|
||||
const core = __importStar(__webpack_require__(42186));
|
||||
const zlib = __importStar(__webpack_require__(78761));
|
||||
class AWSBuildRunner {
|
||||
static runTask(taskDef, ECS, CF, environment, buildUid) {
|
||||
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s;
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
|
|
@ -1418,12 +1485,6 @@ class AWS {
|
|||
cluster,
|
||||
}).promise()).tasks) === null || _r === void 0 ? void 0 : _r[0].containers) === null || _s === void 0 ? void 0 : _s[0].exitCode;
|
||||
if (exitCode !== 0) {
|
||||
try {
|
||||
yield this.cleanupResources(CF, taskDef);
|
||||
}
|
||||
catch (error) {
|
||||
core.warning(`failed to cleanup ${error}`);
|
||||
}
|
||||
core.error(`job failed with exit code ${exitCode}`);
|
||||
throw new Error(`job failed with exit code ${exitCode}`);
|
||||
}
|
||||
|
|
@ -1436,7 +1497,7 @@ class AWS {
|
|||
var _a;
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
// watching logs
|
||||
const kinesis = new SDK.Kinesis();
|
||||
const kinesis = new AWS.Kinesis();
|
||||
const getTaskData = () => __awaiter(this, void 0, void 0, function* () {
|
||||
var _b;
|
||||
const tasks = yield ECS.describeTasks({
|
||||
|
|
@ -1459,7 +1520,7 @@ class AWS {
|
|||
.promise()).ShardIterator || '';
|
||||
yield CF.waitFor('stackCreateComplete', { StackName: taskDef.taskDefStackNameTTL }).promise();
|
||||
core.info(`Task status is ${(_a = (yield getTaskData())) === null || _a === void 0 ? void 0 : _a.lastStatus}`);
|
||||
const logBaseUrl = `https://${SDK.config.region}.console.aws.amazon.com/cloudwatch/home?region=${SDK.config.region}#logsV2:log-groups/log-group/${taskDef.taskDefStackName}`;
|
||||
const logBaseUrl = `https://${AWS.config.region}.console.aws.amazon.com/cloudwatch/home?region=${AWS.config.region}#logsV2:log-groups/log-group/${taskDef.taskDefStackName}`;
|
||||
core.info(`You can also see the logs at AWS Cloud Watch: ${logBaseUrl}`);
|
||||
let readingLogs = true;
|
||||
let timestamp = 0;
|
||||
|
|
@ -1501,26 +1562,8 @@ class AWS {
|
|||
}
|
||||
});
|
||||
}
|
||||
static cleanupResources(CF, taskDef) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
yield CF.deleteStack({
|
||||
StackName: taskDef.taskDefStackName,
|
||||
}).promise();
|
||||
yield CF.deleteStack({
|
||||
StackName: taskDef.taskDefStackNameTTL,
|
||||
}).promise();
|
||||
yield CF.waitFor('stackDeleteComplete', {
|
||||
StackName: taskDef.taskDefStackName,
|
||||
}).promise();
|
||||
// Currently too slow and causes too much waiting
|
||||
yield CF.waitFor('stackDeleteComplete', {
|
||||
StackName: taskDef.taskDefStackNameTTL,
|
||||
}).promise();
|
||||
core.info('Cleanup complete');
|
||||
});
|
||||
}
|
||||
}
|
||||
exports.default = AWS;
|
||||
exports.default = AWSBuildRunner;
|
||||
|
||||
|
||||
/***/ }),
|
||||
|
|
@ -1577,7 +1620,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|||
};
|
||||
Object.defineProperty(exports, "__esModule", ({ value: true }));
|
||||
const nanoid_1 = __webpack_require__(39140);
|
||||
const aws_1 = __importDefault(__webpack_require__(1019));
|
||||
const aws_build_environment_1 = __importDefault(__webpack_require__(50148));
|
||||
const core = __importStar(__webpack_require__(42186));
|
||||
const remote_builder_alphabet_1 = __importDefault(__webpack_require__(41322));
|
||||
const repositoryDirectoryName = 'repo';
|
||||
|
|
@ -1602,7 +1645,7 @@ class RemoteBuilder {
|
|||
},
|
||||
];
|
||||
core.info('Starting part 1/4 (clone from github and restore cache)');
|
||||
yield aws_1.default.run(buildUid, buildParameters.awsStackName, 'alpine/git', [
|
||||
yield aws_build_environment_1.default.run(buildUid, buildParameters.awsStackName, 'alpine/git', [
|
||||
'-c',
|
||||
`apk update;
|
||||
apk add unzip;
|
||||
|
|
@ -1706,7 +1749,7 @@ class RemoteBuilder {
|
|||
EnvironmentVariable: 'AWS_ACCESS_KEY_ALIAS_PASS',
|
||||
ParameterValue: buildParameters.androidKeyaliasPass,
|
||||
});
|
||||
yield aws_1.default.run(buildUid, buildParameters.awsStackName, baseImage.toString(), [
|
||||
yield aws_build_environment_1.default.run(buildUid, buildParameters.awsStackName, baseImage.toString(), [
|
||||
'-c',
|
||||
`
|
||||
cp -r /${efsDirectoryName}/${buildUid}/builder/dist/default-build-script/ /UnityBuilderAction;
|
||||
|
|
@ -1772,7 +1815,7 @@ class RemoteBuilder {
|
|||
], buildSecrets);
|
||||
core.info('Starting part 3/4 (zip unity build and Library for caching)');
|
||||
// Cleanup
|
||||
yield aws_1.default.run(buildUid, buildParameters.awsStackName, 'alpine', [
|
||||
yield aws_build_environment_1.default.run(buildUid, buildParameters.awsStackName, 'alpine', [
|
||||
'-c',
|
||||
`
|
||||
apk update
|
||||
|
|
@ -1791,7 +1834,7 @@ class RemoteBuilder {
|
|||
},
|
||||
], defaultSecretsArray);
|
||||
core.info('Starting part 4/4 (upload build to s3)');
|
||||
yield aws_1.default.run(buildUid, buildParameters.awsStackName, 'amazon/aws-cli', [
|
||||
yield aws_build_environment_1.default.run(buildUid, buildParameters.awsStackName, 'amazon/aws-cli', [
|
||||
'-c',
|
||||
`
|
||||
aws s3 cp ${buildUid}/build-${buildUid}.zip s3://game-ci-storage/
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -4,11 +4,11 @@ import RemoteBuilderSecret from './remote-builder-secret';
|
|||
import RemoteBuilderEnvironmentVariable from './remote-builder-environment-variable';
|
||||
import * as fs from 'fs';
|
||||
import * as core from '@actions/core';
|
||||
import * as zlib from 'zlib';
|
||||
import RemoteBuilderTaskDef from './remote-builder-task-def';
|
||||
import RemoteBuilderAlphabet from './remote-builder-alphabet';
|
||||
import AWSBuildRunner from './aws-build-runner';
|
||||
|
||||
class AWS {
|
||||
class AWSBuildEnvironment {
|
||||
static async run(
|
||||
buildId: string,
|
||||
stackName: string,
|
||||
|
|
@ -34,11 +34,12 @@ class AWS {
|
|||
workingdir,
|
||||
secrets,
|
||||
);
|
||||
|
||||
await this.runTask(taskDef, ECS, CF, environment, buildId);
|
||||
|
||||
try {
|
||||
await AWSBuildRunner.runTask(taskDef, ECS, CF, environment, buildId);
|
||||
} finally {
|
||||
await this.cleanupResources(CF, taskDef);
|
||||
}
|
||||
}
|
||||
|
||||
static async setupCloudFormations(
|
||||
CF: SDK.CloudFormation,
|
||||
|
|
@ -57,7 +58,10 @@ class AWS {
|
|||
`;
|
||||
const taskDefStackName = `${stackName}-${buildUid}`;
|
||||
let taskDefCloudFormation = this.readTaskCloudFormationTemplate();
|
||||
core.info(JSON.stringify(secrets, undefined, 4));
|
||||
|
||||
// Debug secrets
|
||||
// core.info(JSON.stringify(secrets, undefined, 4));
|
||||
|
||||
for (const secret of secrets) {
|
||||
const insertionStringParameters = 'p1 - input';
|
||||
const insertionStringSecrets = 'p2 - secret';
|
||||
|
|
@ -103,7 +107,6 @@ class AWS {
|
|||
taskDefCloudFormation.slice(indexp3),
|
||||
].join('');
|
||||
}
|
||||
core.info(taskDefCloudFormation);
|
||||
const mappedSecrets = secrets.map((x) => {
|
||||
return { ParameterKey: x.ParameterKey.replace(/[^\dA-Za-z]/g, ''), ParameterValue: x.ParameterValue };
|
||||
});
|
||||
|
|
@ -178,6 +181,8 @@ class AWS {
|
|||
|
||||
const events = (await CF.describeStackEvents({ StackName: taskDefStackName }).promise()).StackEvents;
|
||||
const resources = (await CF.describeStackResources({ StackName: taskDefStackName }).promise()).StackResources;
|
||||
|
||||
core.info(taskDefCloudFormation);
|
||||
core.info(JSON.stringify(events, undefined, 4));
|
||||
core.info(JSON.stringify(resources, undefined, 4));
|
||||
|
||||
|
|
@ -210,169 +215,7 @@ class AWS {
|
|||
return fs.readFileSync(`${__dirname}/cloud-formations/task-def-formation.yml`, 'utf8');
|
||||
}
|
||||
|
||||
static async runTask(
|
||||
taskDef: RemoteBuilderTaskDef,
|
||||
ECS: AWS.ECS,
|
||||
CF: AWS.CloudFormation,
|
||||
environment: RemoteBuilderEnvironmentVariable[],
|
||||
buildUid: string,
|
||||
) {
|
||||
const cluster = taskDef.baseResources?.find((x) => x.LogicalResourceId === 'ECSCluster')?.PhysicalResourceId || '';
|
||||
const taskDefinition =
|
||||
taskDef.taskDefResources?.find((x) => x.LogicalResourceId === 'TaskDefinition')?.PhysicalResourceId || '';
|
||||
const SubnetOne =
|
||||
taskDef.baseResources?.find((x) => x.LogicalResourceId === 'PublicSubnetOne')?.PhysicalResourceId || '';
|
||||
const SubnetTwo =
|
||||
taskDef.baseResources?.find((x) => x.LogicalResourceId === 'PublicSubnetTwo')?.PhysicalResourceId || '';
|
||||
const ContainerSecurityGroup =
|
||||
taskDef.baseResources?.find((x) => x.LogicalResourceId === 'ContainerSecurityGroup')?.PhysicalResourceId || '';
|
||||
const streamName =
|
||||
taskDef.taskDefResources?.find((x) => x.LogicalResourceId === 'KinesisStream')?.PhysicalResourceId || '';
|
||||
|
||||
const task = await ECS.runTask({
|
||||
cluster,
|
||||
taskDefinition,
|
||||
platformVersion: '1.4.0',
|
||||
overrides: {
|
||||
containerOverrides: [
|
||||
{
|
||||
name: taskDef.taskDefStackName,
|
||||
environment: [...environment, { name: 'BUILDID', value: buildUid }],
|
||||
},
|
||||
],
|
||||
},
|
||||
launchType: 'FARGATE',
|
||||
networkConfiguration: {
|
||||
awsvpcConfiguration: {
|
||||
subnets: [SubnetOne, SubnetTwo],
|
||||
assignPublicIp: 'ENABLED',
|
||||
securityGroups: [ContainerSecurityGroup],
|
||||
},
|
||||
},
|
||||
}).promise();
|
||||
|
||||
core.info('Task is starting on worker cluster');
|
||||
const taskArn = task.tasks?.[0].taskArn || '';
|
||||
|
||||
try {
|
||||
await ECS.waitFor('tasksRunning', { tasks: [taskArn], cluster }).promise();
|
||||
} catch (error) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||
const describeTasks = await ECS.describeTasks({
|
||||
tasks: [taskArn],
|
||||
cluster,
|
||||
}).promise();
|
||||
core.info(`Task has ended ${describeTasks.tasks?.[0].containers?.[0].lastStatus}`);
|
||||
core.setFailed(error);
|
||||
core.error(error);
|
||||
}
|
||||
core.info(`Task is running on worker cluster`);
|
||||
await this.streamLogsUntilTaskStops(ECS, CF, taskDef, cluster, taskArn, streamName);
|
||||
await ECS.waitFor('tasksStopped', { cluster, tasks: [taskArn] }).promise();
|
||||
const exitCode = (
|
||||
await ECS.describeTasks({
|
||||
tasks: [taskArn],
|
||||
cluster,
|
||||
}).promise()
|
||||
).tasks?.[0].containers?.[0].exitCode;
|
||||
if (exitCode !== 0) {
|
||||
try {
|
||||
await this.cleanupResources(CF, taskDef);
|
||||
} catch (error) {
|
||||
core.warning(`failed to cleanup ${error}`);
|
||||
}
|
||||
core.error(`job failed with exit code ${exitCode}`);
|
||||
throw new Error(`job failed with exit code ${exitCode}`);
|
||||
} else {
|
||||
core.info(`Task has finished successfully`);
|
||||
}
|
||||
}
|
||||
|
||||
static async streamLogsUntilTaskStops(
|
||||
ECS: AWS.ECS,
|
||||
CF: AWS.CloudFormation,
|
||||
taskDef: RemoteBuilderTaskDef,
|
||||
clusterName: string,
|
||||
taskArn: string,
|
||||
kinesisStreamName: string,
|
||||
) {
|
||||
// watching logs
|
||||
const kinesis = new SDK.Kinesis();
|
||||
|
||||
const getTaskData = async () => {
|
||||
const tasks = await ECS.describeTasks({
|
||||
cluster: clusterName,
|
||||
tasks: [taskArn],
|
||||
}).promise();
|
||||
return tasks.tasks?.[0];
|
||||
};
|
||||
|
||||
const stream = await kinesis
|
||||
.describeStream({
|
||||
StreamName: kinesisStreamName,
|
||||
})
|
||||
.promise();
|
||||
|
||||
let iterator =
|
||||
(
|
||||
await kinesis
|
||||
.getShardIterator({
|
||||
ShardIteratorType: 'TRIM_HORIZON',
|
||||
StreamName: stream.StreamDescription.StreamName,
|
||||
ShardId: stream.StreamDescription.Shards[0].ShardId,
|
||||
})
|
||||
.promise()
|
||||
).ShardIterator || '';
|
||||
|
||||
await CF.waitFor('stackCreateComplete', { StackName: taskDef.taskDefStackNameTTL }).promise();
|
||||
|
||||
core.info(`Task status is ${(await getTaskData())?.lastStatus}`);
|
||||
|
||||
const logBaseUrl = `https://${SDK.config.region}.console.aws.amazon.com/cloudwatch/home?region=${SDK.config.region}#logsV2:log-groups/log-group/${taskDef.taskDefStackName}`;
|
||||
core.info(`You can also see the logs at AWS Cloud Watch: ${logBaseUrl}`);
|
||||
|
||||
let readingLogs = true;
|
||||
let timestamp: number = 0;
|
||||
while (readingLogs) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 1500));
|
||||
const taskData = await getTaskData();
|
||||
if (taskData?.lastStatus !== 'RUNNING') {
|
||||
if (timestamp === 0) {
|
||||
core.info('Task stopped, streaming end of logs');
|
||||
timestamp = Date.now();
|
||||
}
|
||||
if (timestamp !== 0 && Date.now() - timestamp < 30000) {
|
||||
core.info('Task status is not RUNNING for 30 seconds, last query for logs');
|
||||
readingLogs = false;
|
||||
}
|
||||
}
|
||||
const records = await kinesis
|
||||
.getRecords({
|
||||
ShardIterator: iterator,
|
||||
})
|
||||
.promise();
|
||||
iterator = records.NextShardIterator || '';
|
||||
if (records.Records.length > 0 && iterator) {
|
||||
for (let index = 0; index < records.Records.length; index++) {
|
||||
const json = JSON.parse(
|
||||
zlib.gunzipSync(Buffer.from(records.Records[index].Data as string, 'base64')).toString('utf8'),
|
||||
);
|
||||
if (json.messageType === 'DATA_MESSAGE') {
|
||||
for (let logEventsIndex = 0; logEventsIndex < json.logEvents.length; logEventsIndex++) {
|
||||
if (json.logEvents[logEventsIndex].message.includes(taskDef.logid)) {
|
||||
core.info('End of task logs');
|
||||
readingLogs = false;
|
||||
} else {
|
||||
core.info(json.logEvents[logEventsIndex].message);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static async cleanupResources(CF: AWS.CloudFormation, taskDef: RemoteBuilderTaskDef) {
|
||||
static async cleanupResources(CF: SDK.CloudFormation, taskDef: RemoteBuilderTaskDef) {
|
||||
await CF.deleteStack({
|
||||
StackName: taskDef.taskDefStackName,
|
||||
}).promise();
|
||||
|
|
@ -393,4 +236,4 @@ class AWS {
|
|||
core.info('Cleanup complete');
|
||||
}
|
||||
}
|
||||
export default AWS;
|
||||
export default AWSBuildEnvironment;
|
||||
|
|
@ -0,0 +1,165 @@
|
|||
import * as AWS from 'aws-sdk';
|
||||
import RemoteBuilderEnvironmentVariable from './remote-builder-environment-variable';
|
||||
import * as core from '@actions/core';
|
||||
import RemoteBuilderTaskDef from './remote-builder-task-def';
|
||||
import * as zlib from 'zlib';
|
||||
|
||||
class AWSBuildRunner {
|
||||
static async runTask(
|
||||
taskDef: RemoteBuilderTaskDef,
|
||||
ECS: AWS.ECS,
|
||||
CF: AWS.CloudFormation,
|
||||
environment: RemoteBuilderEnvironmentVariable[],
|
||||
buildUid: string,
|
||||
) {
|
||||
const cluster = taskDef.baseResources?.find((x) => x.LogicalResourceId === 'ECSCluster')?.PhysicalResourceId || '';
|
||||
const taskDefinition =
|
||||
taskDef.taskDefResources?.find((x) => x.LogicalResourceId === 'TaskDefinition')?.PhysicalResourceId || '';
|
||||
const SubnetOne =
|
||||
taskDef.baseResources?.find((x) => x.LogicalResourceId === 'PublicSubnetOne')?.PhysicalResourceId || '';
|
||||
const SubnetTwo =
|
||||
taskDef.baseResources?.find((x) => x.LogicalResourceId === 'PublicSubnetTwo')?.PhysicalResourceId || '';
|
||||
const ContainerSecurityGroup =
|
||||
taskDef.baseResources?.find((x) => x.LogicalResourceId === 'ContainerSecurityGroup')?.PhysicalResourceId || '';
|
||||
const streamName =
|
||||
taskDef.taskDefResources?.find((x) => x.LogicalResourceId === 'KinesisStream')?.PhysicalResourceId || '';
|
||||
|
||||
const task = await ECS.runTask({
|
||||
cluster,
|
||||
taskDefinition,
|
||||
platformVersion: '1.4.0',
|
||||
overrides: {
|
||||
containerOverrides: [
|
||||
{
|
||||
name: taskDef.taskDefStackName,
|
||||
environment: [...environment, { name: 'BUILDID', value: buildUid }],
|
||||
},
|
||||
],
|
||||
},
|
||||
launchType: 'FARGATE',
|
||||
networkConfiguration: {
|
||||
awsvpcConfiguration: {
|
||||
subnets: [SubnetOne, SubnetTwo],
|
||||
assignPublicIp: 'ENABLED',
|
||||
securityGroups: [ContainerSecurityGroup],
|
||||
},
|
||||
},
|
||||
}).promise();
|
||||
|
||||
core.info('Task is starting on worker cluster');
|
||||
const taskArn = task.tasks?.[0].taskArn || '';
|
||||
|
||||
try {
|
||||
await ECS.waitFor('tasksRunning', { tasks: [taskArn], cluster }).promise();
|
||||
} catch (error) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||
const describeTasks = await ECS.describeTasks({
|
||||
tasks: [taskArn],
|
||||
cluster,
|
||||
}).promise();
|
||||
core.info(`Task has ended ${describeTasks.tasks?.[0].containers?.[0].lastStatus}`);
|
||||
core.setFailed(error);
|
||||
core.error(error);
|
||||
}
|
||||
core.info(`Task is running on worker cluster`);
|
||||
await this.streamLogsUntilTaskStops(ECS, CF, taskDef, cluster, taskArn, streamName);
|
||||
await ECS.waitFor('tasksStopped', { cluster, tasks: [taskArn] }).promise();
|
||||
const exitCode = (
|
||||
await ECS.describeTasks({
|
||||
tasks: [taskArn],
|
||||
cluster,
|
||||
}).promise()
|
||||
).tasks?.[0].containers?.[0].exitCode;
|
||||
if (exitCode !== 0) {
|
||||
core.error(`job failed with exit code ${exitCode}`);
|
||||
throw new Error(`job failed with exit code ${exitCode}`);
|
||||
} else {
|
||||
core.info(`Task has finished successfully`);
|
||||
}
|
||||
}
|
||||
|
||||
static async streamLogsUntilTaskStops(
|
||||
ECS: AWS.ECS,
|
||||
CF: AWS.CloudFormation,
|
||||
taskDef: RemoteBuilderTaskDef,
|
||||
clusterName: string,
|
||||
taskArn: string,
|
||||
kinesisStreamName: string,
|
||||
) {
|
||||
// watching logs
|
||||
const kinesis = new AWS.Kinesis();
|
||||
|
||||
const getTaskData = async () => {
|
||||
const tasks = await ECS.describeTasks({
|
||||
cluster: clusterName,
|
||||
tasks: [taskArn],
|
||||
}).promise();
|
||||
return tasks.tasks?.[0];
|
||||
};
|
||||
|
||||
const stream = await kinesis
|
||||
.describeStream({
|
||||
StreamName: kinesisStreamName,
|
||||
})
|
||||
.promise();
|
||||
|
||||
let iterator =
|
||||
(
|
||||
await kinesis
|
||||
.getShardIterator({
|
||||
ShardIteratorType: 'TRIM_HORIZON',
|
||||
StreamName: stream.StreamDescription.StreamName,
|
||||
ShardId: stream.StreamDescription.Shards[0].ShardId,
|
||||
})
|
||||
.promise()
|
||||
).ShardIterator || '';
|
||||
|
||||
await CF.waitFor('stackCreateComplete', { StackName: taskDef.taskDefStackNameTTL }).promise();
|
||||
|
||||
core.info(`Task status is ${(await getTaskData())?.lastStatus}`);
|
||||
|
||||
const logBaseUrl = `https://${AWS.config.region}.console.aws.amazon.com/cloudwatch/home?region=${AWS.config.region}#logsV2:log-groups/log-group/${taskDef.taskDefStackName}`;
|
||||
core.info(`You can also see the logs at AWS Cloud Watch: ${logBaseUrl}`);
|
||||
|
||||
let readingLogs = true;
|
||||
let timestamp: number = 0;
|
||||
while (readingLogs) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 1500));
|
||||
const taskData = await getTaskData();
|
||||
if (taskData?.lastStatus !== 'RUNNING') {
|
||||
if (timestamp === 0) {
|
||||
core.info('Task stopped, streaming end of logs');
|
||||
timestamp = Date.now();
|
||||
}
|
||||
if (timestamp !== 0 && Date.now() - timestamp < 30000) {
|
||||
core.info('Task status is not RUNNING for 30 seconds, last query for logs');
|
||||
readingLogs = false;
|
||||
}
|
||||
}
|
||||
const records = await kinesis
|
||||
.getRecords({
|
||||
ShardIterator: iterator,
|
||||
})
|
||||
.promise();
|
||||
iterator = records.NextShardIterator || '';
|
||||
if (records.Records.length > 0 && iterator) {
|
||||
for (let index = 0; index < records.Records.length; index++) {
|
||||
const json = JSON.parse(
|
||||
zlib.gunzipSync(Buffer.from(records.Records[index].Data as string, 'base64')).toString('utf8'),
|
||||
);
|
||||
if (json.messageType === 'DATA_MESSAGE') {
|
||||
for (let logEventsIndex = 0; logEventsIndex < json.logEvents.length; logEventsIndex++) {
|
||||
if (json.logEvents[logEventsIndex].message.includes(taskDef.logid)) {
|
||||
core.info('End of task logs');
|
||||
readingLogs = false;
|
||||
} else {
|
||||
core.info(json.logEvents[logEventsIndex].message);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
export default AWSBuildRunner;
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import { customAlphabet } from 'nanoid';
|
||||
import AWS from './aws';
|
||||
import AWSBuildEnvironment from './aws-build-environment';
|
||||
import * as core from '@actions/core';
|
||||
import RemoteBuilderAlphabet from './remote-builder-alphabet';
|
||||
import { BuildParameters } from '..';
|
||||
|
|
@ -25,7 +25,7 @@ class RemoteBuilder {
|
|||
];
|
||||
|
||||
core.info('Starting part 1/4 (clone from github and restore cache)');
|
||||
await AWS.run(
|
||||
await AWSBuildEnvironment.run(
|
||||
buildUid,
|
||||
buildParameters.awsStackName,
|
||||
'alpine/git',
|
||||
|
|
@ -149,7 +149,7 @@ class RemoteBuilder {
|
|||
ParameterValue: buildParameters.androidKeyaliasPass,
|
||||
});
|
||||
|
||||
await AWS.run(
|
||||
await AWSBuildEnvironment.run(
|
||||
buildUid,
|
||||
buildParameters.awsStackName,
|
||||
baseImage.toString(),
|
||||
|
|
@ -224,7 +224,7 @@ class RemoteBuilder {
|
|||
);
|
||||
core.info('Starting part 3/4 (zip unity build and Library for caching)');
|
||||
// Cleanup
|
||||
await AWS.run(
|
||||
await AWSBuildEnvironment.run(
|
||||
buildUid,
|
||||
buildParameters.awsStackName,
|
||||
'alpine',
|
||||
|
|
@ -253,7 +253,7 @@ class RemoteBuilder {
|
|||
);
|
||||
|
||||
core.info('Starting part 4/4 (upload build to s3)');
|
||||
await AWS.run(
|
||||
await AWSBuildEnvironment.run(
|
||||
buildUid,
|
||||
buildParameters.awsStackName,
|
||||
'amazon/aws-cli',
|
||||
|
|
|
|||
Loading…
Reference in New Issue