cleanup
mdugdale
parent
c1f304fce0
commit
cf9cddb2b8
|
|
@ -234,12 +234,23 @@ class AWS {
|
|||
ls;
|
||||
git clone https://${process.env.GITHUB_TOKEN}@github.com/${process.env.GITHUB_REPOSITORY}.git ${buildUid}/repo;
|
||||
git clone https://${process.env.GITHUB_TOKEN}@github.com/game-ci/unity-builder.git ${buildUid}/builder;
|
||||
if [ -f "./${process.env.GITHUB_REF}/lib.zip" ]; then
|
||||
|
||||
if [ ! -d "cache" ]; then
|
||||
mkdir "cache"
|
||||
fi
|
||||
|
||||
cd cache
|
||||
|
||||
latest=$(ls -t | head -1)
|
||||
if [ -f $latest ]; then
|
||||
echo "Cache exists"
|
||||
zip -r ./${process.env.GITHUB_REF}/lib.zip ./${buildUid}/repo/Library/.
|
||||
zip -r ./cache/lib.zip ./${buildUid}/repo/Library/.
|
||||
else
|
||||
echo "Cache does not exist"
|
||||
fi
|
||||
|
||||
cd ..
|
||||
|
||||
cd ${buildUid}/repo;
|
||||
git checkout $GITHUB_SHA;
|
||||
`,
|
||||
|
|
@ -404,140 +415,15 @@ class AWS {
|
|||
});
|
||||
}
|
||||
static run(buildUid, stackName, image, entrypoint, commands, mountdir, workingdir, environment, secrets) {
|
||||
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t, _u, _v;
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
const ECS = new SDK.ECS();
|
||||
const CF = new SDK.CloudFormation();
|
||||
const taskDef = yield this.setupClusters(ECS, CF, buildUid, stackName, image, entrypoint, commands, mountdir, workingdir, environment, secrets);
|
||||
core.info('Build cluster created successfully (skipping waiting for cleanup cluster to start)');
|
||||
const taskDefResources = yield CF.describeStackResources({
|
||||
StackName: taskDef.taskDefStackName,
|
||||
}).promise();
|
||||
const baseResources = yield CF.describeStackResources({ StackName: stackName }).promise();
|
||||
const clusterName = ((_b = (_a = baseResources.StackResources) === null || _a === void 0 ? void 0 : _a.find((x) => x.LogicalResourceId === 'ECSCluster')) === null || _b === void 0 ? void 0 : _b.PhysicalResourceId) || '';
|
||||
const task = yield ECS.runTask({
|
||||
cluster: clusterName,
|
||||
taskDefinition: ((_d = (_c = taskDefResources.StackResources) === null || _c === void 0 ? void 0 : _c.find((x) => x.LogicalResourceId === 'TaskDefinition')) === null || _d === void 0 ? void 0 : _d.PhysicalResourceId) ||
|
||||
'',
|
||||
platformVersion: '1.4.0',
|
||||
overrides: {
|
||||
containerOverrides: [
|
||||
{
|
||||
name: taskDef.taskDefStackName,
|
||||
environment: [...environment, { name: 'BUILDID', value: buildUid }],
|
||||
},
|
||||
],
|
||||
},
|
||||
launchType: 'FARGATE',
|
||||
networkConfiguration: {
|
||||
awsvpcConfiguration: {
|
||||
subnets: [
|
||||
((_f = (_e = baseResources.StackResources) === null || _e === void 0 ? void 0 : _e.find((x) => x.LogicalResourceId === 'PublicSubnetOne')) === null || _f === void 0 ? void 0 : _f.PhysicalResourceId) ||
|
||||
'',
|
||||
((_h = (_g = baseResources.StackResources) === null || _g === void 0 ? void 0 : _g.find((x) => x.LogicalResourceId === 'PublicSubnetTwo')) === null || _h === void 0 ? void 0 : _h.PhysicalResourceId) ||
|
||||
'',
|
||||
],
|
||||
assignPublicIp: 'ENABLED',
|
||||
securityGroups: [
|
||||
((_k = (_j = baseResources.StackResources) === null || _j === void 0 ? void 0 : _j.find((x) => x.LogicalResourceId === 'ContainerSecurityGroup')) === null || _k === void 0 ? void 0 : _k.PhysicalResourceId) || '',
|
||||
],
|
||||
},
|
||||
},
|
||||
}).promise();
|
||||
core.info('Build job is starting');
|
||||
try {
|
||||
yield ECS.waitFor('tasksRunning', { tasks: [((_l = task.tasks) === null || _l === void 0 ? void 0 : _l[0].taskArn) || ''], cluster: clusterName }).promise();
|
||||
}
|
||||
catch (error) {
|
||||
yield new Promise((resolve) => setTimeout(resolve, 3000));
|
||||
const describeTasks = yield ECS.describeTasks({
|
||||
tasks: [((_m = task.tasks) === null || _m === void 0 ? void 0 : _m[0].taskArn) || ''],
|
||||
cluster: clusterName,
|
||||
}).promise();
|
||||
core.info(`Build job has ended ${(_p = (_o = describeTasks.tasks) === null || _o === void 0 ? void 0 : _o[0].containers) === null || _p === void 0 ? void 0 : _p[0].lastStatus}`);
|
||||
core.setFailed(error);
|
||||
core.error(error);
|
||||
}
|
||||
core.info(`Build job is running`);
|
||||
// watching logs
|
||||
const kinesis = new SDK.Kinesis();
|
||||
const getTaskStatus = () => __awaiter(this, void 0, void 0, function* () {
|
||||
var _w, _x;
|
||||
const tasks = yield ECS.describeTasks({
|
||||
cluster: clusterName,
|
||||
tasks: [((_w = task.tasks) === null || _w === void 0 ? void 0 : _w[0].taskArn) || ''],
|
||||
}).promise();
|
||||
return (_x = tasks.tasks) === null || _x === void 0 ? void 0 : _x[0].lastStatus;
|
||||
});
|
||||
const stream = yield kinesis
|
||||
.describeStream({
|
||||
StreamName: ((_r = (_q = taskDefResources.StackResources) === null || _q === void 0 ? void 0 : _q.find((x) => x.LogicalResourceId === 'KinesisStream')) === null || _r === void 0 ? void 0 : _r.PhysicalResourceId) ||
|
||||
'',
|
||||
})
|
||||
.promise();
|
||||
let iterator = (yield kinesis
|
||||
.getShardIterator({
|
||||
ShardIteratorType: 'TRIM_HORIZON',
|
||||
StreamName: stream.StreamDescription.StreamName,
|
||||
ShardId: stream.StreamDescription.Shards[0].ShardId,
|
||||
})
|
||||
.promise()).ShardIterator || '';
|
||||
yield CF.waitFor('stackCreateComplete', { StackName: taskDef.taskDefStackNameTTL }).promise();
|
||||
core.info(`Task status is ${yield getTaskStatus()}`);
|
||||
const logBaseUrl = `https://console.aws.amazon.com/cloudwatch/home?region=${SDK.config.region}#logsV2:log-groups/${taskDef.taskDefStackName}`;
|
||||
core.info(`You can also watch the logs at AWS Cloud Watch: ${logBaseUrl}`);
|
||||
let readingLogs = true;
|
||||
while (readingLogs) {
|
||||
yield new Promise((resolve) => setTimeout(resolve, 1500));
|
||||
if ((yield getTaskStatus()) !== 'RUNNING') {
|
||||
readingLogs = false;
|
||||
yield new Promise((resolve) => setTimeout(resolve, 35000));
|
||||
}
|
||||
const records = yield kinesis
|
||||
.getRecords({
|
||||
ShardIterator: iterator,
|
||||
})
|
||||
.promise();
|
||||
iterator = records.NextShardIterator || '';
|
||||
if (records.Records.length > 0) {
|
||||
for (let index = 0; index < records.Records.length; index++) {
|
||||
const json = JSON.parse(zlib.gunzipSync(Buffer.from(records.Records[index].Data, 'base64')).toString('utf8'));
|
||||
if (json.messageType === 'DATA_MESSAGE') {
|
||||
for (let logEventsIndex = 0; logEventsIndex < json.logEvents.length; logEventsIndex++) {
|
||||
core.info(json.logEvents[logEventsIndex].message);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
yield ECS.waitFor('tasksStopped', { cluster: clusterName, tasks: [((_s = task.tasks) === null || _s === void 0 ? void 0 : _s[0].taskArn) || ''] }).promise();
|
||||
const exitCode = (_v = (_u = (yield ECS.describeTasks({
|
||||
tasks: [((_t = task.tasks) === null || _t === void 0 ? void 0 : _t[0].taskArn) || ''],
|
||||
cluster: clusterName,
|
||||
}).promise()).tasks) === null || _u === void 0 ? void 0 : _u[0].containers) === null || _v === void 0 ? void 0 : _v[0].exitCode;
|
||||
if (exitCode !== 0) {
|
||||
core.error(`job finished with exit code ${exitCode}`);
|
||||
}
|
||||
else {
|
||||
core.info(`Build job has finished with exit code 0`);
|
||||
}
|
||||
yield CF.deleteStack({
|
||||
StackName: taskDef.taskDefStackName,
|
||||
}).promise();
|
||||
yield CF.deleteStack({
|
||||
StackName: taskDef.taskDefStackNameTTL,
|
||||
}).promise();
|
||||
yield CF.waitFor('stackDeleteComplete', {
|
||||
StackName: taskDef.taskDefStackName,
|
||||
}).promise();
|
||||
// Currently too slow and causes too much waiting
|
||||
yield CF.waitFor('stackDeleteComplete', {
|
||||
StackName: taskDef.taskDefStackNameTTL,
|
||||
}).promise();
|
||||
core.info('Cleanup complete');
|
||||
const taskDef = yield this.setupCloudFormations(CF, buildUid, stackName, image, entrypoint, commands, mountdir, workingdir, secrets);
|
||||
yield this.runTask(taskDef, ECS, CF, environment, buildUid);
|
||||
yield this.cleanupResources(CF, taskDef);
|
||||
});
|
||||
}
|
||||
static setupClusters(ECS, CF, buildUid, stackName, image, entrypoint, commands, mountdir, workingdir, environment, secrets) {
|
||||
static setupCloudFormations(CF, buildUid, stackName, image, entrypoint, commands, mountdir, workingdir, secrets) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
const taskDefStackName = `${stackName}-taskDef-${image}-${buildUid}`.toString().replace(/[^\da-z]/gi, '');
|
||||
const taskDefCloudFormation = fs.readFileSync(`${__dirname}/task-def-formation.yml`, 'utf8');
|
||||
|
|
@ -609,7 +495,151 @@ class AWS {
|
|||
catch (error) {
|
||||
core.error(error);
|
||||
}
|
||||
return { taskDefStackName, taskDefCloudFormation, taskDefStackNameTTL, ttlCloudFormation };
|
||||
const taskDefResources = yield CF.describeStackResources({
|
||||
StackName: taskDefStackName,
|
||||
}).promise();
|
||||
const baseResources = yield CF.describeStackResources({ StackName: stackName }).promise();
|
||||
core.info('Build cluster created successfully (skipping waiting for cleanup cluster to start)');
|
||||
return {
|
||||
taskDefStackName,
|
||||
taskDefCloudFormation,
|
||||
taskDefStackNameTTL,
|
||||
ttlCloudFormation,
|
||||
taskDefResources,
|
||||
baseResources,
|
||||
};
|
||||
});
|
||||
}
|
||||
static runTask(taskDef, ECS, CF, environment, buildUid) {
|
||||
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s;
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
const clusterName = ((_b = (_a = taskDef.baseResources.StackResources) === null || _a === void 0 ? void 0 : _a.find((x) => x.LogicalResourceId === 'ECSCluster')) === null || _b === void 0 ? void 0 : _b.PhysicalResourceId) || '';
|
||||
const taskDefinition = ((_d = (_c = taskDef.taskDefResources.StackResources) === null || _c === void 0 ? void 0 : _c.find((x) => x.LogicalResourceId === 'TaskDefinition')) === null || _d === void 0 ? void 0 : _d.PhysicalResourceId) || '';
|
||||
const SubnetOne = ((_f = (_e = taskDef.baseResources.StackResources) === null || _e === void 0 ? void 0 : _e.find((x) => x.LogicalResourceId === 'PublicSubnetOne')) === null || _f === void 0 ? void 0 : _f.PhysicalResourceId) || '';
|
||||
const SubnetTwo = ((_h = (_g = taskDef.baseResources.StackResources) === null || _g === void 0 ? void 0 : _g.find((x) => x.LogicalResourceId === 'PublicSubnetTwo')) === null || _h === void 0 ? void 0 : _h.PhysicalResourceId) || '';
|
||||
const ContainerSecurityGroup = ((_k = (_j = taskDef.baseResources.StackResources) === null || _j === void 0 ? void 0 : _j.find((x) => x.LogicalResourceId === 'ContainerSecurityGroup')) === null || _k === void 0 ? void 0 : _k.PhysicalResourceId) || '';
|
||||
const streamName = ((_m = (_l = taskDef.taskDefResources.StackResources) === null || _l === void 0 ? void 0 : _l.find((x) => x.LogicalResourceId === 'KinesisStream')) === null || _m === void 0 ? void 0 : _m.PhysicalResourceId) || '';
|
||||
const task = yield ECS.runTask({
|
||||
cluster: clusterName,
|
||||
taskDefinition: taskDefinition,
|
||||
platformVersion: '1.4.0',
|
||||
overrides: {
|
||||
containerOverrides: [
|
||||
{
|
||||
name: taskDef.taskDefStackName,
|
||||
environment: [...environment, { name: 'BUILDID', value: buildUid }],
|
||||
},
|
||||
],
|
||||
},
|
||||
launchType: 'FARGATE',
|
||||
networkConfiguration: {
|
||||
awsvpcConfiguration: {
|
||||
subnets: [SubnetOne, SubnetTwo],
|
||||
assignPublicIp: 'ENABLED',
|
||||
securityGroups: [ContainerSecurityGroup],
|
||||
},
|
||||
},
|
||||
}).promise();
|
||||
core.info('Build job is starting');
|
||||
const taskArn = ((_o = task.tasks) === null || _o === void 0 ? void 0 : _o[0].taskArn) || '';
|
||||
try {
|
||||
yield ECS.waitFor('tasksRunning', { tasks: [taskArn], cluster: clusterName }).promise();
|
||||
}
|
||||
catch (error) {
|
||||
yield new Promise((resolve) => setTimeout(resolve, 3000));
|
||||
const describeTasks = yield ECS.describeTasks({
|
||||
tasks: [taskArn],
|
||||
cluster: clusterName,
|
||||
}).promise();
|
||||
core.info(`Build job has ended ${(_q = (_p = describeTasks.tasks) === null || _p === void 0 ? void 0 : _p[0].containers) === null || _q === void 0 ? void 0 : _q[0].lastStatus}`);
|
||||
core.setFailed(error);
|
||||
core.error(error);
|
||||
}
|
||||
core.info(`Build job is running`);
|
||||
yield this.streamLogsUntilTaskStops(ECS, CF, taskDef, clusterName, taskArn, streamName);
|
||||
yield ECS.waitFor('tasksStopped', { cluster: clusterName, tasks: [taskArn] }).promise();
|
||||
const exitCode = (_s = (_r = (yield ECS.describeTasks({
|
||||
tasks: [taskArn],
|
||||
cluster: clusterName,
|
||||
}).promise()).tasks) === null || _r === void 0 ? void 0 : _r[0].containers) === null || _s === void 0 ? void 0 : _s[0].exitCode;
|
||||
if (exitCode !== 0) {
|
||||
core.error(`job finished with exit code ${exitCode}`);
|
||||
}
|
||||
else {
|
||||
core.info(`Build job has finished with exit code 0`);
|
||||
}
|
||||
});
|
||||
}
|
||||
static streamLogsUntilTaskStops(ECS, CF, taskDef, clusterName, taskArn, kinesisStreamName) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
// watching logs
|
||||
const kinesis = new SDK.Kinesis();
|
||||
const getTaskStatus = () => __awaiter(this, void 0, void 0, function* () {
|
||||
var _a;
|
||||
const tasks = yield ECS.describeTasks({
|
||||
cluster: clusterName,
|
||||
tasks: [taskArn],
|
||||
}).promise();
|
||||
return (_a = tasks.tasks) === null || _a === void 0 ? void 0 : _a[0].lastStatus;
|
||||
});
|
||||
const stream = yield kinesis
|
||||
.describeStream({
|
||||
StreamName: kinesisStreamName,
|
||||
})
|
||||
.promise();
|
||||
let iterator = (yield kinesis
|
||||
.getShardIterator({
|
||||
ShardIteratorType: 'TRIM_HORIZON',
|
||||
StreamName: stream.StreamDescription.StreamName,
|
||||
ShardId: stream.StreamDescription.Shards[0].ShardId,
|
||||
})
|
||||
.promise()).ShardIterator || '';
|
||||
yield CF.waitFor('stackCreateComplete', { StackName: taskDef.taskDefStackNameTTL }).promise();
|
||||
core.info(`Task status is ${yield getTaskStatus()}`);
|
||||
const logBaseUrl = `https://console.aws.amazon.com/cloudwatch/home?region=${SDK.config.region}#logsV2:log-groups/${taskDef.taskDefStackName}`;
|
||||
core.info(`You can also watch the logs at AWS Cloud Watch: ${logBaseUrl}`);
|
||||
let readingLogs = true;
|
||||
while (readingLogs) {
|
||||
yield new Promise((resolve) => setTimeout(resolve, 1500));
|
||||
if ((yield getTaskStatus()) !== 'RUNNING') {
|
||||
readingLogs = false;
|
||||
yield new Promise((resolve) => setTimeout(resolve, 35000));
|
||||
}
|
||||
const records = yield kinesis
|
||||
.getRecords({
|
||||
ShardIterator: iterator,
|
||||
})
|
||||
.promise();
|
||||
iterator = records.NextShardIterator || '';
|
||||
if (records.Records.length > 0) {
|
||||
for (let index = 0; index < records.Records.length; index++) {
|
||||
const json = JSON.parse(zlib.gunzipSync(Buffer.from(records.Records[index].Data, 'base64')).toString('utf8'));
|
||||
if (json.messageType === 'DATA_MESSAGE') {
|
||||
for (let logEventsIndex = 0; logEventsIndex < json.logEvents.length; logEventsIndex++) {
|
||||
core.info(json.logEvents[logEventsIndex].message);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
static cleanupResources(CF, taskDef) {
|
||||
return __awaiter(this, void 0, void 0, function* () {
|
||||
yield CF.deleteStack({
|
||||
StackName: taskDef.taskDefStackName,
|
||||
}).promise();
|
||||
yield CF.deleteStack({
|
||||
StackName: taskDef.taskDefStackNameTTL,
|
||||
}).promise();
|
||||
yield CF.waitFor('stackDeleteComplete', {
|
||||
StackName: taskDef.taskDefStackName,
|
||||
}).promise();
|
||||
// Currently too slow and causes too much waiting
|
||||
yield CF.waitFor('stackDeleteComplete', {
|
||||
StackName: taskDef.taskDefStackNameTTL,
|
||||
}).promise();
|
||||
core.info('Cleanup complete');
|
||||
});
|
||||
}
|
||||
static onlog(batch) {
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
291
src/model/aws.ts
291
src/model/aws.ts
|
|
@ -22,12 +22,23 @@ class AWS {
|
|||
ls;
|
||||
git clone https://${process.env.GITHUB_TOKEN}@github.com/${process.env.GITHUB_REPOSITORY}.git ${buildUid}/repo;
|
||||
git clone https://${process.env.GITHUB_TOKEN}@github.com/game-ci/unity-builder.git ${buildUid}/builder;
|
||||
if [ -f "./${process.env.GITHUB_REF}/lib.zip" ]; then
|
||||
|
||||
if [ ! -d "cache" ]; then
|
||||
mkdir "cache"
|
||||
fi
|
||||
|
||||
cd cache
|
||||
|
||||
latest=$(ls -t | head -1)
|
||||
if [ -f $latest ]; then
|
||||
echo "Cache exists"
|
||||
zip -r ./${process.env.GITHUB_REF}/lib.zip ./${buildUid}/repo/Library/.
|
||||
zip -r ./cache/lib.zip ./${buildUid}/repo/Library/.
|
||||
else
|
||||
echo "Cache does not exist"
|
||||
fi
|
||||
|
||||
cd ..
|
||||
|
||||
cd ${buildUid}/repo;
|
||||
git checkout $GITHUB_SHA;
|
||||
`,
|
||||
|
|
@ -239,23 +250,141 @@ class AWS {
|
|||
const ECS = new SDK.ECS();
|
||||
const CF = new SDK.CloudFormation();
|
||||
|
||||
const taskDef = await this.setupClusters(ECS, CF, buildUid, stackName, image, entrypoint, commands, mountdir, workingdir, environment, secrets);
|
||||
const taskDef = await this.setupCloudFormations(
|
||||
CF,
|
||||
buildUid,
|
||||
stackName,
|
||||
image,
|
||||
entrypoint,
|
||||
commands,
|
||||
mountdir,
|
||||
workingdir,
|
||||
secrets,
|
||||
);
|
||||
|
||||
core.info('Build cluster created successfully (skipping waiting for cleanup cluster to start)');
|
||||
await this.runTask(taskDef, ECS, CF, environment, buildUid);
|
||||
|
||||
await this.cleanupResources(CF, taskDef);
|
||||
}
|
||||
|
||||
static async setupCloudFormations(
|
||||
CF,
|
||||
buildUid: string,
|
||||
stackName: string,
|
||||
image: string,
|
||||
entrypoint: string[],
|
||||
commands,
|
||||
mountdir,
|
||||
workingdir,
|
||||
secrets,
|
||||
) {
|
||||
const taskDefStackName = `${stackName}-taskDef-${image}-${buildUid}`.toString().replace(/[^\da-z]/gi, '');
|
||||
const taskDefCloudFormation = fs.readFileSync(`${__dirname}/task-def-formation.yml`, 'utf8');
|
||||
await CF.createStack({
|
||||
StackName: taskDefStackName,
|
||||
TemplateBody: taskDefCloudFormation,
|
||||
Parameters: [
|
||||
{
|
||||
ParameterKey: 'ImageUrl',
|
||||
ParameterValue: image,
|
||||
},
|
||||
{
|
||||
ParameterKey: 'ServiceName',
|
||||
ParameterValue: taskDefStackName,
|
||||
},
|
||||
{
|
||||
ParameterKey: 'Command',
|
||||
ParameterValue: commands.join(','),
|
||||
},
|
||||
{
|
||||
ParameterKey: 'EntryPoint',
|
||||
ParameterValue: entrypoint.join(','),
|
||||
},
|
||||
{
|
||||
ParameterKey: 'WorkingDirectory',
|
||||
ParameterValue: workingdir,
|
||||
},
|
||||
{
|
||||
ParameterKey: 'EFSMountDirectory',
|
||||
ParameterValue: mountdir,
|
||||
},
|
||||
{
|
||||
ParameterKey: 'BUILDID',
|
||||
ParameterValue: buildUid,
|
||||
},
|
||||
...secrets,
|
||||
],
|
||||
}).promise();
|
||||
core.info('Creating build cluster...');
|
||||
|
||||
const taskDefStackNameTTL = `${taskDefStackName}-ttl`;
|
||||
const ttlCloudFormation = fs.readFileSync(`${__dirname}/cloudformation-stack-ttl.yml`, 'utf8');
|
||||
await CF.createStack({
|
||||
StackName: taskDefStackNameTTL,
|
||||
TemplateBody: ttlCloudFormation,
|
||||
Capabilities: ['CAPABILITY_IAM'],
|
||||
Parameters: [
|
||||
{
|
||||
ParameterKey: 'StackName',
|
||||
ParameterValue: taskDefStackName,
|
||||
},
|
||||
{
|
||||
ParameterKey: 'DeleteStackName',
|
||||
ParameterValue: taskDefStackNameTTL,
|
||||
},
|
||||
{
|
||||
ParameterKey: 'TTL',
|
||||
ParameterValue: '100',
|
||||
},
|
||||
{
|
||||
ParameterKey: 'BUILDID',
|
||||
ParameterValue: buildUid,
|
||||
},
|
||||
],
|
||||
}).promise();
|
||||
core.info('Creating cleanup cluster...');
|
||||
|
||||
try {
|
||||
await CF.waitFor('stackCreateComplete', { StackName: taskDefStackName }).promise();
|
||||
} catch (error) {
|
||||
core.error(error);
|
||||
}
|
||||
const taskDefResources = await CF.describeStackResources({
|
||||
StackName: taskDef.taskDefStackName,
|
||||
StackName: taskDefStackName,
|
||||
}).promise();
|
||||
|
||||
const baseResources = await CF.describeStackResources({ StackName: stackName }).promise();
|
||||
|
||||
core.info('Build cluster created successfully (skipping waiting for cleanup cluster to start)');
|
||||
|
||||
return {
|
||||
taskDefStackName,
|
||||
taskDefCloudFormation,
|
||||
taskDefStackNameTTL,
|
||||
ttlCloudFormation,
|
||||
taskDefResources,
|
||||
baseResources,
|
||||
};
|
||||
}
|
||||
|
||||
static async runTask(taskDef, ECS, CF, environment, buildUid) {
|
||||
const clusterName =
|
||||
baseResources.StackResources?.find((x) => x.LogicalResourceId === 'ECSCluster')?.PhysicalResourceId || '';
|
||||
taskDef.baseResources.StackResources?.find((x) => x.LogicalResourceId === 'ECSCluster')?.PhysicalResourceId || '';
|
||||
const taskDefinition = taskDef.taskDefResources.StackResources?.find((x) => x.LogicalResourceId === 'TaskDefinition')
|
||||
?.PhysicalResourceId || '';
|
||||
const SubnetOne = taskDef.baseResources.StackResources?.find((x) => x.LogicalResourceId === 'PublicSubnetOne')
|
||||
?.PhysicalResourceId || '';
|
||||
const SubnetTwo = taskDef.baseResources.StackResources?.find((x) => x.LogicalResourceId === 'PublicSubnetTwo')
|
||||
?.PhysicalResourceId || '';
|
||||
const ContainerSecurityGroup = taskDef.baseResources.StackResources?.find((x) => x.LogicalResourceId === 'ContainerSecurityGroup')
|
||||
?.PhysicalResourceId || '';
|
||||
const streamName =
|
||||
taskDef.taskDefResources.StackResources?.find((x) => x.LogicalResourceId === 'KinesisStream')
|
||||
?.PhysicalResourceId || '';
|
||||
|
||||
const task = await ECS.runTask({
|
||||
cluster: clusterName,
|
||||
taskDefinition:
|
||||
taskDefResources.StackResources?.find((x) => x.LogicalResourceId === 'TaskDefinition')?.PhysicalResourceId ||
|
||||
'',
|
||||
taskDefinition: taskDefinition,
|
||||
platformVersion: '1.4.0',
|
||||
overrides: {
|
||||
containerOverrides: [
|
||||
|
|
@ -268,54 +397,59 @@ class AWS {
|
|||
launchType: 'FARGATE',
|
||||
networkConfiguration: {
|
||||
awsvpcConfiguration: {
|
||||
subnets: [
|
||||
baseResources.StackResources?.find((x) => x.LogicalResourceId === 'PublicSubnetOne')?.PhysicalResourceId ||
|
||||
'',
|
||||
baseResources.StackResources?.find((x) => x.LogicalResourceId === 'PublicSubnetTwo')?.PhysicalResourceId ||
|
||||
'',
|
||||
],
|
||||
subnets: [ SubnetOne, SubnetTwo ],
|
||||
assignPublicIp: 'ENABLED',
|
||||
securityGroups: [
|
||||
baseResources.StackResources?.find((x) => x.LogicalResourceId === 'ContainerSecurityGroup')
|
||||
?.PhysicalResourceId || '',
|
||||
],
|
||||
securityGroups: [ ContainerSecurityGroup ],
|
||||
},
|
||||
},
|
||||
}).promise();
|
||||
|
||||
core.info('Build job is starting');
|
||||
const taskArn = task.tasks?.[0].taskArn || '';
|
||||
|
||||
try {
|
||||
await ECS.waitFor('tasksRunning', { tasks: [task.tasks?.[0].taskArn || ''], cluster: clusterName }).promise();
|
||||
await ECS.waitFor('tasksRunning', { tasks: [taskArn], cluster: clusterName }).promise();
|
||||
} catch (error) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||
const describeTasks = await ECS.describeTasks({
|
||||
tasks: [task.tasks?.[0].taskArn || ''],
|
||||
tasks: [taskArn],
|
||||
cluster: clusterName,
|
||||
}).promise();
|
||||
core.info(`Build job has ended ${describeTasks.tasks?.[0].containers?.[0].lastStatus}`);
|
||||
core.setFailed(error);
|
||||
core.error(error);
|
||||
}
|
||||
|
||||
core.info(`Build job is running`);
|
||||
await this.streamLogsUntilTaskStops(ECS, CF, taskDef, clusterName, taskArn, streamName);
|
||||
await ECS.waitFor('tasksStopped', { cluster: clusterName, tasks: [taskArn] }).promise();
|
||||
const exitCode = (
|
||||
await ECS.describeTasks({
|
||||
tasks: [taskArn],
|
||||
cluster: clusterName,
|
||||
}).promise()
|
||||
).tasks?.[0].containers?.[0].exitCode;
|
||||
if (exitCode !== 0) {
|
||||
core.error(`job finished with exit code ${exitCode}`);
|
||||
} else {
|
||||
core.info(`Build job has finished with exit code 0`);
|
||||
}
|
||||
}
|
||||
|
||||
static async streamLogsUntilTaskStops(ECS, CF, taskDef, clusterName, taskArn, kinesisStreamName) {
|
||||
// watching logs
|
||||
const kinesis = new SDK.Kinesis();
|
||||
|
||||
const getTaskStatus = async () => {
|
||||
const tasks = await ECS.describeTasks({
|
||||
cluster: clusterName,
|
||||
tasks: [task.tasks?.[0].taskArn || ''],
|
||||
tasks: [taskArn],
|
||||
}).promise();
|
||||
return tasks.tasks?.[0].lastStatus;
|
||||
};
|
||||
|
||||
const stream = await kinesis
|
||||
.describeStream({
|
||||
StreamName:
|
||||
taskDefResources.StackResources?.find((x) => x.LogicalResourceId === 'KinesisStream')?.PhysicalResourceId ||
|
||||
'',
|
||||
StreamName: kinesisStreamName,
|
||||
})
|
||||
.promise();
|
||||
|
||||
|
|
@ -363,22 +497,9 @@ class AWS {
|
|||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await ECS.waitFor('tasksStopped', { cluster: clusterName, tasks: [task.tasks?.[0].taskArn || ''] }).promise();
|
||||
|
||||
const exitCode = (
|
||||
await ECS.describeTasks({
|
||||
tasks: [task.tasks?.[0].taskArn || ''],
|
||||
cluster: clusterName,
|
||||
}).promise()
|
||||
).tasks?.[0].containers?.[0].exitCode;
|
||||
|
||||
if (exitCode !== 0) {
|
||||
core.error(`job finished with exit code ${exitCode}`);
|
||||
} else {
|
||||
core.info(`Build job has finished with exit code 0`);
|
||||
}
|
||||
|
||||
static async cleanupResources(CF, taskDef) {
|
||||
await CF.deleteStack({
|
||||
StackName: taskDef.taskDefStackName,
|
||||
}).promise();
|
||||
|
|
@ -399,94 +520,6 @@ class AWS {
|
|||
core.info('Cleanup complete');
|
||||
}
|
||||
|
||||
static async setupClusters(
|
||||
ECS,
|
||||
CF,
|
||||
buildUid: string,
|
||||
stackName: string,
|
||||
image: string,
|
||||
entrypoint: string[],
|
||||
commands,
|
||||
mountdir,
|
||||
workingdir,
|
||||
environment,
|
||||
secrets) {
|
||||
|
||||
const taskDefStackName = `${stackName}-taskDef-${image}-${buildUid}`.toString().replace(/[^\da-z]/gi, '');
|
||||
const taskDefCloudFormation = fs.readFileSync(`${__dirname}/task-def-formation.yml`, 'utf8');
|
||||
await CF.createStack({
|
||||
StackName: taskDefStackName,
|
||||
TemplateBody: taskDefCloudFormation,
|
||||
Parameters: [
|
||||
{
|
||||
ParameterKey: 'ImageUrl',
|
||||
ParameterValue: image,
|
||||
},
|
||||
{
|
||||
ParameterKey: 'ServiceName',
|
||||
ParameterValue: taskDefStackName,
|
||||
},
|
||||
{
|
||||
ParameterKey: 'Command',
|
||||
ParameterValue: commands.join(','),
|
||||
},
|
||||
{
|
||||
ParameterKey: 'EntryPoint',
|
||||
ParameterValue: entrypoint.join(','),
|
||||
},
|
||||
{
|
||||
ParameterKey: 'WorkingDirectory',
|
||||
ParameterValue: workingdir,
|
||||
},
|
||||
{
|
||||
ParameterKey: 'EFSMountDirectory',
|
||||
ParameterValue: mountdir,
|
||||
},
|
||||
{
|
||||
ParameterKey: 'BUILDID',
|
||||
ParameterValue: buildUid,
|
||||
},
|
||||
...secrets,
|
||||
],
|
||||
}).promise();
|
||||
core.info('Creating build cluster...');
|
||||
|
||||
const taskDefStackNameTTL = `${taskDefStackName}-ttl`;
|
||||
const ttlCloudFormation = fs.readFileSync(`${__dirname}/cloudformation-stack-ttl.yml`, 'utf8');
|
||||
await CF.createStack({
|
||||
StackName: taskDefStackNameTTL,
|
||||
TemplateBody: ttlCloudFormation,
|
||||
Capabilities: ['CAPABILITY_IAM'],
|
||||
Parameters: [
|
||||
{
|
||||
ParameterKey: 'StackName',
|
||||
ParameterValue: taskDefStackName,
|
||||
},
|
||||
{
|
||||
ParameterKey: 'DeleteStackName',
|
||||
ParameterValue: taskDefStackNameTTL,
|
||||
},
|
||||
{
|
||||
ParameterKey: 'TTL',
|
||||
ParameterValue: '100',
|
||||
},
|
||||
{
|
||||
ParameterKey: 'BUILDID',
|
||||
ParameterValue: buildUid,
|
||||
},
|
||||
],
|
||||
}).promise();
|
||||
core.info('Creating cleanup cluster...');
|
||||
|
||||
try {
|
||||
await CF.waitFor('stackCreateComplete', { StackName: taskDefStackName }).promise();
|
||||
} catch (error) {
|
||||
core.error(error);
|
||||
}
|
||||
|
||||
return { taskDefStackName, taskDefCloudFormation, taskDefStackNameTTL, ttlCloudFormation };
|
||||
}
|
||||
|
||||
static onlog(batch) {
|
||||
for (const log of batch) {
|
||||
core.info(`log: ${log}`);
|
||||
|
|
|
|||
Loading…
Reference in New Issue