cleanup
mdugdale
parent
c1f304fce0
commit
cf9cddb2b8
|
|
@ -234,12 +234,23 @@ class AWS {
|
||||||
ls;
|
ls;
|
||||||
git clone https://${process.env.GITHUB_TOKEN}@github.com/${process.env.GITHUB_REPOSITORY}.git ${buildUid}/repo;
|
git clone https://${process.env.GITHUB_TOKEN}@github.com/${process.env.GITHUB_REPOSITORY}.git ${buildUid}/repo;
|
||||||
git clone https://${process.env.GITHUB_TOKEN}@github.com/game-ci/unity-builder.git ${buildUid}/builder;
|
git clone https://${process.env.GITHUB_TOKEN}@github.com/game-ci/unity-builder.git ${buildUid}/builder;
|
||||||
if [ -f "./${process.env.GITHUB_REF}/lib.zip" ]; then
|
|
||||||
|
if [ ! -d "cache" ]; then
|
||||||
|
mkdir "cache"
|
||||||
|
fi
|
||||||
|
|
||||||
|
cd cache
|
||||||
|
|
||||||
|
latest=$(ls -t | head -1)
|
||||||
|
if [ -f $latest ]; then
|
||||||
echo "Cache exists"
|
echo "Cache exists"
|
||||||
zip -r ./${process.env.GITHUB_REF}/lib.zip ./${buildUid}/repo/Library/.
|
zip -r ./cache/lib.zip ./${buildUid}/repo/Library/.
|
||||||
else
|
else
|
||||||
echo "Cache does not exist"
|
echo "Cache does not exist"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
cd ..
|
||||||
|
|
||||||
cd ${buildUid}/repo;
|
cd ${buildUid}/repo;
|
||||||
git checkout $GITHUB_SHA;
|
git checkout $GITHUB_SHA;
|
||||||
`,
|
`,
|
||||||
|
|
@ -404,140 +415,15 @@ class AWS {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
static run(buildUid, stackName, image, entrypoint, commands, mountdir, workingdir, environment, secrets) {
|
static run(buildUid, stackName, image, entrypoint, commands, mountdir, workingdir, environment, secrets) {
|
||||||
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t, _u, _v;
|
|
||||||
return __awaiter(this, void 0, void 0, function* () {
|
return __awaiter(this, void 0, void 0, function* () {
|
||||||
const ECS = new SDK.ECS();
|
const ECS = new SDK.ECS();
|
||||||
const CF = new SDK.CloudFormation();
|
const CF = new SDK.CloudFormation();
|
||||||
const taskDef = yield this.setupClusters(ECS, CF, buildUid, stackName, image, entrypoint, commands, mountdir, workingdir, environment, secrets);
|
const taskDef = yield this.setupCloudFormations(CF, buildUid, stackName, image, entrypoint, commands, mountdir, workingdir, secrets);
|
||||||
core.info('Build cluster created successfully (skipping waiting for cleanup cluster to start)');
|
yield this.runTask(taskDef, ECS, CF, environment, buildUid);
|
||||||
const taskDefResources = yield CF.describeStackResources({
|
yield this.cleanupResources(CF, taskDef);
|
||||||
StackName: taskDef.taskDefStackName,
|
|
||||||
}).promise();
|
|
||||||
const baseResources = yield CF.describeStackResources({ StackName: stackName }).promise();
|
|
||||||
const clusterName = ((_b = (_a = baseResources.StackResources) === null || _a === void 0 ? void 0 : _a.find((x) => x.LogicalResourceId === 'ECSCluster')) === null || _b === void 0 ? void 0 : _b.PhysicalResourceId) || '';
|
|
||||||
const task = yield ECS.runTask({
|
|
||||||
cluster: clusterName,
|
|
||||||
taskDefinition: ((_d = (_c = taskDefResources.StackResources) === null || _c === void 0 ? void 0 : _c.find((x) => x.LogicalResourceId === 'TaskDefinition')) === null || _d === void 0 ? void 0 : _d.PhysicalResourceId) ||
|
|
||||||
'',
|
|
||||||
platformVersion: '1.4.0',
|
|
||||||
overrides: {
|
|
||||||
containerOverrides: [
|
|
||||||
{
|
|
||||||
name: taskDef.taskDefStackName,
|
|
||||||
environment: [...environment, { name: 'BUILDID', value: buildUid }],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
launchType: 'FARGATE',
|
|
||||||
networkConfiguration: {
|
|
||||||
awsvpcConfiguration: {
|
|
||||||
subnets: [
|
|
||||||
((_f = (_e = baseResources.StackResources) === null || _e === void 0 ? void 0 : _e.find((x) => x.LogicalResourceId === 'PublicSubnetOne')) === null || _f === void 0 ? void 0 : _f.PhysicalResourceId) ||
|
|
||||||
'',
|
|
||||||
((_h = (_g = baseResources.StackResources) === null || _g === void 0 ? void 0 : _g.find((x) => x.LogicalResourceId === 'PublicSubnetTwo')) === null || _h === void 0 ? void 0 : _h.PhysicalResourceId) ||
|
|
||||||
'',
|
|
||||||
],
|
|
||||||
assignPublicIp: 'ENABLED',
|
|
||||||
securityGroups: [
|
|
||||||
((_k = (_j = baseResources.StackResources) === null || _j === void 0 ? void 0 : _j.find((x) => x.LogicalResourceId === 'ContainerSecurityGroup')) === null || _k === void 0 ? void 0 : _k.PhysicalResourceId) || '',
|
|
||||||
],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}).promise();
|
|
||||||
core.info('Build job is starting');
|
|
||||||
try {
|
|
||||||
yield ECS.waitFor('tasksRunning', { tasks: [((_l = task.tasks) === null || _l === void 0 ? void 0 : _l[0].taskArn) || ''], cluster: clusterName }).promise();
|
|
||||||
}
|
|
||||||
catch (error) {
|
|
||||||
yield new Promise((resolve) => setTimeout(resolve, 3000));
|
|
||||||
const describeTasks = yield ECS.describeTasks({
|
|
||||||
tasks: [((_m = task.tasks) === null || _m === void 0 ? void 0 : _m[0].taskArn) || ''],
|
|
||||||
cluster: clusterName,
|
|
||||||
}).promise();
|
|
||||||
core.info(`Build job has ended ${(_p = (_o = describeTasks.tasks) === null || _o === void 0 ? void 0 : _o[0].containers) === null || _p === void 0 ? void 0 : _p[0].lastStatus}`);
|
|
||||||
core.setFailed(error);
|
|
||||||
core.error(error);
|
|
||||||
}
|
|
||||||
core.info(`Build job is running`);
|
|
||||||
// watching logs
|
|
||||||
const kinesis = new SDK.Kinesis();
|
|
||||||
const getTaskStatus = () => __awaiter(this, void 0, void 0, function* () {
|
|
||||||
var _w, _x;
|
|
||||||
const tasks = yield ECS.describeTasks({
|
|
||||||
cluster: clusterName,
|
|
||||||
tasks: [((_w = task.tasks) === null || _w === void 0 ? void 0 : _w[0].taskArn) || ''],
|
|
||||||
}).promise();
|
|
||||||
return (_x = tasks.tasks) === null || _x === void 0 ? void 0 : _x[0].lastStatus;
|
|
||||||
});
|
|
||||||
const stream = yield kinesis
|
|
||||||
.describeStream({
|
|
||||||
StreamName: ((_r = (_q = taskDefResources.StackResources) === null || _q === void 0 ? void 0 : _q.find((x) => x.LogicalResourceId === 'KinesisStream')) === null || _r === void 0 ? void 0 : _r.PhysicalResourceId) ||
|
|
||||||
'',
|
|
||||||
})
|
|
||||||
.promise();
|
|
||||||
let iterator = (yield kinesis
|
|
||||||
.getShardIterator({
|
|
||||||
ShardIteratorType: 'TRIM_HORIZON',
|
|
||||||
StreamName: stream.StreamDescription.StreamName,
|
|
||||||
ShardId: stream.StreamDescription.Shards[0].ShardId,
|
|
||||||
})
|
|
||||||
.promise()).ShardIterator || '';
|
|
||||||
yield CF.waitFor('stackCreateComplete', { StackName: taskDef.taskDefStackNameTTL }).promise();
|
|
||||||
core.info(`Task status is ${yield getTaskStatus()}`);
|
|
||||||
const logBaseUrl = `https://console.aws.amazon.com/cloudwatch/home?region=${SDK.config.region}#logsV2:log-groups/${taskDef.taskDefStackName}`;
|
|
||||||
core.info(`You can also watch the logs at AWS Cloud Watch: ${logBaseUrl}`);
|
|
||||||
let readingLogs = true;
|
|
||||||
while (readingLogs) {
|
|
||||||
yield new Promise((resolve) => setTimeout(resolve, 1500));
|
|
||||||
if ((yield getTaskStatus()) !== 'RUNNING') {
|
|
||||||
readingLogs = false;
|
|
||||||
yield new Promise((resolve) => setTimeout(resolve, 35000));
|
|
||||||
}
|
|
||||||
const records = yield kinesis
|
|
||||||
.getRecords({
|
|
||||||
ShardIterator: iterator,
|
|
||||||
})
|
|
||||||
.promise();
|
|
||||||
iterator = records.NextShardIterator || '';
|
|
||||||
if (records.Records.length > 0) {
|
|
||||||
for (let index = 0; index < records.Records.length; index++) {
|
|
||||||
const json = JSON.parse(zlib.gunzipSync(Buffer.from(records.Records[index].Data, 'base64')).toString('utf8'));
|
|
||||||
if (json.messageType === 'DATA_MESSAGE') {
|
|
||||||
for (let logEventsIndex = 0; logEventsIndex < json.logEvents.length; logEventsIndex++) {
|
|
||||||
core.info(json.logEvents[logEventsIndex].message);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
yield ECS.waitFor('tasksStopped', { cluster: clusterName, tasks: [((_s = task.tasks) === null || _s === void 0 ? void 0 : _s[0].taskArn) || ''] }).promise();
|
|
||||||
const exitCode = (_v = (_u = (yield ECS.describeTasks({
|
|
||||||
tasks: [((_t = task.tasks) === null || _t === void 0 ? void 0 : _t[0].taskArn) || ''],
|
|
||||||
cluster: clusterName,
|
|
||||||
}).promise()).tasks) === null || _u === void 0 ? void 0 : _u[0].containers) === null || _v === void 0 ? void 0 : _v[0].exitCode;
|
|
||||||
if (exitCode !== 0) {
|
|
||||||
core.error(`job finished with exit code ${exitCode}`);
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
core.info(`Build job has finished with exit code 0`);
|
|
||||||
}
|
|
||||||
yield CF.deleteStack({
|
|
||||||
StackName: taskDef.taskDefStackName,
|
|
||||||
}).promise();
|
|
||||||
yield CF.deleteStack({
|
|
||||||
StackName: taskDef.taskDefStackNameTTL,
|
|
||||||
}).promise();
|
|
||||||
yield CF.waitFor('stackDeleteComplete', {
|
|
||||||
StackName: taskDef.taskDefStackName,
|
|
||||||
}).promise();
|
|
||||||
// Currently too slow and causes too much waiting
|
|
||||||
yield CF.waitFor('stackDeleteComplete', {
|
|
||||||
StackName: taskDef.taskDefStackNameTTL,
|
|
||||||
}).promise();
|
|
||||||
core.info('Cleanup complete');
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
static setupClusters(ECS, CF, buildUid, stackName, image, entrypoint, commands, mountdir, workingdir, environment, secrets) {
|
static setupCloudFormations(CF, buildUid, stackName, image, entrypoint, commands, mountdir, workingdir, secrets) {
|
||||||
return __awaiter(this, void 0, void 0, function* () {
|
return __awaiter(this, void 0, void 0, function* () {
|
||||||
const taskDefStackName = `${stackName}-taskDef-${image}-${buildUid}`.toString().replace(/[^\da-z]/gi, '');
|
const taskDefStackName = `${stackName}-taskDef-${image}-${buildUid}`.toString().replace(/[^\da-z]/gi, '');
|
||||||
const taskDefCloudFormation = fs.readFileSync(`${__dirname}/task-def-formation.yml`, 'utf8');
|
const taskDefCloudFormation = fs.readFileSync(`${__dirname}/task-def-formation.yml`, 'utf8');
|
||||||
|
|
@ -609,7 +495,151 @@ class AWS {
|
||||||
catch (error) {
|
catch (error) {
|
||||||
core.error(error);
|
core.error(error);
|
||||||
}
|
}
|
||||||
return { taskDefStackName, taskDefCloudFormation, taskDefStackNameTTL, ttlCloudFormation };
|
const taskDefResources = yield CF.describeStackResources({
|
||||||
|
StackName: taskDefStackName,
|
||||||
|
}).promise();
|
||||||
|
const baseResources = yield CF.describeStackResources({ StackName: stackName }).promise();
|
||||||
|
core.info('Build cluster created successfully (skipping waiting for cleanup cluster to start)');
|
||||||
|
return {
|
||||||
|
taskDefStackName,
|
||||||
|
taskDefCloudFormation,
|
||||||
|
taskDefStackNameTTL,
|
||||||
|
ttlCloudFormation,
|
||||||
|
taskDefResources,
|
||||||
|
baseResources,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
|
static runTask(taskDef, ECS, CF, environment, buildUid) {
|
||||||
|
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s;
|
||||||
|
return __awaiter(this, void 0, void 0, function* () {
|
||||||
|
const clusterName = ((_b = (_a = taskDef.baseResources.StackResources) === null || _a === void 0 ? void 0 : _a.find((x) => x.LogicalResourceId === 'ECSCluster')) === null || _b === void 0 ? void 0 : _b.PhysicalResourceId) || '';
|
||||||
|
const taskDefinition = ((_d = (_c = taskDef.taskDefResources.StackResources) === null || _c === void 0 ? void 0 : _c.find((x) => x.LogicalResourceId === 'TaskDefinition')) === null || _d === void 0 ? void 0 : _d.PhysicalResourceId) || '';
|
||||||
|
const SubnetOne = ((_f = (_e = taskDef.baseResources.StackResources) === null || _e === void 0 ? void 0 : _e.find((x) => x.LogicalResourceId === 'PublicSubnetOne')) === null || _f === void 0 ? void 0 : _f.PhysicalResourceId) || '';
|
||||||
|
const SubnetTwo = ((_h = (_g = taskDef.baseResources.StackResources) === null || _g === void 0 ? void 0 : _g.find((x) => x.LogicalResourceId === 'PublicSubnetTwo')) === null || _h === void 0 ? void 0 : _h.PhysicalResourceId) || '';
|
||||||
|
const ContainerSecurityGroup = ((_k = (_j = taskDef.baseResources.StackResources) === null || _j === void 0 ? void 0 : _j.find((x) => x.LogicalResourceId === 'ContainerSecurityGroup')) === null || _k === void 0 ? void 0 : _k.PhysicalResourceId) || '';
|
||||||
|
const streamName = ((_m = (_l = taskDef.taskDefResources.StackResources) === null || _l === void 0 ? void 0 : _l.find((x) => x.LogicalResourceId === 'KinesisStream')) === null || _m === void 0 ? void 0 : _m.PhysicalResourceId) || '';
|
||||||
|
const task = yield ECS.runTask({
|
||||||
|
cluster: clusterName,
|
||||||
|
taskDefinition: taskDefinition,
|
||||||
|
platformVersion: '1.4.0',
|
||||||
|
overrides: {
|
||||||
|
containerOverrides: [
|
||||||
|
{
|
||||||
|
name: taskDef.taskDefStackName,
|
||||||
|
environment: [...environment, { name: 'BUILDID', value: buildUid }],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
launchType: 'FARGATE',
|
||||||
|
networkConfiguration: {
|
||||||
|
awsvpcConfiguration: {
|
||||||
|
subnets: [SubnetOne, SubnetTwo],
|
||||||
|
assignPublicIp: 'ENABLED',
|
||||||
|
securityGroups: [ContainerSecurityGroup],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}).promise();
|
||||||
|
core.info('Build job is starting');
|
||||||
|
const taskArn = ((_o = task.tasks) === null || _o === void 0 ? void 0 : _o[0].taskArn) || '';
|
||||||
|
try {
|
||||||
|
yield ECS.waitFor('tasksRunning', { tasks: [taskArn], cluster: clusterName }).promise();
|
||||||
|
}
|
||||||
|
catch (error) {
|
||||||
|
yield new Promise((resolve) => setTimeout(resolve, 3000));
|
||||||
|
const describeTasks = yield ECS.describeTasks({
|
||||||
|
tasks: [taskArn],
|
||||||
|
cluster: clusterName,
|
||||||
|
}).promise();
|
||||||
|
core.info(`Build job has ended ${(_q = (_p = describeTasks.tasks) === null || _p === void 0 ? void 0 : _p[0].containers) === null || _q === void 0 ? void 0 : _q[0].lastStatus}`);
|
||||||
|
core.setFailed(error);
|
||||||
|
core.error(error);
|
||||||
|
}
|
||||||
|
core.info(`Build job is running`);
|
||||||
|
yield this.streamLogsUntilTaskStops(ECS, CF, taskDef, clusterName, taskArn, streamName);
|
||||||
|
yield ECS.waitFor('tasksStopped', { cluster: clusterName, tasks: [taskArn] }).promise();
|
||||||
|
const exitCode = (_s = (_r = (yield ECS.describeTasks({
|
||||||
|
tasks: [taskArn],
|
||||||
|
cluster: clusterName,
|
||||||
|
}).promise()).tasks) === null || _r === void 0 ? void 0 : _r[0].containers) === null || _s === void 0 ? void 0 : _s[0].exitCode;
|
||||||
|
if (exitCode !== 0) {
|
||||||
|
core.error(`job finished with exit code ${exitCode}`);
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
core.info(`Build job has finished with exit code 0`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
static streamLogsUntilTaskStops(ECS, CF, taskDef, clusterName, taskArn, kinesisStreamName) {
|
||||||
|
return __awaiter(this, void 0, void 0, function* () {
|
||||||
|
// watching logs
|
||||||
|
const kinesis = new SDK.Kinesis();
|
||||||
|
const getTaskStatus = () => __awaiter(this, void 0, void 0, function* () {
|
||||||
|
var _a;
|
||||||
|
const tasks = yield ECS.describeTasks({
|
||||||
|
cluster: clusterName,
|
||||||
|
tasks: [taskArn],
|
||||||
|
}).promise();
|
||||||
|
return (_a = tasks.tasks) === null || _a === void 0 ? void 0 : _a[0].lastStatus;
|
||||||
|
});
|
||||||
|
const stream = yield kinesis
|
||||||
|
.describeStream({
|
||||||
|
StreamName: kinesisStreamName,
|
||||||
|
})
|
||||||
|
.promise();
|
||||||
|
let iterator = (yield kinesis
|
||||||
|
.getShardIterator({
|
||||||
|
ShardIteratorType: 'TRIM_HORIZON',
|
||||||
|
StreamName: stream.StreamDescription.StreamName,
|
||||||
|
ShardId: stream.StreamDescription.Shards[0].ShardId,
|
||||||
|
})
|
||||||
|
.promise()).ShardIterator || '';
|
||||||
|
yield CF.waitFor('stackCreateComplete', { StackName: taskDef.taskDefStackNameTTL }).promise();
|
||||||
|
core.info(`Task status is ${yield getTaskStatus()}`);
|
||||||
|
const logBaseUrl = `https://console.aws.amazon.com/cloudwatch/home?region=${SDK.config.region}#logsV2:log-groups/${taskDef.taskDefStackName}`;
|
||||||
|
core.info(`You can also watch the logs at AWS Cloud Watch: ${logBaseUrl}`);
|
||||||
|
let readingLogs = true;
|
||||||
|
while (readingLogs) {
|
||||||
|
yield new Promise((resolve) => setTimeout(resolve, 1500));
|
||||||
|
if ((yield getTaskStatus()) !== 'RUNNING') {
|
||||||
|
readingLogs = false;
|
||||||
|
yield new Promise((resolve) => setTimeout(resolve, 35000));
|
||||||
|
}
|
||||||
|
const records = yield kinesis
|
||||||
|
.getRecords({
|
||||||
|
ShardIterator: iterator,
|
||||||
|
})
|
||||||
|
.promise();
|
||||||
|
iterator = records.NextShardIterator || '';
|
||||||
|
if (records.Records.length > 0) {
|
||||||
|
for (let index = 0; index < records.Records.length; index++) {
|
||||||
|
const json = JSON.parse(zlib.gunzipSync(Buffer.from(records.Records[index].Data, 'base64')).toString('utf8'));
|
||||||
|
if (json.messageType === 'DATA_MESSAGE') {
|
||||||
|
for (let logEventsIndex = 0; logEventsIndex < json.logEvents.length; logEventsIndex++) {
|
||||||
|
core.info(json.logEvents[logEventsIndex].message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
static cleanupResources(CF, taskDef) {
|
||||||
|
return __awaiter(this, void 0, void 0, function* () {
|
||||||
|
yield CF.deleteStack({
|
||||||
|
StackName: taskDef.taskDefStackName,
|
||||||
|
}).promise();
|
||||||
|
yield CF.deleteStack({
|
||||||
|
StackName: taskDef.taskDefStackNameTTL,
|
||||||
|
}).promise();
|
||||||
|
yield CF.waitFor('stackDeleteComplete', {
|
||||||
|
StackName: taskDef.taskDefStackName,
|
||||||
|
}).promise();
|
||||||
|
// Currently too slow and causes too much waiting
|
||||||
|
yield CF.waitFor('stackDeleteComplete', {
|
||||||
|
StackName: taskDef.taskDefStackNameTTL,
|
||||||
|
}).promise();
|
||||||
|
core.info('Cleanup complete');
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
static onlog(batch) {
|
static onlog(batch) {
|
||||||
|
|
|
||||||
File diff suppressed because one or more lines are too long
363
src/model/aws.ts
363
src/model/aws.ts
|
|
@ -22,12 +22,23 @@ class AWS {
|
||||||
ls;
|
ls;
|
||||||
git clone https://${process.env.GITHUB_TOKEN}@github.com/${process.env.GITHUB_REPOSITORY}.git ${buildUid}/repo;
|
git clone https://${process.env.GITHUB_TOKEN}@github.com/${process.env.GITHUB_REPOSITORY}.git ${buildUid}/repo;
|
||||||
git clone https://${process.env.GITHUB_TOKEN}@github.com/game-ci/unity-builder.git ${buildUid}/builder;
|
git clone https://${process.env.GITHUB_TOKEN}@github.com/game-ci/unity-builder.git ${buildUid}/builder;
|
||||||
if [ -f "./${process.env.GITHUB_REF}/lib.zip" ]; then
|
|
||||||
|
if [ ! -d "cache" ]; then
|
||||||
|
mkdir "cache"
|
||||||
|
fi
|
||||||
|
|
||||||
|
cd cache
|
||||||
|
|
||||||
|
latest=$(ls -t | head -1)
|
||||||
|
if [ -f $latest ]; then
|
||||||
echo "Cache exists"
|
echo "Cache exists"
|
||||||
zip -r ./${process.env.GITHUB_REF}/lib.zip ./${buildUid}/repo/Library/.
|
zip -r ./cache/lib.zip ./${buildUid}/repo/Library/.
|
||||||
else
|
else
|
||||||
echo "Cache does not exist"
|
echo "Cache does not exist"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
cd ..
|
||||||
|
|
||||||
cd ${buildUid}/repo;
|
cd ${buildUid}/repo;
|
||||||
git checkout $GITHUB_SHA;
|
git checkout $GITHUB_SHA;
|
||||||
`,
|
`,
|
||||||
|
|
@ -239,168 +250,24 @@ class AWS {
|
||||||
const ECS = new SDK.ECS();
|
const ECS = new SDK.ECS();
|
||||||
const CF = new SDK.CloudFormation();
|
const CF = new SDK.CloudFormation();
|
||||||
|
|
||||||
const taskDef = await this.setupClusters(ECS, CF, buildUid, stackName, image, entrypoint, commands, mountdir, workingdir, environment, secrets);
|
const taskDef = await this.setupCloudFormations(
|
||||||
|
CF,
|
||||||
core.info('Build cluster created successfully (skipping waiting for cleanup cluster to start)');
|
buildUid,
|
||||||
|
stackName,
|
||||||
const taskDefResources = await CF.describeStackResources({
|
image,
|
||||||
StackName: taskDef.taskDefStackName,
|
entrypoint,
|
||||||
}).promise();
|
commands,
|
||||||
|
mountdir,
|
||||||
const baseResources = await CF.describeStackResources({ StackName: stackName }).promise();
|
workingdir,
|
||||||
|
secrets,
|
||||||
const clusterName =
|
|
||||||
baseResources.StackResources?.find((x) => x.LogicalResourceId === 'ECSCluster')?.PhysicalResourceId || '';
|
|
||||||
const task = await ECS.runTask({
|
|
||||||
cluster: clusterName,
|
|
||||||
taskDefinition:
|
|
||||||
taskDefResources.StackResources?.find((x) => x.LogicalResourceId === 'TaskDefinition')?.PhysicalResourceId ||
|
|
||||||
'',
|
|
||||||
platformVersion: '1.4.0',
|
|
||||||
overrides: {
|
|
||||||
containerOverrides: [
|
|
||||||
{
|
|
||||||
name: taskDef.taskDefStackName,
|
|
||||||
environment: [...environment, { name: 'BUILDID', value: buildUid }],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
launchType: 'FARGATE',
|
|
||||||
networkConfiguration: {
|
|
||||||
awsvpcConfiguration: {
|
|
||||||
subnets: [
|
|
||||||
baseResources.StackResources?.find((x) => x.LogicalResourceId === 'PublicSubnetOne')?.PhysicalResourceId ||
|
|
||||||
'',
|
|
||||||
baseResources.StackResources?.find((x) => x.LogicalResourceId === 'PublicSubnetTwo')?.PhysicalResourceId ||
|
|
||||||
'',
|
|
||||||
],
|
|
||||||
assignPublicIp: 'ENABLED',
|
|
||||||
securityGroups: [
|
|
||||||
baseResources.StackResources?.find((x) => x.LogicalResourceId === 'ContainerSecurityGroup')
|
|
||||||
?.PhysicalResourceId || '',
|
|
||||||
],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}).promise();
|
|
||||||
|
|
||||||
core.info('Build job is starting');
|
|
||||||
|
|
||||||
try {
|
|
||||||
await ECS.waitFor('tasksRunning', { tasks: [task.tasks?.[0].taskArn || ''], cluster: clusterName }).promise();
|
|
||||||
} catch (error) {
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
|
||||||
const describeTasks = await ECS.describeTasks({
|
|
||||||
tasks: [task.tasks?.[0].taskArn || ''],
|
|
||||||
cluster: clusterName,
|
|
||||||
}).promise();
|
|
||||||
core.info(`Build job has ended ${describeTasks.tasks?.[0].containers?.[0].lastStatus}`);
|
|
||||||
core.setFailed(error);
|
|
||||||
core.error(error);
|
|
||||||
}
|
|
||||||
|
|
||||||
core.info(`Build job is running`);
|
|
||||||
|
|
||||||
// watching logs
|
|
||||||
const kinesis = new SDK.Kinesis();
|
|
||||||
|
|
||||||
const getTaskStatus = async () => {
|
|
||||||
const tasks = await ECS.describeTasks({
|
|
||||||
cluster: clusterName,
|
|
||||||
tasks: [task.tasks?.[0].taskArn || ''],
|
|
||||||
}).promise();
|
|
||||||
return tasks.tasks?.[0].lastStatus;
|
|
||||||
};
|
|
||||||
|
|
||||||
const stream = await kinesis
|
|
||||||
.describeStream({
|
|
||||||
StreamName:
|
|
||||||
taskDefResources.StackResources?.find((x) => x.LogicalResourceId === 'KinesisStream')?.PhysicalResourceId ||
|
|
||||||
'',
|
|
||||||
})
|
|
||||||
.promise();
|
|
||||||
|
|
||||||
let iterator =
|
|
||||||
(
|
|
||||||
await kinesis
|
|
||||||
.getShardIterator({
|
|
||||||
ShardIteratorType: 'TRIM_HORIZON',
|
|
||||||
StreamName: stream.StreamDescription.StreamName,
|
|
||||||
ShardId: stream.StreamDescription.Shards[0].ShardId,
|
|
||||||
})
|
|
||||||
.promise()
|
|
||||||
).ShardIterator || '';
|
|
||||||
|
|
||||||
await CF.waitFor('stackCreateComplete', { StackName: taskDef.taskDefStackNameTTL }).promise();
|
|
||||||
|
|
||||||
core.info(`Task status is ${await getTaskStatus()}`);
|
|
||||||
|
|
||||||
const logBaseUrl = `https://console.aws.amazon.com/cloudwatch/home?region=${SDK.config.region}#logsV2:log-groups/${taskDef.taskDefStackName}`;
|
|
||||||
core.info(`You can also watch the logs at AWS Cloud Watch: ${logBaseUrl}`);
|
|
||||||
|
|
||||||
let readingLogs = true;
|
|
||||||
while (readingLogs) {
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, 1500));
|
|
||||||
if ((await getTaskStatus()) !== 'RUNNING') {
|
|
||||||
readingLogs = false;
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, 35000));
|
|
||||||
}
|
|
||||||
const records = await kinesis
|
|
||||||
.getRecords({
|
|
||||||
ShardIterator: iterator,
|
|
||||||
})
|
|
||||||
.promise();
|
|
||||||
iterator = records.NextShardIterator || '';
|
|
||||||
if (records.Records.length > 0) {
|
|
||||||
for (let index = 0; index < records.Records.length; index++) {
|
|
||||||
const json = JSON.parse(
|
|
||||||
zlib.gunzipSync(Buffer.from(records.Records[index].Data as string, 'base64')).toString('utf8'),
|
|
||||||
);
|
);
|
||||||
if (json.messageType === 'DATA_MESSAGE') {
|
|
||||||
for (let logEventsIndex = 0; logEventsIndex < json.logEvents.length; logEventsIndex++) {
|
await this.runTask(taskDef, ECS, CF, environment, buildUid);
|
||||||
core.info(json.logEvents[logEventsIndex].message);
|
|
||||||
}
|
await this.cleanupResources(CF, taskDef);
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
await ECS.waitFor('tasksStopped', { cluster: clusterName, tasks: [task.tasks?.[0].taskArn || ''] }).promise();
|
static async setupCloudFormations(
|
||||||
|
|
||||||
const exitCode = (
|
|
||||||
await ECS.describeTasks({
|
|
||||||
tasks: [task.tasks?.[0].taskArn || ''],
|
|
||||||
cluster: clusterName,
|
|
||||||
}).promise()
|
|
||||||
).tasks?.[0].containers?.[0].exitCode;
|
|
||||||
|
|
||||||
if (exitCode !== 0) {
|
|
||||||
core.error(`job finished with exit code ${exitCode}`);
|
|
||||||
} else {
|
|
||||||
core.info(`Build job has finished with exit code 0`);
|
|
||||||
}
|
|
||||||
|
|
||||||
await CF.deleteStack({
|
|
||||||
StackName: taskDef.taskDefStackName,
|
|
||||||
}).promise();
|
|
||||||
|
|
||||||
await CF.deleteStack({
|
|
||||||
StackName: taskDef.taskDefStackNameTTL,
|
|
||||||
}).promise();
|
|
||||||
|
|
||||||
await CF.waitFor('stackDeleteComplete', {
|
|
||||||
StackName: taskDef.taskDefStackName,
|
|
||||||
}).promise();
|
|
||||||
|
|
||||||
// Currently too slow and causes too much waiting
|
|
||||||
await CF.waitFor('stackDeleteComplete', {
|
|
||||||
StackName: taskDef.taskDefStackNameTTL,
|
|
||||||
}).promise();
|
|
||||||
|
|
||||||
core.info('Cleanup complete');
|
|
||||||
}
|
|
||||||
|
|
||||||
static async setupClusters(
|
|
||||||
ECS,
|
|
||||||
CF,
|
CF,
|
||||||
buildUid: string,
|
buildUid: string,
|
||||||
stackName: string,
|
stackName: string,
|
||||||
|
|
@ -409,9 +276,8 @@ class AWS {
|
||||||
commands,
|
commands,
|
||||||
mountdir,
|
mountdir,
|
||||||
workingdir,
|
workingdir,
|
||||||
environment,
|
secrets,
|
||||||
secrets) {
|
) {
|
||||||
|
|
||||||
const taskDefStackName = `${stackName}-taskDef-${image}-${buildUid}`.toString().replace(/[^\da-z]/gi, '');
|
const taskDefStackName = `${stackName}-taskDef-${image}-${buildUid}`.toString().replace(/[^\da-z]/gi, '');
|
||||||
const taskDefCloudFormation = fs.readFileSync(`${__dirname}/task-def-formation.yml`, 'utf8');
|
const taskDefCloudFormation = fs.readFileSync(`${__dirname}/task-def-formation.yml`, 'utf8');
|
||||||
await CF.createStack({
|
await CF.createStack({
|
||||||
|
|
@ -483,8 +349,175 @@ class AWS {
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
core.error(error);
|
core.error(error);
|
||||||
}
|
}
|
||||||
|
const taskDefResources = await CF.describeStackResources({
|
||||||
|
StackName: taskDefStackName,
|
||||||
|
}).promise();
|
||||||
|
|
||||||
return { taskDefStackName, taskDefCloudFormation, taskDefStackNameTTL, ttlCloudFormation };
|
const baseResources = await CF.describeStackResources({ StackName: stackName }).promise();
|
||||||
|
|
||||||
|
core.info('Build cluster created successfully (skipping waiting for cleanup cluster to start)');
|
||||||
|
|
||||||
|
return {
|
||||||
|
taskDefStackName,
|
||||||
|
taskDefCloudFormation,
|
||||||
|
taskDefStackNameTTL,
|
||||||
|
ttlCloudFormation,
|
||||||
|
taskDefResources,
|
||||||
|
baseResources,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
static async runTask(taskDef, ECS, CF, environment, buildUid) {
|
||||||
|
const clusterName =
|
||||||
|
taskDef.baseResources.StackResources?.find((x) => x.LogicalResourceId === 'ECSCluster')?.PhysicalResourceId || '';
|
||||||
|
const taskDefinition = taskDef.taskDefResources.StackResources?.find((x) => x.LogicalResourceId === 'TaskDefinition')
|
||||||
|
?.PhysicalResourceId || '';
|
||||||
|
const SubnetOne = taskDef.baseResources.StackResources?.find((x) => x.LogicalResourceId === 'PublicSubnetOne')
|
||||||
|
?.PhysicalResourceId || '';
|
||||||
|
const SubnetTwo = taskDef.baseResources.StackResources?.find((x) => x.LogicalResourceId === 'PublicSubnetTwo')
|
||||||
|
?.PhysicalResourceId || '';
|
||||||
|
const ContainerSecurityGroup = taskDef.baseResources.StackResources?.find((x) => x.LogicalResourceId === 'ContainerSecurityGroup')
|
||||||
|
?.PhysicalResourceId || '';
|
||||||
|
const streamName =
|
||||||
|
taskDef.taskDefResources.StackResources?.find((x) => x.LogicalResourceId === 'KinesisStream')
|
||||||
|
?.PhysicalResourceId || '';
|
||||||
|
|
||||||
|
const task = await ECS.runTask({
|
||||||
|
cluster: clusterName,
|
||||||
|
taskDefinition: taskDefinition,
|
||||||
|
platformVersion: '1.4.0',
|
||||||
|
overrides: {
|
||||||
|
containerOverrides: [
|
||||||
|
{
|
||||||
|
name: taskDef.taskDefStackName,
|
||||||
|
environment: [...environment, { name: 'BUILDID', value: buildUid }],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
launchType: 'FARGATE',
|
||||||
|
networkConfiguration: {
|
||||||
|
awsvpcConfiguration: {
|
||||||
|
subnets: [ SubnetOne, SubnetTwo ],
|
||||||
|
assignPublicIp: 'ENABLED',
|
||||||
|
securityGroups: [ ContainerSecurityGroup ],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}).promise();
|
||||||
|
|
||||||
|
core.info('Build job is starting');
|
||||||
|
const taskArn = task.tasks?.[0].taskArn || '';
|
||||||
|
|
||||||
|
try {
|
||||||
|
await ECS.waitFor('tasksRunning', { tasks: [taskArn], cluster: clusterName }).promise();
|
||||||
|
} catch (error) {
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||||
|
const describeTasks = await ECS.describeTasks({
|
||||||
|
tasks: [taskArn],
|
||||||
|
cluster: clusterName,
|
||||||
|
}).promise();
|
||||||
|
core.info(`Build job has ended ${describeTasks.tasks?.[0].containers?.[0].lastStatus}`);
|
||||||
|
core.setFailed(error);
|
||||||
|
core.error(error);
|
||||||
|
}
|
||||||
|
core.info(`Build job is running`);
|
||||||
|
await this.streamLogsUntilTaskStops(ECS, CF, taskDef, clusterName, taskArn, streamName);
|
||||||
|
await ECS.waitFor('tasksStopped', { cluster: clusterName, tasks: [taskArn] }).promise();
|
||||||
|
const exitCode = (
|
||||||
|
await ECS.describeTasks({
|
||||||
|
tasks: [taskArn],
|
||||||
|
cluster: clusterName,
|
||||||
|
}).promise()
|
||||||
|
).tasks?.[0].containers?.[0].exitCode;
|
||||||
|
if (exitCode !== 0) {
|
||||||
|
core.error(`job finished with exit code ${exitCode}`);
|
||||||
|
} else {
|
||||||
|
core.info(`Build job has finished with exit code 0`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static async streamLogsUntilTaskStops(ECS, CF, taskDef, clusterName, taskArn, kinesisStreamName) {
|
||||||
|
// watching logs
|
||||||
|
const kinesis = new SDK.Kinesis();
|
||||||
|
|
||||||
|
const getTaskStatus = async () => {
|
||||||
|
const tasks = await ECS.describeTasks({
|
||||||
|
cluster: clusterName,
|
||||||
|
tasks: [taskArn],
|
||||||
|
}).promise();
|
||||||
|
return tasks.tasks?.[0].lastStatus;
|
||||||
|
};
|
||||||
|
|
||||||
|
const stream = await kinesis
|
||||||
|
.describeStream({
|
||||||
|
StreamName: kinesisStreamName,
|
||||||
|
})
|
||||||
|
.promise();
|
||||||
|
|
||||||
|
let iterator =
|
||||||
|
(
|
||||||
|
await kinesis
|
||||||
|
.getShardIterator({
|
||||||
|
ShardIteratorType: 'TRIM_HORIZON',
|
||||||
|
StreamName: stream.StreamDescription.StreamName,
|
||||||
|
ShardId: stream.StreamDescription.Shards[0].ShardId,
|
||||||
|
})
|
||||||
|
.promise()
|
||||||
|
).ShardIterator || '';
|
||||||
|
|
||||||
|
await CF.waitFor('stackCreateComplete', { StackName: taskDef.taskDefStackNameTTL }).promise();
|
||||||
|
|
||||||
|
core.info(`Task status is ${await getTaskStatus()}`);
|
||||||
|
|
||||||
|
const logBaseUrl = `https://console.aws.amazon.com/cloudwatch/home?region=${SDK.config.region}#logsV2:log-groups/${taskDef.taskDefStackName}`;
|
||||||
|
core.info(`You can also watch the logs at AWS Cloud Watch: ${logBaseUrl}`);
|
||||||
|
|
||||||
|
let readingLogs = true;
|
||||||
|
while (readingLogs) {
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 1500));
|
||||||
|
if ((await getTaskStatus()) !== 'RUNNING') {
|
||||||
|
readingLogs = false;
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 35000));
|
||||||
|
}
|
||||||
|
const records = await kinesis
|
||||||
|
.getRecords({
|
||||||
|
ShardIterator: iterator,
|
||||||
|
})
|
||||||
|
.promise();
|
||||||
|
iterator = records.NextShardIterator || '';
|
||||||
|
if (records.Records.length > 0) {
|
||||||
|
for (let index = 0; index < records.Records.length; index++) {
|
||||||
|
const json = JSON.parse(
|
||||||
|
zlib.gunzipSync(Buffer.from(records.Records[index].Data as string, 'base64')).toString('utf8'),
|
||||||
|
);
|
||||||
|
if (json.messageType === 'DATA_MESSAGE') {
|
||||||
|
for (let logEventsIndex = 0; logEventsIndex < json.logEvents.length; logEventsIndex++) {
|
||||||
|
core.info(json.logEvents[logEventsIndex].message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static async cleanupResources(CF, taskDef) {
|
||||||
|
await CF.deleteStack({
|
||||||
|
StackName: taskDef.taskDefStackName,
|
||||||
|
}).promise();
|
||||||
|
|
||||||
|
await CF.deleteStack({
|
||||||
|
StackName: taskDef.taskDefStackNameTTL,
|
||||||
|
}).promise();
|
||||||
|
|
||||||
|
await CF.waitFor('stackDeleteComplete', {
|
||||||
|
StackName: taskDef.taskDefStackName,
|
||||||
|
}).promise();
|
||||||
|
|
||||||
|
// Currently too slow and causes too much waiting
|
||||||
|
await CF.waitFor('stackDeleteComplete', {
|
||||||
|
StackName: taskDef.taskDefStackNameTTL,
|
||||||
|
}).promise();
|
||||||
|
|
||||||
|
core.info('Cleanup complete');
|
||||||
}
|
}
|
||||||
|
|
||||||
static onlog(batch) {
|
static onlog(batch) {
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue