Remove shell scripts in favour of internal cli calls to remote-cli

pull/310/head
Frostebite 2021-12-24 01:13:33 +00:00
parent c21ae5d424
commit 639bab8d90
5 changed files with 177 additions and 71 deletions

View File

@ -0,0 +1,90 @@
[AWS][game-ci-3-test-0-windows64-vbld] androidKeystorePass=
[AWS][game-ci-3-test-0-windows64-vbld] androidKeystoreBase64=
[AWS][game-ci-3-test-0-windows64-vbld] ANDROID_VERSION_CODE=1
[AWS][game-ci-3-test-0-windows64-vbld] kubeConfig=
[AWS][game-ci-3-test-0-windows64-vbld] testCustomSecret=VALUEXXX
[AWS][game-ci-3-test-0-windows64-vbld] HOSTNAME=ip-10-0-0-112.eu-west-2.compute.internal
[AWS][game-ci-3-test-0-windows64-vbld] customImage=false
[AWS][game-ci-3-test-0-windows64-vbld] BUILDID=0-windows64-vbld
[AWS][game-ci-3-test-0-windows64-vbld] androidKeyaliasPass=
[AWS][game-ci-3-test-0-windows64-vbld] SHLVL=1
[AWS][game-ci-3-test-0-windows64-vbld] branch=remote-builder/unified-providers
[AWS][game-ci-3-test-0-windows64-vbld] githubRepo=game-ci/unity-builder
[AWS][game-ci-3-test-0-windows64-vbld] HOME=/root
[AWS][game-ci-3-test-0-windows64-vbld] cloudRunnerCpu=1.0
[AWS][game-ci-3-test-0-windows64-vbld] platform=StandaloneWindows64
[AWS][game-ci-3-test-0-windows64-vbld] versioningStrategy=None
[AWS][game-ci-3-test-0-windows64-vbld] version=2019.2.11f1
[AWS][game-ci-3-test-0-windows64-vbld] githubEnabled=false
[AWS][game-ci-3-test-0-windows64-vbld] ANDROID_KEYSTORE_NAME=
[AWS][game-ci-3-test-0-windows64-vbld] buildPathFull=/data/0-windows64-vbld
[AWS][game-ci-3-test-0-windows64-vbld] AWS_EXECUTION_ENV=AWS_ECS_FARGATE
[AWS][game-ci-3-test-0-windows64-vbld] region=eu-west-2
[AWS][game-ci-3-test-0-windows64-vbld] specifiedVersion=
[AWS][game-ci-3-test-0-windows64-vbld] runNumber=0
[AWS][game-ci-3-test-0-windows64-vbld] BUILD_FILE=StandaloneWindows64.exe
[AWS][game-ci-3-test-0-windows64-vbld] chownFilesTo=
[AWS][game-ci-3-test-0-windows64-vbld] buildMethod=
[AWS][game-ci-3-test-0-windows64-vbld] ANDROID_KEYALIAS_NAME=
[AWS][game-ci-3-test-0-windows64-vbld] BUILD_NAME=StandaloneWindows64
[AWS][game-ci-3-test-0-windows64-vbld] VERSION=none
[AWS][game-ci-3-test-0-windows64-vbld] AWS_DEFAULT_REGION=eu-west-2
[AWS][game-ci-3-test-0-windows64-vbld] logToFile=true
[AWS][game-ci-3-test-0-windows64-vbld] runnerTempPath=undefined
[AWS][game-ci-3-test-0-windows64-vbld] sshAgent=
[AWS][game-ci-3-test-0-windows64-vbld] androidVersionCode=false
[AWS][game-ci-3-test-0-windows64-vbld] ECS_CONTAINER_METADATA_URI_V4=http://169.254.170.2/v4/64f6d6f4e91042739407585a1ecf6bc5-3072300965
[AWS][game-ci-3-test-0-windows64-vbld] androidKeystoreName=
[AWS][game-ci-3-test-0-windows64-vbld] BUILD_METHOD=
[AWS][game-ci-3-test-0-windows64-vbld] steamPathFull=/data/0-windows64-vbld/steam
[AWS][game-ci-3-test-0-windows64-vbld] CUSTOM_PARAMETERS=
[AWS][game-ci-3-test-0-windows64-vbld] kubeVolumeSize=5Gi
[AWS][game-ci-3-test-0-windows64-vbld] ECS_CONTAINER_METADATA_URI=http://169.254.170.2/v3/64f6d6f4e91042739407585a1ecf6bc5-3072300965
[AWS][game-ci-3-test-0-windows64-vbld] customParameters=
[AWS][game-ci-3-test-0-windows64-vbld] cloudRunnerCluster=
[AWS][game-ci-3-test-0-windows64-vbld] repoPathFull=/data/0-windows64-vbld/repo
[AWS][game-ci-3-test-0-windows64-vbld] buildParameters={"version":"2019.2.11f1","customImage":false,"platform":"StandaloneWindows64","projectPath":"test-project","buildName":"StandaloneWindows64","buildPath":"build/StandaloneWindows64","buildFile":"StandaloneWindows64.exe","buildMethod":"","buildVersion":"none","androidVersionCode":1,"androidKeystoreName":"","androidKeystoreBase64":"","androidKeystorePass":"","androidKeyaliasName":"","androidKeyaliasPass":"","customParameters":"","sshAgent":"","chownFilesTo":"","cloudRunnerCluster":"","awsBaseStackName":"game-ci-3-test","kubeConfig":"","githubToken":"","cloudRunnerMemory":"750M","cloudRunnerCpu":"1.0","kubeVolumeSize":"5Gi","kubeVolume":"","postBuildSteps":"","preBuildSteps":"","customBuildSteps":"\n - name: 'step 1'\n image: 'alpine'\n commands: ['printenv']\n secrets:\n - name: 'testCustomSecret'\n value: 'VALUEXXX'\n ","runNumber":"0","branch":"remote-builder/unified-providers","githubRepo":"game-ci/unity-builder","logToFile":true}
[AWS][game-ci-3-test-0-windows64-vbld] PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
[AWS][game-ci-3-test-0-windows64-vbld] libraryFolderFull=/data/0-windows64-vbld/repo/test-project/Library
[AWS][game-ci-3-test-0-windows64-vbld] androidAppBundle=false
[AWS][game-ci-3-test-0-windows64-vbld] preBuildSteps=
[AWS][game-ci-3-test-0-windows64-vbld] androidKeyaliasName=
[AWS][game-ci-3-test-0-windows64-vbld] projectPath=test-project
[AWS][game-ci-3-test-0-windows64-vbld] buildFile=StandaloneWindows64.exe
[AWS][game-ci-3-test-0-windows64-vbld] ContainerMemory=750M
[AWS][game-ci-3-test-0-windows64-vbld] BUILD_TARGET=StandaloneWindows64
[AWS][game-ci-3-test-0-windows64-vbld] buildName=StandaloneWindows64
[AWS][game-ci-3-test-0-windows64-vbld] unityVersion=auto
[AWS][game-ci-3-test-0-windows64-vbld] targetPlatform=StandaloneWindows64
[AWS][game-ci-3-test-0-windows64-vbld] ALLOW_EMPTY_PASSWORD=yes
[AWS][game-ci-3-test-0-windows64-vbld] BUILD_PATH=build/StandaloneWindows64
[AWS][game-ci-3-test-0-windows64-vbld] PROJECT_PATH=test-project
[AWS][game-ci-3-test-0-windows64-vbld] AWS_REGION=eu-west-2
[AWS][game-ci-3-test-0-windows64-vbld] awsBaseStackName=game-ci-3-test
[AWS][game-ci-3-test-0-windows64-vbld] GITHUB_WORKSPACE=/github/workspace
[AWS][game-ci-3-test-0-windows64-vbld] length=0
[AWS][game-ci-3-test-0-windows64-vbld] builderPathFull=/data/0-windows64-vbld/builder
[AWS][game-ci-3-test-0-windows64-vbld] buildVersion=none
[AWS][game-ci-3-test-0-windows64-vbld] cliOptions=[object Object]
[AWS][game-ci-3-test-0-windows64-vbld] postBuildSteps=
[AWS][game-ci-3-test-0-windows64-vbld] cloudRunnerMemory=750M
[AWS][game-ci-3-test-0-windows64-vbld] name=Input
[AWS][game-ci-3-test-0-windows64-vbld] PWD=/data
[AWS][game-ci-3-test-0-windows64-vbld] buildsPath=build
[AWS][game-ci-3-test-0-windows64-vbld] allowDirtyBuild=false
[AWS][game-ci-3-test-0-windows64-vbld] UNITY_VERSION=2019.2.11f1
[AWS][game-ci-3-test-0-windows64-vbld] githubToken=
[AWS][game-ci-3-test-0-windows64-vbld] buildPath=build/StandaloneWindows64
[AWS][game-ci-3-test-0-windows64-vbld] projectPathFull=/data/0-windows64-vbld/repo/test-project
[AWS][game-ci-3-test-0-windows64-vbld] prototype=[object Object]
[AWS][game-ci-3-test-0-windows64-vbld] ContainerCpu=1.0
[AWS][game-ci-3-test-0-windows64-vbld] customBuildSteps=
[AWS][game-ci-3-test-0-windows64-vbld] - name: 'step 1'
[AWS][game-ci-3-test-0-windows64-vbld] image: 'alpine'
[AWS][game-ci-3-test-0-windows64-vbld] commands: ['printenv']
[AWS][game-ci-3-test-0-windows64-vbld] secrets:
[AWS][game-ci-3-test-0-windows64-vbld] - name: 'testCustomSecret'
[AWS][game-ci-3-test-0-windows64-vbld] value: 'VALUEXXX'
[AWS][game-ci-3-test-0-windows64-vbld]
[AWS][game-ci-3-test-0-windows64-vbld] kubeVolume=
[AWS][game-ci-3-test-0-windows64-vbld] remoteBuilderIntegrationTests=true

78
dist/index.js vendored
View File

@ -660,7 +660,6 @@ class AWSBuildRunner {
else if (message.includes('Rebuilding Library because the asset database could not be found!')) {
core.warning('LIBRARY NOT FOUND!');
}
//message = `[AWS][${taskDef.taskDefStackName}] ${message}`;
if (cloud_runner_state_1.CloudRunnerState.buildParams.logToFile) {
fs_1.default.appendFileSync(`${cloud_runner_state_1.CloudRunnerState.buildGuid}-outputfile.txt`, `${message}\r\n`);
}
@ -2048,77 +2047,86 @@ class DownloadRepository {
ls ${cloud_runner_state_1.CloudRunnerState.projectPathFull}
echo ' '
`);
const lfsCacheFolder = `${cloud_runner_state_1.CloudRunnerState.cacheFolderFull}/lfs`;
const libraryCacheFolder = `${cloud_runner_state_1.CloudRunnerState.cacheFolderFull}/lib`;
yield run_cli_1.RunCli.RunCli(`
tree ${cloud_runner_state_1.CloudRunnerState.builderPathFull}
echo 'Starting checks of cache for the Unity project Library and git LFS files'
cacheFolderFull=${cloud_runner_state_1.CloudRunnerState.cacheFolderFull}
libraryFolderFull=${cloud_runner_state_1.CloudRunnerState.libraryFolderFull}
gitLFSDestinationFolder=${cloud_runner_state_1.CloudRunnerState.lfsDirectory}
purgeCloudRunnerCache=${cloud_runner_state_1.CloudRunnerState.purgeRemoteCaching}
cacheFolderWithBranch="$cacheFolderFull"
lfsCacheFolder="$cacheFolderFull/lfs"
libraryCacheFolder="$cacheFolderFull/lib"
mkdir -p "$lfsCacheFolder"
mkdir -p "$libraryCacheFolder"
mkdir -p "${lfsCacheFolder}"
mkdir -p "${libraryCacheFolder}"
echo 'Library Caching'
`);
yield run_cli_1.RunCli.RunCli(`
# if the unity git project has included the library delete it and echo a warning
if [ -d "$libraryFolderFull" ]; then
rm -r "$libraryFolderFull"
if [ -d "${cloud_runner_state_1.CloudRunnerState.libraryFolderFull}" ]; then
rm -r "${cloud_runner_state_1.CloudRunnerState.libraryFolderFull}"
echo "!Warning!: The Unity library was included in the git repository (this isn't usually a good practice)"
fi
`);
yield run_cli_1.RunCli.RunCli(`
# Restore library cache
ls -lh "$libraryCacheFolder"
latestLibraryCacheFile=$(ls -t "$libraryCacheFolder" | grep .zip$ | head -1)
echo "Checking if Library cache $libraryCacheFolder/$latestLibraryCacheFile exists"
cd $libraryCacheFolder
ls -lh "${libraryCacheFolder}"
latestLibraryCacheFile=$(ls -t "${libraryCacheFolder}" | grep .zip$ | head -1)
echo "Checking if Library cache ${libraryCacheFolder}/$latestLibraryCacheFile exists"
cd ${libraryCacheFolder}
if [ -f "$latestLibraryCacheFile" ]; then
echo "Library cache exists"
unzip -q "$libraryCacheFolder/$latestLibraryCacheFile" -d "$projectPathFull"
tree "$libraryFolderFull"
unzip -q "${libraryCacheFolder}/$latestLibraryCacheFile" -d "$projectPathFull"
tree "${cloud_runner_state_1.CloudRunnerState.libraryFolderFull}"
fi
`);
yield run_cli_1.RunCli.RunCli(`
echo ' '
echo 'Large File Caching'
echo "Checking large file cache exists ($lfsCacheFolder/$LFS_ASSETS_HASH.zip)"
cd $lfsCacheFolder
echo "Checking large file cache exists (${lfsCacheFolder}/$LFS_ASSETS_HASH.zip)"
cd ${lfsCacheFolder}
if [ -f "$LFS_ASSETS_HASH.zip" ]; then
echo "Match found: using large file hash match $LFS_ASSETS_HASH.zip"
latestLFSCacheFile="$LFS_ASSETS_HASH"
else
latestLFSCacheFile=$(ls -t "$lfsCacheFolder" | grep .zip$ | head -1)
latestLFSCacheFile=$(ls -t "${lfsCacheFolder}" | grep .zip$ | head -1)
echo "Match not found: using latest large file cache $latestLFSCacheFile"
fi
if [ ! -f "$latestLFSCacheFile" ]; then
echo "LFS cache exists from build $latestLFSCacheFile from $branch"
rm -r "$gitLFSDestinationFolder"
unzip -q "$lfsCacheFolder/$latestLFSCacheFile" -d "$repoPathFull/.git"
rm -r "${cloud_runner_state_1.CloudRunnerState.lfsDirectory}"
unzip -q "${lfsCacheFolder}/$latestLFSCacheFile" -d "$repoPathFull/.git"
echo "git LFS folder, (should not contain $latestLFSCacheFile)"
ls -lh "$gitLFSDestinationFolder/"
ls -lh "${cloud_runner_state_1.CloudRunnerState.lfsDirectory}/"
fi
`);
yield run_cli_1.RunCli.RunCli(`
echo ' '
echo "LFS cache for $branch"
du -sch "$lfsCacheFolder/"
du -sch "${lfsCacheFolder}/"
echo '**'
echo "Library cache for $branch"
du -sch "$libraryCacheFolder/"
du -sch "${libraryCacheFolder}/"
echo '**'
echo "Branch: $branch"
du -sch "$cacheFolderWithBranch/"
du -sch "${cloud_runner_state_1.CloudRunnerState.cacheFolderFull}/"
echo '**'
echo 'Full cache'
du -sch "$cacheFolderFull/"
du -sch "${cloud_runner_state_1.CloudRunnerState.cacheFolderFull}/"
echo ' '
cd "$repoPathFull"
`);
yield run_cli_1.RunCli.RunCli(`
cd "${cloud_runner_state_1.CloudRunnerState.repoPathFull}"
git lfs pull
echo 'pulled latest LFS files'
cd "$gitLFSDestinationFolder/.."
`);
yield run_cli_1.RunCli.RunCli(`
cd "${cloud_runner_state_1.CloudRunnerState.lfsDirectory}/.."
zip -q -r "$LFS_ASSETS_HASH.zip" "./lfs"
cp "$LFS_ASSETS_HASH.zip" "$lfsCacheFolder"
echo "copied $LFS_ASSETS_HASH to $lfsCacheFolder"
cp "$LFS_ASSETS_HASH.zip" "${lfsCacheFolder}"
echo "copied $LFS_ASSETS_HASH to ${lfsCacheFolder}"
`);
yield run_cli_1.RunCli.RunCli(`
# purge cache
if [ -z "$purgeCloudRunnerCache" ]; then
if [ -z "${cloud_runner_state_1.CloudRunnerState.purgeRemoteCaching}" ]; then
echo ' '
echo "purging $purgeCloudRunnerCache"
rm -r "$purgeCloudRunnerCache"
echo "purging ${cloud_runner_state_1.CloudRunnerState.purgeRemoteCaching}"
rm -r "${cloud_runner_state_1.CloudRunnerState.purgeRemoteCaching}"
echo ' '
fi
`);

2
dist/index.js.map vendored

File diff suppressed because one or more lines are too long

View File

@ -178,7 +178,6 @@ class AWSBuildRunner {
} else if (message.includes('Rebuilding Library because the asset database could not be found!')) {
core.warning('LIBRARY NOT FOUND!');
}
//message = `[AWS][${taskDef.taskDefStackName}] ${message}`;
if (CloudRunnerState.buildParams.logToFile) {
fs.appendFileSync(`${CloudRunnerState.buildGuid}-outputfile.txt`, `${message}\r\n`);
}

View File

@ -43,77 +43,86 @@ export class DownloadRepository {
ls ${CloudRunnerState.projectPathFull}
echo ' '
`);
const lfsCacheFolder = `${CloudRunnerState.cacheFolderFull}/lfs`;
const libraryCacheFolder = `${CloudRunnerState.cacheFolderFull}/lib`;
await RunCli.RunCli(`
tree ${CloudRunnerState.builderPathFull}
echo 'Starting checks of cache for the Unity project Library and git LFS files'
cacheFolderFull=${CloudRunnerState.cacheFolderFull}
libraryFolderFull=${CloudRunnerState.libraryFolderFull}
gitLFSDestinationFolder=${CloudRunnerState.lfsDirectory}
purgeCloudRunnerCache=${CloudRunnerState.purgeRemoteCaching}
cacheFolderWithBranch="$cacheFolderFull"
lfsCacheFolder="$cacheFolderFull/lfs"
libraryCacheFolder="$cacheFolderFull/lib"
mkdir -p "$lfsCacheFolder"
mkdir -p "$libraryCacheFolder"
mkdir -p "${lfsCacheFolder}"
mkdir -p "${libraryCacheFolder}"
echo 'Library Caching'
`);
await RunCli.RunCli(`
# if the unity git project has included the library delete it and echo a warning
if [ -d "$libraryFolderFull" ]; then
rm -r "$libraryFolderFull"
if [ -d "${CloudRunnerState.libraryFolderFull}" ]; then
rm -r "${CloudRunnerState.libraryFolderFull}"
echo "!Warning!: The Unity library was included in the git repository (this isn't usually a good practice)"
fi
`);
await RunCli.RunCli(`
# Restore library cache
ls -lh "$libraryCacheFolder"
latestLibraryCacheFile=$(ls -t "$libraryCacheFolder" | grep .zip$ | head -1)
echo "Checking if Library cache $libraryCacheFolder/$latestLibraryCacheFile exists"
cd $libraryCacheFolder
ls -lh "${libraryCacheFolder}"
latestLibraryCacheFile=$(ls -t "${libraryCacheFolder}" | grep .zip$ | head -1)
echo "Checking if Library cache ${libraryCacheFolder}/$latestLibraryCacheFile exists"
cd ${libraryCacheFolder}
if [ -f "$latestLibraryCacheFile" ]; then
echo "Library cache exists"
unzip -q "$libraryCacheFolder/$latestLibraryCacheFile" -d "$projectPathFull"
tree "$libraryFolderFull"
unzip -q "${libraryCacheFolder}/$latestLibraryCacheFile" -d "$projectPathFull"
tree "${CloudRunnerState.libraryFolderFull}"
fi
`);
await RunCli.RunCli(`
echo ' '
echo 'Large File Caching'
echo "Checking large file cache exists ($lfsCacheFolder/$LFS_ASSETS_HASH.zip)"
cd $lfsCacheFolder
echo "Checking large file cache exists (${lfsCacheFolder}/$LFS_ASSETS_HASH.zip)"
cd ${lfsCacheFolder}
if [ -f "$LFS_ASSETS_HASH.zip" ]; then
echo "Match found: using large file hash match $LFS_ASSETS_HASH.zip"
latestLFSCacheFile="$LFS_ASSETS_HASH"
else
latestLFSCacheFile=$(ls -t "$lfsCacheFolder" | grep .zip$ | head -1)
latestLFSCacheFile=$(ls -t "${lfsCacheFolder}" | grep .zip$ | head -1)
echo "Match not found: using latest large file cache $latestLFSCacheFile"
fi
if [ ! -f "$latestLFSCacheFile" ]; then
echo "LFS cache exists from build $latestLFSCacheFile from $branch"
rm -r "$gitLFSDestinationFolder"
unzip -q "$lfsCacheFolder/$latestLFSCacheFile" -d "$repoPathFull/.git"
rm -r "${CloudRunnerState.lfsDirectory}"
unzip -q "${lfsCacheFolder}/$latestLFSCacheFile" -d "$repoPathFull/.git"
echo "git LFS folder, (should not contain $latestLFSCacheFile)"
ls -lh "$gitLFSDestinationFolder/"
ls -lh "${CloudRunnerState.lfsDirectory}/"
fi
`);
await RunCli.RunCli(`
echo ' '
echo "LFS cache for $branch"
du -sch "$lfsCacheFolder/"
du -sch "${lfsCacheFolder}/"
echo '**'
echo "Library cache for $branch"
du -sch "$libraryCacheFolder/"
du -sch "${libraryCacheFolder}/"
echo '**'
echo "Branch: $branch"
du -sch "$cacheFolderWithBranch/"
du -sch "${CloudRunnerState.cacheFolderFull}/"
echo '**'
echo 'Full cache'
du -sch "$cacheFolderFull/"
du -sch "${CloudRunnerState.cacheFolderFull}/"
echo ' '
cd "$repoPathFull"
`);
await RunCli.RunCli(`
cd "${CloudRunnerState.repoPathFull}"
git lfs pull
echo 'pulled latest LFS files'
cd "$gitLFSDestinationFolder/.."
`);
await RunCli.RunCli(`
cd "${CloudRunnerState.lfsDirectory}/.."
zip -q -r "$LFS_ASSETS_HASH.zip" "./lfs"
cp "$LFS_ASSETS_HASH.zip" "$lfsCacheFolder"
echo "copied $LFS_ASSETS_HASH to $lfsCacheFolder"
cp "$LFS_ASSETS_HASH.zip" "${lfsCacheFolder}"
echo "copied $LFS_ASSETS_HASH to ${lfsCacheFolder}"
`);
await RunCli.RunCli(`
# purge cache
if [ -z "$purgeCloudRunnerCache" ]; then
if [ -z "${CloudRunnerState.purgeRemoteCaching}" ]; then
echo ' '
echo "purging $purgeCloudRunnerCache"
rm -r "$purgeCloudRunnerCache"
echo "purging ${CloudRunnerState.purgeRemoteCaching}"
rm -r "${CloudRunnerState.purgeRemoteCaching}"
echo ' '
fi
`);