Extract remote client code into better functions
parent
92812497f1
commit
f56193215b
|
|
@ -447,7 +447,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
||||||
Object.defineProperty(exports, "__esModule", ({ value: true }));
|
Object.defineProperty(exports, "__esModule", ({ value: true }));
|
||||||
exports.RemoteClient = void 0;
|
exports.RemoteClient = void 0;
|
||||||
const cloud_runner_state_1 = __webpack_require__(70912);
|
const cloud_runner_state_1 = __webpack_require__(70912);
|
||||||
const setup_repo_1 = __webpack_require__(42959);
|
const setup_remote_repository_1 = __webpack_require__(62100);
|
||||||
class RemoteClient {
|
class RemoteClient {
|
||||||
static Run(options) {
|
static Run(options) {
|
||||||
return __awaiter(this, void 0, void 0, function* () {
|
return __awaiter(this, void 0, void 0, function* () {
|
||||||
|
|
@ -455,7 +455,7 @@ class RemoteClient {
|
||||||
cloud_runner_state_1.CloudRunnerState.setup(buildParameter);
|
cloud_runner_state_1.CloudRunnerState.setup(buildParameter);
|
||||||
switch (options.remoteClientState) {
|
switch (options.remoteClientState) {
|
||||||
default:
|
default:
|
||||||
yield setup_repo_1.DownloadRepository.run();
|
yield setup_remote_repository_1.SetupRemoteRepository.run();
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
@ -512,7 +512,7 @@ exports.RemoteClientSystem = RemoteClientSystem;
|
||||||
|
|
||||||
/***/ }),
|
/***/ }),
|
||||||
|
|
||||||
/***/ 42959:
|
/***/ 62100:
|
||||||
/***/ (function(__unused_webpack_module, exports, __webpack_require__) {
|
/***/ (function(__unused_webpack_module, exports, __webpack_require__) {
|
||||||
|
|
||||||
"use strict";
|
"use strict";
|
||||||
|
|
@ -530,38 +530,34 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
||||||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||||||
};
|
};
|
||||||
Object.defineProperty(exports, "__esModule", ({ value: true }));
|
Object.defineProperty(exports, "__esModule", ({ value: true }));
|
||||||
exports.DownloadRepository = void 0;
|
exports.SetupRemoteRepository = void 0;
|
||||||
const fs_1 = __importDefault(__webpack_require__(35747));
|
const fs_1 = __importDefault(__webpack_require__(35747));
|
||||||
const path_1 = __importDefault(__webpack_require__(85622));
|
const path_1 = __importDefault(__webpack_require__(85622));
|
||||||
const cloud_runner_logger_1 = __importDefault(__webpack_require__(22855));
|
const cloud_runner_logger_1 = __importDefault(__webpack_require__(22855));
|
||||||
const cloud_runner_state_1 = __webpack_require__(70912);
|
const cloud_runner_state_1 = __webpack_require__(70912);
|
||||||
const remote_client_system_1 = __webpack_require__(91269);
|
const remote_client_system_1 = __webpack_require__(91269);
|
||||||
class DownloadRepository {
|
class SetupRemoteRepository {
|
||||||
static run() {
|
static run() {
|
||||||
return __awaiter(this, void 0, void 0, function* () {
|
return __awaiter(this, void 0, void 0, function* () {
|
||||||
fs_1.default.mkdirSync(cloud_runner_state_1.CloudRunnerState.buildPathFull);
|
fs_1.default.mkdirSync(cloud_runner_state_1.CloudRunnerState.buildPathFull);
|
||||||
fs_1.default.mkdirSync(cloud_runner_state_1.CloudRunnerState.repoPathFull);
|
fs_1.default.mkdirSync(cloud_runner_state_1.CloudRunnerState.repoPathFull);
|
||||||
cloud_runner_logger_1.default.logRemoteCli(`Initializing source repository for cloning with caching of LFS files`);
|
yield SetupRemoteRepository.cloneRepoWithoutLFSFiles();
|
||||||
process.chdir(cloud_runner_state_1.CloudRunnerState.repoPathFull);
|
yield SetupRemoteRepository.createLFSHashFiles();
|
||||||
// stop annoying git detatched head info
|
|
||||||
yield remote_client_system_1.RemoteClientSystem.Run(`git config --global advice.detachedHead false`);
|
|
||||||
cloud_runner_logger_1.default.logRemoteCli(`Cloning the repository being built:`);
|
|
||||||
yield remote_client_system_1.RemoteClientSystem.Run(`git lfs install --skip-smudge`);
|
|
||||||
cloud_runner_logger_1.default.logRemoteCli(cloud_runner_state_1.CloudRunnerState.targetBuildRepoUrl);
|
|
||||||
yield remote_client_system_1.RemoteClientSystem.Run(`
|
|
||||||
git clone --progress --verbose ${cloud_runner_state_1.CloudRunnerState.targetBuildRepoUrl} ${cloud_runner_state_1.CloudRunnerState.repoPathFull}
|
|
||||||
`);
|
|
||||||
yield remote_client_system_1.RemoteClientSystem.Run(`
|
|
||||||
git checkout ${cloud_runner_state_1.CloudRunnerState.buildParams.gitSha}
|
|
||||||
`);
|
|
||||||
cloud_runner_logger_1.default.logRemoteCli(`Checked out ${process.env.GITHUB_SHA}`);
|
|
||||||
yield remote_client_system_1.RemoteClientSystem.Run(`
|
|
||||||
git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-guid
|
|
||||||
`);
|
|
||||||
yield remote_client_system_1.RemoteClientSystem.Run(`
|
|
||||||
md5sum .lfs-assets-guid > .lfs-assets-guid-sum
|
|
||||||
`);
|
|
||||||
const LFS_ASSETS_HASH = fs_1.default.readFileSync(`${path_1.default.join(cloud_runner_state_1.CloudRunnerState.repoPathFull, `.lfs-assets-guid`)}`, 'utf8');
|
const LFS_ASSETS_HASH = fs_1.default.readFileSync(`${path_1.default.join(cloud_runner_state_1.CloudRunnerState.repoPathFull, `.lfs-assets-guid`)}`, 'utf8');
|
||||||
|
yield SetupRemoteRepository.printLFSHashState();
|
||||||
|
const lfsCacheFolder = path_1.default.join(cloud_runner_state_1.CloudRunnerState.cacheFolderFull, `lfs`);
|
||||||
|
const libraryCacheFolder = path_1.default.join(cloud_runner_state_1.CloudRunnerState.cacheFolderFull, `lib`);
|
||||||
|
yield remote_client_system_1.RemoteClientSystem.Run(`tree ${cloud_runner_state_1.CloudRunnerState.builderPathFull}`);
|
||||||
|
yield SetupRemoteRepository.libraryCaching(lfsCacheFolder, libraryCacheFolder);
|
||||||
|
yield SetupRemoteRepository.lfsCaching(lfsCacheFolder, LFS_ASSETS_HASH);
|
||||||
|
yield SetupRemoteRepository.printCacheState(lfsCacheFolder, libraryCacheFolder);
|
||||||
|
yield SetupRemoteRepository.pullLatestLFS();
|
||||||
|
yield SetupRemoteRepository.cacheLatestLFSFiles(LFS_ASSETS_HASH, lfsCacheFolder);
|
||||||
|
SetupRemoteRepository.handleCachePurging();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
static printLFSHashState() {
|
||||||
|
return __awaiter(this, void 0, void 0, function* () {
|
||||||
yield remote_client_system_1.RemoteClientSystem.Run(`
|
yield remote_client_system_1.RemoteClientSystem.Run(`
|
||||||
echo ' '
|
echo ' '
|
||||||
echo 'Contents of .lfs-assets-guid file:'
|
echo 'Contents of .lfs-assets-guid file:'
|
||||||
|
|
@ -574,9 +570,73 @@ class DownloadRepository {
|
||||||
ls ${cloud_runner_state_1.CloudRunnerState.projectPathFull}
|
ls ${cloud_runner_state_1.CloudRunnerState.projectPathFull}
|
||||||
echo ' '
|
echo ' '
|
||||||
`);
|
`);
|
||||||
const lfsCacheFolder = path_1.default.join(cloud_runner_state_1.CloudRunnerState.cacheFolderFull, `lfs`);
|
});
|
||||||
const libraryCacheFolder = path_1.default.join(cloud_runner_state_1.CloudRunnerState.cacheFolderFull, `lib`);
|
}
|
||||||
yield remote_client_system_1.RemoteClientSystem.Run(`tree ${cloud_runner_state_1.CloudRunnerState.builderPathFull}`);
|
static printCacheState(lfsCacheFolder, libraryCacheFolder) {
|
||||||
|
return __awaiter(this, void 0, void 0, function* () {
|
||||||
|
yield remote_client_system_1.RemoteClientSystem.Run(`
|
||||||
|
echo ' '
|
||||||
|
echo "LFS cache for $branch"
|
||||||
|
du -sch "${lfsCacheFolder}/"
|
||||||
|
echo '**'
|
||||||
|
echo "Library cache for $branch"
|
||||||
|
du -sch "${libraryCacheFolder}/"
|
||||||
|
echo '**'
|
||||||
|
echo "Branch: $branch"
|
||||||
|
du -sch "${cloud_runner_state_1.CloudRunnerState.cacheFolderFull}/"
|
||||||
|
echo '**'
|
||||||
|
echo 'Full cache'
|
||||||
|
du -sch "${cloud_runner_state_1.CloudRunnerState.cacheFolderFull}/"
|
||||||
|
echo ' '
|
||||||
|
`);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
static handleCachePurging() {
|
||||||
|
if (process.env.purgeRemoteCaching !== undefined) {
|
||||||
|
cloud_runner_logger_1.default.logRemoteCli(`purging ${cloud_runner_state_1.CloudRunnerState.purgeRemoteCaching}`);
|
||||||
|
fs_1.default.rmdirSync(cloud_runner_state_1.CloudRunnerState.cacheFolder, { recursive: true });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
static cacheLatestLFSFiles(LFS_ASSETS_HASH, lfsCacheFolder) {
|
||||||
|
return __awaiter(this, void 0, void 0, function* () {
|
||||||
|
process.chdir(`${cloud_runner_state_1.CloudRunnerState.lfsDirectory}/..`);
|
||||||
|
yield remote_client_system_1.RemoteClientSystem.Run(`zip -r "${LFS_ASSETS_HASH}.zip" "./lfs"`);
|
||||||
|
fs_1.default.copyFileSync(`${LFS_ASSETS_HASH}.zip`, lfsCacheFolder);
|
||||||
|
cloud_runner_logger_1.default.logRemoteCli(`copied ${LFS_ASSETS_HASH} to ${lfsCacheFolder}`);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
static pullLatestLFS() {
|
||||||
|
return __awaiter(this, void 0, void 0, function* () {
|
||||||
|
process.chdir(cloud_runner_state_1.CloudRunnerState.repoPathFull);
|
||||||
|
yield remote_client_system_1.RemoteClientSystem.Run(`git lfs pull`);
|
||||||
|
cloud_runner_logger_1.default.logRemoteCli(`pulled latest LFS files`);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
static lfsCaching(lfsCacheFolder, LFS_ASSETS_HASH) {
|
||||||
|
return __awaiter(this, void 0, void 0, function* () {
|
||||||
|
cloud_runner_logger_1.default.logRemoteCli(` `);
|
||||||
|
cloud_runner_logger_1.default.logRemoteCli(`LFS Caching`);
|
||||||
|
process.chdir(lfsCacheFolder);
|
||||||
|
let latestLFSCacheFile;
|
||||||
|
if (fs_1.default.existsSync(`${LFS_ASSETS_HASH}.zip`)) {
|
||||||
|
cloud_runner_logger_1.default.logRemoteCli(`Match found: using large file hash match ${LFS_ASSETS_HASH}.zip`);
|
||||||
|
latestLFSCacheFile = `${LFS_ASSETS_HASH}.zip`;
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
latestLFSCacheFile = yield remote_client_system_1.RemoteClientSystem.Run(`ls -t "${lfsCacheFolder}" | grep .zip$ | head -1`);
|
||||||
|
}
|
||||||
|
if (fs_1.default.existsSync(latestLFSCacheFile)) {
|
||||||
|
cloud_runner_logger_1.default.logRemoteCli(`LFS cache exists`);
|
||||||
|
fs_1.default.rmdirSync(cloud_runner_state_1.CloudRunnerState.lfsDirectory, { recursive: true });
|
||||||
|
cloud_runner_logger_1.default.logRemoteCli(`LFS cache exists from build $latestLFSCacheFile from $branch`);
|
||||||
|
yield remote_client_system_1.RemoteClientSystem.Run(`unzip -q "${lfsCacheFolder}/${latestLFSCacheFile}" -d "${path_1.default.join(cloud_runner_state_1.CloudRunnerState.repoPathFull, `.git`)}"`);
|
||||||
|
yield remote_client_system_1.RemoteClientSystem.Run(`ls -lh "${cloud_runner_state_1.CloudRunnerState.lfsDirectory}"`);
|
||||||
|
cloud_runner_logger_1.default.logRemoteCli(`git LFS folder, (should not contain $latestLFSCacheFile)`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
static libraryCaching(lfsCacheFolder, libraryCacheFolder) {
|
||||||
|
return __awaiter(this, void 0, void 0, function* () {
|
||||||
cloud_runner_logger_1.default.logRemoteCli(`Starting checks of cache for the Unity project Library and git LFS files`);
|
cloud_runner_logger_1.default.logRemoteCli(`Starting checks of cache for the Unity project Library and git LFS files`);
|
||||||
if (!fs_1.default.existsSync(lfsCacheFolder)) {
|
if (!fs_1.default.existsSync(lfsCacheFolder)) {
|
||||||
fs_1.default.mkdirSync(lfsCacheFolder);
|
fs_1.default.mkdirSync(lfsCacheFolder);
|
||||||
|
|
@ -601,55 +661,38 @@ class DownloadRepository {
|
||||||
tree "${cloud_runner_state_1.CloudRunnerState.libraryFolderFull}"
|
tree "${cloud_runner_state_1.CloudRunnerState.libraryFolderFull}"
|
||||||
`);
|
`);
|
||||||
}
|
}
|
||||||
cloud_runner_logger_1.default.logRemoteCli(` `);
|
});
|
||||||
cloud_runner_logger_1.default.logRemoteCli(`LFS Caching`);
|
|
||||||
process.chdir(lfsCacheFolder);
|
|
||||||
let latestLFSCacheFile;
|
|
||||||
if (fs_1.default.existsSync(`${LFS_ASSETS_HASH}.zip`)) {
|
|
||||||
cloud_runner_logger_1.default.logRemoteCli(`Match found: using large file hash match ${LFS_ASSETS_HASH}.zip`);
|
|
||||||
latestLFSCacheFile = `${LFS_ASSETS_HASH}.zip`;
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
latestLFSCacheFile = yield remote_client_system_1.RemoteClientSystem.Run(`ls -t "${lfsCacheFolder}" | grep .zip$ | head -1`);
|
|
||||||
}
|
|
||||||
if (fs_1.default.existsSync(latestLFSCacheFile)) {
|
|
||||||
cloud_runner_logger_1.default.logRemoteCli(`LFS cache exists`);
|
|
||||||
fs_1.default.rmdirSync(cloud_runner_state_1.CloudRunnerState.lfsDirectory, { recursive: true });
|
|
||||||
cloud_runner_logger_1.default.logRemoteCli(`LFS cache exists from build $latestLFSCacheFile from $branch`);
|
|
||||||
yield remote_client_system_1.RemoteClientSystem.Run(`unzip -q "${lfsCacheFolder}/${latestLFSCacheFile}" -d "${path_1.default.join(cloud_runner_state_1.CloudRunnerState.repoPathFull, `.git`)}"`);
|
|
||||||
yield remote_client_system_1.RemoteClientSystem.Run(`ls -lh "${cloud_runner_state_1.CloudRunnerState.lfsDirectory}"`);
|
|
||||||
cloud_runner_logger_1.default.logRemoteCli(`git LFS folder, (should not contain $latestLFSCacheFile)`);
|
|
||||||
}
|
}
|
||||||
|
static createLFSHashFiles() {
|
||||||
|
return __awaiter(this, void 0, void 0, function* () {
|
||||||
yield remote_client_system_1.RemoteClientSystem.Run(`
|
yield remote_client_system_1.RemoteClientSystem.Run(`
|
||||||
echo ' '
|
git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-guid
|
||||||
echo "LFS cache for $branch"
|
|
||||||
du -sch "${lfsCacheFolder}/"
|
|
||||||
echo '**'
|
|
||||||
echo "Library cache for $branch"
|
|
||||||
du -sch "${libraryCacheFolder}/"
|
|
||||||
echo '**'
|
|
||||||
echo "Branch: $branch"
|
|
||||||
du -sch "${cloud_runner_state_1.CloudRunnerState.cacheFolderFull}/"
|
|
||||||
echo '**'
|
|
||||||
echo 'Full cache'
|
|
||||||
du -sch "${cloud_runner_state_1.CloudRunnerState.cacheFolderFull}/"
|
|
||||||
echo ' '
|
|
||||||
`);
|
`);
|
||||||
process.chdir(cloud_runner_state_1.CloudRunnerState.repoPathFull);
|
yield remote_client_system_1.RemoteClientSystem.Run(`
|
||||||
yield remote_client_system_1.RemoteClientSystem.Run(`git lfs pull`);
|
md5sum .lfs-assets-guid > .lfs-assets-guid-sum
|
||||||
cloud_runner_logger_1.default.logRemoteCli(`pulled latest LFS files`);
|
`);
|
||||||
process.chdir(`${cloud_runner_state_1.CloudRunnerState.lfsDirectory}/..`);
|
});
|
||||||
yield remote_client_system_1.RemoteClientSystem.Run(`zip -r "${LFS_ASSETS_HASH}.zip" "./lfs"`);
|
|
||||||
fs_1.default.copyFileSync(`${LFS_ASSETS_HASH}.zip`, lfsCacheFolder);
|
|
||||||
cloud_runner_logger_1.default.logRemoteCli(`copied ${LFS_ASSETS_HASH} to ${lfsCacheFolder}`);
|
|
||||||
if (process.env.purgeRemoteCaching !== undefined) {
|
|
||||||
cloud_runner_logger_1.default.logRemoteCli(`purging ${cloud_runner_state_1.CloudRunnerState.purgeRemoteCaching}`);
|
|
||||||
fs_1.default.rmdirSync(cloud_runner_state_1.CloudRunnerState.cacheFolder, { recursive: true });
|
|
||||||
}
|
}
|
||||||
|
static cloneRepoWithoutLFSFiles() {
|
||||||
|
return __awaiter(this, void 0, void 0, function* () {
|
||||||
|
cloud_runner_logger_1.default.logRemoteCli(`Initializing source repository for cloning with caching of LFS files`);
|
||||||
|
process.chdir(cloud_runner_state_1.CloudRunnerState.repoPathFull);
|
||||||
|
// stop annoying git detatched head info
|
||||||
|
yield remote_client_system_1.RemoteClientSystem.Run(`git config --global advice.detachedHead false`);
|
||||||
|
cloud_runner_logger_1.default.logRemoteCli(`Cloning the repository being built:`);
|
||||||
|
yield remote_client_system_1.RemoteClientSystem.Run(`git lfs install --skip-smudge`);
|
||||||
|
cloud_runner_logger_1.default.logRemoteCli(cloud_runner_state_1.CloudRunnerState.targetBuildRepoUrl);
|
||||||
|
yield remote_client_system_1.RemoteClientSystem.Run(`
|
||||||
|
git clone --progress --verbose ${cloud_runner_state_1.CloudRunnerState.targetBuildRepoUrl} ${cloud_runner_state_1.CloudRunnerState.repoPathFull}
|
||||||
|
`);
|
||||||
|
yield remote_client_system_1.RemoteClientSystem.Run(`
|
||||||
|
git checkout ${cloud_runner_state_1.CloudRunnerState.buildParams.gitSha}
|
||||||
|
`);
|
||||||
|
cloud_runner_logger_1.default.logRemoteCli(`Checked out ${process.env.GITHUB_SHA}`);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
exports.DownloadRepository = DownloadRepository;
|
exports.SetupRemoteRepository = SetupRemoteRepository;
|
||||||
|
|
||||||
|
|
||||||
/***/ }),
|
/***/ }),
|
||||||
|
|
|
||||||
File diff suppressed because one or more lines are too long
|
|
@ -1,5 +1,5 @@
|
||||||
import { CloudRunnerState } from '../../cloud-runner/state/cloud-runner-state';
|
import { CloudRunnerState } from '../../cloud-runner/state/cloud-runner-state';
|
||||||
import { DownloadRepository } from './setup-repo';
|
import { SetupRemoteRepository } from './setup-remote-repository';
|
||||||
|
|
||||||
export class RemoteClient {
|
export class RemoteClient {
|
||||||
static async Run(options) {
|
static async Run(options) {
|
||||||
|
|
@ -7,7 +7,7 @@ export class RemoteClient {
|
||||||
CloudRunnerState.setup(buildParameter);
|
CloudRunnerState.setup(buildParameter);
|
||||||
switch (options.remoteClientState) {
|
switch (options.remoteClientState) {
|
||||||
default:
|
default:
|
||||||
await DownloadRepository.run();
|
await SetupRemoteRepository.run();
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -4,31 +4,28 @@ import CloudRunnerLogger from '../../cloud-runner/services/cloud-runner-logger';
|
||||||
import { CloudRunnerState } from '../../cloud-runner/state/cloud-runner-state';
|
import { CloudRunnerState } from '../../cloud-runner/state/cloud-runner-state';
|
||||||
import { RemoteClientSystem } from './remote-client-system';
|
import { RemoteClientSystem } from './remote-client-system';
|
||||||
|
|
||||||
export class DownloadRepository {
|
export class SetupRemoteRepository {
|
||||||
public static async run() {
|
public static async run() {
|
||||||
fs.mkdirSync(CloudRunnerState.buildPathFull);
|
fs.mkdirSync(CloudRunnerState.buildPathFull);
|
||||||
fs.mkdirSync(CloudRunnerState.repoPathFull);
|
fs.mkdirSync(CloudRunnerState.repoPathFull);
|
||||||
CloudRunnerLogger.logRemoteCli(`Initializing source repository for cloning with caching of LFS files`);
|
await SetupRemoteRepository.cloneRepoWithoutLFSFiles();
|
||||||
process.chdir(CloudRunnerState.repoPathFull);
|
|
||||||
// stop annoying git detatched head info
|
await SetupRemoteRepository.createLFSHashFiles();
|
||||||
await RemoteClientSystem.Run(`git config --global advice.detachedHead false`);
|
|
||||||
CloudRunnerLogger.logRemoteCli(`Cloning the repository being built:`);
|
|
||||||
await RemoteClientSystem.Run(`git lfs install --skip-smudge`);
|
|
||||||
CloudRunnerLogger.logRemoteCli(CloudRunnerState.targetBuildRepoUrl);
|
|
||||||
await RemoteClientSystem.Run(`
|
|
||||||
git clone --progress --verbose ${CloudRunnerState.targetBuildRepoUrl} ${CloudRunnerState.repoPathFull}
|
|
||||||
`);
|
|
||||||
await RemoteClientSystem.Run(`
|
|
||||||
git checkout ${CloudRunnerState.buildParams.gitSha}
|
|
||||||
`);
|
|
||||||
CloudRunnerLogger.logRemoteCli(`Checked out ${process.env.GITHUB_SHA}`);
|
|
||||||
await RemoteClientSystem.Run(`
|
|
||||||
git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-guid
|
|
||||||
`);
|
|
||||||
await RemoteClientSystem.Run(`
|
|
||||||
md5sum .lfs-assets-guid > .lfs-assets-guid-sum
|
|
||||||
`);
|
|
||||||
const LFS_ASSETS_HASH = fs.readFileSync(`${path.join(CloudRunnerState.repoPathFull, `.lfs-assets-guid`)}`, 'utf8');
|
const LFS_ASSETS_HASH = fs.readFileSync(`${path.join(CloudRunnerState.repoPathFull, `.lfs-assets-guid`)}`, 'utf8');
|
||||||
|
await SetupRemoteRepository.printLFSHashState();
|
||||||
|
const lfsCacheFolder = path.join(CloudRunnerState.cacheFolderFull, `lfs`);
|
||||||
|
const libraryCacheFolder = path.join(CloudRunnerState.cacheFolderFull, `lib`);
|
||||||
|
await RemoteClientSystem.Run(`tree ${CloudRunnerState.builderPathFull}`);
|
||||||
|
await SetupRemoteRepository.libraryCaching(lfsCacheFolder, libraryCacheFolder);
|
||||||
|
await SetupRemoteRepository.lfsCaching(lfsCacheFolder, LFS_ASSETS_HASH);
|
||||||
|
|
||||||
|
await SetupRemoteRepository.printCacheState(lfsCacheFolder, libraryCacheFolder);
|
||||||
|
await SetupRemoteRepository.pullLatestLFS();
|
||||||
|
await SetupRemoteRepository.cacheLatestLFSFiles(LFS_ASSETS_HASH, lfsCacheFolder);
|
||||||
|
SetupRemoteRepository.handleCachePurging();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async printLFSHashState() {
|
||||||
await RemoteClientSystem.Run(`
|
await RemoteClientSystem.Run(`
|
||||||
echo ' '
|
echo ' '
|
||||||
echo 'Contents of .lfs-assets-guid file:'
|
echo 'Contents of .lfs-assets-guid file:'
|
||||||
|
|
@ -41,9 +38,70 @@ export class DownloadRepository {
|
||||||
ls ${CloudRunnerState.projectPathFull}
|
ls ${CloudRunnerState.projectPathFull}
|
||||||
echo ' '
|
echo ' '
|
||||||
`);
|
`);
|
||||||
const lfsCacheFolder = path.join(CloudRunnerState.cacheFolderFull, `lfs`);
|
}
|
||||||
const libraryCacheFolder = path.join(CloudRunnerState.cacheFolderFull, `lib`);
|
|
||||||
await RemoteClientSystem.Run(`tree ${CloudRunnerState.builderPathFull}`);
|
private static async printCacheState(lfsCacheFolder: string, libraryCacheFolder: string) {
|
||||||
|
await RemoteClientSystem.Run(`
|
||||||
|
echo ' '
|
||||||
|
echo "LFS cache for $branch"
|
||||||
|
du -sch "${lfsCacheFolder}/"
|
||||||
|
echo '**'
|
||||||
|
echo "Library cache for $branch"
|
||||||
|
du -sch "${libraryCacheFolder}/"
|
||||||
|
echo '**'
|
||||||
|
echo "Branch: $branch"
|
||||||
|
du -sch "${CloudRunnerState.cacheFolderFull}/"
|
||||||
|
echo '**'
|
||||||
|
echo 'Full cache'
|
||||||
|
du -sch "${CloudRunnerState.cacheFolderFull}/"
|
||||||
|
echo ' '
|
||||||
|
`);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static handleCachePurging() {
|
||||||
|
if (process.env.purgeRemoteCaching !== undefined) {
|
||||||
|
CloudRunnerLogger.logRemoteCli(`purging ${CloudRunnerState.purgeRemoteCaching}`);
|
||||||
|
fs.rmdirSync(CloudRunnerState.cacheFolder, { recursive: true });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async cacheLatestLFSFiles(LFS_ASSETS_HASH: string, lfsCacheFolder: string) {
|
||||||
|
process.chdir(`${CloudRunnerState.lfsDirectory}/..`);
|
||||||
|
await RemoteClientSystem.Run(`zip -r "${LFS_ASSETS_HASH}.zip" "./lfs"`);
|
||||||
|
fs.copyFileSync(`${LFS_ASSETS_HASH}.zip`, lfsCacheFolder);
|
||||||
|
CloudRunnerLogger.logRemoteCli(`copied ${LFS_ASSETS_HASH} to ${lfsCacheFolder}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async pullLatestLFS() {
|
||||||
|
process.chdir(CloudRunnerState.repoPathFull);
|
||||||
|
await RemoteClientSystem.Run(`git lfs pull`);
|
||||||
|
CloudRunnerLogger.logRemoteCli(`pulled latest LFS files`);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async lfsCaching(lfsCacheFolder: string, LFS_ASSETS_HASH: string) {
|
||||||
|
CloudRunnerLogger.logRemoteCli(` `);
|
||||||
|
CloudRunnerLogger.logRemoteCli(`LFS Caching`);
|
||||||
|
process.chdir(lfsCacheFolder);
|
||||||
|
let latestLFSCacheFile;
|
||||||
|
if (fs.existsSync(`${LFS_ASSETS_HASH}.zip`)) {
|
||||||
|
CloudRunnerLogger.logRemoteCli(`Match found: using large file hash match ${LFS_ASSETS_HASH}.zip`);
|
||||||
|
latestLFSCacheFile = `${LFS_ASSETS_HASH}.zip`;
|
||||||
|
} else {
|
||||||
|
latestLFSCacheFile = await RemoteClientSystem.Run(`ls -t "${lfsCacheFolder}" | grep .zip$ | head -1`);
|
||||||
|
}
|
||||||
|
if (fs.existsSync(latestLFSCacheFile)) {
|
||||||
|
CloudRunnerLogger.logRemoteCli(`LFS cache exists`);
|
||||||
|
fs.rmdirSync(CloudRunnerState.lfsDirectory, { recursive: true });
|
||||||
|
CloudRunnerLogger.logRemoteCli(`LFS cache exists from build $latestLFSCacheFile from $branch`);
|
||||||
|
await RemoteClientSystem.Run(
|
||||||
|
`unzip -q "${lfsCacheFolder}/${latestLFSCacheFile}" -d "${path.join(CloudRunnerState.repoPathFull, `.git`)}"`,
|
||||||
|
);
|
||||||
|
await RemoteClientSystem.Run(`ls -lh "${CloudRunnerState.lfsDirectory}"`);
|
||||||
|
CloudRunnerLogger.logRemoteCli(`git LFS folder, (should not contain $latestLFSCacheFile)`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async libraryCaching(lfsCacheFolder: string, libraryCacheFolder: string) {
|
||||||
CloudRunnerLogger.logRemoteCli(`Starting checks of cache for the Unity project Library and git LFS files`);
|
CloudRunnerLogger.logRemoteCli(`Starting checks of cache for the Unity project Library and git LFS files`);
|
||||||
if (!fs.existsSync(lfsCacheFolder)) {
|
if (!fs.existsSync(lfsCacheFolder)) {
|
||||||
fs.mkdirSync(lfsCacheFolder);
|
fs.mkdirSync(lfsCacheFolder);
|
||||||
|
|
@ -70,52 +128,31 @@ export class DownloadRepository {
|
||||||
tree "${CloudRunnerState.libraryFolderFull}"
|
tree "${CloudRunnerState.libraryFolderFull}"
|
||||||
`);
|
`);
|
||||||
}
|
}
|
||||||
CloudRunnerLogger.logRemoteCli(` `);
|
|
||||||
CloudRunnerLogger.logRemoteCli(`LFS Caching`);
|
|
||||||
process.chdir(lfsCacheFolder);
|
|
||||||
let latestLFSCacheFile;
|
|
||||||
if (fs.existsSync(`${LFS_ASSETS_HASH}.zip`)) {
|
|
||||||
CloudRunnerLogger.logRemoteCli(`Match found: using large file hash match ${LFS_ASSETS_HASH}.zip`);
|
|
||||||
latestLFSCacheFile = `${LFS_ASSETS_HASH}.zip`;
|
|
||||||
} else {
|
|
||||||
latestLFSCacheFile = await RemoteClientSystem.Run(`ls -t "${lfsCacheFolder}" | grep .zip$ | head -1`);
|
|
||||||
}
|
|
||||||
if (fs.existsSync(latestLFSCacheFile)) {
|
|
||||||
CloudRunnerLogger.logRemoteCli(`LFS cache exists`);
|
|
||||||
fs.rmdirSync(CloudRunnerState.lfsDirectory, { recursive: true });
|
|
||||||
CloudRunnerLogger.logRemoteCli(`LFS cache exists from build $latestLFSCacheFile from $branch`);
|
|
||||||
await RemoteClientSystem.Run(
|
|
||||||
`unzip -q "${lfsCacheFolder}/${latestLFSCacheFile}" -d "${path.join(CloudRunnerState.repoPathFull, `.git`)}"`,
|
|
||||||
);
|
|
||||||
await RemoteClientSystem.Run(`ls -lh "${CloudRunnerState.lfsDirectory}"`);
|
|
||||||
CloudRunnerLogger.logRemoteCli(`git LFS folder, (should not contain $latestLFSCacheFile)`);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static async createLFSHashFiles() {
|
||||||
await RemoteClientSystem.Run(`
|
await RemoteClientSystem.Run(`
|
||||||
echo ' '
|
git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-guid
|
||||||
echo "LFS cache for $branch"
|
|
||||||
du -sch "${lfsCacheFolder}/"
|
|
||||||
echo '**'
|
|
||||||
echo "Library cache for $branch"
|
|
||||||
du -sch "${libraryCacheFolder}/"
|
|
||||||
echo '**'
|
|
||||||
echo "Branch: $branch"
|
|
||||||
du -sch "${CloudRunnerState.cacheFolderFull}/"
|
|
||||||
echo '**'
|
|
||||||
echo 'Full cache'
|
|
||||||
du -sch "${CloudRunnerState.cacheFolderFull}/"
|
|
||||||
echo ' '
|
|
||||||
`);
|
`);
|
||||||
|
await RemoteClientSystem.Run(`
|
||||||
|
md5sum .lfs-assets-guid > .lfs-assets-guid-sum
|
||||||
|
`);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async cloneRepoWithoutLFSFiles() {
|
||||||
|
CloudRunnerLogger.logRemoteCli(`Initializing source repository for cloning with caching of LFS files`);
|
||||||
process.chdir(CloudRunnerState.repoPathFull);
|
process.chdir(CloudRunnerState.repoPathFull);
|
||||||
await RemoteClientSystem.Run(`git lfs pull`);
|
// stop annoying git detatched head info
|
||||||
CloudRunnerLogger.logRemoteCli(`pulled latest LFS files`);
|
await RemoteClientSystem.Run(`git config --global advice.detachedHead false`);
|
||||||
process.chdir(`${CloudRunnerState.lfsDirectory}/..`);
|
CloudRunnerLogger.logRemoteCli(`Cloning the repository being built:`);
|
||||||
await RemoteClientSystem.Run(`zip -r "${LFS_ASSETS_HASH}.zip" "./lfs"`);
|
await RemoteClientSystem.Run(`git lfs install --skip-smudge`);
|
||||||
fs.copyFileSync(`${LFS_ASSETS_HASH}.zip`, lfsCacheFolder);
|
CloudRunnerLogger.logRemoteCli(CloudRunnerState.targetBuildRepoUrl);
|
||||||
CloudRunnerLogger.logRemoteCli(`copied ${LFS_ASSETS_HASH} to ${lfsCacheFolder}`);
|
await RemoteClientSystem.Run(`
|
||||||
if (process.env.purgeRemoteCaching !== undefined) {
|
git clone --progress --verbose ${CloudRunnerState.targetBuildRepoUrl} ${CloudRunnerState.repoPathFull}
|
||||||
CloudRunnerLogger.logRemoteCli(`purging ${CloudRunnerState.purgeRemoteCaching}`);
|
`);
|
||||||
fs.rmdirSync(CloudRunnerState.cacheFolder, { recursive: true });
|
await RemoteClientSystem.Run(`
|
||||||
}
|
git checkout ${CloudRunnerState.buildParams.gitSha}
|
||||||
|
`);
|
||||||
|
CloudRunnerLogger.logRemoteCli(`Checked out ${process.env.GITHUB_SHA}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Loading…
Reference in New Issue