actions
/
unity-builder
mirror of https://github.com/game-ci/unity-builder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix lib caching

pull/310/head
Frostebite 2021-12-27 21:25:46 +00:00
parent 1416d19c78
commit bb3065f8b5
5 changed files with 151 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

137
dist/index.js vendored
View File

@ -454,6 +454,7 @@ const console_1 = __webpack_require__(57082);
const fs_1 = __importDefault(__webpack_require__(35747)); const fs_1 = __importDefault(__webpack_require__(35747));
const path_1 = __importDefault(__webpack_require__(85622)); const path_1 = __importDefault(__webpack_require__(85622));
const cloud_runner_logger_1 = __importDefault(__webpack_require__(22855)); const cloud_runner_logger_1 = __importDefault(__webpack_require__(22855));
const cloud_runner_state_1 = __webpack_require__(70912);
const remote_client_system_1 = __webpack_require__(91269); const remote_client_system_1 = __webpack_require__(91269);
class Caching { class Caching {
static PushToCache(cacheFolder, destinationFolder, artifactName) { static PushToCache(cacheFolder, destinationFolder, artifactName) {
@ -496,6 +497,29 @@ class Caching {
} }
}); });
} }
static handleCachePurging() {
if (process.env.purgeRemoteCaching !== undefined) {
cloud_runner_logger_1.default.logCli(`purging ${cloud_runner_state_1.CloudRunnerState.purgeRemoteCaching}`);
fs_1.default.rmdirSync(cloud_runner_state_1.CloudRunnerState.cacheFolder, { recursive: true });
}
}
static printCacheState(lfsCacheFolder, libraryCacheFolder) {
return __awaiter(this, void 0, void 0, function* () {
yield remote_client_system_1.RemoteClientSystem.Run(`echo ' '
echo "LFS cache for $branch"
du -sch "${lfsCacheFolder}/"
echo '**'
echo "Library cache for $branch"
du -sch "${libraryCacheFolder}/"
echo '**'
echo "Branch: $branch"
du -sch "${cloud_runner_state_1.CloudRunnerState.cacheFolderFull}/"
echo '**'
echo 'Full cache'
du -sch "${cloud_runner_state_1.CloudRunnerState.cacheFolderFull}/"
echo ' '`);
});
}
} }
exports.Caching = Caching; exports.Caching = Caching;
@ -536,6 +560,57 @@ class RemoteClient {
exports.RemoteClient = RemoteClient; exports.RemoteClient = RemoteClient;
/***/ }),
/***/ 47011:
/***/ (function(__unused_webpack_module, exports, __webpack_require__) {
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.LFSHashing = void 0;
const path_1 = __importDefault(__webpack_require__(85622));
const cloud_runner_state_1 = __webpack_require__(70912);
const remote_client_system_1 = __webpack_require__(91269);
const fs_1 = __importDefault(__webpack_require__(35747));
class LFSHashing {
static printLFSHashState() {
return __awaiter(this, void 0, void 0, function* () {
yield remote_client_system_1.RemoteClientSystem.Run(`echo ' '
echo 'Contents of .lfs-assets-guid file:'
cat .lfs-assets-guid
echo ' '
echo 'Contents of .lfs-assets-guid-sum file:'
cat .lfs-assets-guid-sum
echo ' '
echo 'Source repository initialized'
ls ${cloud_runner_state_1.CloudRunnerState.projectPathFull}
echo ' '`);
});
}
static createLFSHashFiles() {
return __awaiter(this, void 0, void 0, function* () {
yield remote_client_system_1.RemoteClientSystem.Run(`git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-guid`);
yield remote_client_system_1.RemoteClientSystem.Run(`md5sum .lfs-assets-guid > .lfs-assets-guid-sum`);
return fs_1.default.readFileSync(`${path_1.default.join(cloud_runner_state_1.CloudRunnerState.repoPathFull, `.lfs-assets-guid`)}`, 'utf8');
});
}
}
exports.LFSHashing = LFSHashing;
/***/ }), /***/ }),
/***/ 91269: /***/ 91269:
@ -620,6 +695,7 @@ const path_1 = __importDefault(__webpack_require__(85622));
const cloud_runner_logger_1 = __importDefault(__webpack_require__(22855)); const cloud_runner_logger_1 = __importDefault(__webpack_require__(22855));
const cloud_runner_state_1 = __webpack_require__(70912); const cloud_runner_state_1 = __webpack_require__(70912);
const caching_1 = __webpack_require__(35010); const caching_1 = __webpack_require__(35010);
const lfs_hashing_1 = __webpack_require__(47011);
const remote_client_system_1 = __webpack_require__(91269); const remote_client_system_1 = __webpack_require__(91269);
class SetupRemoteRepository { class SetupRemoteRepository {
static run() { static run() {
@ -629,65 +705,26 @@ class SetupRemoteRepository {
fs_1.default.mkdirSync(cloud_runner_state_1.CloudRunnerState.repoPathFull); fs_1.default.mkdirSync(cloud_runner_state_1.CloudRunnerState.repoPathFull);
yield SetupRemoteRepository.cloneRepoWithoutLFSFiles(); yield SetupRemoteRepository.cloneRepoWithoutLFSFiles();
yield SetupRemoteRepository.createLFSHashFiles(); yield SetupRemoteRepository.createLFSHashFiles();
yield SetupRemoteRepository.printLFSHashState(); yield lfs_hashing_1.LFSHashing.printLFSHashState();
const lfsCacheFolder = path_1.default.join(cloud_runner_state_1.CloudRunnerState.cacheFolderFull, `lfs`); const lfsCacheFolder = path_1.default.join(cloud_runner_state_1.CloudRunnerState.cacheFolderFull, `lfs`);
const libraryCacheFolder = path_1.default.join(cloud_runner_state_1.CloudRunnerState.cacheFolderFull, `lib`); const libraryCacheFolder = path_1.default.join(cloud_runner_state_1.CloudRunnerState.cacheFolderFull, `lib`);
yield remote_client_system_1.RemoteClientSystem.Run(`tree ${libraryCacheFolder}`); yield remote_client_system_1.RemoteClientSystem.Run(`tree ${cloud_runner_state_1.CloudRunnerState.repoPathFull}`);
yield remote_client_system_1.RemoteClientSystem.Run(`tree ${cloud_runner_state_1.CloudRunnerState.builderPathFull}`);
yield SetupRemoteRepository.libraryCaching(libraryCacheFolder); yield SetupRemoteRepository.libraryCaching(libraryCacheFolder);
yield SetupRemoteRepository.lfsCaching(lfsCacheFolder); yield SetupRemoteRepository.lfsCaching(lfsCacheFolder);
yield SetupRemoteRepository.printCacheState(lfsCacheFolder, libraryCacheFolder); yield remote_client_system_1.RemoteClientSystem.Run(`tree ${cloud_runner_state_1.CloudRunnerState.repoPathFull}`);
yield caching_1.Caching.printCacheState(lfsCacheFolder, libraryCacheFolder);
yield SetupRemoteRepository.pullLatestLFS(); yield SetupRemoteRepository.pullLatestLFS();
yield SetupRemoteRepository.cacheLatestLFSFiles(lfsCacheFolder); yield remote_client_system_1.RemoteClientSystem.Run(`tree ${cloud_runner_state_1.CloudRunnerState.repoPathFull}`);
SetupRemoteRepository.handleCachePurging(); yield remote_client_system_1.RemoteClientSystem.Run(`tree ${cloud_runner_state_1.CloudRunnerState.cacheFolderFull}`);
yield caching_1.Caching.PushToCache(lfsCacheFolder, cloud_runner_state_1.CloudRunnerState.lfsDirectory, SetupRemoteRepository.LFS_ASSETS_HASH);
yield remote_client_system_1.RemoteClientSystem.Run(`tree ${cloud_runner_state_1.CloudRunnerState.cacheFolderFull}`);
caching_1.Caching.handleCachePurging();
} }
catch (error) { catch (error) {
throw error; throw error;
} }
}); });
} }
static printLFSHashState() {
return __awaiter(this, void 0, void 0, function* () {
yield remote_client_system_1.RemoteClientSystem.Run(`echo ' '
echo 'Contents of .lfs-assets-guid file:'
cat .lfs-assets-guid
echo ' '
echo 'Contents of .lfs-assets-guid-sum file:'
cat .lfs-assets-guid-sum
echo ' '
echo 'Source repository initialized'
ls ${cloud_runner_state_1.CloudRunnerState.projectPathFull}
echo ' '`);
});
}
static printCacheState(lfsCacheFolder, libraryCacheFolder) {
return __awaiter(this, void 0, void 0, function* () {
yield remote_client_system_1.RemoteClientSystem.Run(`echo ' '
echo "LFS cache for $branch"
du -sch "${lfsCacheFolder}/"
echo '**'
echo "Library cache for $branch"
du -sch "${libraryCacheFolder}/"
echo '**'
echo "Branch: $branch"
du -sch "${cloud_runner_state_1.CloudRunnerState.cacheFolderFull}/"
echo '**'
echo 'Full cache'
du -sch "${cloud_runner_state_1.CloudRunnerState.cacheFolderFull}/"
echo ' '`);
});
}
static handleCachePurging() {
if (process.env.purgeRemoteCaching !== undefined) {
cloud_runner_logger_1.default.logCli(`purging ${cloud_runner_state_1.CloudRunnerState.purgeRemoteCaching}`);
fs_1.default.rmdirSync(cloud_runner_state_1.CloudRunnerState.cacheFolder, { recursive: true });
}
}
static cacheLatestLFSFiles(lfsCacheFolder) {
return __awaiter(this, void 0, void 0, function* () {
yield caching_1.Caching.PushToCache(lfsCacheFolder, cloud_runner_state_1.CloudRunnerState.lfsDirectory, SetupRemoteRepository.LFS_ASSETS_HASH);
});
}
static pullLatestLFS() { static pullLatestLFS() {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
process.chdir(cloud_runner_state_1.CloudRunnerState.repoPathFull); process.chdir(cloud_runner_state_1.CloudRunnerState.repoPathFull);
@ -714,9 +751,7 @@ class SetupRemoteRepository {
} }
static createLFSHashFiles() { static createLFSHashFiles() {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
yield remote_client_system_1.RemoteClientSystem.Run(`git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-guid`); SetupRemoteRepository.LFS_ASSETS_HASH = yield lfs_hashing_1.LFSHashing.createLFSHashFiles();
yield remote_client_system_1.RemoteClientSystem.Run(`md5sum .lfs-assets-guid > .lfs-assets-guid-sum`);
SetupRemoteRepository.LFS_ASSETS_HASH = fs_1.default.readFileSync(`${path_1.default.join(cloud_runner_state_1.CloudRunnerState.repoPathFull, `.lfs-assets-guid`)}`, 'utf8');
cloud_runner_logger_1.default.logCli(SetupRemoteRepository.LFS_ASSETS_HASH); cloud_runner_logger_1.default.logCli(SetupRemoteRepository.LFS_ASSETS_HASH);
}); });
} }

2
dist/index.js.map vendored
View File

File diff suppressed because one or more lines are too long

26
src/model/cli/remote-client/caching.ts
View File

@ -2,6 +2,7 @@ import { assert } from 'console';
import fs from 'fs'; import fs from 'fs';
import path from 'path'; import path from 'path';
import CloudRunnerLogger from '../../cloud-runner/services/cloud-runner-logger'; import CloudRunnerLogger from '../../cloud-runner/services/cloud-runner-logger';
import { CloudRunnerState } from '../../cloud-runner/state/cloud-runner-state';
import { RemoteClientSystem } from './remote-client-system'; import { RemoteClientSystem } from './remote-client-system';
export class Caching { export class Caching {
@ -46,4 +47,29 @@ export class Caching {
} }
} }
} }
public static handleCachePurging() {
if (process.env.purgeRemoteCaching !== undefined) {
CloudRunnerLogger.logCli(`purging ${CloudRunnerState.purgeRemoteCaching}`);
fs.rmdirSync(CloudRunnerState.cacheFolder, { recursive: true });
}
}
public static async printCacheState(lfsCacheFolder: string, libraryCacheFolder: string) {
await RemoteClientSystem.Run(
`echo ' '
echo "LFS cache for $branch"
du -sch "${lfsCacheFolder}/"
echo '**'
echo "Library cache for $branch"
du -sch "${libraryCacheFolder}/"
echo '**'
echo "Branch: $branch"
du -sch "${CloudRunnerState.cacheFolderFull}/"
echo '**'
echo 'Full cache'
du -sch "${CloudRunnerState.cacheFolderFull}/"
echo ' '`,
);
}
} }

27
src/model/cli/remote-client/lfs-hashing.ts 100644
View File

@ -0,0 +1,27 @@
import path from 'path';
import { CloudRunnerState } from '../../cloud-runner/state/cloud-runner-state';
import { RemoteClientSystem } from './remote-client-system';
import fs from 'fs';
export class LFSHashing {
public static async printLFSHashState() {
await RemoteClientSystem.Run(
`echo ' '
echo 'Contents of .lfs-assets-guid file:'
cat .lfs-assets-guid
echo ' '
echo 'Contents of .lfs-assets-guid-sum file:'
cat .lfs-assets-guid-sum
echo ' '
echo 'Source repository initialized'
ls ${CloudRunnerState.projectPathFull}
echo ' '`,
);
}
public static async createLFSHashFiles() {
await RemoteClientSystem.Run(`git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-guid`);
await RemoteClientSystem.Run(`md5sum .lfs-assets-guid > .lfs-assets-guid-sum`);
return fs.readFileSync(`${path.join(CloudRunnerState.repoPathFull, `.lfs-assets-guid`)}`, 'utf8');
}
}

68
src/model/cli/remote-client/setup-remote-repository.ts
View File

@ -3,6 +3,7 @@ import path from 'path';
import CloudRunnerLogger from '../../cloud-runner/services/cloud-runner-logger'; import CloudRunnerLogger from '../../cloud-runner/services/cloud-runner-logger';
import { CloudRunnerState } from '../../cloud-runner/state/cloud-runner-state'; import { CloudRunnerState } from '../../cloud-runner/state/cloud-runner-state';
import { Caching } from './caching'; import { Caching } from './caching';
import { LFSHashing } from './lfs-hashing';
import { RemoteClientSystem } from './remote-client-system'; import { RemoteClientSystem } from './remote-client-system';
export class SetupRemoteRepository { export class SetupRemoteRepository {
@ -14,67 +15,25 @@ export class SetupRemoteRepository {
await SetupRemoteRepository.cloneRepoWithoutLFSFiles(); await SetupRemoteRepository.cloneRepoWithoutLFSFiles();
await SetupRemoteRepository.createLFSHashFiles(); await SetupRemoteRepository.createLFSHashFiles();
await SetupRemoteRepository.printLFSHashState(); await LFSHashing.printLFSHashState();
const lfsCacheFolder = path.join(CloudRunnerState.cacheFolderFull, `lfs`); const lfsCacheFolder = path.join(CloudRunnerState.cacheFolderFull, `lfs`);
const libraryCacheFolder = path.join(CloudRunnerState.cacheFolderFull, `lib`); const libraryCacheFolder = path.join(CloudRunnerState.cacheFolderFull, `lib`);
await RemoteClientSystem.Run(`tree ${libraryCacheFolder}`); await RemoteClientSystem.Run(`tree ${CloudRunnerState.repoPathFull}`);
await RemoteClientSystem.Run(`tree ${CloudRunnerState.builderPathFull}`);
await SetupRemoteRepository.libraryCaching(libraryCacheFolder); await SetupRemoteRepository.libraryCaching(libraryCacheFolder);
await SetupRemoteRepository.lfsCaching(lfsCacheFolder); await SetupRemoteRepository.lfsCaching(lfsCacheFolder);
await RemoteClientSystem.Run(`tree ${CloudRunnerState.repoPathFull}`);
await SetupRemoteRepository.printCacheState(lfsCacheFolder, libraryCacheFolder); await Caching.printCacheState(lfsCacheFolder, libraryCacheFolder);
await SetupRemoteRepository.pullLatestLFS(); await SetupRemoteRepository.pullLatestLFS();
await SetupRemoteRepository.cacheLatestLFSFiles(lfsCacheFolder); await RemoteClientSystem.Run(`tree ${CloudRunnerState.repoPathFull}`);
SetupRemoteRepository.handleCachePurging(); await RemoteClientSystem.Run(`tree ${CloudRunnerState.cacheFolderFull}`);
await Caching.PushToCache(lfsCacheFolder, CloudRunnerState.lfsDirectory, SetupRemoteRepository.LFS_ASSETS_HASH);
await RemoteClientSystem.Run(`tree ${CloudRunnerState.cacheFolderFull}`);
Caching.handleCachePurging();
} catch (error) { } catch (error) {
throw error; throw error;
} }
} }
private static async printLFSHashState() {
await RemoteClientSystem.Run(
`echo ' '
echo 'Contents of .lfs-assets-guid file:'
cat .lfs-assets-guid
echo ' '
echo 'Contents of .lfs-assets-guid-sum file:'
cat .lfs-assets-guid-sum
echo ' '
echo 'Source repository initialized'
ls ${CloudRunnerState.projectPathFull}
echo ' '`,
);
}
private static async printCacheState(lfsCacheFolder: string, libraryCacheFolder: string) {
await RemoteClientSystem.Run(
`echo ' '
echo "LFS cache for $branch"
du -sch "${lfsCacheFolder}/"
echo '**'
echo "Library cache for $branch"
du -sch "${libraryCacheFolder}/"
echo '**'
echo "Branch: $branch"
du -sch "${CloudRunnerState.cacheFolderFull}/"
echo '**'
echo 'Full cache'
du -sch "${CloudRunnerState.cacheFolderFull}/"
echo ' '`,
);
}
private static handleCachePurging() {
if (process.env.purgeRemoteCaching !== undefined) {
CloudRunnerLogger.logCli(`purging ${CloudRunnerState.purgeRemoteCaching}`);
fs.rmdirSync(CloudRunnerState.cacheFolder, { recursive: true });
}
}
private static async cacheLatestLFSFiles(lfsCacheFolder: string) {
await Caching.PushToCache(lfsCacheFolder, CloudRunnerState.lfsDirectory, SetupRemoteRepository.LFS_ASSETS_HASH);
}
private static async pullLatestLFS() { private static async pullLatestLFS() {
process.chdir(CloudRunnerState.repoPathFull); process.chdir(CloudRunnerState.repoPathFull);
await RemoteClientSystem.Run(`git lfs pull`); await RemoteClientSystem.Run(`git lfs pull`);
@ -101,12 +60,7 @@ export class SetupRemoteRepository {
} }
private static async createLFSHashFiles() { private static async createLFSHashFiles() {
await RemoteClientSystem.Run(`git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-guid`); SetupRemoteRepository.LFS_ASSETS_HASH = await LFSHashing.createLFSHashFiles();
await RemoteClientSystem.Run(`md5sum .lfs-assets-guid > .lfs-assets-guid-sum`);
SetupRemoteRepository.LFS_ASSETS_HASH = fs.readFileSync(
`${path.join(CloudRunnerState.repoPathFull, `.lfs-assets-guid`)}`,
'utf8',
);
CloudRunnerLogger.logCli(SetupRemoteRepository.LFS_ASSETS_HASH); CloudRunnerLogger.logCli(SetupRemoteRepository.LFS_ASSETS_HASH);
} }