import fs from 'fs'; import CloudRunner from '../cloud-runner'; import { CloudRunnerFolders } from '../services/cloud-runner-folders'; import { Caching } from './caching'; import { LfsHashing } from '../services/lfs-hashing'; import { RemoteClientLogger } from './remote-client-logger'; import path from 'path'; import { assert } from 'console'; import CloudRunnerLogger from '../services/cloud-runner-logger'; import { CliFunction } from '../../cli/cli-functions-repository'; import { CloudRunnerSystem } from '../services/cloud-runner-system'; export class RemoteClient { public static async bootstrapRepository() { try { await CloudRunnerSystem.Run( `mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`, ); await CloudRunnerSystem.Run(`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute)}`); await CloudRunnerSystem.Run(`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderFull)}`); process.chdir(CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute)); await RemoteClient.cloneRepoWithoutLFSFiles(); RemoteClient.replaceLargePackageReferencesWithSharedReferences(); await RemoteClient.sizeOfFolder( 'repo before lfs cache pull', CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute), ); const lfsHashes = await LfsHashing.createLFSHashFiles(); if (fs.existsSync(CloudRunnerFolders.libraryFolderAbsolute)) { RemoteClientLogger.logWarning(`!Warning!: The Unity library was included in the git repository`); } await Caching.PullFromCache( CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull), CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute), `${lfsHashes.lfsGuidSum}`, ); await RemoteClient.sizeOfFolder('repo after lfs cache pull', CloudRunnerFolders.repoPathAbsolute); await RemoteClient.pullLatestLFS(); await RemoteClient.sizeOfFolder('repo before lfs git pull', CloudRunnerFolders.repoPathAbsolute); await Caching.PushToCache( CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull), CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute), `${lfsHashes.lfsGuidSum}`, ); await Caching.PullFromCache( CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryCacheFolderFull), CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute), ); await RemoteClient.sizeOfFolder('repo after library cache pull', CloudRunnerFolders.repoPathAbsolute); await Caching.handleCachePurging(); } catch (error) { throw error; } } private static async sizeOfFolder(message: string, folder: string) { if (CloudRunner.buildParameters.cloudRunnerIntegrationTests) { CloudRunnerLogger.log(`Size of ${message}`); await CloudRunnerSystem.Run(`du -sh ${folder}`); } } private static async cloneRepoWithoutLFSFiles() { if (fs.existsSync(path.basename(CloudRunnerFolders.repoPathAbsolute))) { RemoteClientLogger.log( `${CloudRunnerFolders.repoPathAbsolute} repo exists - skipping clone - retained workspace mode ${CloudRunner.buildParameters.retainWorkspace}`, ); return; } try { process.chdir(`${CloudRunnerFolders.repoPathAbsolute}`); RemoteClientLogger.log(`Initializing source repository for cloning with caching of LFS files`); await CloudRunnerSystem.Run(`git config --global advice.detachedHead false`); RemoteClientLogger.log(`Cloning the repository being built:`); await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge --skip -- %f"`); await CloudRunnerSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process --skip"`); await CloudRunnerSystem.Run( `git clone -q ${CloudRunnerFolders.targetBuildRepoUrl} ${path.resolve( `..`, path.basename(CloudRunnerFolders.repoPathAbsolute), )}`, ); await CloudRunnerSystem.Run(`git lfs install`); assert(fs.existsSync(`.git`), 'git folder exists'); RemoteClientLogger.log(`${CloudRunner.buildParameters.branch}`); await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.branch}`); assert(fs.existsSync(path.join(`.git`, `lfs`)), 'LFS folder should not exist before caching'); RemoteClientLogger.log(`Checked out ${CloudRunner.buildParameters.branch}`); } catch (error) { throw error; } } static replaceLargePackageReferencesWithSharedReferences() { if (CloudRunner.buildParameters.cloudRunnerIntegrationTests) { CloudRunnerLogger.log( fs.readFileSync(path.join(CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`), 'utf8'), ); } } private static async pullLatestLFS() { process.chdir(CloudRunnerFolders.repoPathAbsolute); await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge -- %f"`); await CloudRunnerSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process"`); await CloudRunnerSystem.Run(`git lfs pull`); RemoteClientLogger.log(`pulled latest LFS files`); assert(fs.existsSync(CloudRunnerFolders.lfsFolderAbsolute)); } @CliFunction(`remote-cli`, `sets up a repository, usually before a game-ci build`) static async runRemoteClientJob() { RemoteClient.handleRetainedWorkspace(); await RemoteClient.bootstrapRepository(); } static handleRetainedWorkspace() { if (!CloudRunner.buildParameters.retainWorkspace) { return; } RemoteClientLogger.log(`Retained Workspace: ${process.env.LOCKED_WORKSPACE}`); } }