e2e LFS and Library caching

dist/index.js

@@ -2153,13 +2153,14 @@ class RemoteBuilder {
             const repoPathFull = `${buildPathFull}/${repositoryFolder}`;
             const projectPathFull = `${repoPathFull}/${buildParameters.projectPath}`;
             const libraryFolderFull = `${projectPathFull}/Library`;
-            const testLFSFile = 'test-project/Assets/LFS_Test_File.jpg';
+            const lfsDirectory = `${repoPathFull}/.git/lfs`;
+            const testLFSFile = `${repoPathFull}/test-project/Assets/LFS_Test_File.jpg`;
             const repo = `https://${buildParameters.githubToken}@github.com/game-ci/unity-builder.git`;
             const repo2 = `https://${buildParameters.githubToken}@github.com/game-ci/steam-deploy.git`;
             const repo3 = `https://${buildParameters.githubToken}@github.com/${process.env.GITHUB_REPOSITORY}.git`;
             const purgeRemoteCache = process.env.PURGE_REMOTE_BUILDER_CACHE === undefined;
             const initializeSourceRepoForCaching = `${builderPathFull}/dist/remote-builder/cloneNoLFS.sh ${repoPathFull} ${repo3} $GITHUB_SHA ${testLFSFile}`;
-            const handleCaching = `${builderPathFull}/dist/remote-builder/handleCaching.sh ${cacheFolderFull} ${branchName} ${libraryFolderFull} ${purgeRemoteCache}`;
+            const handleCaching = `${builderPathFull}/dist/remote-builder/handleCaching.sh ${cacheFolderFull} ${branchName} ${libraryFolderFull} ${lfsDirectory} ${purgeRemoteCache}`;
             yield this.RemoteBuilderProviderPlatform.runBuildTask(buildUid, 'alpine/git', [
                 ` apk update -q
           apk add unzip -q
@@ -2325,7 +2326,8 @@ class RemoteBuilder {
             apk add zip
             cd Library
             zip -r lib-${buildUid}.zip .*
-            mv lib-${buildUid}.zip /${buildVolumeFolder}/${cacheFolder}/${branchName}/lib-${buildUid}.zip
+            zip -r lib-${buildUid}.zip .*
+            mv lib-${buildUid}.zip /${buildVolumeFolder}/${cacheFolder}/${branchName}/lib/lib-${buildUid}.zip
             cd ../../
             ls
             echo ' '

dist/remote-builder/cloneNoLFS.sh

@@ -13,7 +13,7 @@ git clone --filter=blob:none --no-checkout $cloneUrl $repoPathFull
 git checkout $githubSha
 echo "Checked out $githubSha"
 
-ls -l "$repoPathFull/$testLFSFile"
+ls -l "$testLFSFile"
 
 echo ' '
 echo 'Tree of cloned target repository:'

dist/remote-builder/handleCaching.sh

@@ -3,7 +3,8 @@
 cacheFolderFull=$1
 branchName=$2
 libraryFolderFull=$3
-purgeRemoteBuilderCache=$4
+gitLFSDestinationFolder=$4
+purgeRemoteBuilderCache=$5
 
 echo " "
 echo "Caching starting, parameters:"
@@ -15,22 +16,14 @@ echo "$purgeRemoteBuilderCache"
 cacheFolderWithBranch="$cacheFolderFull/$branchName"
 
 echo " "
-# handle library cache
+
-if [ ! -d "$cacheFolderFull" ]; then
+mkdir -p "$cacheFolderWithBranch/lib"
-  echo "creating new cache folder $cacheFolderFull"
+mkdir -p "$cacheFolderWithBranch/lfs"
-  mkdir "$cacheFolderFull"
-  if [ ! -d "$cacheFolderWithBranch" ]; then
-    echo "creating new cache branch folder for: $cacheFolderWithBranch"
-    mkdir "$cacheFolderWithBranch"
-  else
-    echo "cache branch folder already exists for: $cacheFolderWithBranch"
-  fi
-else
-  echo "cache folder already exists $cacheFolderFull"
-fi
 
 echo "Library cache for branch: $branchName"
 ls -lh "$cacheFolderWithBranch"
+ls -lh "$cacheFolderWithBranch/lib"
+ls -lh "$cacheFolderWithBranch/lfs"
 echo ''
 
 if [ -d "$libraryFolderFull" ]; then
@@ -42,16 +35,24 @@ fi
 echo "Checking cache"
 
 # Restore library cache
-latest=$(ls -t "$cacheFolderWithBranch" | egrep -i -e '\\.zip$' | head -1)
+latest=$(ls -t "$cacheFolderWithBranch/lib" | egrep -i -e '\\.zip$' | head -1)
 
 if [ ! -z "$latest" ]; then
   echo "Library cache exists from build $latest from $branchName"
   echo 'Creating empty Library folder for cache'
   mkdir "$libraryFolderFull"
-  unzip -q "$cacheFolderWithBranch/$latest" -d "$libraryFolderFull"
+  unzip -q "$cacheFolderWithBranch/lib/$latest" -d "$libraryFolderFull"
 fi
 
 # Restore LFS cache
+latest=$(ls -t "$cacheFolderWithBranch/lfs" | egrep -i -e '\\.zip$' | head -1)
+
+if [ ! -z "$latest" ]; then
+  echo "Library cache exists from build $latest from $branchName"
+  echo 'Creating empty Library folder for cache'
+  mkdir "$libraryFolderFull"
+  unzip -q "$cacheFolderWithBranch/lfs/$latest" -d "$gitLFSDestinationFolder"
+fi
 
 # purge cache
 if [ "$purgeRemoteBuilderCache" == "true" ]; then

src/model/remote-builder/remote-builder.ts

@@ -95,7 +95,8 @@ class RemoteBuilder {
     const projectPathFull = `${repoPathFull}/${buildParameters.projectPath}`;
     const libraryFolderFull = `${projectPathFull}/Library`;
 
-    const testLFSFile = 'test-project/Assets/LFS_Test_File.jpg';
+    const lfsDirectory = `${repoPathFull}/.git/lfs`;
+    const testLFSFile = `${repoPathFull}/test-project/Assets/LFS_Test_File.jpg`;
 
     const repo = `https://${buildParameters.githubToken}@github.com/game-ci/unity-builder.git`;
     const repo2 = `https://${buildParameters.githubToken}@github.com/game-ci/steam-deploy.git`;
@@ -103,7 +104,7 @@ class RemoteBuilder {
 
     const purgeRemoteCache = process.env.PURGE_REMOTE_BUILDER_CACHE === undefined;
     const initializeSourceRepoForCaching = `${builderPathFull}/dist/remote-builder/cloneNoLFS.sh ${repoPathFull} ${repo3} $GITHUB_SHA ${testLFSFile}`;
-    const handleCaching = `${builderPathFull}/dist/remote-builder/handleCaching.sh ${cacheFolderFull} ${branchName} ${libraryFolderFull} ${purgeRemoteCache}`;
+    const handleCaching = `${builderPathFull}/dist/remote-builder/handleCaching.sh ${cacheFolderFull} ${branchName} ${libraryFolderFull} ${lfsDirectory} ${purgeRemoteCache}`;
     await this.RemoteBuilderProviderPlatform.runBuildTask(
       buildUid,
       'alpine/git',
@@ -304,7 +305,7 @@ class RemoteBuilder {
             apk add zip
             cd Library
             zip -r lib-${buildUid}.zip .*
-            mv lib-${buildUid}.zip /${buildVolumeFolder}/${cacheFolder}/${branchName}/lib-${buildUid}.zip
+            mv lib-${buildUid}.zip /${buildVolumeFolder}/${cacheFolder}/${branchName}/lib/lib-${buildUid}.zip
             cd ../../
             ls
             echo ' '