Cleanup

dist/cloud-runner/cloneNoLFS.sh

@@ -2,7 +2,6 @@
 
 repoPathFull=$1
 cloneUrl=$2
-testLFSFile=$3
 
 githubSha=$GITHUB_SHA
 

dist/index.js

@@ -961,7 +961,6 @@ class CloudRunner {
         this.libraryFolderFull = `${this.projectPathFull}/Library`;
         this.cacheFolderFull = `/${buildVolumeFolder}/${cacheFolder}/${this.branchName}`;
         this.lfsDirectory = `${this.repoPathFull}/.git/lfs`;
-        this.testLFSFile = `${this.projectPathFull}/Assets/LFS_Test_File.jpg`;
         this.purgeRemoteCaching = process.env.PURGE_REMOTE_BUILDER_CACHE !== undefined;
         this.CloudRunnerBranch = process.env.CloudRunnerBranch ? `--branch "${process.env.CloudRunnerBranch}"` : '';
         this.unityBuilderRepoUrl = `https://${this.buildParams.githubToken}@github.com/game-ci/unity-builder.git`;
@@ -971,7 +970,7 @@ class CloudRunner {
         return `${this.builderPathFull}/dist/cloud-runner/handleCaching.sh "${this.cacheFolderFull}" "${this.libraryFolderFull}" "${this.lfsDirectory}" "${this.purgeRemoteCaching}"`;
     }
     static getCloneNoLFSCommand() {
-        return `${this.builderPathFull}/dist/cloud-runner/cloneNoLFS.sh "${this.repoPathFull}" "${this.targetBuildRepoUrl}" "${this.testLFSFile}"`;
+        return `${this.builderPathFull}/dist/cloud-runner/cloneNoLFS.sh "${this.repoPathFull}" "${this.targetBuildRepoUrl}"`;
     }
     static getCloneBuilder() {
         return `git clone -q ${this.CloudRunnerBranch} ${this.unityBuilderRepoUrl} ${this.builderPathFull}`;
@@ -1885,6 +1884,26 @@ const async_wait_until_1 = __importDefault(__webpack_require__(41299));
 const core = __importStar(__webpack_require__(42186));
 const k8s = __importStar(__webpack_require__(89679));
 class KubernetesStorage {
+    static createPersistentVolumeClaim(buildParameters, pvcName, kubeClient, namespace) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (buildParameters.kubeVolume) {
+                core.info(buildParameters.kubeVolume);
+                pvcName = buildParameters.kubeVolume;
+                return;
+            }
+            const pvcList = (yield kubeClient.listNamespacedPersistentVolumeClaim(namespace)).body.items.map((x) => { var _a; return (_a = x.metadata) === null || _a === void 0 ? void 0 : _a.name; });
+            core.info(`Current PVCs in namespace ${namespace}`);
+            core.info(JSON.stringify(pvcList, undefined, 4));
+            if (pvcList.includes(pvcName)) {
+                core.info(`pvc ${pvcName} already exists`);
+                core.setOutput('volume', pvcName);
+                return;
+            }
+            core.info(`Creating PVC ${pvcName} (does not exist)`);
+            const result = yield KubernetesStorage.createPVC(pvcName, buildParameters, kubeClient, namespace);
+            yield KubernetesStorage.handleResult(result, kubeClient, namespace, pvcName);
+        });
+    }
     static getPVCPhase(kubeClient, name, namespace) {
         var _a;
         return __awaiter(this, void 0, void 0, function* () {
@@ -1901,23 +1920,8 @@ class KubernetesStorage {
             });
         });
     }
-    static createPersistentVolumeClaim(buildParameters, pvcName, kubeClient, namespace) {
+    static createPVC(pvcName, buildParameters, kubeClient, namespace) {
-        var _a;
         return __awaiter(this, void 0, void 0, function* () {
-            if (buildParameters.kubeVolume) {
-                core.info(buildParameters.kubeVolume);
-                pvcName = buildParameters.kubeVolume;
-                return;
-            }
-            const pvcList = (yield kubeClient.listNamespacedPersistentVolumeClaim(namespace)).body.items.map((x) => { var _a; return (_a = x.metadata) === null || _a === void 0 ? void 0 : _a.name; });
-            core.info(`Current PVCs in namespace ${namespace}`);
-            core.info(JSON.stringify(pvcList, undefined, 4));
-            if (pvcList.includes(pvcName)) {
-                core.info(`pvc ${pvcName} already exists`);
-                core.setOutput('volume', pvcName);
-                return;
-            }
-            core.info(`Creating PVC ${pvcName} (does not exist)`);
             const pvc = new k8s.V1PersistentVolumeClaim();
             pvc.apiVersion = 'v1';
             pvc.kind = 'PersistentVolumeClaim';
@@ -1926,7 +1930,7 @@ class KubernetesStorage {
             };
             pvc.spec = {
                 accessModes: ['ReadWriteMany'],
-                volumeMode: 'Filesystem',
+                storageClassName: 'fileserver',
                 resources: {
                     requests: {
                         storage: buildParameters.kubeVolumeSize,
@@ -1934,6 +1938,12 @@ class KubernetesStorage {
                 },
             };
             const result = yield kubeClient.createNamespacedPersistentVolumeClaim(namespace, pvc);
+            return result;
+        });
+    }
+    static handleResult(result, kubeClient, namespace, pvcName) {
+        var _a;
+        return __awaiter(this, void 0, void 0, function* () {
             const name = (_a = result.body.metadata) === null || _a === void 0 ? void 0 : _a.name;
             if (!name)
                 throw new Error('failed to create PVC');
@@ -1946,6 +1956,66 @@ class KubernetesStorage {
     }
 }
 exports.default = KubernetesStorage;
+/*
+It's possible now with Cloud Filestore.
+
+First create a Filestore instance.
+
+gcloud filestore instances create nfs-server
+    --project=[PROJECT_ID]
+    --zone=us-central1-c
+    --tier=STANDARD
+    --file-share=name="vol1",capacity=1TB
+    --network=name="default",reserved-ip-range="10.0.0.0/29"
+Then create a persistent volume in GKE.
+
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: fileserver
+spec:
+  capacity:
+    storage: 1T
+  accessModes:
+  - ReadWriteMany
+  nfs:
+    path: /vol1
+    server: [IP_ADDRESS]
+[IP_ADDRESS] is available in filestore instance details.
+
+You can now request a persistent volume claim.
+
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: fileserver-claim
+spec:
+  accessModes:
+  - ReadWriteMany
+  storageClassName: "fileserver"
+  resources:
+    requests:
+      storage: 100G
+Finally, mount the volume in your pod.
+
+apiVersion: v1
+kind: Pod
+metadata:
+  name: my-pod
+spec:
+  containers:
+  - name: my container
+    image: nginx:latest
+    volumeMounts:
+    - mountPath: /workdir
+      name: mypvc
+  volumes:
+  - name: mypvc
+    persistentVolumeClaim:
+      claimName: fileserver-claim
+      readOnly: false
+Solution is detailed here : https://cloud.google.com/filestore/docs/accessing-fileshares
+*/
 
 
 /***/ }),

src/model/cloud-runner/cloud-runner.ts

@@ -26,7 +26,6 @@ class CloudRunner {
   private static libraryFolderFull: string;
   private static cacheFolderFull: string;
   private static lfsDirectory: string;
-  private static testLFSFile: string;
   private static purgeRemoteCaching: boolean;
   private static CloudRunnerBranch: string;
   private static unityBuilderRepoUrl: string;
@@ -112,7 +111,6 @@ class CloudRunner {
     this.libraryFolderFull = `${this.projectPathFull}/Library`;
     this.cacheFolderFull = `/${buildVolumeFolder}/${cacheFolder}/${this.branchName}`;
     this.lfsDirectory = `${this.repoPathFull}/.git/lfs`;
-    this.testLFSFile = `${this.projectPathFull}/Assets/LFS_Test_File.jpg`;
     this.purgeRemoteCaching = process.env.PURGE_REMOTE_BUILDER_CACHE !== undefined;
     this.CloudRunnerBranch = process.env.CloudRunnerBranch ? `--branch "${process.env.CloudRunnerBranch}"` : '';
     this.unityBuilderRepoUrl = `https://${this.buildParams.githubToken}@github.com/game-ci/unity-builder.git`;
@@ -124,7 +122,7 @@ class CloudRunner {
   }
 
   private static getCloneNoLFSCommand() {
-    return `${this.builderPathFull}/dist/cloud-runner/cloneNoLFS.sh "${this.repoPathFull}" "${this.targetBuildRepoUrl}" "${this.testLFSFile}"`;
+    return `${this.builderPathFull}/dist/cloud-runner/cloneNoLFS.sh "${this.repoPathFull}" "${this.targetBuildRepoUrl}"`;
   }
 
   private static getCloneBuilder() {

src/model/cloud-runner/kubernetes-storage.ts

@@ -4,18 +4,6 @@ import * as k8s from '@kubernetes/client-node';
 import BuildParameters from '../build-parameters';
 
 class KubernetesStorage {
-  public static async getPVCPhase(kubeClient: k8s.CoreV1Api, name: string, namespace: string) {
-    return (await kubeClient.readNamespacedPersistentVolumeClaim(name, namespace)).body.status?.phase;
-  }
-  public static async watchUntilPVCNotPending(kubeClient: k8s.CoreV1Api, name: string, namespace: string) {
-    core.info(`watch Until PVC Not Pending ${name} ${namespace}`);
-    core.info(`${await this.getPVCPhase(kubeClient, name, namespace)}`);
-    await waitUntil(async () => (await this.getPVCPhase(kubeClient, name, namespace)) !== 'Pending', {
-      timeout: 500000,
-      intervalBetweenAttempts: 15000,
-    });
-  }
-
   public static async createPersistentVolumeClaim(
     buildParameters: BuildParameters,
     pvcName: string,
@@ -38,6 +26,29 @@ class KubernetesStorage {
       return;
     }
     core.info(`Creating PVC ${pvcName} (does not exist)`);
+    const result = await KubernetesStorage.createPVC(pvcName, buildParameters, kubeClient, namespace);
+    await KubernetesStorage.handleResult(result, kubeClient, namespace, pvcName);
+  }
+
+  public static async getPVCPhase(kubeClient: k8s.CoreV1Api, name: string, namespace: string) {
+    return (await kubeClient.readNamespacedPersistentVolumeClaim(name, namespace)).body.status?.phase;
+  }
+
+  public static async watchUntilPVCNotPending(kubeClient: k8s.CoreV1Api, name: string, namespace: string) {
+    core.info(`watch Until PVC Not Pending ${name} ${namespace}`);
+    core.info(`${await this.getPVCPhase(kubeClient, name, namespace)}`);
+    await waitUntil(async () => (await this.getPVCPhase(kubeClient, name, namespace)) !== 'Pending', {
+      timeout: 500000,
+      intervalBetweenAttempts: 15000,
+    });
+  }
+
+  private static async createPVC(
+    pvcName: string,
+    buildParameters: BuildParameters,
+    kubeClient: k8s.CoreV1Api,
+    namespace: string,
+  ) {
     const pvc = new k8s.V1PersistentVolumeClaim();
     pvc.apiVersion = 'v1';
     pvc.kind = 'PersistentVolumeClaim';
@@ -46,7 +57,7 @@ class KubernetesStorage {
     };
     pvc.spec = {
       accessModes: ['ReadWriteMany'],
-      volumeMode: 'Filesystem',
+      storageClassName: 'fileserver',
       resources: {
         requests: {
           storage: buildParameters.kubeVolumeSize,
@@ -54,6 +65,15 @@ class KubernetesStorage {
       },
     };
     const result = await kubeClient.createNamespacedPersistentVolumeClaim(namespace, pvc);
+    return result;
+  }
+
+  private static async handleResult(
+    result: { response: import('http').IncomingMessage; body: k8s.V1PersistentVolumeClaim },
+    kubeClient: k8s.CoreV1Api,
+    namespace: string,
+    pvcName: string,
+  ) {
     const name = result.body.metadata?.name;
     if (!name) throw new Error('failed to create PVC');
     core.info(JSON.stringify(await kubeClient.readNamespacedPersistentVolumeClaim(name, namespace), undefined, 4));
@@ -65,3 +85,64 @@ class KubernetesStorage {
 }
 
 export default KubernetesStorage;
+
+/*
+It's possible now with Cloud Filestore.
+
+First create a Filestore instance.
+
+gcloud filestore instances create nfs-server
+    --project=[PROJECT_ID]
+    --zone=us-central1-c
+    --tier=STANDARD
+    --file-share=name="vol1",capacity=1TB
+    --network=name="default",reserved-ip-range="10.0.0.0/29"
+Then create a persistent volume in GKE.
+
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: fileserver
+spec:
+  capacity:
+    storage: 1T
+  accessModes:
+  - ReadWriteMany
+  nfs:
+    path: /vol1
+    server: [IP_ADDRESS]
+[IP_ADDRESS] is available in filestore instance details.
+
+You can now request a persistent volume claim.
+
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: fileserver-claim
+spec:
+  accessModes:
+  - ReadWriteMany
+  storageClassName: "fileserver"
+  resources:
+    requests:
+      storage: 100G
+Finally, mount the volume in your pod.
+
+apiVersion: v1
+kind: Pod
+metadata:
+  name: my-pod
+spec:
+  containers:
+  - name: my container
+    image: nginx:latest
+    volumeMounts:
+    - mountPath: /workdir
+      name: mypvc
+  volumes:
+  - name: mypvc
+    persistentVolumeClaim:
+      claimName: fileserver-claim
+      readOnly: false
+Solution is detailed here : https://cloud.google.com/filestore/docs/accessing-fileshares
+*/