Code Coverage Support (#182)

* Added basic framework for enable code coverage

* Added basic coverage results building and combination

* fixed ENABLE_CODE_COVERAGE to be `true` or `false`

* Added code coverage project to manifest

* Updated to add more tests for code coverage build

* Updated coverage parameter documentation

* Fixed small syntax error

* Enabled code coverage flag for code coverage tests

* Fixed error in test file build

* Updated project run settings

* Fixed error when creating combined code coverage results

* Updated testing workflows

* updated test workflows

* Updated parameters and added tests

* Updated tests and bash script for running

* Updated run_tests.sh script to simplfy some parameters

* Updated run_tests to remove incorrect ';'

* Updated run_tests script

* Fixed small syntax error

* Fixed for loop in run_tests.sh

* Updated run_tests syntax error for '=' operator

* Fixed runTests variable assignment

* Fixed parameters for running tests via bash

* Corrected bash arguments

* Updated test cases in main.yml

* Updated parameter names and default values for code coverage

* Fixed broken paths for coverage results upload in main.yml

* Corrected names of coverage results artifacts

.github/workflows/main.yml

@@ -105,6 +105,7 @@ jobs:
           projectPath: ${{ matrix.projectPath }}
           unityVersion: ${{ matrix.unityVersion }}
           testMode: all
+          coverageOptions: 'enableCyclomaticComplexity;generateHtmlReport;generateBadgeReport;assemblyFilters:+MyScripts'
           # Test implicit artifactsPath, by not setting it
 
       # Upload artifacts
@@ -115,6 +116,14 @@ jobs:
           path: ${{ steps.allTests.outputs.artifactsPath }}
           retention-days: 14
 
+      # Upload coverage
+      - name: Upload coverage results
+        uses: actions/upload-artifact@v3
+        with:
+          name: Coverage results (all)
+          path: ${{ steps.allTests.outputs.coveragePath }}
+          retention-days: 14
+
   testRunnerInEditMode:
     name: Test edit mode 📝
     runs-on: ubuntu-latest
@@ -162,6 +171,14 @@ jobs:
           path: ${{ steps.editMode.outputs.artifactsPath }}
           retention-days: 14
 
+      # Upload coverage
+      - name: Upload coverage results
+        uses: actions/upload-artifact@v3
+        with:
+          name: Coverage results (edit mode)
+          path: ${{ steps.editMode.outputs.coveragePath }}
+          retention-days: 14
+
   testRunnerInPlayMode:
     name: Test play mode 📺
     runs-on: ubuntu-latest
@@ -209,6 +226,14 @@ jobs:
           path: ${{ steps.playMode.outputs.artifactsPath }}
           retention-days: 14
 
+      # Upload coverage
+      - name: Upload coverage results
+        uses: actions/upload-artifact@v3
+        with:
+          name: Coverage results (play mode)
+          path: ${{ steps.playMode.outputs.coveragePath }}
+          retention-days: 14
+
   testEachModeSequentially:
     name: Test each mode sequentially 👩‍👩‍👧‍👦 # don't try this at home (it's much slower)
     runs-on: ubuntu-latest

action.yml

@@ -20,6 +20,10 @@ inputs:
     required: false
     default: 'all'
     description: 'The type of tests to be run by the test runner.'
+  coverageOptions:
+    required: false
+    default: 'enableCyclomaticComplexity;generateHtmlReport;generateBadgeReport'
+    description: 'Optional coverage parameters for the -coverageOptions argument.'
   artifactsPath:
     required: false
     default: 'artifacts'
@@ -46,7 +50,9 @@ inputs:
     description: 'Name for the check run that is created when a github token is provided.'
 outputs:
   artifactsPath:
-    description: 'Path where the artifacts are stored'
+    description: 'Path where the artifacts are stored.'
+  coveragePath:
+    description: 'Path where the code coverage results are stored.'
 branding:
   icon: 'box'
   color: 'gray-dark'

dist/index.js

@@ -42,7 +42,7 @@ function run() {
         try {
             model_1.Action.checkCompatibility();
             const { workspace, actionFolder } = model_1.Action;
-            const { editorVersion, customImage, projectPath, customParameters, testMode, artifactsPath, useHostNetwork, sshAgent, gitPrivateToken, githubToken, checkName, } = model_1.Input.getFromUser();
+            const { editorVersion, customImage, projectPath, customParameters, testMode, coverageOptions, artifactsPath, useHostNetwork, sshAgent, gitPrivateToken, githubToken, checkName, } = model_1.Input.getFromUser();
             const baseImage = new model_1.ImageTag({ editorVersion, customImage });
             const runnerTemporaryPath = process.env.RUNNER_TEMP;
             try {
@@ -53,6 +53,7 @@ function run() {
                     projectPath,
                     customParameters,
                     testMode,
+                    coverageOptions,
                     artifactsPath,
                     useHostNetwork,
                     sshAgent,
@@ -63,6 +64,7 @@ function run() {
             }
             finally {
                 yield model_1.Output.setArtifactsPath(artifactsPath);
+                yield model_1.Output.setCoveragePath('CodeCoverage');
             }
             if (githubToken) {
                 const failedTestCount = yield model_1.ResultsCheck.createCheck(artifactsPath, githubToken, checkName);
@@ -152,13 +154,14 @@ const path_1 = __importDefault(__nccwpck_require__(1017));
 const Docker = {
     run(image, parameters, silent = false) {
         return __awaiter(this, void 0, void 0, function* () {
-            const { actionFolder, editorVersion, workspace, projectPath, customParameters, testMode, artifactsPath, useHostNetwork, sshAgent, gitPrivateToken, githubToken, runnerTemporaryPath, } = parameters;
+            const { actionFolder, editorVersion, workspace, projectPath, customParameters, testMode, coverageOptions, artifactsPath, useHostNetwork, sshAgent, gitPrivateToken, githubToken, runnerTemporaryPath, } = parameters;
             const githubHome = path_1.default.join(runnerTemporaryPath, '_github_home');
             if (!(0, fs_1.existsSync)(githubHome))
                 (0, fs_1.mkdirSync)(githubHome);
             const githubWorkflow = path_1.default.join(runnerTemporaryPath, '_github_workflow');
             if (!(0, fs_1.existsSync)(githubWorkflow))
                 (0, fs_1.mkdirSync)(githubWorkflow);
+            const testPlatforms = (testMode === 'all' ? ['playmode', 'editmode', 'COMBINE_RESULTS'] : [testMode]).join(';');
             const command = `docker run \
         --workdir /github/workspace \
         --rm \
@@ -170,7 +173,9 @@ const Docker = {
         --env UNITY_VERSION="${editorVersion}" \
         --env PROJECT_PATH="${projectPath}" \
         --env CUSTOM_PARAMETERS="${customParameters}" \
-        --env TEST_MODE="${testMode}" \
+        --env TEST_PLATFORMS="${testPlatforms}" \
+        --env COVERAGE_OPTIONS="${coverageOptions}" \
+        --env COVERAGE_RESULTS_PATH="CodeCoverage" \
         --env ARTIFACTS_PATH="${artifactsPath}" \
         --env GITHUB_REF \
         --env GITHUB_SHA \
@@ -385,6 +390,7 @@ const Input = {
         const rawProjectPath = (0, core_1.getInput)('projectPath') || '.';
         const customParameters = (0, core_1.getInput)('customParameters') || '';
         const testMode = ((0, core_1.getInput)('testMode') || 'all').toLowerCase();
+        const coverageOptions = (0, core_1.getInput)('coverageOptions') || '';
         const rawArtifactsPath = (0, core_1.getInput)('artifactsPath') || 'artifacts';
         const rawUseHostNetwork = (0, core_1.getInput)('useHostNetwork') || 'false';
         const sshAgent = (0, core_1.getInput)('sshAgent') || '';
@@ -416,6 +422,7 @@ const Input = {
             projectPath,
             customParameters,
             testMode,
+            coverageOptions,
             artifactsPath,
             useHostNetwork,
             sshAgent,
@@ -471,6 +478,11 @@ const Output = {
             yield core.setOutput('artifactsPath', artifactsPath);
         });
     },
+    setCoveragePath(coveragePath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield core.setOutput('coveragePath', coveragePath);
+        });
+    },
 };
 exports["default"] = Output;
 

dist/steps/run_tests.sh

@@ -14,27 +14,18 @@ echo "Using project path \"$UNITY_PROJECT_PATH\"."
 echo "Using artifacts path \"$ARTIFACTS_PATH\" to save test results."
 FULL_ARTIFACTS_PATH=$GITHUB_WORKSPACE/$ARTIFACTS_PATH
 
+#
+# Set and display the coverage results path
+#
+
+echo "Using coverage results path \"$COVERAGE_RESULTS_PATH\" to save test coverage results."
+FULL_COVERAGE_RESULTS_PATH=$GITHUB_WORKSPACE/$COVERAGE_RESULTS_PATH
+
 #
 # Display custom parameters
 #
-echo "Using custom parameters $CUSTOM_PARAMETERS."
 
-# Set the modes for testing
+echo "Using custom parameters $CUSTOM_PARAMETERS."
-case $TEST_MODE in
-  editmode)
-    echo "Edit mode selected for testing."
-    EDIT_MODE=true
-    ;;
-  playmode)
-    echo "Play mode selected for testing."
-    PLAY_MODE=true
-    ;;
-  *)
-    echo "All modes selected for testing."
-    EDIT_MODE=true
-    PLAY_MODE=true
-    ;;
-esac
 
 # The following tests are 2019 mode (requires Unity 2019.2.11f1 or later)
 # Reference: https://docs.unity3d.com/2019.3/Documentation/Manual/CommandLineArguments.html
@@ -65,118 +56,61 @@ echo ""
 ls -alh $UNITY_PROJECT_PATH
 
 #
-# Testing in EditMode
+# Testing for each platform
 #
-EDIT_MODE_EXIT_CODE=0
+for platform in ${TEST_PLATFORMS//;/ }; do
-if [ "$EDIT_MODE" = "true" ]; then
   echo ""
   echo "###########################"
-  echo "#   Testing in EditMode   #"
+  echo "#   Testing in $platform  #"
   echo "###########################"
   echo ""
+
+  if [[ "$platform" != "COMBINE_RESULTS" ]]; then
+    runTests="-runTests -testPlatform $platform -testResults $FULL_ARTIFACTS_PATH/$platform-results.xml"
+  else
+    runTests="-quit"
+  fi
+
   unity-editor \
     -batchmode \
-    -logFile "$FULL_ARTIFACTS_PATH/editmode.log" \
+    -logFile "$FULL_ARTIFACTS_PATH/$platform.log" \
     -projectPath "$UNITY_PROJECT_PATH" \
-    -runTests \
+    -coverageResultsPath "$FULL_COVERAGE_RESULTS_PATH" \
-    -testPlatform editmode \
+    $runTests \
-    -testResults "$FULL_ARTIFACTS_PATH/editmode-results.xml" \
+    -enableCodeCoverage \
+    -debugCodeOptimization \
+    -coverageOptions "$COVERAGE_OPTIONS" \
     $CUSTOM_PARAMETERS
 
   # Catch exit code
-  EDIT_MODE_EXIT_CODE=$?
+  TEST_EXIT_CODE=$?
 
   # Print unity log output
-  cat "$FULL_ARTIFACTS_PATH/editmode.log"
+  cat "$FULL_ARTIFACTS_PATH/$platform.log"
 
   # Display results
-  if [ $EDIT_MODE_EXIT_CODE -eq 0 ]; then
+  if [ $TEST_EXIT_CODE -eq 0 ]; then
     echo "Run succeeded, no failures occurred";
-  elif [ $EDIT_MODE_EXIT_CODE -eq 2 ]; then
+  elif [ $TEST_EXIT_CODE -eq 2 ]; then
     echo "Run succeeded, some tests failed";
-  elif [ $EDIT_MODE_EXIT_CODE -eq 3 ]; then
+  elif [ $TEST_EXIT_CODE -eq 3 ]; then
     echo "Run failure (other failure)";
   else
-    echo "Unexpected exit code $EDIT_MODE_EXIT_CODE";
+    echo "Unexpected exit code $TEST_EXIT_CODE";
   fi
-fi
 
-#
+  if [ $TEST_EXIT_CODE -ne 0 ]; then
-# Testing in PlayMode
+    TEST_RUNNER_EXIT_CODE=$TEST_EXIT_CODE
-#
-PLAY_MODE_EXIT_CODE=0
-if [ "$PLAY_MODE" = "true" ]; then
-  echo ""
-  echo "###########################"
-  echo "#   Testing in PlayMode   #"
-  echo "###########################"
-  echo ""
-  unity-editor \
-    -batchmode \
-    -logFile "$FULL_ARTIFACTS_PATH/playmode.log" \
-    -projectPath "$UNITY_PROJECT_PATH" \
-    -runTests \
-    -testPlatform playmode \
-    -testResults "$FULL_ARTIFACTS_PATH/playmode-results.xml" \
-    $CUSTOM_PARAMETERS
-
-  # Catch exit code
-  PLAY_MODE_EXIT_CODE=$?
-
-  # Print unity log output
-  cat "$FULL_ARTIFACTS_PATH/playmode.log"
-
-  # Display results
-  if [ $PLAY_MODE_EXIT_CODE -eq 0 ]; then
-    echo "Run succeeded, no failures occurred";
-  elif [ $PLAY_MODE_EXIT_CODE -eq 2 ]; then
-    echo "Run succeeded, some tests failed";
-  elif [ $PLAY_MODE_EXIT_CODE -eq 3 ]; then
-    echo "Run failure (other failure)";
-  else
-    echo "Unexpected exit code $PLAY_MODE_EXIT_CODE";
   fi
-fi
 
-#
-# Results
-#
-
-echo ""
-echo "###########################"
-echo "#    Project directory    #"
-echo "###########################"
-echo ""
-ls -alh $UNITY_PROJECT_PATH
-
-if [ "$EDIT_MODE" = "true" ]; then
   echo ""
   echo "###########################"
-  echo "#    Edit Mode Results    #"
+  echo "#    $platform Results    #"
   echo "###########################"
   echo ""
-  cat "$FULL_ARTIFACTS_PATH/editmode-results.xml"
-  cat "$FULL_ARTIFACTS_PATH/editmode-results.xml" | grep test-run | grep Passed
-fi
 
-if [ "$PLAY_MODE" = "true" ]; then
+  if [[ "$platform" != "COMBINE_RESULTS" ]]; then
-  echo ""
+    cat "$FULL_ARTIFACTS_PATH/$platform-results.xml"
-  echo "###########################"
+    cat "$FULL_ARTIFACTS_PATH/$platform-results.xml" | grep test-run | grep Passed
-  echo "#    Play Mode Results    #"
+  fi
-  echo "###########################"
+done
-  echo ""
-  cat "$FULL_ARTIFACTS_PATH/playmode-results.xml"
-  cat "$FULL_ARTIFACTS_PATH/playmode-results.xml" | grep test-run | grep Passed
-fi
-
-#
-# Exit
-#
-
-if [ $EDIT_MODE_EXIT_CODE -gt 0 ]; then
-  TEST_RUNNER_EXIT_CODE=$EDIT_MODE_EXIT_CODE
-fi
-
-if [ $PLAY_MODE_EXIT_CODE -gt 0 ]; then
-  TEST_RUNNER_EXIT_CODE=$PLAY_MODE_EXIT_CODE
-fi

src/index.ts

@@ -12,6 +12,7 @@ async function run() {
       projectPath,
       customParameters,
       testMode,
+      coverageOptions,
       artifactsPath,
       useHostNetwork,
       sshAgent,
@@ -30,6 +31,7 @@ async function run() {
         projectPath,
         customParameters,
         testMode,
+        coverageOptions,
         artifactsPath,
         useHostNetwork,
         sshAgent,
@@ -39,6 +41,7 @@ async function run() {
       });
     } finally {
       await Output.setArtifactsPath(artifactsPath);
+      await Output.setCoveragePath('CodeCoverage');
     }
 
     if (githubToken) {

src/model/docker.ts

@@ -11,6 +11,7 @@ const Docker = {
       projectPath,
       customParameters,
       testMode,
+      coverageOptions,
       artifactsPath,
       useHostNetwork,
       sshAgent,
@@ -23,6 +24,9 @@ const Docker = {
     if (!existsSync(githubHome)) mkdirSync(githubHome);
     const githubWorkflow = path.join(runnerTemporaryPath, '_github_workflow');
     if (!existsSync(githubWorkflow)) mkdirSync(githubWorkflow);
+    const testPlatforms = (
+      testMode === 'all' ? ['playmode', 'editmode', 'COMBINE_RESULTS'] : [testMode]
+    ).join(';');
 
     const command = `docker run \
         --workdir /github/workspace \
@@ -35,7 +39,9 @@ const Docker = {
         --env UNITY_VERSION="${editorVersion}" \
         --env PROJECT_PATH="${projectPath}" \
         --env CUSTOM_PARAMETERS="${customParameters}" \
-        --env TEST_MODE="${testMode}" \
+        --env TEST_PLATFORMS="${testPlatforms}" \
+        --env COVERAGE_OPTIONS="${coverageOptions}" \
+        --env COVERAGE_RESULTS_PATH="CodeCoverage" \
         --env ARTIFACTS_PATH="${artifactsPath}" \
         --env GITHUB_REF \
         --env GITHUB_SHA \

src/model/input.ts

@@ -19,6 +19,7 @@ const Input = {
     const rawProjectPath = getInput('projectPath') || '.';
     const customParameters = getInput('customParameters') || '';
     const testMode = (getInput('testMode') || 'all').toLowerCase();
+    const coverageOptions = getInput('coverageOptions') || '';
     const rawArtifactsPath = getInput('artifactsPath') || 'artifacts';
     const rawUseHostNetwork = getInput('useHostNetwork') || 'false';
     const sshAgent = getInput('sshAgent') || '';
@@ -57,6 +58,7 @@ const Input = {
       projectPath,
       customParameters,
       testMode,
+      coverageOptions,
       artifactsPath,
       useHostNetwork,
       sshAgent,

src/model/output.test.ts

@@ -6,4 +6,19 @@ describe('Output', () => {
       await expect(Output.setArtifactsPath('')).resolves.not.toThrow();
     });
   });
+  describe('setCoveragePath', () => {
+    it('does not throw', async () => {
+      await expect(Output.setCoveragePath('')).resolves.not.toThrow();
+      await expect(Output.setCoveragePath('artifacts')).resolves.not.toThrow();
+      await expect(Output.setCoveragePath('coverage')).resolves.not.toThrow();
+      await expect(Output.setCoveragePath('CodeCoverage')).resolves.not.toThrow();
+      await expect(Output.setCoveragePath('./artifacts')).resolves.not.toThrow();
+      await expect(Output.setCoveragePath('./coverage')).resolves.not.toThrow();
+      await expect(Output.setCoveragePath('./CodeCoverage')).resolves.not.toThrow();
+      await expect(Output.setCoveragePath('./artifacts/coverage')).resolves.not.toThrow();
+      await expect(Output.setCoveragePath('./coverage/')).resolves.not.toThrow();
+      await expect(Output.setCoveragePath('./CodeCoverage/')).resolves.not.toThrow();
+      await expect(Output.setCoveragePath('./artifacts/coverage/')).resolves.not.toThrow();
+    });
+  });
 });

src/model/output.ts

@@ -4,6 +4,9 @@ const Output = {
   async setArtifactsPath(artifactsPath) {
     await core.setOutput('artifactsPath', artifactsPath);
   },
+  async setCoveragePath(coveragePath) {
+    await core.setOutput('coveragePath', coveragePath);
+  },
 };
 
 export default Output;

unity-project-with-correct-tests/Packages/manifest.json

@@ -6,6 +6,7 @@
     "com.unity.ide.vscode": "1.1.2",
     "com.unity.package-manager-ui": "2.2.0",
     "com.unity.test-framework": "1.0.13",
+    "com.unity.testtools.codecoverage": "1.0.1",
     "com.unity.textmeshpro": "2.0.1",
     "com.unity.timeline": "1.1.0",
     "com.unity.ugui": "1.0.0",