actions
/
unity-test-runner
mirror of https://github.com/game-ci/unity-test-runner.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Small results-check refactor for debugging (#104) 

* Small results-check refactor for debugging

* Fix #103
pull/109/head
David Finol 2021-03-06 00:55:03 -06:00 committed by GitHub
parent a195213baa
commit fffbf11b2a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 254 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
action/index.js
View File

File diff suppressed because one or more lines are too long

81
artifacts/editmode-results.xml 100644
View File

@ -0,0 +1,81 @@
<?xml version="1.0" encoding="utf-8"?>
<test-run id="2" testcasecount="6" result="Failed(Child)" total="6" passed="2" failed="2" inconclusive="0" skipped="2" asserts="0" engine-version="3.5.0.0" clr-version="4.0.30319.42000" start-time="2021-01-19 20:51:34Z" end-time="2021-01-19 20:51:34Z" duration="0.1168037">
<test-suite type="TestSuite" id="1012" name="sample-project" fullname="sample-project" runstate="Runnable" testcasecount="6" result="Failed" site="Child" start-time="2021-01-19 20:51:34Z" end-time="2021-01-19 20:51:34Z" duration="0.116804" total="6" passed="2" failed="2" inconclusive="0" skipped="2" asserts="0">
<properties />
<failure>
<message><![CDATA[One or more child tests had errors]]></message>
</failure>
<test-suite type="Assembly" id="1020" name="Editor.dll" fullname="/github/workspace/unity-project/Library/ScriptAssemblies/Editor.dll" runstate="Runnable" testcasecount="6" result="Failed" site="Child" start-time="2021-01-19 20:51:34Z" end-time="2021-01-19 20:51:34Z" duration="0.087946" total="6" passed="2" failed="2" inconclusive="0" skipped="2" asserts="0">
<properties>
<property name="_PID" value="78" />
<property name="_APPDOMAIN" value="Unity Child Domain" />
<property name="platform" value="EditMode" />
</properties>
<failure>
<message><![CDATA[One or more child tests had errors]]></message>
</failure>
<test-suite type="TestSuite" id="1021" name="Editor" fullname="Editor" runstate="Runnable" testcasecount="6" result="Failed" site="Child" start-time="2021-01-19 20:51:34Z" end-time="2021-01-19 20:51:34Z" duration="0.086212" total="6" passed="2" failed="2" inconclusive="0" skipped="2" asserts="0">
<properties />
<failure>
<message><![CDATA[One or more child tests had errors]]></message>
</failure>
<test-suite type="TestFixture" id="1013" name="EditorModeTest" fullname="Editor.EditorModeTest" classname="Editor.EditorModeTest" runstate="Runnable" testcasecount="6" result="Failed" site="Child" start-time="2021-01-19 20:51:34Z" end-time="2021-01-19 20:51:34Z" duration="0.076416" total="6" passed="2" failed="2" inconclusive="0" skipped="2" asserts="0">
<properties />
<failure>
<message><![CDATA[One or more child tests had errors]]></message>
</failure>
<output><![CDATA[You are trying to create a MonoBehaviour using the 'new' keyword. This is not allowed. MonoBehaviours can only be added using AddComponent(). Alternatively, your script can inherit from ScriptableObject or no base class at all
]]></output>
<test-case id="1016" name="FailedTest" fullname="Editor.EditorModeTest.FailedTest" methodname="FailedTest" classname="Editor.EditorModeTest" runstate="Runnable" seed="330279882" result="Failed" start-time="2021-01-19 20:51:34Z" end-time="2021-01-19 20:51:34Z" duration="0.023093" asserts="0">
<properties />
<failure>
<message><![CDATA[ Expected: True
But was: False
]]></message>
<stack-trace><![CDATA[at Editor.EditorModeTest.FailedTest () [0x00000] in /github/workspace/unity-project/Assets/Editor/EditorModeTest.cs:21
]]></stack-trace>
</failure>
</test-case>
<test-case id="1019" name="FailedUnityTest" fullname="Editor.EditorModeTest.FailedUnityTest" methodname="FailedUnityTest" classname="Editor.EditorModeTest" runstate="Runnable" seed="347277877" result="Failed" start-time="2021-01-19 20:51:34Z" end-time="2021-01-19 20:51:34Z" duration="0.014203" asserts="0">
<properties>
<property name="_JOINTYPE" value="UnityCombinatorial" />
</properties>
<failure>
<message><![CDATA[ Expected: True
But was: False
]]></message>
<stack-trace><![CDATA[at Editor.EditorModeTest+<FailedUnityTest>d__5.MoveNext () [0x0002e] in /github/workspace/unity-project/Assets/Editor/EditorModeTest.cs:40
at UnityEngine.TestTools.TestEnumerator+<Execute>d__6.MoveNext () [0x00038] in /github/workspace/unity-project/Library/PackageCache/com.unity.test-framework@1.1.19/UnityEngine.TestRunner/NUnitExtensions/Attributes/TestEnumerator.cs:36
]]></stack-trace>
</failure>
</test-case>
<test-case id="1015" name="IgnoredTest" fullname="Editor.EditorModeTest.IgnoredTest" methodname="IgnoredTest" classname="Editor.EditorModeTest" runstate="Ignored" seed="1319288303" result="Skipped" label="Ignored" start-time="2021-01-19 20:51:34Z" end-time="2021-01-19 20:51:34Z" duration="0.000218" asserts="0">
<properties>
<property name="_SKIPREASON" value="ignore" />
</properties>
<reason>
<message><![CDATA[ignore]]></message>
</reason>
</test-case>
<test-case id="1018" name="IgnoredUnityTest" fullname="Editor.EditorModeTest.IgnoredUnityTest" methodname="IgnoredUnityTest" classname="Editor.EditorModeTest" runstate="Ignored" seed="2034877647" result="Skipped" label="Ignored" start-time="2021-01-19 20:51:34Z" end-time="2021-01-19 20:51:34Z" duration="0.000004" asserts="0">
<properties>
<property name="_JOINTYPE" value="UnityCombinatorial" />
<property name="_SKIPREASON" value="ignore" />
</properties>
<reason>
<message><![CDATA[ignore]]></message>
</reason>
</test-case>
<test-case id="1014" name="PassedTest" fullname="Editor.EditorModeTest.PassedTest" methodname="PassedTest" classname="Editor.EditorModeTest" runstate="Runnable" seed="387558551" result="Passed" start-time="2021-01-19 20:51:34Z" end-time="2021-01-19 20:51:34Z" duration="0.000366" asserts="0">
<properties />
</test-case>
<test-case id="1017" name="PassedUnityTest" fullname="Editor.EditorModeTest.PassedUnityTest" methodname="PassedUnityTest" classname="Editor.EditorModeTest" runstate="Runnable" seed="1069930397" result="Passed" start-time="2021-01-19 20:51:34Z" end-time="2021-01-19 20:51:34Z" duration="0.007903" asserts="0">
<properties>
<property name="_JOINTYPE" value="UnityCombinatorial" />
</properties>
</test-case>
</test-suite>
</test-suite>
</test-suite>
</test-suite>
</test-run>

111
artifacts/playmode-results.xml 100644
View File

@ -0,0 +1,111 @@
<?xml version="1.0" encoding="utf-8"?>
<test-run id="2" testcasecount="8" result="Failed(Child)" total="8" passed="2" failed="4" inconclusive="0" skipped="2" asserts="0" engine-version="3.5.0.0" clr-version="4.0.30319.42000" start-time="2021-01-19 20:51:44Z" end-time="2021-01-19 20:51:44Z" duration="0.1055695">
<test-suite type="TestSuite" id="1000" name="sample-project" fullname="sample-project" runstate="Runnable" testcasecount="8" result="Failed" site="Child" start-time="2021-01-19 20:51:44Z" end-time="2021-01-19 20:51:44Z" duration="0.105570" total="8" passed="2" failed="4" inconclusive="0" skipped="2" asserts="0">
<properties />
<failure>
<message><![CDATA[One or more child tests had errors]]></message>
</failure>
<test-suite type="Assembly" id="1012" name="Assembly-CSharp.dll" fullname="/github/workspace/unity-project/Library/ScriptAssemblies/Assembly-CSharp.dll" runstate="Runnable" testcasecount="8" result="Failed" site="Child" start-time="2021-01-19 20:51:44Z" end-time="2021-01-19 20:51:44Z" duration="0.088008" total="8" passed="2" failed="4" inconclusive="0" skipped="2" asserts="0">
<properties>
<property name="_PID" value="474" />
<property name="_APPDOMAIN" value="Unity Child Domain" />
<property name="platform" value="PlayMode" />
</properties>
<failure>
<message><![CDATA[One or more child tests had errors]]></message>
</failure>
<test-suite type="TestSuite" id="1013" name="Tests" fullname="Tests" runstate="Runnable" testcasecount="8" result="Failed" site="Child" start-time="2021-01-19 20:51:44Z" end-time="2021-01-19 20:51:44Z" duration="0.084864" total="8" passed="2" failed="4" inconclusive="0" skipped="2" asserts="0">
<properties />
<failure>
<message><![CDATA[One or more child tests had errors]]></message>
</failure>
<test-suite type="TestFixture" id="1001" name="PlayModeTest" fullname="Tests.PlayModeTest" classname="Tests.PlayModeTest" runstate="Runnable" testcasecount="6" result="Failed" site="Child" start-time="2021-01-19 20:51:44Z" end-time="2021-01-19 20:51:44Z" duration="0.074407" total="6" passed="2" failed="2" inconclusive="0" skipped="2" asserts="0">
<properties />
<failure>
<message><![CDATA[One or more child tests had errors]]></message>
</failure>
<test-case id="1004" name="FailedTest" fullname="Tests.PlayModeTest.FailedTest" methodname="FailedTest" classname="Tests.PlayModeTest" runstate="Runnable" seed="1067965392" result="Failed" start-time="2021-01-19 20:51:44Z" end-time="2021-01-19 20:51:44Z" duration="0.034377" asserts="0">
<properties />
<failure>
<message><![CDATA[ Expected: True
But was: False
]]></message>
<stack-trace><![CDATA[at Tests.PlayModeTest.FailedTest () [0x00000] in /github/workspace/unity-project/Assets/Tests/PlayModeTest.cs:20
]]></stack-trace>
</failure>
</test-case>
<test-case id="1007" name="FailedUnityTest" fullname="Tests.PlayModeTest.FailedUnityTest" methodname="FailedUnityTest" classname="Tests.PlayModeTest" runstate="Runnable" seed="318345342" result="Failed" start-time="2021-01-19 20:51:44Z" end-time="2021-01-19 20:51:44Z" duration="0.009524" asserts="0">
<properties>
<property name="_JOINTYPE" value="UnityCombinatorial" />
</properties>
<failure>
<message><![CDATA[ Expected: True
But was: False
]]></message>
<stack-trace><![CDATA[at Tests.PlayModeTest+<FailedUnityTest>d__5.MoveNext () [0x0002e] in /github/workspace/unity-project/Assets/Tests/PlayModeTest.cs:39
at UnityEngine.TestTools.TestEnumerator+<Execute>d__6.MoveNext () [0x00038] in /github/workspace/unity-project/Library/PackageCache/com.unity.test-framework@1.1.19/UnityEngine.TestRunner/NUnitExtensions/Attributes/TestEnumerator.cs:36
]]></stack-trace>
</failure>
</test-case>
<test-case id="1003" name="IgnoredTest" fullname="Tests.PlayModeTest.IgnoredTest" methodname="IgnoredTest" classname="Tests.PlayModeTest" runstate="Ignored" seed="1914466070" result="Skipped" label="Ignored" start-time="2021-01-19 20:51:44Z" end-time="2021-01-19 20:51:44Z" duration="0.000211" asserts="0">
<properties>
<property name="_SKIPREASON" value="ignore" />
</properties>
<reason>
<message><![CDATA[ignore]]></message>
</reason>
</test-case>
<test-case id="1006" name="IgnoredUnityTest" fullname="Tests.PlayModeTest.IgnoredUnityTest" methodname="IgnoredUnityTest" classname="Tests.PlayModeTest" runstate="Ignored" seed="475291067" result="Skipped" label="Ignored" start-time="2021-01-19 20:51:44Z" end-time="2021-01-19 20:51:44Z" duration="0.000004" asserts="0">
<properties>
<property name="_JOINTYPE" value="UnityCombinatorial" />
<property name="_SKIPREASON" value="ignore" />
</properties>
<reason>
<message><![CDATA[ignore]]></message>
</reason>
</test-case>
<test-case id="1002" name="PassedTest" fullname="Tests.PlayModeTest.PassedTest" methodname="PassedTest" classname="Tests.PlayModeTest" runstate="Runnable" seed="635988114" result="Passed" start-time="2021-01-19 20:51:44Z" end-time="2021-01-19 20:51:44Z" duration="0.000400" asserts="0">
<properties />
</test-case>
<test-case id="1005" name="PassedUnityTest" fullname="Tests.PlayModeTest.PassedUnityTest" methodname="PassedUnityTest" classname="Tests.PlayModeTest" runstate="Runnable" seed="881217608" result="Passed" start-time="2021-01-19 20:51:44Z" end-time="2021-01-19 20:51:44Z" duration="0.000736" asserts="0">
<properties>
<property name="_JOINTYPE" value="UnityCombinatorial" />
</properties>
</test-case>
</test-suite>
<test-suite type="TestFixture" id="1008" name="SetupFailedTest" fullname="Tests.SetupFailedTest" classname="Tests.SetupFailedTest" runstate="Runnable" testcasecount="1" result="Failed" site="Child" start-time="2021-01-19 20:51:44Z" end-time="2021-01-19 20:51:44Z" duration="0.004318" total="1" passed="0" failed="1" inconclusive="0" skipped="0" asserts="0">
<properties />
<failure>
<message><![CDATA[One or more child tests had errors]]></message>
</failure>
<test-case id="1009" name="PassedTest" fullname="Tests.SetupFailedTest.PassedTest" methodname="PassedTest" classname="Tests.SetupFailedTest" runstate="Runnable" seed="1423699315" result="Failed" label="Error" start-time="2021-01-19 20:51:44Z" end-time="2021-01-19 20:51:44Z" duration="0.003206" asserts="0">
<properties />
<failure>
<message><![CDATA[SetUp : System.NullReferenceException : Object reference not set to an instance of an object]]></message>
<stack-trace><![CDATA[--SetUp
at Tests.SetupFailedTest.SetUp () [0x00000] in /github/workspace/unity-project/Assets/Tests/SetupFailedTest.cs:10
at (wrapper managed-to-native) System.Reflection.MonoMethod.InternalInvoke(System.Reflection.MonoMethod,object,object[],System.Exception&)
at System.Reflection.MonoMethod.Invoke (System.Object obj, System.Reflection.BindingFlags invokeAttr, System.Reflection.Binder binder, System.Object[] parameters, System.Globalization.CultureInfo culture) [0x00032] in <9577ac7a62ef43179789031239ba8798>:0 ]]></stack-trace>
</failure>
</test-case>
</test-suite>
<test-suite type="TestFixture" id="1010" name="TearDownFailedTest" fullname="Tests.TearDownFailedTest" classname="Tests.TearDownFailedTest" runstate="Runnable" testcasecount="1" result="Failed" site="Child" start-time="2021-01-19 20:51:44Z" end-time="2021-01-19 20:51:44Z" duration="0.001857" total="1" passed="0" failed="1" inconclusive="0" skipped="0" asserts="0">
<properties />
<failure>
<message><![CDATA[One or more child tests had errors]]></message>
</failure>
<test-case id="1011" name="PassedTest" fullname="Tests.TearDownFailedTest.PassedTest" methodname="PassedTest" classname="Tests.TearDownFailedTest" runstate="Runnable" seed="1384928637" result="Failed" label="Error" start-time="2021-01-19 20:51:44Z" end-time="2021-01-19 20:51:44Z" duration="0.000755" asserts="0">
<properties />
<failure>
<message><![CDATA[TearDown : System.NullReferenceException : Object reference not set to an instance of an object]]></message>
<stack-trace><![CDATA[--TearDown
at Tests.TearDownFailedTest.TearDown () [0x00000] in /github/workspace/unity-project/Assets/Tests/TearDownFailedTest.cs:10
at (wrapper managed-to-native) System.Reflection.MonoMethod.InternalInvoke(System.Reflection.MonoMethod,object,object[],System.Exception&)
at System.Reflection.MonoMethod.Invoke (System.Object obj, System.Reflection.BindingFlags invokeAttr, System.Reflection.Binder binder, System.Object[] parameters, System.Globalization.CultureInfo culture) [0x00032] in <9577ac7a62ef43179789031239ba8798>:0 ]]></stack-trace>
</failure>
</test-case>
</test-suite>
</test-suite>
</test-suite>
</test-suite>
</test-run>

38
src/model/results-check.js
View File

@ -9,7 +9,7 @@ import { RunMeta } from './ts/results-meta.ts';
class ResultsCheck { class ResultsCheck {
static async createCheck(artifactsPath, githubToken, checkName) { static async createCheck(artifactsPath, githubToken, checkName) {
// Validate input // Validate input
if (!artifactsPath || !checkName || !githubToken) { if (!fs.existsSync(artifactsPath) || !githubToken || !checkName) {
throw new Error( throw new Error(
`Missing input! {"artifactsPath": "${artifactsPath}", "githubToken": "${githubToken}, "checkName": "${checkName}"`, `Missing input! {"artifactsPath": "${artifactsPath}", "githubToken": "${githubToken}, "checkName": "${checkName}"`,
); );
@ -46,24 +46,35 @@ class ResultsCheck {
core.info('Analyze result:'); core.info('Analyze result:');
core.info(runSummary.summary); core.info(runSummary.summary);
// Call GitHub API // Format output
await ResultsCheck.requestGitHubCheck(checkName, githubToken, runs, runSummary);
return runSummary.failed;
}
static async requestGitHubCheck(checkName, githubToken, runs, runSummary) {
const pullRequest = github.context.payload.pull_request;
const headSha = (pullRequest && pullRequest.head.sha) || github.context.sha;
const title = runSummary.summary; const title = runSummary.summary;
const summary = await ResultsCheck.renderSummary(runs); const summary = await ResultsCheck.renderSummary(runs);
core.debug(`Summary view: ${summary}`);
const details = await ResultsCheck.renderDetails(runs); const details = await ResultsCheck.renderDetails(runs);
core.debug(`Details view: ${details}`);
const rawAnnotations = runSummary.extractAnnotations(); const rawAnnotations = runSummary.extractAnnotations();
core.debug(`Raw annotations: ${rawAnnotations}`);
const annotations = rawAnnotations.map(rawAnnotation => { const annotations = rawAnnotations.map(rawAnnotation => {
const annotation = rawAnnotation; const annotation = rawAnnotation;
annotation.path = rawAnnotation.path.replace('/github/workspace/', ''); annotation.path = rawAnnotation.path.replace('/github/workspace/', '');
return annotation; return annotation;
}); });
core.debug(`Annotations: ${annotations}`);
const output = {
title,
summary,
text: details,
annotations: annotations.slice(0, 50),
};
// Call GitHub API
await ResultsCheck.requestGitHubCheck(githubToken, checkName, output);
return runSummary.failed;
}
static async requestGitHubCheck(githubToken, checkName, output) {
const pullRequest = github.context.payload.pull_request;
const headSha = (pullRequest && pullRequest.head.sha) || github.context.sha;
core.info(`Posting results for ${headSha}`); core.info(`Posting results for ${headSha}`);
const createCheckRequest = { const createCheckRequest = {
@ -72,12 +83,7 @@ class ResultsCheck {
head_sha: headSha, head_sha: headSha,
status: 'completed', status: 'completed',
conclusion: 'neutral', conclusion: 'neutral',
output: { output,
title,
summary,
text: details,
annotations: annotations.slice(0, 50),
},
}; };
const octokit = github.getOctokit(githubToken); const octokit = github.getOctokit(githubToken);

6
src/model/results-parser.js
View File

@ -23,13 +23,15 @@ class ResultsParser {
const run = filedata['test-run']; const run = filedata['test-run'];
const runMeta = new RunMeta(filename); const runMeta = new RunMeta(filename);
const tests = ResultsParser.convertSuite(run['test-suite']);
core.debug(tests);
runMeta.total = Number(run._attributes.total); runMeta.total = Number(run._attributes.total);
runMeta.failed = Number(run._attributes.failed); runMeta.failed = Number(run._attributes.failed);
runMeta.skipped = Number(run._attributes.skipped); runMeta.skipped = Number(run._attributes.skipped);
runMeta.passed = Number(run._attributes.passed); runMeta.passed = Number(run._attributes.passed);
runMeta.duration = Number(run._attributes.duration); runMeta.duration = Number(run._attributes.duration);
runMeta.addTests(ResultsParser.convertSuite(run['test-suite'])); runMeta.addTests(tests);
return runMeta; return runMeta;
} }
@ -38,7 +40,7 @@ class ResultsParser {
if (Array.isArray(suites)) { if (Array.isArray(suites)) {
const innerResult = []; const innerResult = [];
suites.forEach(suite => { suites.forEach(suite => {
innerResult.push(ResultsParser.convertSuite(suite)); innerResult.push(...ResultsParser.convertSuite(suite));
}); });
return innerResult; return innerResult;
} }

42
src/model/results-parser.test.js
View File

@ -1,3 +1,5 @@
import * as xmljs from 'xml-js';
import * as fs from 'fs';
import ResultsParser from './results-parser'; import ResultsParser from './results-parser';
describe('ResultsParser', () => { describe('ResultsParser', () => {
@ -5,6 +7,30 @@ describe('ResultsParser', () => {
it('throws for missing file', () => { it('throws for missing file', () => {
expect(() => ResultsParser.parseResults('')).rejects.toEqual(Error); expect(() => ResultsParser.parseResults('')).rejects.toEqual(Error);
}); });
it('parses editmode-results.xml', () => {
expect(() => ResultsParser.parseResults('./artifacts/editmode-results.xml')).not.toThrow();
});
it('parses playmode-results.xml', () => {
expect(() => ResultsParser.parseResults('./artifacts/playmode-results.xml')).not.toThrow();
});
});
describe('convertResults', () => {
it('converts editmode-results.xml', () => {
const file = fs.readFileSync('./artifacts/editmode-results.xml');
const filedata = xmljs.xml2js(file, { compact: true });
const result = ResultsParser.convertResults('editmode-results.xml', filedata);
expect(result.suites.length).toEqual(1);
});
it('converts playmode-results.xml', () => {
const file = fs.readFileSync('./artifacts/playmode-results.xml');
const filedata = xmljs.xml2js(file, { compact: true });
const result = ResultsParser.convertResults('playmode-results.xml', filedata);
expect(result.suites.length).toEqual(3);
});
}); });
describe('convertSuite', () => { describe('convertSuite', () => {
@ -27,15 +53,13 @@ describe('ResultsParser', () => {
const result = ResultsParser.convertSuite(targetSuite); const result = ResultsParser.convertSuite(targetSuite);
expect(result).toMatchObject([ expect(result).toMatchObject([
[ {
{ annotation: undefined,
annotation: undefined, duration: Number.NaN,
duration: Number.NaN, result: undefined,
result: undefined, suite: 'Inner Suite Full Name',
suite: 'Inner Suite Full Name', title: 'testC',
title: 'testC', },
},
],
{ {
annotation: undefined, annotation: undefined,
duration: Number.NaN, duration: Number.NaN,

4
src/model/ts/results-meta.ts
View File

@ -41,8 +41,8 @@ export class RunMeta extends Meta {
return result; return result;
} }
addTests(testsToAdd: TestMeta[]): void { addTests(testSuite: TestMeta[]): void {
testsToAdd.forEach(test => { testSuite.forEach(test => {
this.addTest(test); this.addTest(test);
}); });
} }