ninjdai/pokeemerald
1
0
Fork 0
mirror of https://github.com/Ninjdai1/pokeemerald.git synced 2024-12-25 19:24:16 +01:00
Code Issues Packages Projects Releases Wiki Activity

More detailed hydra test results (#2722)

Browse Source
This commit is contained in:
Eduardo Quezada D'Ottone 2023-03-08 10:11:11 -03:00 committed by GitHub
commit dd686ee606
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 71 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
test/test.h
View File

@ -9,7 +9,7 @@ enum TestResult
{
TEST_RESULT_FAIL,
TEST_RESULT_PASS,
TEST_RESULT_SKIP,
TEST_RESULT_ASSUMPTION_FAIL,
TEST_RESULT_INVALID,
TEST_RESULT_ERROR,
TEST_RESULT_TIMEOUT,
@ -77,7 +77,7 @@ s32 MgbaPrintf_(const char *fmt, ...);
do \
{ \
if (!(c)) \
Test_ExitWithResult(TEST_RESULT_SKIP, "%s:%d: ASSUME failed", gTestRunnerState.test->filename, __LINE__); \
Test_ExitWithResult(TEST_RESULT_ASSUMPTION_FAIL, "%s:%d: ASSUME failed", gTestRunnerState.test->filename, __LINE__); \
} while (0)
#define EXPECT(c) \

35
test/test_runner.c
View File

@ -121,7 +121,7 @@ void CB2_TestRunner(void)
// NOTE: Assumes that the compiler interns __FILE__.
if (gTestRunnerState.skipFilename == gTestRunnerState.test->filename)
{
gTestRunnerState.result = TEST_RESULT_SKIP;
gTestRunnerState.result = TEST_RESULT_ASSUMPTION_FAIL;
}
else
{
@ -157,7 +157,7 @@ void CB2_TestRunner(void)
color = "\e[32m";
MgbaPrintf_(":N%s", gTestRunnerState.test->name);
}
else if (gTestRunnerState.result != TEST_RESULT_SKIP || gTestRunnerSkipIsFail)
else if (gTestRunnerState.result != TEST_RESULT_ASSUMPTION_FAIL || gTestRunnerSkipIsFail)
{
gTestRunnerState.exitCode = 1;
color = "\e[31m";
@ -186,16 +186,33 @@ void CB2_TestRunner(void)
result = "FAIL";
}
break;
case TEST_RESULT_PASS: result = "PASS"; break;
case TEST_RESULT_SKIP: result = "SKIP"; break;
case TEST_RESULT_INVALID: result = "INVALID"; break;
case TEST_RESULT_ERROR: result = "ERROR"; break;
case TEST_RESULT_TIMEOUT: result = "TIMEOUT"; break;
default: result = "UNKNOWN"; break;
case TEST_RESULT_PASS:
result = "PASS";
break;
case TEST_RESULT_ASSUMPTION_FAIL:
result = "ASSUMPTION_FAIL";
color = "\e[33m";
break;
case TEST_RESULT_INVALID:
result = "INVALID";
break;
case TEST_RESULT_ERROR:
result = "ERROR";
break;
case TEST_RESULT_TIMEOUT:
result = "TIMEOUT";
break;
default:
result = "UNKNOWN";
break;
}
if (gTestRunnerState.expectedResult == gTestRunnerState.result)
if (gTestRunnerState.result == TEST_RESULT_PASS)
MgbaPrintf_(":P%s%s\e[0m", color, result);
else if (gTestRunnerState.result == TEST_RESULT_ASSUMPTION_FAIL)
MgbaPrintf_(":A%s%s\e[0m", color, result);
else if (gTestRunnerState.expectedResult == gTestRunnerState.result)
MgbaPrintf_(":K%s%s\e[0m", color, result);
else
MgbaPrintf_(":F%s%s\e[0m", color, result);
}

2
test/test_runner_battle.c
View File

@ -720,7 +720,7 @@ static void CB2_BattleTest_NextTrial(void)
case TEST_RESULT_PASS:
STATE->observedPasses++;
break;
case TEST_RESULT_SKIP:
case TEST_RESULT_ASSUMPTION_FAIL:
STATE->skippedTrials++;
if (STATE->skippedTrials > STATE->trials / 4)
Test_ExitWithResult(TEST_RESULT_INVALID, "25%% of the trials were SKIPed");

46
tools/mgba-rom-test-hydra/main.c
View File

@ -9,7 +9,11 @@
* COMMANDS
* N: Sets the test name to the remainder of the line.
* R: Sets the result to the remainder of the line, and flushes any
* output buffered since the previous R. */
* output buffered since the previous R.
* P/K/F/A: Sets the result to the remaining of the line, flushes any
* output since the previous P/K/F/A and increment the number of
* passes/known fails/assumption fails/fails.
*/
#include <fcntl.h>
#include <poll.h>
#include <signal.h>
@ -39,6 +43,9 @@ struct Runner
size_t output_buffer_capacity;
char *output_buffer;
int passes;
int knownFails;
int assumptionFails;
int fails;
int results;
};
@ -75,9 +82,17 @@ static void handle_read(struct Runner *runner)
break;
case 'P':
runner->passes++;
goto add_to_results;
case 'K':
runner->knownFails++;
goto add_to_results;
case 'A':
runner->assumptionFails++;
goto add_to_results;
case 'F':
if (soc[1] == 'P')
runner->passes++;
runner->fails++;
add_to_results:
runner->results++;
soc += 2;
fprintf(stdout, "%s: ", runner->test_name);
@ -411,6 +426,9 @@ int main(int argc, char *argv[])
// Reap test runners and collate exit codes.
int exit_code = 0;
int passes = 0;
int knownFails = 0;
int assumptionFails = 0;
int fails = 0;
int results = 0;
for (int i = 0; i < nrunners; i++)
{
@ -425,9 +443,29 @@ int main(int argc, char *argv[])
if (WIFEXITED(wstatus) && WEXITSTATUS(wstatus) > exit_code)
exit_code = WEXITSTATUS(wstatus);
passes += runners[i].passes;
knownFails += runners[i].knownFails;
assumptionFails += runners[i].assumptionFails;
fails += runners[i].fails;
results += runners[i].results;
}
fprintf(stdout, "%d/%d \e[32mPASS\e[0med\n", passes, results);
if (results == 0)
{
fprintf(stdout, "\nNo tests found.\n");
}
else
{
fprintf(stdout, "\n- Tests TOTAL: %d\n", results);
fprintf(stdout, "- Tests \e[32mPASSED\e[0m: %d\n", passes);
if (knownFails > 0)
fprintf(stdout, "- Tests \e[33mKNOWN_FAILING\e[0m: %d\n", knownFails);
if (fails > 0)
fprintf(stdout, "- Tests \e[31mFAILED\e[0m : %d\n", fails);
if (assumptionFails > 0)
fprintf(stdout, "- \e[33mASSUMPTIONS_FAILED\e[0m: %d\n", assumptionFails);
}
fprintf(stdout, "\n");
fflush(stdout);
return exit_code;
}