ninjdai/pokeemerald
1
0
Fork 0
mirror of https://github.com/Ninjdai1/pokeemerald.git synced 2024-11-16 11:37:40 +01:00
Code Issues Packages Projects Releases Wiki Activity

More detailed hydra test results (#2722)

Browse Source
This commit is contained in:
Eduardo Quezada D'Ottone 2023-03-08 10:11:11 -03:00 committed by GitHub
commit dd686ee606
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 71 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
test/test.h
View File

@ -9,7 +9,7 @@ enum TestResult
{ {
TEST_RESULT_FAIL, TEST_RESULT_FAIL,
TEST_RESULT_PASS, TEST_RESULT_PASS,
TEST_RESULT_SKIP, TEST_RESULT_ASSUMPTION_FAIL,
TEST_RESULT_INVALID, TEST_RESULT_INVALID,
TEST_RESULT_ERROR, TEST_RESULT_ERROR,
TEST_RESULT_TIMEOUT, TEST_RESULT_TIMEOUT,
@ -77,7 +77,7 @@ s32 MgbaPrintf_(const char *fmt, ...);
do \ do \
{ \ { \
if (!(c)) \ if (!(c)) \
Test_ExitWithResult(TEST_RESULT_SKIP, "%s:%d: ASSUME failed", gTestRunnerState.test->filename, __LINE__); \ Test_ExitWithResult(TEST_RESULT_ASSUMPTION_FAIL, "%s:%d: ASSUME failed", gTestRunnerState.test->filename, __LINE__); \
} while (0) } while (0)
#define EXPECT(c) \ #define EXPECT(c) \

35
test/test_runner.c
View File

@ -121,7 +121,7 @@ void CB2_TestRunner(void)
// NOTE: Assumes that the compiler interns __FILE__. // NOTE: Assumes that the compiler interns __FILE__.
if (gTestRunnerState.skipFilename == gTestRunnerState.test->filename) if (gTestRunnerState.skipFilename == gTestRunnerState.test->filename)
{ {
gTestRunnerState.result = TEST_RESULT_SKIP; gTestRunnerState.result = TEST_RESULT_ASSUMPTION_FAIL;
} }
else else
{ {
@ -157,7 +157,7 @@ void CB2_TestRunner(void)
color = "\e[32m"; color = "\e[32m";
MgbaPrintf_(":N%s", gTestRunnerState.test->name); MgbaPrintf_(":N%s", gTestRunnerState.test->name);
} }
else if (gTestRunnerState.result != TEST_RESULT_SKIP || gTestRunnerSkipIsFail) else if (gTestRunnerState.result != TEST_RESULT_ASSUMPTION_FAIL || gTestRunnerSkipIsFail)
{ {
gTestRunnerState.exitCode = 1; gTestRunnerState.exitCode = 1;
color = "\e[31m"; color = "\e[31m";
@ -186,16 +186,33 @@ void CB2_TestRunner(void)
result = "FAIL"; result = "FAIL";
} }
break; break;
case TEST_RESULT_PASS: result = "PASS"; break; case TEST_RESULT_PASS:
case TEST_RESULT_SKIP: result = "SKIP"; break; result = "PASS";
case TEST_RESULT_INVALID: result = "INVALID"; break; break;
case TEST_RESULT_ERROR: result = "ERROR"; break; case TEST_RESULT_ASSUMPTION_FAIL:
case TEST_RESULT_TIMEOUT: result = "TIMEOUT"; break; result = "ASSUMPTION_FAIL";
default: result = "UNKNOWN"; break; color = "\e[33m";
break;
case TEST_RESULT_INVALID:
result = "INVALID";
break;
case TEST_RESULT_ERROR:
result = "ERROR";
break;
case TEST_RESULT_TIMEOUT:
result = "TIMEOUT";
break;
default:
result = "UNKNOWN";
break;
} }
if (gTestRunnerState.expectedResult == gTestRunnerState.result) if (gTestRunnerState.result == TEST_RESULT_PASS)
MgbaPrintf_(":P%s%s\e[0m", color, result); MgbaPrintf_(":P%s%s\e[0m", color, result);
else if (gTestRunnerState.result == TEST_RESULT_ASSUMPTION_FAIL)
MgbaPrintf_(":A%s%s\e[0m", color, result);
else if (gTestRunnerState.expectedResult == gTestRunnerState.result)
MgbaPrintf_(":K%s%s\e[0m", color, result);
else else
MgbaPrintf_(":F%s%s\e[0m", color, result); MgbaPrintf_(":F%s%s\e[0m", color, result);
} }

2
test/test_runner_battle.c
View File

@ -720,7 +720,7 @@ static void CB2_BattleTest_NextTrial(void)
case TEST_RESULT_PASS: case TEST_RESULT_PASS:
STATE->observedPasses++; STATE->observedPasses++;
break; break;
case TEST_RESULT_SKIP: case TEST_RESULT_ASSUMPTION_FAIL:
STATE->skippedTrials++; STATE->skippedTrials++;
if (STATE->skippedTrials > STATE->trials / 4) if (STATE->skippedTrials > STATE->trials / 4)
Test_ExitWithResult(TEST_RESULT_INVALID, "25%% of the trials were SKIPed"); Test_ExitWithResult(TEST_RESULT_INVALID, "25%% of the trials were SKIPed");

46
tools/mgba-rom-test-hydra/main.c
View File

@ -9,7 +9,11 @@
* COMMANDS * COMMANDS
* N: Sets the test name to the remainder of the line. * N: Sets the test name to the remainder of the line.
* R: Sets the result to the remainder of the line, and flushes any * R: Sets the result to the remainder of the line, and flushes any
* output buffered since the previous R. */ * output buffered since the previous R.
* P/K/F/A: Sets the result to the remaining of the line, flushes any
* output since the previous P/K/F/A and increment the number of
* passes/known fails/assumption fails/fails.
*/
#include <fcntl.h> #include <fcntl.h>
#include <poll.h> #include <poll.h>
#include <signal.h> #include <signal.h>
@ -39,6 +43,9 @@ struct Runner
size_t output_buffer_capacity; size_t output_buffer_capacity;
char *output_buffer; char *output_buffer;
int passes; int passes;
int knownFails;
int assumptionFails;
int fails;
int results; int results;
}; };
@ -75,9 +82,17 @@ static void handle_read(struct Runner *runner)
break; break;
case 'P': case 'P':
runner->passes++;
goto add_to_results;
case 'K':
runner->knownFails++;
goto add_to_results;
case 'A':
runner->assumptionFails++;
goto add_to_results;
case 'F': case 'F':
if (soc[1] == 'P') runner->fails++;
runner->passes++; add_to_results:
runner->results++; runner->results++;
soc += 2; soc += 2;
fprintf(stdout, "%s: ", runner->test_name); fprintf(stdout, "%s: ", runner->test_name);
@ -411,6 +426,9 @@ int main(int argc, char *argv[])
// Reap test runners and collate exit codes. // Reap test runners and collate exit codes.
int exit_code = 0; int exit_code = 0;
int passes = 0; int passes = 0;
int knownFails = 0;
int assumptionFails = 0;
int fails = 0;
int results = 0; int results = 0;
for (int i = 0; i < nrunners; i++) for (int i = 0; i < nrunners; i++)
{ {
@ -425,9 +443,29 @@ int main(int argc, char *argv[])
if (WIFEXITED(wstatus) && WEXITSTATUS(wstatus) > exit_code) if (WIFEXITED(wstatus) && WEXITSTATUS(wstatus) > exit_code)
exit_code = WEXITSTATUS(wstatus); exit_code = WEXITSTATUS(wstatus);
passes += runners[i].passes; passes += runners[i].passes;
knownFails += runners[i].knownFails;
assumptionFails += runners[i].assumptionFails;
fails += runners[i].fails;
results += runners[i].results; results += runners[i].results;
} }
fprintf(stdout, "%d/%d \e[32mPASS\e[0med\n", passes, results);
if (results == 0)
{
fprintf(stdout, "\nNo tests found.\n");
}
else
{
fprintf(stdout, "\n- Tests TOTAL: %d\n", results);
fprintf(stdout, "- Tests \e[32mPASSED\e[0m: %d\n", passes);
if (knownFails > 0)
fprintf(stdout, "- Tests \e[33mKNOWN_FAILING\e[0m: %d\n", knownFails);
if (fails > 0)
fprintf(stdout, "- Tests \e[31mFAILED\e[0m : %d\n", fails);
if (assumptionFails > 0)
fprintf(stdout, "- \e[33mASSUMPTIONS_FAILED\e[0m: %d\n", assumptionFails);
}
fprintf(stdout, "\n");
fflush(stdout); fflush(stdout);
return exit_code; return exit_code;
} }