From f3c6b647c089d75c53dbb26cf5cfccd5fade73ee Mon Sep 17 00:00:00 2001 From: Martin Griffin Date: Tue, 21 Feb 2023 15:30:42 +0000 Subject: [PATCH] Hydra fixes and improvements (#2718) * Build tools for check * Display PASSes when Hydra exits * Print buffered output at exit * Remove unused test summaries * Show SKIP if ASSUMPTIONS fails --- Makefile | 2 +- test/test.h | 1 - test/test_runner.c | 27 +++++++++------------------ tools/mgba-rom-test-hydra/main.c | 16 +++++++++++++++- 4 files changed, 25 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 8e3ff2975..2b997fee5 100644 --- a/Makefile +++ b/Makefile @@ -464,7 +464,7 @@ LD_SCRIPT_TEST := ld_script_test.txt $(OBJ_DIR)/ld_script_test.ld: $(LD_SCRIPT_TEST) $(LD_SCRIPT_DEPS) cd $(OBJ_DIR) && sed "s#tools/#../../tools/#g" ../../$(LD_SCRIPT_TEST) > ld_script_test.ld -$(TESTELF): $(OBJ_DIR)/ld_script_test.ld $(OBJS) $(TEST_OBJS) libagbsyscall check-tools +$(TESTELF): $(OBJ_DIR)/ld_script_test.ld $(OBJS) $(TEST_OBJS) libagbsyscall tools check-tools @echo "cd $(OBJ_DIR) && $(LD) -T ld_script_test.ld -o ../../$@ " @cd $(OBJ_DIR) && $(LD) $(TESTLDFLAGS) -T ld_script_test.ld -o ../../$@ $(OBJS_REL) $(TEST_OBJS_REL) $(LIB) $(FIX) $@ -t"$(TITLE)" -c$(GAME_CODE) -m$(MAKER_CODE) -r$(REVISION) --silent diff --git a/test/test.h b/test/test.h index cbcf11717..c69a4fa92 100644 --- a/test/test.h +++ b/test/test.h @@ -39,7 +39,6 @@ struct TestRunnerState u8 exitCode; s32 tests; s32 passes; - s32 skips; const char *skipFilename; const struct Test *test; u32 processCosts[MAX_PROCESSES]; diff --git a/test/test_runner.c b/test/test_runner.c index 3cb1f7c21..0be163431 100644 --- a/test/test_runner.c +++ b/test/test_runner.c @@ -60,7 +60,6 @@ void CB2_TestRunner(void) gTestRunnerState.exitCode = 0; gTestRunnerState.tests = 0; gTestRunnerState.passes = 0; - gTestRunnerState.skips = 0; gTestRunnerState.skipFilename = NULL; gTestRunnerState.test = __start_tests - 1; break; @@ -70,20 +69,15 @@ void CB2_TestRunner(void) if (gTestRunnerState.test == __stop_tests) { - MgbaPrintf_("%s%d/%d PASSED\e[0m", gTestRunnerState.exitCode == 0 ? "\e[32m" : "\e[31m", gTestRunnerState.passes, gTestRunnerState.tests); - if (gTestRunnerState.skips) - { - if (gTestRunnerSkipIsFail) - MgbaPrintf_("\e[31m%d SKIPPED\e[0m", gTestRunnerState.skips); - else - MgbaPrintf_("%d SKIPPED", gTestRunnerState.skips); - } gTestRunnerState.state = STATE_EXIT; return; } - if (!PrefixMatch(gTestRunnerArgv, gTestRunnerState.test->name)) + if (gTestRunnerState.test->runner != &gAssumptionsRunner + && !PrefixMatch(gTestRunnerArgv, gTestRunnerState.test->name)) + { return; + } // Greedily assign tests to processes based on estimated cost. // TODO: Make processCosts a min heap. @@ -111,6 +105,7 @@ void CB2_TestRunner(void) return; } + MgbaPrintf_(":N%s", gTestRunnerState.test->name); gTestRunnerState.state = STATE_REPORT_RESULT; gTestRunnerState.result = TEST_RESULT_PASS; gTestRunnerState.expectedResult = TEST_RESULT_PASS; @@ -130,7 +125,6 @@ void CB2_TestRunner(void) } else { - MgbaPrintf_(":N%s", gTestRunnerState.test->name); if (gTestRunnerState.test->runner->setUp) gTestRunnerState.test->runner->setUp(gTestRunnerState.test->data); gTestRunnerState.test->runner->run(gTestRunnerState.test->data); @@ -150,12 +144,6 @@ void CB2_TestRunner(void) if (gTestRunnerState.result != TEST_RESULT_PASS) gTestRunnerState.skipFilename = gTestRunnerState.test->filename; } - else if (gTestRunnerState.result == TEST_RESULT_SKIP) - { - gTestRunnerState.skips++; - if (gTestRunnerSkipIsFail) - gTestRunnerState.exitCode = 1; - } else { const char *color; @@ -206,7 +194,10 @@ void CB2_TestRunner(void) default: result = "UNKNOWN"; break; } - MgbaPrintf_(":R%s%s\e[0m", color, result); + if (gTestRunnerState.expectedResult == gTestRunnerState.result) + MgbaPrintf_(":P%s%s\e[0m", color, result); + else + MgbaPrintf_(":F%s%s\e[0m", color, result); } break; diff --git a/tools/mgba-rom-test-hydra/main.c b/tools/mgba-rom-test-hydra/main.c index 5dfc99517..4ce9b09bd 100644 --- a/tools/mgba-rom-test-hydra/main.c +++ b/tools/mgba-rom-test-hydra/main.c @@ -38,6 +38,8 @@ struct Runner size_t output_buffer_size; size_t output_buffer_capacity; char *output_buffer; + int passes; + int results; }; static unsigned nrunners = 0; @@ -72,7 +74,11 @@ static void handle_read(struct Runner *runner) runner->test_name[eol - soc - 1] = '\0'; break; - case 'R': + case 'P': + case 'F': + if (soc[1] == 'P') + runner->passes++; + runner->results++; soc += 2; fprintf(stdout, "%s: ", runner->test_name); fwrite(soc, 1, eol - soc, stdout); @@ -404,6 +410,8 @@ int main(int argc, char *argv[]) // Reap test runners and collate exit codes. int exit_code = 0; + int passes = 0; + int results = 0; for (int i = 0; i < nrunners; i++) { int wstatus; @@ -412,8 +420,14 @@ int main(int argc, char *argv[]) perror("waitpid runners[i] failed"); exit(2); } + if (runners[i].output_buffer_size > 0) + fwrite(runners[i].output_buffer, 1, runners[i].output_buffer_size, stdout); if (WIFEXITED(wstatus) && WEXITSTATUS(wstatus) > exit_code) exit_code = WEXITSTATUS(wstatus); + passes += runners[i].passes; + results += runners[i].results; } + fprintf(stdout, "%d/%d \e[32mPASS\e[0med\n", passes, results); + fflush(stdout); return exit_code; }