Hydra fixes and improvements (#2718)

* Build tools for check

* Display PASSes when Hydra exits

* Print buffered output at exit

* Remove unused test summaries

* Show SKIP if ASSUMPTIONS fails

Makefile

@@ -464,7 +464,7 @@ LD_SCRIPT_TEST := ld_script_test.txt
 $(OBJ_DIR)/ld_script_test.ld: $(LD_SCRIPT_TEST) $(LD_SCRIPT_DEPS)
 	cd $(OBJ_DIR) && sed "s#tools/#../../tools/#g" ../../$(LD_SCRIPT_TEST) > ld_script_test.ld
 
-$(TESTELF): $(OBJ_DIR)/ld_script_test.ld $(OBJS) $(TEST_OBJS) libagbsyscall check-tools
+$(TESTELF): $(OBJ_DIR)/ld_script_test.ld $(OBJS) $(TEST_OBJS) libagbsyscall tools check-tools
 	@echo "cd $(OBJ_DIR) && $(LD) -T ld_script_test.ld -o ../../$@ <objects> <test-objects> <lib>"
 	@cd $(OBJ_DIR) && $(LD) $(TESTLDFLAGS) -T ld_script_test.ld -o ../../$@ $(OBJS_REL) $(TEST_OBJS_REL) $(LIB)
 	$(FIX) $@ -t"$(TITLE)" -c$(GAME_CODE) -m$(MAKER_CODE) -r$(REVISION) --silent

test/test.h

@@ -39,7 +39,6 @@ struct TestRunnerState
     u8 exitCode;
     s32 tests;
     s32 passes;
-    s32 skips;
     const char *skipFilename;
     const struct Test *test;
     u32 processCosts[MAX_PROCESSES];

test/test_runner.c

@@ -60,7 +60,6 @@ void CB2_TestRunner(void)
         gTestRunnerState.exitCode = 0;
         gTestRunnerState.tests = 0;
         gTestRunnerState.passes = 0;
-        gTestRunnerState.skips = 0;
         gTestRunnerState.skipFilename = NULL;
         gTestRunnerState.test = __start_tests - 1;
         break;
@@ -70,20 +69,15 @@ void CB2_TestRunner(void)
 
         if (gTestRunnerState.test == __stop_tests)
         {
-            MgbaPrintf_("%s%d/%d PASSED\e[0m", gTestRunnerState.exitCode == 0 ? "\e[32m" : "\e[31m", gTestRunnerState.passes, gTestRunnerState.tests);
-            if (gTestRunnerState.skips)
-            {
-                if (gTestRunnerSkipIsFail)
-                    MgbaPrintf_("\e[31m%d SKIPPED\e[0m", gTestRunnerState.skips);
-                else
-                    MgbaPrintf_("%d SKIPPED", gTestRunnerState.skips);
-            }
             gTestRunnerState.state = STATE_EXIT;
             return;
         }
 
-        if (!PrefixMatch(gTestRunnerArgv, gTestRunnerState.test->name))
+        if (gTestRunnerState.test->runner != &gAssumptionsRunner
+          && !PrefixMatch(gTestRunnerArgv, gTestRunnerState.test->name))
+        {
             return;
+        }
 
         // Greedily assign tests to processes based on estimated cost.
         // TODO: Make processCosts a min heap.
@@ -111,6 +105,7 @@ void CB2_TestRunner(void)
                 return;
         }
 
+        MgbaPrintf_(":N%s", gTestRunnerState.test->name);
         gTestRunnerState.state = STATE_REPORT_RESULT;
         gTestRunnerState.result = TEST_RESULT_PASS;
         gTestRunnerState.expectedResult = TEST_RESULT_PASS;
@@ -130,7 +125,6 @@ void CB2_TestRunner(void)
         }
         else
         {
-            MgbaPrintf_(":N%s", gTestRunnerState.test->name);
             if (gTestRunnerState.test->runner->setUp)
                 gTestRunnerState.test->runner->setUp(gTestRunnerState.test->data);
             gTestRunnerState.test->runner->run(gTestRunnerState.test->data);
@@ -150,12 +144,6 @@ void CB2_TestRunner(void)
             if (gTestRunnerState.result != TEST_RESULT_PASS)
                 gTestRunnerState.skipFilename = gTestRunnerState.test->filename;
         }
-        else if (gTestRunnerState.result == TEST_RESULT_SKIP)
-        {
-            gTestRunnerState.skips++;
-            if (gTestRunnerSkipIsFail)
-                gTestRunnerState.exitCode = 1;
-        }
         else
         {
             const char *color;
@@ -206,7 +194,10 @@ void CB2_TestRunner(void)
             default: result = "UNKNOWN"; break;
             }
 
-            MgbaPrintf_(":R%s%s\e[0m", color, result);
+            if (gTestRunnerState.expectedResult == gTestRunnerState.result)
+                MgbaPrintf_(":P%s%s\e[0m", color, result);
+            else
+                MgbaPrintf_(":F%s%s\e[0m", color, result);
         }
 
         break;

tools/mgba-rom-test-hydra/main.c

@@ -38,6 +38,8 @@ struct Runner
     size_t output_buffer_size;
     size_t output_buffer_capacity;
     char *output_buffer;
+    int passes;
+    int results;
 };
 
 static unsigned nrunners = 0;
@@ -72,7 +74,11 @@ static void handle_read(struct Runner *runner)
                     runner->test_name[eol - soc - 1] = '\0';
                     break;
 
-                case 'R':
+                case 'P':
+                case 'F':
+                    if (soc[1] == 'P')
+                        runner->passes++;
+                    runner->results++;
                     soc += 2;
                     fprintf(stdout, "%s: ", runner->test_name);
                     fwrite(soc, 1, eol - soc, stdout);
@@ -404,6 +410,8 @@ int main(int argc, char *argv[])
 
     // Reap test runners and collate exit codes.
     int exit_code = 0;
+    int passes = 0;
+    int results = 0;
     for (int i = 0; i < nrunners; i++)
     {
         int wstatus;
@@ -412,8 +420,14 @@ int main(int argc, char *argv[])
             perror("waitpid runners[i] failed");
             exit(2);
         }
+        if (runners[i].output_buffer_size > 0)
+            fwrite(runners[i].output_buffer, 1, runners[i].output_buffer_size, stdout);
         if (WIFEXITED(wstatus) && WEXITSTATUS(wstatus) > exit_code)
             exit_code = WEXITSTATUS(wstatus);
+        passes += runners[i].passes;
+        results += runners[i].results;
     }
+    fprintf(stdout, "%d/%d \e[32mPASS\e[0med\n", passes, results);
+    fflush(stdout);
     return exit_code;
 }