Merge pull request #309 from mkmkl93/main

Add flag to capture stderr in chkwer

Author: MasloMaslane

src/sinol_make/commands/chkwer/__init__.py

@@ -34,6 +34,7 @@ def configure_subparser(self, subparser):
         )
         parser.add_argument('-t', '--tests', type=str, nargs='+',
                             help='test to run, for example in/abc{0,1}*')
+        parser.add_argument('--cerr', action='store_true', help='capture cerr output')
         parsers.add_cpus_argument(parser, 'number of cpus to use when verifying tests')
         parsers.add_compilation_arguments(parser)
         return parser
@@ -59,11 +60,11 @@ def run_test(self, execution: ChkwerExecution) -> RunResult:
         with open(execution.in_test_path, 'r') as inf, open(output_file, 'w') as outf:
             process = subprocess.Popen([execution.model_exe], stdin=inf, stdout=outf)
             process.wait()
-        ok, points, comment = self.task_type.check_output(execution.in_test_path, output_file, execution.out_test_path)
+        ok, points, comment, stderr = self.task_type.check_output(execution.in_test_path, output_file, execution.out_test_path)
 
-        return RunResult(execution.in_test_path, ok, int(points), comment)
+        return RunResult(execution.in_test_path, ok, int(points), comment, stderr)
 
-    def run_and_print_table(self) -> Dict[str, TestResult]:
+    def run_and_print_table(self, args) -> Dict[str, TestResult]:
         results = {}
         sorted_tests = sorted(self.tests, key=lambda test: package_util.get_group(test, self.task_id))
         executions: List[ChkwerExecution] = []
@@ -77,14 +78,14 @@ def run_and_print_table(self) -> Dict[str, TestResult]:
         if has_terminal:
             run_event = threading.Event()
             run_event.set()
-            thr = threading.Thread(target=printer.printer_thread, args=(run_event, chkwer_util.print_view, table_data))
+            thr = threading.Thread(target=printer.printer_thread, args=(run_event, chkwer_util.print_view, table_data, args))
             thr.start()
 
         keyboard_interrupt = False
         try:
             with mp.Pool(self.cpus) as pool:
                 for i, result in enumerate(pool.imap(self.run_test, executions)):
-                    table_data.results[result.test_path].set_results(result.points, result.ok, result.comment)
+                    table_data.results[result.test_path].set_results(result.points, result.ok, result.comment, result.stderr)
                     table_data.i = i
         except KeyboardInterrupt:
             keyboard_interrupt = True
@@ -93,7 +94,7 @@ def run_and_print_table(self) -> Dict[str, TestResult]:
             run_event.clear()
             thr.join()
 
-        print("\n".join(chkwer_util.print_view(terminal_width, terminal_height, table_data)[0]))
+        print("\n".join(chkwer_util.print_view(terminal_width, terminal_height, table_data, args)[0]))
         if keyboard_interrupt:
             util.exit_with_error("Keyboard interrupt.")
         return results
@@ -129,7 +130,7 @@ def run(self, args):
                                              "model solution", args.compile_mode)
         print()
 
-        results = self.run_and_print_table()
+        results = self.run_and_print_table(args)
         for result in results.values():
             if not result.ok or result.points != self.contest_type.max_score_per_test():
                 util.exit_with_error("Model solution didn't score maximum points.")

src/sinol_make/commands/chkwer/chkwer_util.py

@@ -5,8 +5,7 @@
 from sinol_make.commands.inwer.inwer_util import sort_tests
 from sinol_make.structs.chkwer_structs import TableData
 
-
-def print_view(term_width, term_height, table_data: TableData):
+def print_view(term_width, term_height, table_data: TableData, args):
     """
     Prints current results of test verification.
     """
@@ -65,6 +64,11 @@ def print_line_separator():
                 print(util.color_gray("No comment"))
 
     print_line_separator()
+
+    if args.cerr:
+        for test_path in tests:
+            result = results[test_path]
+            print(util.bold(f"Stderr on {result.test_name}: ") + result.stderr)
     print()
     print()
 

src/sinol_make/structs/chkwer_structs.py

@@ -14,6 +14,7 @@ class TestResult:
     points: int
     ok: bool
     comment: str
+    stderr: str
 
     def __init__(self, test_path, task_id):
         self.test_path = test_path
@@ -24,12 +25,14 @@ def __init__(self, test_path, task_id):
         self.points = 0
         self.ok = False
         self.run = False
+        self.stderr = ""
 
-    def set_results(self, points, ok, output):
+    def set_results(self, points, ok, output, stderr):
         self.run = True
         self.points = points
         self.ok = ok
         self.comment = output
+        self.stderr = stderr
 
 
 @dataclass
@@ -66,3 +69,4 @@ class RunResult:
     ok: bool
     points: int
     comment: str
+    stderr: str

src/sinol_make/task_type/__init__.py

@@ -135,51 +135,51 @@ def _output_to_fraction(self, output_str):
         except TypeError:
             raise CheckerException(f'Invalid checker output "{output_str}"')
 
-    def _parse_checker_output(self, output: List[str]) -> Tuple[bool, Fraction, str]:
+    def _parse_checker_output(self, output: List[str], stderr: str) -> Tuple[bool, Fraction, str, str]:
         while len(output) < 3:
             output.append('')
 
         if output[0].strip() == "OK":
             points = self._output_to_fraction(output[2])
-            return True, points, output[1].strip()
+            return True, points, output[1].strip(), stderr
         else:
-            return False, Fraction(0, 1), output[1].strip()
+            return False, Fraction(0, 1), output[1].strip(), stderr
 
-    def _run_checker(self, input_file_path, output_file_path, answer_file_path) -> Tuple[bool, Fraction, str]:
+    def _run_checker(self, input_file_path, output_file_path, answer_file_path) -> Tuple[bool, Fraction, str, str]:
         proc = subprocess.Popen([self.checker_path, input_file_path, output_file_path, answer_file_path],
                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         proc.wait()
         output, stderr = proc.communicate()
         if proc.returncode > 2:
             return False, Fraction(0, 1), (f"Checker returned with code {proc.returncode}, "
                                            f"stderr: '{stderr.decode('utf-8')}'")
-        return self._parse_checker_output(output.decode('utf-8').split('\n'))
+        return self._parse_checker_output(output.decode('utf-8').split('\n'), stderr.decode('utf-8'))
 
-    def _run_diff(self, output_file_path, answer_file_path) -> Tuple[bool, Fraction, str]:
+    def _run_diff(self, output_file_path, answer_file_path) -> Tuple[bool, Fraction, str, str]:
         same = oicompare.compare(output_file_path, answer_file_path)
         if same:
-            return True, Fraction(100, 1), ""
+            return True, Fraction(100, 1), "", ""
         else:
-            return False, Fraction(0, 1), ""
+            return False, Fraction(0, 1), "", ""
 
-    def _run_oicompare(self, output_file_path, answer_file_path) -> Tuple[bool, Fraction, str]:
+    def _run_oicompare(self, output_file_path, answer_file_path) -> Tuple[bool, Fraction, str, str]:
         path = oicompare.get_path()
         proc = subprocess.Popen([path, output_file_path, answer_file_path, 'english_abbreviated'],
                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         proc.wait()
         output, stderr = proc.communicate()
         if proc.returncode == 0:
-            return True, Fraction(100, 1), ""
+            return True, Fraction(100, 1), "", ""
         elif proc.returncode == 1:
-            return False, Fraction(0, 1), output.decode('utf-8').strip()
+            return False, Fraction(0, 1), output.decode('utf-8').strip(), stderr.decode('utf-8')
         else:
             raise CheckerException(f"!!! oicompare failed with code {proc.returncode}. This is a huge bug, please report"
                                    f" it here https://github.com/sio2project/sinol-make/issues/new/choose and provide "
                                    f"these files: {output_file_path}, {answer_file_path}.\n"
                                    f"Output: {output.decode('utf-8').strip()}\n"
                                    f"Stderr: {stderr.decode('utf-8').strip()}")
 
-    def check_output(self, input_file_path, output_file_path, answer_file_path) -> Tuple[bool, Fraction, str]:
+    def check_output(self, input_file_path, output_file_path, answer_file_path) -> Tuple[bool, Fraction, str, str]:
         """
         Runs the checker (or runs diff) and returns a tuple of three values:
         - bool: whether the solution is correct

src/sinol_make/task_type/interactive.py

@@ -116,7 +116,7 @@ def _fill_result(self, result: ExecutionResult, iresult: ExecutionResult, intera
                 result.Error = None
             else:
                 try:
-                    ok, points, comment = self._parse_checker_output(interactor_output)
+                    ok, points, comment, _ = self._parse_checker_output(interactor_output, '')
                 except CheckerException as e:
                     result.Status = Status.RE
                     result.Error = str(e)

src/sinol_make/task_type/normal.py

@@ -25,7 +25,7 @@ def run(self, time_limit, hard_time_limit, memory_limit, input_file_path, output
             result.Status = Status.ML
         elif result.Status == Status.OK:
             try:
-                correct, points, comment = self.check_output(input_file_path, output_file_path, answer_file_path)
+                correct, points, comment, _ = self.check_output(input_file_path, output_file_path, answer_file_path)
                 result.Points = float(points)
                 result.Comment = comment
                 if not correct: