Introduce Test Case Configuration .yaml file, move .hint and .desc to there

Author: mpsijm

bin/config.py

@@ -72,9 +72,7 @@
     *KNOWN_TESTCASE_EXTENSIONS,
     *KNOWN_SAMPLE_TESTCASE_EXTENSIONS,
     ".interaction",
-    ".hint",
-    ".desc",
-    #'.args',
+    ".yaml",
 ]
 
 KNOWN_DATA_EXTENSIONS: Final[Sequence[str]] = [

bin/export.py

@@ -298,7 +298,7 @@ def add_testcase(in_file: Path) -> None:
             ryaml_filter(limits, "time_limit")
         # validator_flags
         validator_flags = " ".join(
-            problem.get_test_group_yaml(
+            problem.get_test_case_yaml(
                 problem.path / "data",
                 OutputValidator.args_key,
                 PrintBar("Getting validator_flags for legacy export"),

bin/generate.py

@@ -394,7 +394,14 @@ def __init__(self, problem, key, name, yaml, parent):
 
 class TestcaseRule(Rule):
     def __init__(
-        self, problem: Problem, generator_config, key, name: str, yaml, parent, count_index
+        self,
+        problem: Problem,
+        generator_config,
+        key,
+        name: str,
+        yaml: dict[str, Any],
+        parent,
+        count_index,
     ):
         assert is_testcase(yaml)
 
@@ -558,11 +565,16 @@ def __init__(
                         if self.copy.with_suffix(ext).is_file():
                             hashes[ext] = hash_file_content(self.copy.with_suffix(ext))
 
-                # 3. hardcoded
+                # 3. hardcoded strings (or, for the Test Case Configuration, a yaml mapping)
                 for ext in config.KNOWN_TEXT_DATA_EXTENSIONS:
                     if ext[1:] in yaml:
                         value = yaml[ext[1:]]
-                        assert_type(ext, value, str)
+                        if ext == ".yaml":
+                            assert_type(ext, value, dict)
+                            value = write_yaml(value)
+                            assert value is not None
+                        else:
+                            assert_type(ext, value, str)
                         if len(value) > 0 and value[-1] != "\n":
                             value += "\n"
                         self.hardcoded[ext] = value
@@ -1045,7 +1057,7 @@ def use_feedback_image(feedbackdir: Path, source: str) -> None:
                 use_feedback_image(feedbackdir, "validator")
                 return True
 
-            visualizer_args = testcase.test_group_yaml_args(visualizer, bar)
+            visualizer_args = testcase.test_case_yaml_args(visualizer, bar)
             visualizer_hash = {
                 "visualizer_hash": visualizer.hash,
                 "visualizer_args": visualizer_args,

bin/interactive.py

@@ -56,7 +56,7 @@ def get_validator_command():
                 run.testcase.ans_path.resolve(),
                 run.feedbackdir.resolve(),
             ]
-            + run.testcase.test_group_yaml_args(
+            + run.testcase.test_case_yaml_args(
                 output_validator,
                 bar or PrintBar("Run interactive test case"),
             )

bin/problem.py

@@ -365,7 +365,7 @@ def __init__(self, path: Path, tmpdir: Path, label: Optional[str] = None):
         self._program_callbacks = dict[Path, list[Callable[["Program"], None]]]()
         # Dictionary from path to parsed file contents.
         # TODO #102: Add type for test_group.yaml (typed Namespace?)
-        self._test_group_yamls = dict[Path, dict[str, Any]]()
+        self._test_case_yamls = dict[Path, dict[str, Any]]()
         self._test_group_lock = threading.Lock()
 
         # The label for the problem: A, B, A1, A2, X, ...
@@ -460,83 +460,80 @@ def _read_settings(self):
         self.custom_output: bool = self.settings.custom_output
 
     # TODO #102 move to a new TestGroup class
-    def _parse_test_group_yaml(p, path, bar):
+    def _parse_test_case_and_groups_yaml(p, path: Path, bar: BAR_TYPE):
         assert path.is_relative_to(p.path / "data")
-        for dir in [path] + list(path.parents):
+        for f in [path] + list(path.parents):
             # Do not go above the data directory.
-            if dir == p.path:
+            if f == p.path:
                 return
 
-            f = dir / "test_group.yaml"
-            if not f.is_file() or f in p._test_group_yamls:
-                continue
+            if f.suffix != ".yaml":
+                f = f / "test_group.yaml"
             with p._test_group_lock:
-                if f not in p._test_group_yamls:
-                    raw = substitute(
-                        f.read_text(),
-                        p.settings.constants,
-                        pattern=config.CONSTANT_SUBSTITUTE_REGEX,
-                    )
-                    p._test_group_yamls[f] = flags = parse_yaml(raw, path=f, plain=True)
+                if not f.is_file() or f in p._test_case_yamls:
+                    continue
+                raw = substitute(
+                    f.read_text(),
+                    p.settings.constants,
+                    pattern=config.CONSTANT_SUBSTITUTE_REGEX,
+                )
+                p._test_case_yamls[f] = flags = parse_yaml(raw, path=f, plain=True)
 
-                    parse_deprecated_setting(
-                        flags, "output_validator_flags", validate.OutputValidator.args_key
-                    )
-                    parse_deprecated_setting(
-                        flags, "input_validator_flags", validate.InputValidator.args_key
-                    )
+                parse_deprecated_setting(
+                    flags, "output_validator_flags", validate.OutputValidator.args_key
+                )
+                parse_deprecated_setting(
+                    flags, "input_validator_flags", validate.InputValidator.args_key
+                )
 
-                    # Verify test_group.yaml
-                    for k in flags:
-                        match k:
-                            case (
-                                validate.OutputValidator.args_key
-                                | validate.AnswerValidator.args_key
-                                | visualize.TestCaseVisualizer.args_key
-                                | visualize.OutputVisualizer.args_key
-                            ):
-                                if not isinstance(flags[k], list):
-                                    bar.error(
-                                        f"{k} must be a list of strings",
-                                        resume=True,
-                                        print_item=False,
-                                    )
-                            case validate.InputValidator.args_key:
-                                if not isinstance(flags[k], (list, dict)):
-                                    bar.error(
-                                        f"{k} must be list or map",
-                                        resume=True,
-                                        print_item=False,
-                                    )
-                                if isinstance(flags[k], dict):
-                                    input_validator_names = set(
-                                        val.name for val in p.validators(validate.InputValidator)
-                                    )
-                                    for name in set(flags[k]) - input_validator_names:
-                                        bar.warn(
-                                            f"Unknown input validator {name}; expected {input_validator_names}",
-                                            print_item=False,
-                                        )
-                            case (
-                                "args"
-                                | "description"
-                                | "full_feedback"
-                                | "hint"
-                                | "scoring"
-                                | "static_validation"
-                            ):
-                                bar.warn(
-                                    f"{k} in test_group.yaml not implemented in BAPCtools",
-                                    print_item=False,
+                # Use variable kwargs so the type checker does not complain when passing them to a PrintBar (nothing happens in that case anyway)
+                bar_kwargs = {"resume": True, "print_item": False}
+
+                # Verify test_group.yaml
+                for k in flags:
+                    match k:
+                        case (
+                            validate.OutputValidator.args_key
+                            | validate.AnswerValidator.args_key
+                            | visualize.TestCaseVisualizer.args_key
+                            | visualize.OutputVisualizer.args_key
+                        ):
+                            if not isinstance(flags[k], list):
+                                bar.error(
+                                    f"{k} must be a list of strings",
+                                    None,
+                                    **bar_kwargs,
                                 )
-                            case _:
-                                path = f.relative_to(p.path / "data")
-                                bar.warn(f'Unknown key "{k}" in {path}', print_item=False)
-            # Do not go above the data directory.
-            if dir == p.path / "data":
-                break
-
-    def get_test_group_yaml(
+                        case validate.InputValidator.args_key:
+                            if not isinstance(flags[k], (list, dict)):
+                                bar.error(
+                                    f"{k} must be list or map",
+                                    None,
+                                    **bar_kwargs,
+                                )
+                            if isinstance(flags[k], dict):
+                                input_validator_names = set(
+                                    val.name for val in p.validators(validate.InputValidator)
+                                )
+                                for name in set(flags[k]) - input_validator_names:
+                                    bar.warn(
+                                        f"Unknown input validator {name}; expected {input_validator_names}",
+                                        None,
+                                        **bar_kwargs,
+                                    )
+                        case "description" | "hint":
+                            pass  # We don't do anything with hint or description in BAPCtools, but no need to warn about this
+                        case "args" | "full_feedback" | "scoring" | "static_validation":
+                            bar.warn(
+                                f"{k} in test_group.yaml not implemented in BAPCtools",
+                                None,
+                                **bar_kwargs,
+                            )
+                        case _:
+                            path = f.relative_to(p.path / "data")
+                            bar.warn(f'Unknown key "{k}" in {path}', None, **bar_kwargs)
+
+    def get_test_case_yaml(
         p,
         path: Path,
         key: str,
@@ -574,19 +571,21 @@ def get_test_group_yaml(
                 f"Only input validators support flags by validator name, got {key} and {name}"
             )
 
-        # parse and cache test_group.yaml
-        p._parse_test_group_yaml(path, bar)
+        # parse and cache <test_case>.yaml and test_group.yaml
+        path = path.with_suffix(".yaml")
+        p._parse_test_case_and_groups_yaml(path, bar)
 
         # extract the flags
-        for dir in [path] + list(path.parents):
+        for f in [path] + list(path.parents):
             # Do not go above the data directory.
-            if dir == p.path:
+            if f == p.path:
                 return []
 
-            f = dir / "test_group.yaml"
-            if f not in p._test_group_yamls:
+            if f.suffix != ".yaml":
+                f = f / "test_group.yaml"
+            if f not in p._test_case_yamls:
                 continue
-            flags = p._test_group_yamls[f]
+            flags = p._test_case_yamls[f]
             if key in flags:
                 args = flags[key]
                 if key == validate.InputValidator.args_key:
@@ -1333,7 +1332,7 @@ def validate_valid_extra_data(p) -> bool:
         if not p.validators(validate.OutputValidator, strict=True, print_warn=False):
             return True
 
-        args = p.get_test_group_yaml(
+        args = p.get_test_case_yaml(
             p.path / "data" / "valid_output",
             "output_validator_args",
             PrintBar("Generic Output Validation"),

bin/run.py

@@ -228,7 +228,7 @@ def _validate_output(self, bar: BAR_TYPE) -> Optional[ExecResult]:
         return output_validator.run(
             self.testcase,
             self,
-            args=self.testcase.test_group_yaml_args(output_validator, bar),
+            args=self.testcase.test_case_yaml_args(output_validator, bar),
         )
 
     def _visualize_output(self, bar: BAR_TYPE) -> Optional[ExecResult]:
@@ -242,7 +242,7 @@ def _visualize_output(self, bar: BAR_TYPE) -> Optional[ExecResult]:
             self.testcase.ans_path.resolve(),
             self.out_path if not self.problem.interactive else None,
             self.feedbackdir,
-            args=self.testcase.test_group_yaml_args(output_visualizer, bar),
+            args=self.testcase.test_case_yaml_args(output_visualizer, bar),
         )
 
 

bin/testcase.py

@@ -116,7 +116,7 @@ def __repr__(self) -> str:
     def with_suffix(self, ext: str) -> Path:
         return self.in_path.with_suffix(ext)
 
-    def test_group_yaml_args(
+    def test_case_yaml_args(
         self,
         program: "validate.AnyValidator | visualize.AnyVisualizer",
         bar: BAR_TYPE,
@@ -131,9 +131,8 @@ def test_group_yaml_args(
         or ["--max_N", "50"] or even [""].
         """
 
-        path = self.problem.path / "data" / self.short_path
-        return self.problem.get_test_group_yaml(
-            path,
+        return self.problem.get_test_case_yaml(
+            self.problem.path / "data" / self.short_path,
             type(program).args_key,
             bar,
             name=program.name if isinstance(program, validate.InputValidator) else None,
@@ -157,7 +156,7 @@ def validator_hashes(
         d = dict()
 
         for validator in validators:
-            flags = self.test_group_yaml_args(validator, bar)
+            flags = self.test_case_yaml_args(validator, bar)
             flags_string = " ".join(flags)
             h = combine_hashes_dict(
                 {
@@ -288,7 +287,7 @@ def _run_validators(
             if isinstance(validator, validate.OutputValidator) and mode == validate.Mode.ANSWER:
                 args += ["case_sensitive", "space_change_sensitive"]
                 name = f"{name} (ans)"
-            flags = self.test_group_yaml_args(validator, bar)
+            flags = self.test_case_yaml_args(validator, bar)
             flags = flags + args
 
             ret = validator.run(self, mode=mode, constraints=constraints, args=flags)