Do not allow an unnumbered test case to be named 'test_group'

This is also mentioned in https://icpc.io/problem-package-format/spec/2023-07-draft.html#test-data

Author: mpsijm

bin/generate.py

@@ -457,12 +457,17 @@ def __init__(
         # root in /data
         self.root = self.path.parts[0]
 
-        if not config.COMPILED_FILE_NAME_REGEX.fullmatch(name + ".in"):
-            raise ParseException("Testcase does not have a valid name.")
-
         # files to consider for hashing
         hashes = {}
         try:
+            if not config.COMPILED_FILE_NAME_REGEX.fullmatch(name + ".in"):
+                raise ParseException("Test case does not have a valid name.")
+
+            if name == "test_group":
+                raise ParseException(
+                    "Test case must not be named 'test_group', this clashes with the group-level 'test_group.yaml'."
+                )
+
             if yaml is None:
                 raise ParseException(
                     "Empty yaml entry (Testcases must be generated not only mentioned)."
@@ -1678,7 +1683,7 @@ def parse(key: str, name_gen: Callable[[], str], yaml: dict, parent: AnyDirector
 
             if is_testcase(yaml):
                 if isinstance(parent, RootDirectory):
-                    raise ParseException("Testcase must be inside Directory", name)
+                    raise ParseException("Test case must be inside a Directory.", name)
 
                 count = parse_count(yaml, parent.path / name)
 

bin/problem.py

@@ -612,6 +612,15 @@ def get_test_case_yaml(
 
         return []
 
+    # Because Problem.testcases() may be called multiple times (e.g. validating multiple modes, or with `bt all`),
+    # this cache makes sure that some warnings (like malformed test case names) only appear once.
+    _warned_for_test_case = set[str]()
+
+    def _warn_once(p, test_name, msg):
+        if test_name not in p._warned_for_test_case:
+            p._warned_for_test_case.add(test_name)
+            warn(msg)
+
     def testcases(
         p,
         *,
@@ -660,6 +669,15 @@ def testcases(
         testcases = []
         for f in in_paths:
             t = testcase.Testcase(p, f)
+            if not config.COMPILED_FILE_NAME_REGEX.fullmatch(f.name):
+                p._warn_once(t.name, f"Test case name {t.name} is not valid. Skipping.")
+                continue
+            if f.with_suffix("").name == "test_group":
+                p._warn_once(
+                    t.name,
+                    "Test case must not be named 'test_group', this clashes with the group-level 'test_group.yaml'. Skipping.",
+                )
+                continue
             if (
                 (p.interactive or p.multi_pass)
                 and mode in [validate.Mode.INVALID, validate.Mode.VALID_OUTPUT]
@@ -671,7 +689,7 @@ def testcases(
                 continue
             if needans and not t.ans_path.is_file():
                 if t.root != "invalid_input":
-                    warn(f"Found input file {f} without a .ans file. Skipping.")
+                    p._warn_once(t.name, f"Found input file {f} without a .ans file. Skipping.")
                     continue
             if mode == validate.Mode.VALID_OUTPUT:
                 if t.out_path is None:

bin/testcase.py

@@ -87,7 +87,6 @@ def __init__(
 
         self.problem = base_problem
 
-        # TODO add self.out_path
         if short_path is None:
             try:
                 self.short_path: Path = path.relative_to(self.problem.path / "data")

test/yaml/generators/valid_yaml/rich-generators.yaml

@@ -33,6 +33,12 @@ data:
           input_validator_args: [--connected, --max_n, "2000"]
         data:
           'a': my_generator
+          # A test case may not be called 'test_group', but it may appear as part of the name.
+          'test_group_': my_generator
+      'group_with_numbers':
+        data:
+          # A numbered test case may be called 'X-test-group'.
+          - 'test_group': my_generator
   invalid_input:
     data:
     - '':