Tests: unroll 2 player gen, add parametrization helper, add docs (#4648)

* Tests: unroll test_multiworlds.TestTwoPlayerMulti

Also adds a helper function that other tests can use to unroll tests.

* Docs: add more details to docs/tests.md

* Explain parametrization, subtests and link to the new helper
* Mention some performance details and work-arounds
* Mention multithreading / pytest-xdist

* Tests: make param.classvar_matrix accept sets

* CI: add test/param.py to type checking

* Tests: add missing typing to test/param.py

* Tests: fix typo in test/param.py doc comment

Co-authored-by: qwint <qwint.42@gmail.com>

* update docs

* Docs: reword note on performance

---------

Co-authored-by: qwint <qwint.42@gmail.com>

test/multiworld/test_multiworlds.py

@@ -1,5 +1,5 @@
 import unittest
-from typing import List, Tuple
+from typing import ClassVar, List, Tuple
 from unittest import TestCase
 
 from BaseClasses import CollectionState, Location, MultiWorld
@@ -7,6 +7,7 @@ from Fill import distribute_items_restrictive
 from Options import Accessibility
 from worlds.AutoWorld import AutoWorldRegister, call_all, call_single
 from ..general import gen_steps, setup_multiworld
+from ..param import classvar_matrix
 
 
 class MultiworldTestBase(TestCase):
@@ -63,15 +64,18 @@ class TestAllGamesMultiworld(MultiworldTestBase):
             self.assertTrue(self.fulfills_accessibility(), "Collected all locations, but can't beat the game")
 
 
+@classvar_matrix(game=AutoWorldRegister.world_types.keys())
 class TestTwoPlayerMulti(MultiworldTestBase):
+    game: ClassVar[str]
+
     def test_two_player_single_game_fills(self) -> None:
         """Tests that a multiworld of two players for each registered game world can generate."""
-        for world_type in AutoWorldRegister.world_types.values():
-            self.multiworld = setup_multiworld([world_type, world_type], ())
-            for world in self.multiworld.worlds.values():
-                world.options.accessibility.value = Accessibility.option_full
-            self.assertSteps(gen_steps)
-            with self.subTest("filling multiworld", games=world_type.game, seed=self.multiworld.seed):
-                distribute_items_restrictive(self.multiworld)
-                call_all(self.multiworld, "post_fill")
-                self.assertTrue(self.fulfills_accessibility(), "Collected all locations, but can't beat the game")
+        world_type = AutoWorldRegister.world_types[self.game]
+        self.multiworld = setup_multiworld([world_type, world_type], ())
+        for world in self.multiworld.worlds.values():
+            world.options.accessibility.value = Accessibility.option_full
+        self.assertSteps(gen_steps)
+        with self.subTest("filling multiworld", games=world_type.game, seed=self.multiworld.seed):
+            distribute_items_restrictive(self.multiworld)
+            call_all(self.multiworld, "post_fill")
+            self.assertTrue(self.fulfills_accessibility(), "Collected all locations, but can't beat the game")

test/param.py

@@ -0,0 +1,46 @@
+import itertools
+import sys
+from typing import Any, Callable, Iterable
+
+
+def classvar_matrix(**kwargs: Iterable[Any]) -> Callable[[type], None]:
+    """
+    Create a new class for each variation of input, allowing to generate a TestCase matrix / parametrization that
+    supports multi-threading and has better reporting for ``unittest --durations=...`` and ``pytest --durations=...``
+    than subtests.
+
+    The kwargs will be set as ClassVars in the newly created classes. Use as ::
+
+        @classvar_matrix(var_name=[value1, value2])
+        class MyTestCase(unittest.TestCase):
+            var_name: typing.ClassVar[...]
+
+    :param kwargs: A dict of ClassVars to set, where key is the variable name and value is a list of all values.
+    :return: A decorator to be applied to a class.
+    """
+    keys: tuple[str]
+    values: Iterable[Iterable[Any]]
+    keys, values = zip(*kwargs.items())
+    values = map(lambda v: sorted(v) if isinstance(v, (set, frozenset)) else v, values)
+    permutations_dicts = [dict(zip(keys, v)) for v in itertools.product(*values)]
+
+    def decorator(cls: type) -> None:
+        mod = sys.modules[cls.__module__]
+
+        for permutation in permutations_dicts:
+
+            class Unrolled(cls):  # type: ignore
+                pass
+
+            for k, v in permutation.items():
+                setattr(Unrolled, k, v)
+            params = ", ".join([f"{k}={repr(v)}" for k, v in permutation.items()])
+            params = f"{{{params}}}"
+
+            Unrolled.__module__ = cls.__module__
+            Unrolled.__qualname__ = f"{cls.__qualname__}{params}"
+            setattr(mod, f"{cls.__name__}{params}", Unrolled)
+
+        return None
+
+    return decorator