Stardew Valley: Use classvar_matrix to split tests (#4762)

* Unroll tests for better parallelization

* fix ut test

* self review

* bro it's the second time today I have to commit some garbage to have a github action rerun because messenger fails what is this

* my god can the tests plz pass

* code reviews

* code reviews

* move TestRandomWorlds out of long module

worlds/stardew_valley/test/TestOptions.py

@@ -1,12 +1,13 @@
 import itertools
+from typing import ClassVar
 
 from BaseClasses import ItemClassification
-from Options import NamedRange
+from test.param import classvar_matrix
 from . import SVTestCase, solo_multiworld, SVTestBase
 from .assertion import WorldAssertMixin
-from .long.option_names import all_option_choices
+from .options.option_names import all_option_choices
 from .options.presets import allsanity_no_mods_6_x_x, allsanity_mods_6_x_x
-from .. import items_by_group, Group, StardewValleyWorld
+from .. import items_by_group, Group
 from ..locations import locations_by_tag, LocationTags, location_table
 from ..options import ExcludeGingerIsland, ToolProgression, Goal, SeasonRandomization, TrapItems, SpecialOrderLocations, ArcadeMachineLocations
 from ..strings.goal_names import Goal as GoalName
@@ -18,40 +19,34 @@ SEASONS = {Season.spring, Season.summer, Season.fall, Season.winter}
 TOOLS = {"Hoe", "Pickaxe", "Axe", "Watering Can", "Trash Can", "Fishing Rod"}
 
 
+@classvar_matrix(option_and_choice=all_option_choices)
 class TestGenerateDynamicOptions(WorldAssertMixin, SVTestCase):
-    def test_given_special_range_when_generate_then_basic_checks(self):
+    option_and_choice: ClassVar[tuple[str, str]]
-        options = StardewValleyWorld.options_dataclass.type_hints
-        for option_name, option in options.items():
-            if not issubclass(option, NamedRange):
-                continue
-            for value in option.special_range_names:
-                world_options = {option_name: option.special_range_names[value]}
-                with self.solo_world_sub_test(f"{option_name}: {value}", world_options) as (multiworld, _):
-                    self.assert_basic_checks(multiworld)
 
-    def test_given_choice_when_generate_then_basic_checks(self):
+    def test_given_option_and_choice_when_generate_then_basic_checks(self):
-        options = StardewValleyWorld.options_dataclass.type_hints
+        option, choice = self.option_and_choice
-        for option_name, option in options.items():
+        world_options = {option: choice}
-            if not option.options:
+        with solo_multiworld(world_options) as (multiworld, stardew_world):
-                continue
-            for value in option.options:
-                world_options = {option_name: option.options[value]}
-                with self.solo_world_sub_test(f"{option_name}: {value}", world_options) as (multiworld, _):
             self.assert_basic_checks(multiworld)
 
 
-class TestGoal(SVTestCase):
+@classvar_matrix(goal_and_location=[
-    def test_given_goal_when_generate_then_victory_is_in_correct_location(self):
+    ("community_center", GoalName.community_center),
-        for goal, location in [("community_center", GoalName.community_center),
     ("grandpa_evaluation", GoalName.grandpa_evaluation),
     ("bottom_of_the_mines", GoalName.bottom_of_the_mines),
     ("cryptic_note", GoalName.cryptic_note),
     ("master_angler", GoalName.master_angler),
     ("complete_collection", GoalName.complete_museum),
     ("full_house", GoalName.full_house),
-                               ("perfection", GoalName.perfection)]:
+    ("perfection", GoalName.perfection),
-            world_options = {Goal.internal_name: Goal.options[goal]}
+])
-            with self.solo_world_sub_test(f"Goal: {goal}, Location: {location}", world_options) as (multi_world, _):
+class TestGoal(SVTestCase):
+    goal_and_location: ClassVar[tuple[str, str]]
+
+    def test_given_goal_when_generate_then_victory_is_in_correct_location(self):
+        goal, location = self.goal_and_location
+        world_options = {Goal.internal_name: goal}
+        with solo_multiworld(world_options) as (multi_world, _):
             victory = multi_world.find_item("Victory", 1)
             self.assertEqual(victory.name, location)
 
@@ -104,23 +99,25 @@ class TestToolProgression(SVTestBase):
         self.assertEqual(useful_count, 1)
 
 
+@classvar_matrix(option_and_choice=all_option_choices)
 class TestGenerateAllOptionsWithExcludeGingerIsland(WorldAssertMixin, SVTestCase):
+    option_and_choice: ClassVar[tuple[str, str]]
 
     def test_given_choice_when_generate_exclude_ginger_island_then_ginger_island_is_properly_excluded(self):
-        for option, option_choice in all_option_choices:
+        option, option_choice = self.option_and_choice
-            if option is ExcludeGingerIsland:
+
-                continue
+        if option == ExcludeGingerIsland.internal_name:
+            self.skipTest("ExcludeGingerIsland is forced to true")
 
         world_options = {
-                ExcludeGingerIsland: ExcludeGingerIsland.option_true,
+            ExcludeGingerIsland.internal_name: ExcludeGingerIsland.option_true,
             option: option_choice
         }
 
-            with self.solo_world_sub_test(f"{option.internal_name}: {option_choice}", world_options) as (multiworld, stardew_world):
+        with solo_multiworld(world_options) as (multiworld, stardew_world):
 
-                # Some options, like goals, will force Ginger island back in the game. We want to skip testing those.
             if stardew_world.options.exclude_ginger_island != ExcludeGingerIsland.option_true:
-                    continue
+                self.skipTest("Some options, like goals, will force Ginger island back in the game. We want to skip testing those.")
 
             self.assert_basic_checks(multiworld)
             self.assert_no_ginger_island_content(multiworld)

worlds/stardew_valley/test/TestRandomWorlds.py

@@ -0,0 +1,29 @@
+from typing import ClassVar
+
+from BaseClasses import MultiWorld, get_seed
+from test.param import classvar_matrix
+from . import SVTestCase, skip_long_tests, solo_multiworld
+from .assertion import GoalAssertMixin, OptionAssertMixin, WorldAssertMixin
+from .options.option_names import generate_random_world_options
+
+
+@classvar_matrix(n=range(10 if skip_long_tests() else 1000))
+class TestGenerateManyWorlds(GoalAssertMixin, OptionAssertMixin, WorldAssertMixin, SVTestCase):
+    n: ClassVar[int]
+
+    def test_generate_many_worlds_then_check_results(self):
+        seed = get_seed()
+        world_options = generate_random_world_options(seed + self.n)
+
+        print(f"Generating solo multiworld with seed {seed} for Stardew Valley...")
+        with solo_multiworld(world_options, seed=seed, world_caching=False) as (multiworld, _):
+            self.assert_multiworld_is_valid(multiworld)
+
+    def assert_multiworld_is_valid(self, multiworld: MultiWorld):
+        self.assert_victory_exists(multiworld)
+        self.assert_same_number_items_locations(multiworld)
+        self.assert_goal_world_is_valid(multiworld)
+        self.assert_can_reach_island_if_should(multiworld)
+        self.assert_cropsanity_same_number_items_and_locations(multiworld)
+        self.assert_festivals_give_access_to_deluxe_scarecrow(multiworld)
+        self.assert_has_festival_recipes(multiworld)

worlds/stardew_valley/test/__init__.py

@@ -22,21 +22,19 @@ DEFAULT_TEST_SEED = get_seed()
 logger.info(f"Default Test Seed: {DEFAULT_TEST_SEED}")
 
 
-class SVTestCase(unittest.TestCase):
+def skip_default_tests() -> bool:
-    # Set False to not skip some 'extra' tests
+    return not bool(os.environ.get("base", False))
-    skip_base_tests: bool = True
-    # Set False to run tests that take long
-    skip_long_tests: bool = True
 
-    @classmethod
+
-    def setUpClass(cls) -> None:
+def skip_long_tests() -> bool:
-        super().setUpClass()
+    return not bool(os.environ.get("long", False))
-        base_tests_key = "base"
+
-        if base_tests_key in os.environ:
+
-            cls.skip_base_tests = not bool(os.environ[base_tests_key])
+class SVTestCase(unittest.TestCase):
-        long_tests_key = "long"
+    skip_default_tests: bool = skip_default_tests()
-        if long_tests_key in os.environ:
+    """Set False to not skip the base fill tests"""
-            cls.skip_long_tests = not bool(os.environ[long_tests_key])
+    skip_long_tests: bool = skip_long_tests()
+    """Set False to run tests that take long"""
 
     @contextmanager
     def solo_world_sub_test(self, msg: Optional[str] = None,
@@ -94,7 +92,7 @@ class SVTestBase(RuleAssertMixin, WorldTestBase, SVTestCase):
 
     @property
     def run_default_tests(self) -> bool:
-        if self.skip_base_tests:
+        if self.skip_default_tests:
             return False
         return super().run_default_tests
 
@@ -196,6 +194,7 @@ def solo_multiworld(world_options: Optional[Dict[Union[str, StardewValleyOption]
         yield multiworld, multiworld.worlds[1]
     else:
         multiworld = setup_solo_multiworld(world_options, seed)
+        try:
             multiworld.lock.acquire()
             world = multiworld.worlds[1]
 
@@ -209,7 +208,7 @@ def solo_multiworld(world_options: Optional[Dict[Union[str, StardewValleyOption]
             multiworld.itempool = original_itempool
             for location in unfilled_locations:
                 location.item = None
-
+        finally:
             multiworld.lock.release()
 
 

worlds/stardew_valley/test/long/TestModsLong.py

@@ -1,61 +1,19 @@
 import unittest
-from itertools import combinations, product
+from itertools import combinations
+from typing import ClassVar
 
 from BaseClasses import get_seed
-from .option_names import all_option_choices, get_option_choices
+from test.param import classvar_matrix
-from .. import SVTestCase
+from .. import SVTestCase, solo_multiworld, skip_long_tests
 from ..assertion import WorldAssertMixin, ModAssertMixin
+from ..options.option_names import all_option_choices
 from ... import options
 from ...mods.mod_data import ModNames
+from ...options.options import all_mods
 
-assert unittest
-
-
-class TestGenerateModsOptions(WorldAssertMixin, ModAssertMixin, SVTestCase):
-
-    @classmethod
-    def setUpClass(cls) -> None:
-        super().setUpClass()
-        if cls.skip_long_tests:
-            raise unittest.SkipTest("Long tests disabled")
-
-    def test_given_mod_pairs_when_generate_then_basic_checks(self):
-        for mod_pair in combinations(options.Mods.valid_keys, 2):
-            world_options = {
-                options.Mods: frozenset(mod_pair)
-            }
-
-            with self.solo_world_sub_test(f"Mods: {mod_pair}", world_options, world_caching=False) as (multiworld, _):
-                self.assert_basic_checks(multiworld)
-                self.assert_stray_mod_items(list(mod_pair), multiworld)
-
-    def test_given_mod_names_when_generate_paired_with_other_options_then_basic_checks(self):
-        for mod, (option, value) in product(options.Mods.valid_keys, all_option_choices):
-            world_options = {
-                option: value,
-                options.Mods: mod
-            }
-
-            with self.solo_world_sub_test(f"{option.internal_name}: {value}, Mod: {mod}", world_options, world_caching=False) as (multiworld, _):
-                self.assert_basic_checks(multiworld)
-                self.assert_stray_mod_items(mod, multiworld)
-
-    def test_given_no_quest_all_mods_when_generate_with_all_goals_then_basic_checks(self):
-        for goal, (option, value) in product(get_option_choices(options.Goal), all_option_choices):
-            if option is options.QuestLocations:
-                continue
-
-            world_options = {
-                options.Goal: goal,
-                option: value,
-                options.QuestLocations: -1,
-                options.Mods: frozenset(options.Mods.valid_keys),
-            }
-
-            with self.solo_world_sub_test(f"Goal: {goal}, {option.internal_name}: {value}", world_options, world_caching=False) as (multiworld, _):
-                self.assert_basic_checks(multiworld)
 
 @unittest.skip
+class TestTroubleshootMods(WorldAssertMixin, ModAssertMixin, SVTestCase):
     def test_troubleshoot_option(self):
         seed = get_seed(78709133382876990000)
 
@@ -67,3 +25,60 @@ class TestGenerateModsOptions(WorldAssertMixin, ModAssertMixin, SVTestCase):
         with self.solo_world_sub_test(world_options=world_options, seed=seed, world_caching=False) as (multiworld, _):
             self.assert_basic_checks(multiworld)
             self.assert_stray_mod_items(world_options[options.Mods], multiworld)
+
+
+if skip_long_tests():
+    raise unittest.SkipTest("Long tests disabled")
+
+
+@classvar_matrix(mod_pair=combinations(sorted(all_mods), 2))
+class TestGenerateModsPairs(WorldAssertMixin, ModAssertMixin, SVTestCase):
+    mod_pair: ClassVar[tuple[str, str]]
+
+    def test_given_mod_pairs_when_generate_then_basic_checks(self):
+        world_options = {
+            options.Mods.internal_name: frozenset(self.mod_pair)
+        }
+
+        with solo_multiworld(world_options, world_caching=False) as (multiworld, _):
+            self.assert_basic_checks(multiworld)
+            self.assert_stray_mod_items(list(self.mod_pair), multiworld)
+
+
+@classvar_matrix(mod=all_mods, option_and_choice=all_option_choices)
+class TestGenerateModAndOptionChoice(WorldAssertMixin, ModAssertMixin, SVTestCase):
+    mod: ClassVar[str]
+    option_and_choice: ClassVar[tuple[str, str]]
+
+    def test_given_mod_names_when_generate_paired_with_other_options_then_basic_checks(self):
+        option, choice = self.option_and_choice
+
+        world_options = {
+            option: choice,
+            options.Mods.internal_name: self.mod
+        }
+
+        with solo_multiworld(world_options, world_caching=False) as (multiworld, _):
+            self.assert_basic_checks(multiworld)
+            self.assert_stray_mod_items(self.mod, multiworld)
+
+
+@classvar_matrix(goal=options.Goal.options.keys(), option_and_choice=all_option_choices)
+class TestGenerateAllGoalAndAllOptionWithAllModsWithoutQuest(WorldAssertMixin, ModAssertMixin, SVTestCase):
+    goal = ClassVar[str]
+    option_and_choice = ClassVar[tuple[str, str]]
+
+    def test_given_no_quest_all_mods_when_generate_with_all_goals_then_basic_checks(self):
+        option, choice = self.option_and_choice
+        if option == options.QuestLocations.internal_name:
+            self.skipTest("QuestLocations are disabled")
+
+        world_options = {
+            options.Goal.internal_name: self.goal,
+            option: choice,
+            options.QuestLocations.internal_name: -1,
+            options.Mods.internal_name: frozenset(options.Mods.valid_keys),
+        }
+
+        with solo_multiworld(world_options, world_caching=False) as (multiworld, _):
+            self.assert_basic_checks(multiworld)

worlds/stardew_valley/test/long/TestOptionsLong.py

@@ -1,34 +1,16 @@
 import unittest
 from itertools import combinations
+from typing import ClassVar
 
 from BaseClasses import get_seed
-from .option_names import all_option_choices
+from test.param import classvar_matrix
-from .. import SVTestCase, solo_multiworld
+from .. import SVTestCase, solo_multiworld, skip_long_tests
 from ..assertion.world_assert import WorldAssertMixin
+from ..options.option_names import all_option_choices
 from ... import options
 
 
-class TestGenerateDynamicOptions(WorldAssertMixin, SVTestCase):
+@unittest.skip
-    def test_given_option_pair_when_generate_then_basic_checks(self):
-        if self.skip_long_tests:
-            raise unittest.SkipTest("Long tests disabled")
-
-        for (option1, option1_choice), (option2, option2_choice) in combinations(all_option_choices, 2):
-            if option1 is option2:
-                continue
-
-            world_options = {
-                option1: option1_choice,
-                option2: option2_choice
-            }
-
-            with self.solo_world_sub_test(f"{option1.internal_name}: {option1_choice}, {option2.internal_name}: {option2_choice}",
-                                          world_options,
-                                          world_caching=False) \
-                    as (multiworld, _):
-                self.assert_basic_checks(multiworld)
-
-
 class TestDynamicOptionDebug(WorldAssertMixin, SVTestCase):
 
     def test_option_pair_debug(self):
@@ -42,3 +24,23 @@ class TestDynamicOptionDebug(WorldAssertMixin, SVTestCase):
                 print(f"Seed: {seed}")
                 with solo_multiworld(option_dict, seed=seed) as (multiworld, _):
                     self.assert_basic_checks(multiworld)
+
+
+if skip_long_tests():
+    raise unittest.SkipTest("Long tests disabled")
+
+
+@classvar_matrix(options_and_choices=combinations(all_option_choices, 2))
+class TestGenerateDynamicOptions(WorldAssertMixin, SVTestCase):
+    options_and_choices: ClassVar[tuple[tuple[str, str], tuple[str, str]]]
+
+    def test_given_option_pair_when_generate_then_basic_checks(self):
+        (option1, option1_choice), (option2, option2_choice) = self.options_and_choices
+
+        world_options = {
+            option1: option1_choice,
+            option2: option2_choice
+        }
+
+        with solo_multiworld(world_options, world_caching=False) as (multiworld, _):
+            self.assert_basic_checks(multiworld)

worlds/stardew_valley/test/long/TestPreRolledRandomness.py

@@ -1,28 +1,29 @@
 import unittest
+from typing import ClassVar
 
 from BaseClasses import get_seed
-from .. import SVTestCase
+from test.param import classvar_matrix
+from .. import SVTestCase, solo_multiworld, skip_long_tests
 from ..assertion import WorldAssertMixin
 from ... import options
 
-
+if skip_long_tests():
-class TestGeneratePreRolledRandomness(WorldAssertMixin, SVTestCase):
-    def test_given_pre_rolled_difficult_randomness_when_generate_then_basic_checks(self):
-        if self.skip_long_tests:
     raise unittest.SkipTest("Long tests disabled")
 
-        choices = {
+player_options = {
     options.EntranceRandomization.internal_name: options.EntranceRandomization.option_buildings,
     options.BundleRandomization.internal_name: options.BundleRandomization.option_remixed,
     options.BundlePrice.internal_name: options.BundlePrice.option_maximum
 }
 
-        num_tests = 1000
+
-        for i in range(num_tests):
+@classvar_matrix(n=range(1000))
-            seed = get_seed()  # Put seed in parameter to test
+class TestGeneratePreRolledRandomness(WorldAssertMixin, SVTestCase):
-            with self.solo_world_sub_test(f"Entrance Randomizer and Remixed Bundles",
+    n: ClassVar[int]
-                                          choices,
+
-                                          seed=seed,
+    def test_given_pre_rolled_difficult_randomness_when_generate_then_basic_checks(self):
-                                          world_caching=False) \
+        seed = get_seed()
-                    as (multiworld, _):
+
+        print(f"Generating solo multiworld with seed {seed} for Stardew Valley...")
+        with solo_multiworld(player_options, seed=seed, world_caching=False) as (multiworld, _):
             self.assert_basic_checks(multiworld)

worlds/stardew_valley/test/long/TestRandomWorlds.py

@@ -1,86 +0,0 @@
-import random
-import unittest
-from typing import Dict
-
-from BaseClasses import MultiWorld, get_seed
-from Options import NamedRange, Range
-from .option_names import options_to_include
-from .. import SVTestCase
-from ..assertion import GoalAssertMixin, OptionAssertMixin, WorldAssertMixin
-
-
-def get_option_choices(option) -> Dict[str, int]:
-    if issubclass(option, NamedRange):
-        return option.special_range_names
-    if issubclass(option, Range):
-        return {f"{val}": val for val in range(option.range_start, option.range_end + 1)}
-    elif option.options:
-        return option.options
-    return {}
-
-
-def generate_random_world_options(seed: int) -> Dict[str, int]:
-    num_options = len(options_to_include)
-    world_options = dict()
-    rng = random.Random(seed)
-    for option_index in range(0, num_options):
-        option = options_to_include[option_index]
-        option_choices = get_option_choices(option)
-        if not option_choices:
-            continue
-        chosen_option_value = rng.choice(list(option_choices.values()))
-        world_options[option.internal_name] = chosen_option_value
-    return world_options
-
-
-def get_number_log_steps(number_worlds: int) -> int:
-    if number_worlds <= 10:
-        return 2
-    if number_worlds <= 100:
-        return 5
-    if number_worlds <= 500:
-        return 10
-    if number_worlds <= 1000:
-        return 20
-    if number_worlds <= 5000:
-        return 25
-    if number_worlds <= 10000:
-        return 50
-    return 100
-
-
-class TestGenerateManyWorlds(GoalAssertMixin, OptionAssertMixin, WorldAssertMixin, SVTestCase):
-    def test_generate_many_worlds_then_check_results(self):
-        if self.skip_long_tests:
-            raise unittest.SkipTest("Long tests disabled")
-
-        number_worlds = 10 if self.skip_long_tests else 1000
-        seed = get_seed()
-        self.generate_and_check_many_worlds(number_worlds, seed)
-
-    def generate_and_check_many_worlds(self, number_worlds: int, seed: int):
-        num_steps = get_number_log_steps(number_worlds)
-        log_step = number_worlds / num_steps
-
-        print(f"Generating {number_worlds} Solo Multiworlds [Start Seed: {seed}] for Stardew Valley...")
-        for world_number in range(0, number_worlds + 1):
-
-            world_seed = world_number + seed
-            world_options = generate_random_world_options(world_seed)
-
-            with self.solo_world_sub_test(f"Multiworld: {world_seed}", world_options, seed=world_seed, world_caching=False) as (multiworld, _):
-                self.assert_multiworld_is_valid(multiworld)
-
-            if world_number > 0 and world_number % log_step == 0:
-                print(f"Generated and Verified {world_number}/{number_worlds} worlds [{(world_number * 100) // number_worlds}%]")
-
-        print(f"Finished generating and verifying {number_worlds} Solo Multiworlds for Stardew Valley")
-
-    def assert_multiworld_is_valid(self, multiworld: MultiWorld):
-        self.assert_victory_exists(multiworld)
-        self.assert_same_number_items_locations(multiworld)
-        self.assert_goal_world_is_valid(multiworld)
-        self.assert_can_reach_island_if_should(multiworld)
-        self.assert_cropsanity_same_number_items_and_locations(multiworld)
-        self.assert_festivals_give_access_to_deluxe_scarecrow(multiworld)
-        self.assert_has_festival_recipes(multiworld)

worlds/stardew_valley/test/long/option_names.py

@@ -1,30 +0,0 @@
-from typing import Dict
-
-from Options import NamedRange
-from ... import StardewValleyWorld
-
-options_to_exclude = {"profit_margin", "starting_money", "multiple_day_sleep_enabled", "multiple_day_sleep_cost",
-                      "experience_multiplier", "friendship_multiplier", "debris_multiplier",
-                      "quick_start", "gifting", "gift_tax",
-                      "progression_balancing", "accessibility", "start_inventory", "start_hints", "death_link"}
-
-options_to_include = [option
-                      for option_name, option in StardewValleyWorld.options_dataclass.type_hints.items()
-                      if option_name not in options_to_exclude]
-
-
-def get_option_choices(option) -> Dict[str, int]:
-    if issubclass(option, NamedRange):
-        return option.special_range_names
-    elif option.options:
-        return option.options
-    return {}
-
-
-all_option_choices = [(option, value)
-                      for option in options_to_include
-                      if option.options
-                      for value in get_option_choices(option)
-                      if option.default != get_option_choices(option)[value]]
-
-assert all_option_choices

worlds/stardew_valley/test/mods/TestMods.py

@@ -1,88 +1,64 @@
 import random
+from typing import ClassVar
 
 from BaseClasses import get_seed
-from .. import SVTestBase, SVTestCase
+from test.param import classvar_matrix
+from .. import SVTestBase, SVTestCase, solo_multiworld
 from ..TestGeneration import get_all_permanent_progression_items
 from ..assertion import ModAssertMixin, WorldAssertMixin
 from ..options.presets import allsanity_mods_6_x_x
 from ..options.utils import fill_dataclass_with_default
 from ... import options, Group, create_content
 from ...mods.mod_data import ModNames
-from ...options import SkillProgression, Walnutsanity
 from ...options.options import all_mods
 from ...regions import RandomizationFlag, randomize_connections, create_final_connections_and_regions
 
 
-class TestGenerateModsOptions(WorldAssertMixin, ModAssertMixin, SVTestCase):
+class TestCanGenerateAllsanityWithMods(WorldAssertMixin, ModAssertMixin, SVTestCase):
-
-    def test_given_single_mods_when_generate_then_basic_checks(self):
-        for mod in options.Mods.valid_keys:
-            world_options = {options.Mods: mod, options.ExcludeGingerIsland: options.ExcludeGingerIsland.option_false}
-            with self.solo_world_sub_test(f"Mod: {mod}", world_options) as (multi_world, _):
-                self.assert_basic_checks(multi_world)
-                self.assert_stray_mod_items(mod, multi_world)
-
-    # The following tests validate that ER still generates winnable and logically-sane games with given mods.
-    # Mods that do not interact with entrances are skipped
-    # Not all ER settings are tested, because 'buildings' is, essentially, a superset of all others
-    def test_deepwoods_entrance_randomization_buildings(self):
-        self.perform_basic_checks_on_mod_with_er(ModNames.deepwoods, options.EntranceRandomization.option_buildings)
-
-    def test_juna_entrance_randomization_buildings(self):
-        self.perform_basic_checks_on_mod_with_er(ModNames.juna, options.EntranceRandomization.option_buildings)
-
-    def test_jasper_entrance_randomization_buildings(self):
-        self.perform_basic_checks_on_mod_with_er(ModNames.jasper, options.EntranceRandomization.option_buildings)
-
-    def test_alec_entrance_randomization_buildings(self):
-        self.perform_basic_checks_on_mod_with_er(ModNames.alec, options.EntranceRandomization.option_buildings)
-
-    def test_yoba_entrance_randomization_buildings(self):
-        self.perform_basic_checks_on_mod_with_er(ModNames.yoba, options.EntranceRandomization.option_buildings)
-
-    def test_eugene_entrance_randomization_buildings(self):
-        self.perform_basic_checks_on_mod_with_er(ModNames.eugene, options.EntranceRandomization.option_buildings)
-
-    def test_ayeisha_entrance_randomization_buildings(self):
-        self.perform_basic_checks_on_mod_with_er(ModNames.ayeisha, options.EntranceRandomization.option_buildings)
-
-    def test_riley_entrance_randomization_buildings(self):
-        self.perform_basic_checks_on_mod_with_er(ModNames.riley, options.EntranceRandomization.option_buildings)
-
-    def test_sve_entrance_randomization_buildings(self):
-        self.perform_basic_checks_on_mod_with_er(ModNames.sve, options.EntranceRandomization.option_buildings)
-
-    def test_alecto_entrance_randomization_buildings(self):
-        self.perform_basic_checks_on_mod_with_er(ModNames.alecto, options.EntranceRandomization.option_buildings)
-
-    def test_lacey_entrance_randomization_buildings(self):
-        self.perform_basic_checks_on_mod_with_er(ModNames.lacey, options.EntranceRandomization.option_buildings)
-
-    def test_boarding_house_entrance_randomization_buildings(self):
-        self.perform_basic_checks_on_mod_with_er(ModNames.boarding_house, options.EntranceRandomization.option_buildings)
-
-    def test_all_mods_entrance_randomization_buildings(self):
-        self.perform_basic_checks_on_mod_with_er(all_mods, options.EntranceRandomization.option_buildings)
-
-    def perform_basic_checks_on_mod_with_er(self, mods: str | set[str], er_option: int) -> None:
-        if isinstance(mods, str):
-            mods = {mods}
-        world_options = {
-            options.EntranceRandomization: er_option,
-            options.Mods: frozenset(mods),
-            options.ExcludeGingerIsland: options.ExcludeGingerIsland.option_false
-        }
-        with self.solo_world_sub_test(f"entrance_randomization: {er_option}, Mods: {mods}", world_options) as (multi_world, _):
-            self.assert_basic_checks(multi_world)
 
     def test_allsanity_all_mods_when_generate_then_basic_checks(self):
-        with self.solo_world_sub_test(world_options=allsanity_mods_6_x_x()) as (multi_world, _):
+        with solo_multiworld(allsanity_mods_6_x_x()) as (multi_world, _):
             self.assert_basic_checks(multi_world)
 
     def test_allsanity_all_mods_exclude_island_when_generate_then_basic_checks(self):
         world_options = allsanity_mods_6_x_x()
         world_options.update({options.ExcludeGingerIsland.internal_name: options.ExcludeGingerIsland.option_true})
-        with self.solo_world_sub_test(world_options=world_options) as (multi_world, _):
+        with solo_multiworld(world_options) as (multi_world, _):
+            self.assert_basic_checks(multi_world)
+
+
+@classvar_matrix(mod=all_mods)
+class TestCanGenerateWithEachMod(WorldAssertMixin, ModAssertMixin, SVTestCase):
+    mod: ClassVar[str]
+
+    def test_given_single_mods_when_generate_then_basic_checks(self):
+        world_options = {
+            options.Mods: self.mod,
+            options.ExcludeGingerIsland: options.ExcludeGingerIsland.option_false
+        }
+        with solo_multiworld(world_options) as (multi_world, _):
+            self.assert_basic_checks(multi_world)
+            self.assert_stray_mod_items(self.mod, multi_world)
+
+
+@classvar_matrix(mod=all_mods.difference([
+    ModNames.ginger, ModNames.distant_lands, ModNames.skull_cavern_elevator, ModNames.wellwick, ModNames.magic, ModNames.binning_skill, ModNames.big_backpack,
+    ModNames.luck_skill, ModNames.tractor, ModNames.shiko, ModNames.archaeology, ModNames.delores, ModNames.socializing_skill, ModNames.cooking_skill
+]))
+class TestCanGenerateEachModWithEntranceRandomizationBuildings(WorldAssertMixin, SVTestCase):
+    """The following tests validate that ER still generates winnable and logically-sane games with given mods.
+    Mods that do not interact with entrances are skipped
+    Not all ER settings are tested, because 'buildings' is, essentially, a superset of all others
+    """
+    mod: ClassVar[str]
+
+    def test_given_mod_when_generate_then_basic_checks(self) -> None:
+        world_options = {
+            options.EntranceRandomization: options.EntranceRandomization.option_buildings,
+            options.Mods: self.mod,
+            options.ExcludeGingerIsland: options.ExcludeGingerIsland.option_false
+        }
+        with solo_multiworld(world_options, world_caching=False) as (multi_world, _):
             self.assert_basic_checks(multi_world)
 
 
@@ -105,7 +81,7 @@ class TestBaseItemGeneration(SVTestBase):
         options.Chefsanity.internal_name: options.Chefsanity.option_all,
         options.Craftsanity.internal_name: options.Craftsanity.option_all,
         options.Booksanity.internal_name: options.Booksanity.option_all,
-        Walnutsanity.internal_name: Walnutsanity.preset_all,
+        options.Walnutsanity.internal_name: options.Walnutsanity.preset_all,
         options.Mods.internal_name: frozenset(options.Mods.valid_keys)
     }
 
@@ -151,7 +127,7 @@ class TestModEntranceRando(SVTestCase):
             sv_options = fill_dataclass_with_default({
                 options.EntranceRandomization.internal_name: option,
                 options.ExcludeGingerIsland.internal_name: options.ExcludeGingerIsland.option_false,
-                SkillProgression.internal_name: SkillProgression.option_progressive_with_masteries,
+                options.SkillProgression.internal_name: options.SkillProgression.option_progressive_with_masteries,
                 options.Mods.internal_name: frozenset(options.Mods.valid_keys)
             })
             content = create_content(sv_options)

worlds/stardew_valley/test/options/option_names.py

@@ -0,0 +1,51 @@
+import random
+
+from Options import NamedRange, Option, Range
+from ... import StardewValleyWorld
+from ...options import StardewValleyOption
+
+options_to_exclude = {"profit_margin", "starting_money", "multiple_day_sleep_enabled", "multiple_day_sleep_cost",
+                      "experience_multiplier", "friendship_multiplier", "debris_multiplier",
+                      "quick_start", "gifting", "gift_tax",
+                      "progression_balancing", "accessibility", "start_inventory", "start_hints", "death_link"}
+
+options_to_include: list[type[StardewValleyOption | Option]] = [
+    option
+    for option_name, option in StardewValleyWorld.options_dataclass.type_hints.items()
+    if option_name not in options_to_exclude
+]
+
+
+def get_option_choices(option: type[Option]) -> dict[str, int]:
+    if issubclass(option, NamedRange):
+        return option.special_range_names
+    if issubclass(option, Range):
+        return {f"{val}": val for val in range(option.range_start, option.range_end + 1)}
+    elif option.options:
+        return option.options
+    return {}
+
+
+def generate_random_world_options(seed: int) -> dict[str, int]:
+    num_options = len(options_to_include)
+    world_options = dict()
+    rng = random.Random(seed)
+    for option_index in range(0, num_options):
+        option = options_to_include[option_index]
+        option_choices = get_option_choices(option)
+        if not option_choices:
+            continue
+        chosen_option_value = rng.choice(list(option_choices.values()))
+        world_options[option.internal_name] = chosen_option_value
+    return world_options
+
+
+all_option_choices = [
+    (option.internal_name, value)
+    for option in options_to_include
+    if option.options
+    for value in get_option_choices(option)
+    if option.default != get_option_choices(option)[value]
+]
+
+assert all_option_choices

worlds/stardew_valley/test/stability/TestUniversalTracker.py

@@ -1,11 +1,12 @@
 import unittest
 from unittest.mock import Mock
 
-from .. import SVTestBase, fill_namespace_with_default
+from .. import SVTestBase, fill_namespace_with_default, skip_long_tests
 from ..options.presets import allsanity_mods_6_x_x
 from ... import STARDEW_VALLEY, FarmType, BundleRandomization, EntranceRandomization
 
 
+@unittest.skipIf(skip_long_tests(), "Long tests disabled")
 class TestUniversalTrackerGenerationIsStable(SVTestBase):
     options = allsanity_mods_6_x_x()
     options.update({
@@ -16,8 +17,6 @@ class TestUniversalTrackerGenerationIsStable(SVTestBase):
 
     def test_all_locations_and_items_are_the_same_between_two_generations(self):
         # This might open a kivy window temporarily, but it's the only way to test this...
-        if self.skip_long_tests:
-            raise unittest.SkipTest("Long tests disabled")
 
         try:
             # This test only run if UT is present, so no risk of running in the CI.
@@ -30,7 +29,7 @@ class TestUniversalTrackerGenerationIsStable(SVTestBase):
 
         fake_context = Mock()
         fake_context.re_gen_passthrough = {STARDEW_VALLEY: ut_data}
-        args = fill_namespace_with_default({0: self.options})
+        args = fill_namespace_with_default([self.options])
         args.outputpath = None
         args.outputname = None
         args.multi = 1