Ocarina of Time: reduce memory use by 64 MiB for each OoT world past the first

Ocarina of Time: limit parallel output to 2, to not waste memory that doesn't benefit speed
Ocarina of Time: remove swarm of os.chdir()
This commit is contained in:
Fabian Dill
2021-09-03 12:50:26 +02:00
parent 51c38fc628
commit 1b27fc495f
5 changed files with 145 additions and 146 deletions

View File

@@ -1,5 +1,5 @@
import logging
import os
import threading
import copy
from collections import Counter
@@ -33,17 +33,21 @@ from ..AutoWorld import World
location_id_offset = 67000
# OoT's generate_output doesn't benefit from more than 2 threads, instead it uses a lot of memory.
i_o_limiter = threading.Semaphore(2)
class OOTWorld(World):
game: str = "Ocarina of Time"
options: dict = oot_options
topology_present: bool = True
item_name_to_id = {item_name: oot_data_to_ap_id(data, False) for item_name, data in item_table.items() if data[2] is not None}
item_name_to_id = {item_name: oot_data_to_ap_id(data, False) for item_name, data in item_table.items() if
data[2] is not None}
location_name_to_id = location_name_to_id
remote_items: bool = False
data_version = 1
def __new__(cls, world, player):
# Add necessary objects to CollectionState on initialization
orig_init = CollectionState.__init__
@@ -64,9 +68,9 @@ class OOTWorld(World):
ret.adult_reachable_regions = {player: copy.copy(self.adult_reachable_regions[player]) for player in
range(1, self.world.players + 1)}
ret.child_blocked_connections = {player: copy.copy(self.child_blocked_connections[player]) for player in
range(1, self.world.players + 1)}
range(1, self.world.players + 1)}
ret.adult_blocked_connections = {player: copy.copy(self.adult_blocked_connections[player]) for player in
range(1, self.world.players + 1)}
range(1, self.world.players + 1)}
return ret
CollectionState.__init__ = oot_init
@@ -81,17 +85,17 @@ class OOTWorld(World):
return super().__new__(cls)
def generate_early(self):
# Player name MUST be at most 16 bytes ascii-encoded, otherwise won't write to ROM correctly
if len(bytes(self.world.get_player_name(self.player), 'ascii')) > 16:
raise Exception(f"OoT: Player {self.player}'s name ({self.world.get_player_name(self.player)}) must be ASCII-compatible")
raise Exception(
f"OoT: Player {self.player}'s name ({self.world.get_player_name(self.player)}) must be ASCII-compatible")
self.parser = Rule_AST_Transformer(self, self.player)
for (option_name, option) in oot_options.items():
for (option_name, option) in oot_options.items():
result = getattr(self.world, option_name)[self.player]
if isinstance(result, Range):
if isinstance(result, Range):
option_value = int(result)
elif isinstance(result, Toggle):
option_value = bool(result)
@@ -109,17 +113,21 @@ class OOTWorld(World):
self.file_hash = [self.world.random.randint(0, 31) for i in range(5)]
self.item_name_groups = {
"medallions": {"Light Medallion", "Forest Medallion", "Fire Medallion", "Water Medallion", "Shadow Medallion", "Spirit Medallion"},
"medallions": {"Light Medallion", "Forest Medallion", "Fire Medallion", "Water Medallion",
"Shadow Medallion", "Spirit Medallion"},
"stones": {"Kokiri Emerald", "Goron Ruby", "Zora Sapphire"},
"rewards": {"Light Medallion", "Forest Medallion", "Fire Medallion", "Water Medallion", "Shadow Medallion", "Spirit Medallion", \
"Kokiri Emerald", "Goron Ruby", "Zora Sapphire"},
"bottles": {"Bottle", "Bottle with Milk", "Deliver Letter", "Sell Big Poe", "Bottle with Red Potion", "Bottle with Green Potion", \
"Bottle with Blue Potion", "Bottle with Fairy", "Bottle with Fish", "Bottle with Blue Fire", "Bottle with Bugs", "Bottle with Poe"}
"rewards": {"Light Medallion", "Forest Medallion", "Fire Medallion", "Water Medallion", "Shadow Medallion",
"Spirit Medallion", \
"Kokiri Emerald", "Goron Ruby", "Zora Sapphire"},
"bottles": {"Bottle", "Bottle with Milk", "Deliver Letter", "Sell Big Poe", "Bottle with Red Potion",
"Bottle with Green Potion", \
"Bottle with Blue Potion", "Bottle with Fairy", "Bottle with Fish", "Bottle with Blue Fire",
"Bottle with Bugs", "Bottle with Poe"}
}
# Incompatible option handling
# ER and glitched logic are not compatible; glitched takes priority
if self.logic_rules == 'glitched':
if self.logic_rules == 'glitched':
self.shuffle_interior_entrances = False
self.shuffle_grotto_entrances = False
self.shuffle_dungeon_entrances = False
@@ -129,7 +137,7 @@ class OOTWorld(World):
self.spawn_positions = False
# Closed forest and adult start are not compatible; closed forest takes priority
if self.open_forest == 'closed':
if self.open_forest == 'closed':
self.starting_age = 'child'
# Skip child zelda and shuffle egg are not compatible; skip-zelda takes priority
@@ -149,16 +157,16 @@ class OOTWorld(World):
self.dungeon_mq = {item['name']: (item in mq_dungeons) for item in dungeon_table}
# Determine tricks in logic
for trick in self.logic_tricks:
for trick in self.logic_tricks:
normalized_name = trick.casefold()
if normalized_name in normalized_name_tricks:
if normalized_name in normalized_name_tricks:
setattr(self, normalized_name_tricks[normalized_name]['name'], True)
else:
raise Exception(f'Unknown OOT logic trick for player {self.player}: {trick}')
# Not implemented for now, but needed to placate the generator. Remove as they are implemented
self.mq_dungeons_random = False # this will be a deprecated option later
self.ocarina_songs = False # just need to pull in the OcarinaSongs module
self.ocarina_songs = False # just need to pull in the OcarinaSongs module
self.big_poe_count = 1 # disabled due to client-side issues for now
self.correct_chest_sizes = False # will probably never be implemented since multiworld items are always major
# ER options
@@ -171,7 +179,8 @@ class OOTWorld(World):
self.spawn_positions = False
# Set internal names used by the OoT generator
self.keysanity = self.shuffle_smallkeys in ['keysanity', 'remove', 'any_dungeon', 'overworld'] # only 'keysanity' and 'remove' implemented
self.keysanity = self.shuffle_smallkeys in ['keysanity', 'remove', 'any_dungeon',
'overworld'] # only 'keysanity' and 'remove' implemented
# Hint stuff
self.misc_hints = True # this is just always on
@@ -193,7 +202,6 @@ class OOTWorld(World):
self.shopsanity = self.shopsanity.replace('_value', '') # can't set "random" manually
self.shuffle_scrubs = self.shuffle_scrubs.replace('_prices', '')
# Get hint distribution
self.hint_dist_user = read_json(data_path('Hints', f'{self.hint_dist}.json'))
@@ -230,11 +238,9 @@ class OOTWorld(World):
self.always_hints = [hint.name for hint in getRequiredHints(self)]
def load_regions_from_json(self, file_path):
region_json = read_json(file_path)
for region in region_json:
new_region = OOTRegion(region['region_name'], RegionType.Generic, None, self.player)
new_region.world = self.world
@@ -294,7 +300,6 @@ class OOTWorld(World):
self.regions.append(new_region)
self.world._recache()
def set_scrub_prices(self):
# Get Deku Scrub Locations
scrub_locations = [location for location in self.get_locations() if 'Deku Scrub' in location.name]
@@ -323,13 +328,12 @@ class OOTWorld(World):
if location.item is not None:
location.item.price = price
def random_shop_prices(self):
shop_item_indexes = ['7', '5', '8', '6']
self.shop_prices = {}
for region in self.regions:
if self.shopsanity == 'random':
shop_item_count = self.world.random.randint(0,4)
shop_item_count = self.world.random.randint(0, 4)
else:
shop_item_count = int(self.shopsanity)
@@ -338,8 +342,7 @@ class OOTWorld(World):
if location.name[-1:] in shop_item_indexes[:shop_item_count]:
self.shop_prices[location.name] = int(self.world.random.betavariate(1.5, 2) * 60) * 5
def fill_bosses(self, bossCount=9):
def fill_bosses(self, bossCount=9):
rewardlist = (
'Kokiri Emerald',
'Goron Ruby',
@@ -381,13 +384,11 @@ class OOTWorld(World):
loc.locked = True
loc.event = True
def create_item(self, name: str):
if name in item_table:
def create_item(self, name: str):
if name in item_table:
return OOTItem(name, self.player, item_table[name], False)
return OOTItem(name, self.player, ('Event', True, None, None), True)
def make_event_item(self, name, location, item=None):
if item is None:
item = self.create_item(name)
@@ -398,7 +399,6 @@ class OOTWorld(World):
location.internal = True
return item
def create_regions(self): # create and link regions
if self.logic_rules == 'glitchless':
world_type = 'World'
@@ -427,33 +427,31 @@ class OOTWorld(World):
if self.entrance_shuffle:
shuffle_random_entrances(self)
def set_rules(self):
def set_rules(self):
set_rules(self)
def generate_basic(self): # generate item pools, place fixed items
# Generate itempool
generate_itempool(self)
junk_pool = get_junk_pool(self)
# Determine starting items
for item in self.world.precollected_items:
for item in self.world.precollected_items:
if item.player != self.player:
continue
if item.name in self.remove_from_start_inventory:
self.remove_from_start_inventory.remove(item.name)
else:
self.starting_items[item.name] += 1
if item.type == 'Song':
if item.type == 'Song':
self.starting_songs = True
# Call the junk fill and get a replacement
if item in self.itempool:
self.itempool.remove(item)
self.itempool.append(self.create_item(*get_junk_item(pool=junk_pool)))
if self.start_with_consumables:
if self.start_with_consumables:
self.starting_items['Deku Sticks'] = 30
self.starting_items['Deku Nuts'] = 40
if self.start_with_rupees:
if self.start_with_rupees:
self.starting_items['Rupees'] = 999
# Uniquely rename drop locations for each region and erase them from the spoiler
@@ -486,7 +484,7 @@ class OOTWorld(World):
'keysanity': [],
}
any_dungeon_locations = []
for dungeon in self.dungeons:
for dungeon in self.dungeons:
itempools['dungeon'] = []
# Put the dungeon items into their appropriate pools.
# Build in reverse order since we need to fill boss key first and pop() returns the last element
@@ -499,23 +497,29 @@ class OOTWorld(World):
itempools[shufflebk].extend(dungeon.boss_key)
# We can't put a dungeon item on the end of a dungeon if a song is supposed to go there. Make sure not to include it.
dungeon_locations = [loc for region in dungeon.regions for loc in region.locations
if loc.item is None and (self.shuffle_song_items != 'dungeon' or loc.name not in dungeon_song_locations)]
if itempools['dungeon']: # only do this if there's anything to shuffle
dungeon_locations = [loc for region in dungeon.regions for loc in region.locations
if loc.item is None and (
self.shuffle_song_items != 'dungeon' or loc.name not in dungeon_song_locations)]
if itempools['dungeon']: # only do this if there's anything to shuffle
self.world.random.shuffle(dungeon_locations)
fill_restrictive(self.world, self.state_with_items(self.itempool), dungeon_locations, itempools['dungeon'], True, True)
fill_restrictive(self.world, self.state_with_items(self.itempool), dungeon_locations,
itempools['dungeon'], True, True)
any_dungeon_locations.extend(dungeon_locations) # adds only the unfilled locations
# Now fill items that can go into any dungeon. Retrieve the Gerudo Fortress keys from the pool if necessary
if self.shuffle_fortresskeys == 'any_dungeon':
fortresskeys = list(filter(lambda item: item.player == self.player and item.type == 'FortressSmallKey', self.itempool))
if self.shuffle_fortresskeys == 'any_dungeon':
fortresskeys = list(
filter(lambda item: item.player == self.player and item.type == 'FortressSmallKey', self.itempool))
itempools['any_dungeon'].extend(fortresskeys)
for key in fortresskeys:
self.itempool.remove(key)
if itempools['any_dungeon']:
itempools['any_dungeon'].sort(key=lambda item: {'GanonBossKey': 4, 'BossKey': 3, 'SmallKey': 2, 'FortressSmallKey': 1}.get(item.type, 0))
itempools['any_dungeon'].sort(
key=lambda item: {'GanonBossKey': 4, 'BossKey': 3, 'SmallKey': 2, 'FortressSmallKey': 1}.get(item.type,
0))
self.world.random.shuffle(any_dungeon_locations)
fill_restrictive(self.world, self.state_with_items(self.itempool), any_dungeon_locations, itempools['any_dungeon'], True, True)
fill_restrictive(self.world, self.state_with_items(self.itempool), any_dungeon_locations,
itempools['any_dungeon'], True, True)
# If anything is overworld-only, enforce them as local and not in the remaining dungeon locations
if itempools['overworld'] or self.shuffle_fortresskeys == 'overworld':
@@ -535,26 +539,27 @@ class OOTWorld(World):
# Place songs
# 5 built-in retries because this section can fail sometimes
if self.shuffle_song_items != 'any':
if self.shuffle_song_items != 'any':
tries = 5
if self.shuffle_song_items == 'song':
song_locations = list(filter(lambda location: location.type == 'Song',
self.world.get_unfilled_locations(player=self.player)))
elif self.shuffle_song_items == 'dungeon':
song_locations = list(filter(lambda location: location.name in dungeon_song_locations,
self.world.get_unfilled_locations(player=self.player)))
song_locations = list(filter(lambda location: location.type == 'Song',
self.world.get_unfilled_locations(player=self.player)))
elif self.shuffle_song_items == 'dungeon':
song_locations = list(filter(lambda location: location.name in dungeon_song_locations,
self.world.get_unfilled_locations(player=self.player)))
else:
raise Exception(f"Unknown song shuffle type: {self.shuffle_song_items}")
songs = list(filter(lambda item: item.player == self.player and item.type == 'Song', self.itempool))
for song in songs:
for song in songs:
self.itempool.remove(song)
while tries:
try:
self.world.random.shuffle(songs) # shuffling songs makes it less likely to fail by placing ZL last
self.world.random.shuffle(songs) # shuffling songs makes it less likely to fail by placing ZL last
self.world.random.shuffle(song_locations)
fill_restrictive(self.world, self.state_with_items(self.itempool), song_locations[:], songs[:], True, True)
logger.debug(f"Successfully placed songs for player {self.player} after {6-tries} attempt(s)")
fill_restrictive(self.world, self.state_with_items(self.itempool), song_locations[:], songs[:],
True, True)
logger.debug(f"Successfully placed songs for player {self.player} after {6 - tries} attempt(s)")
tries = 0
except FillError as e:
tries -= 1
@@ -572,13 +577,14 @@ class OOTWorld(World):
# Place shop items
# fast fill will fail because there is some logic on the shop items. we'll gather them up and place the shop items
if self.shopsanity != 'off':
if self.shopsanity != 'off':
shop_items = list(filter(lambda item: item.player == self.player and item.type == 'Shop', self.itempool))
shop_locations = list(filter(lambda location: location.type == 'Shop' and location.name not in self.shop_prices,
self.world.get_unfilled_locations(player=self.player)))
shop_locations = list(
filter(lambda location: location.type == 'Shop' and location.name not in self.shop_prices,
self.world.get_unfilled_locations(player=self.player)))
shop_items.sort(key=lambda item: 1 if item.name in ["Buy Goron Tunic", "Buy Zora Tunic"] else 0)
self.world.random.shuffle(shop_locations)
for item in shop_items:
for item in shop_items:
self.itempool.remove(item)
fill_restrictive(self.world, self.state_with_items(self.itempool), shop_locations, shop_items, True, True)
set_shop_rules(self)
@@ -586,8 +592,9 @@ class OOTWorld(World):
# Locations which are not sendable must be converted to events
# This includes all locations for which show_in_spoiler is false, and shuffled shop items.
for loc in self.get_locations():
if loc.address is not None and (not loc.show_in_spoiler or (loc.item is not None and loc.item.type == 'Shop')
or (self.skip_child_zelda and loc.name in ['HC Zeldas Letter', 'Song from Impa'])):
if loc.address is not None and (
not loc.show_in_spoiler or (loc.item is not None and loc.item.type == 'Shop')
or (self.skip_child_zelda and loc.name in ['HC Zeldas Letter', 'Song from Impa'])):
loc.address = None
# Gather items for ice trap appearances
@@ -595,7 +602,8 @@ class OOTWorld(World):
if self.ice_trap_appearance in ['major_only', 'anything']:
self.fake_items.extend([item for item in self.itempool if item.index and self.is_major_item(item)])
if self.ice_trap_appearance in ['junk_only', 'anything']:
self.fake_items.extend([item for item in self.itempool if item.index and not self.is_major_item(item) and item.name != 'Ice Trap'])
self.fake_items.extend([item for item in self.itempool if
item.index and not self.is_major_item(item) and item.name != 'Ice Trap'])
# Put all remaining items into the general itempool
self.world.itempool += self.itempool
@@ -605,8 +613,9 @@ class OOTWorld(World):
all_state = self.state_with_items(self.itempool)
all_locations = [loc for loc in self.world.get_locations() if loc.player == self.player]
reachable = self.world.get_reachable_locations(all_state, self.player)
unreachable = [loc for loc in all_locations if loc.internal and loc.event and loc.locked and loc not in reachable]
for loc in unreachable:
unreachable = [loc for loc in all_locations if
loc.internal and loc.event and loc.locked and loc not in reachable]
for loc in unreachable:
loc.parent_region.locations.remove(loc)
# Exception: Sell Big Poe is an event which is only reachable if Bottle with Big Poe is in the item pool.
# We allow it to be removed only if Bottle with Big Poe is not in the itempool.
@@ -629,30 +638,32 @@ class OOTWorld(World):
impa = self.world.get_location("Song from Impa", self.player)
if self.skip_child_zelda and impa.item is None:
from .SaveContext import SaveContext
item_to_place = self.world.random.choice([item for item in self.world.itempool
if item.player == self.player and item.name in SaveContext.giveable_items])
item_to_place = self.world.random.choice([item for item in self.world.itempool
if
item.player == self.player and item.name in SaveContext.giveable_items])
self.world.push_item(impa, item_to_place, False)
impa.locked = True
impa.event = True
self.world.itempool.remove(item_to_place)
# For now we will always output a patch file.
def generate_output(self, output_directory: str):
# Make ice traps appear as other random items
ice_traps = [loc.item for loc in self.get_locations() if loc.item.name == 'Ice Trap']
for trap in ice_traps:
trap.looks_like_item = self.create_item(self.world.slot_seeds[self.player].choice(self.fake_items).name)
outfile_name = f"AP_{self.world.seed_name}_P{self.player}_{self.world.get_player_name(self.player)}"
rom = Rom(file=get_options()['oot_options']['rom_file']) # a ROM must be provided, cannot produce patches without it
if self.hints != 'none':
buildWorldGossipHints(self)
patch_rom(self, rom)
patch_cosmetics(self, rom)
rom.update_header()
create_patch_file(rom, output_path(output_directory, outfile_name+'.apz5'))
rom.restore()
def generate_output(self, output_directory: str):
with i_o_limiter:
# Make ice traps appear as other random items
ice_traps = [loc.item for loc in self.get_locations() if loc.item.name == 'Ice Trap']
for trap in ice_traps:
trap.looks_like_item = self.create_item(self.world.slot_seeds[self.player].choice(self.fake_items).name)
outfile_name = f"AP_{self.world.seed_name}_P{self.player}_{self.world.get_player_name(self.player)}"
rom = Rom(
file=get_options()['oot_options']['rom_file']) # a ROM must be provided, cannot produce patches without it
if self.hints != 'none':
buildWorldGossipHints(self)
patch_rom(self, rom)
patch_cosmetics(self, rom)
rom.update_header()
create_patch_file(rom, output_path(output_directory, outfile_name + '.apz5'))
rom.restore()
# Helper functions
def get_shuffled_entrances(self):
@@ -662,7 +673,7 @@ class OOTWorld(World):
def get_locations(self):
return [loc for region in self.regions for loc in region.locations]
def get_location(self, location):
def get_location(self, location):
return self.world.get_location(location, self.player)
def get_region(self, region):
@@ -698,7 +709,6 @@ class OOTWorld(World):
return True
# Run this once for to gather up all required locations (for WOTH), barren regions (for foolish), and location of major items.
# required_locations and major_item_locations need to be ordered for deterministic hints.
def gather_hint_data(self):
@@ -710,11 +720,11 @@ class OOTWorld(World):
items_by_region[r.hint_text] = {'dungeon': False, 'weight': 0, 'prog_items': 0}
for d in self.dungeons:
items_by_region[d.hint_text] = {'dungeon': True, 'weight': 0, 'prog_items': 0}
del(items_by_region["Link's Pocket"])
del(items_by_region[None])
del (items_by_region["Link's Pocket"])
del (items_by_region[None])
for loc in self.get_locations():
if loc.item.code: # is a real item
if loc.item.code: # is a real item
hint_area = get_hint_area(loc)
items_by_region[hint_area]['weight'] += 1
if loc.item.advancement and (not loc.locked or loc.item.type == 'Song'):
@@ -726,9 +736,9 @@ class OOTWorld(World):
if not self.world.can_beat_game(state):
self.required_locations.append(loc)
self.empty_areas = {region: info for (region, info) in items_by_region.items() if not info['prog_items']}
for loc in self.world.get_filled_locations():
if (loc.item.player == self.player and self.is_major_item(loc.item)
if (loc.item.player == self.player and self.is_major_item(loc.item)
or (loc.item.player == self.player and loc.item.name in self.item_added_hint_types['item'])
or (loc.name in self.added_hint_types['item'] and loc.player == self.player)):
self.major_item_locations.append(loc)