Core, WebHost: lazy-load worlds in unpickler, WebHost and WebHostLib (#2156)

* Core: lazy-load worlds in unpickler

this should hopefully fix customserver's memory consumption

* WebHost: move imports around to save memory in MP

* MultiServer: prefer loading _speedups without pyximport

This saves ~15MB per MP and speeds up module import if it was built in-place.

* Tests: fix tests for changed WebHost imports

* CustomServer: run GC after setup

* CustomServer: cleanup exception handling
This commit is contained in:
black-sliver
2023-09-20 16:05:56 +02:00
committed by GitHub
parent 4a27fae1ab
commit d471dcc067
8 changed files with 93 additions and 75 deletions

View File

@@ -359,11 +359,13 @@ safe_builtins = frozenset((
class RestrictedUnpickler(pickle.Unpickler):
generic_properties_module: Optional[object]
def __init__(self, *args, **kwargs):
super(RestrictedUnpickler, self).__init__(*args, **kwargs)
self.options_module = importlib.import_module("Options")
self.net_utils_module = importlib.import_module("NetUtils")
self.generic_properties_module = importlib.import_module("worlds.generic")
self.generic_properties_module = None
def find_class(self, module, name):
if module == "builtins" and name in safe_builtins:
@@ -373,6 +375,8 @@ class RestrictedUnpickler(pickle.Unpickler):
return getattr(self.net_utils_module, name)
# Options and Plando are unpickled by WebHost -> Generate
if module == "worlds.generic" and name in {"PlandoItem", "PlandoConnection"}:
if not self.generic_properties_module:
self.generic_properties_module = importlib.import_module("worlds.generic")
return getattr(self.generic_properties_module, name)
# pep 8 specifies that modules should have "all-lowercase names" (options, not Options)
if module.lower().endswith("options"):