Deprecate data_version and introduce checksum for DataPackages. (#684)

Co-authored-by: black-sliver <59490463+black-sliver@users.noreply.github.com>

WebHostLib/api/__init__.py

@@ -39,12 +39,21 @@ def get_datapackage():
 
 @api_endpoints.route('/datapackage_version')
 @cache.cached()
-
 def get_datapackage_versions():
-    from worlds import network_data_package, AutoWorldRegister
+    from worlds import AutoWorldRegister
 
     version_package = {game: world.data_version for game, world in AutoWorldRegister.world_types.items()}
     return version_package
 
 
+@api_endpoints.route('/datapackage_checksum')
+@cache.cached()
+def get_datapackage_checksums():
+    from worlds import network_data_package
+    version_package = {
+        game: game_data["checksum"] for game, game_data in network_data_package["games"].items()
+    }
+    return version_package
+
+
 from . import generate, user  # trigger registration

WebHostLib/customserver.py

@@ -19,7 +19,7 @@ import Utils
 
 from MultiServer import Context, server, auto_shutdown, ServerCommandProcessor, ClientMessageProcessor, load_server_cert
 from Utils import get_public_ipv4, get_public_ipv6, restricted_loads, cache_argsless
-from .models import Room, Command, db
+from .models import Command, GameDataPackage, Room, db
 
 
 class CustomClientMessageProcessor(ClientMessageProcessor):
@@ -92,7 +92,20 @@ class WebHostContext(Context):
         else:
             self.port = get_random_port()
 
-        return self._load(self.decompress(room.seed.multidata), True)
+        multidata = self.decompress(room.seed.multidata)
+        game_data_packages = {}
+        for game in list(multidata["datapackage"]):
+            game_data = multidata["datapackage"][game]
+            if "checksum" in game_data:
+                if self.gamespackage.get(game, {}).get("checksum") == game_data["checksum"]:
+                    # non-custom. remove from multidata
+                    # games package could be dropped from static data once all rooms embed data package
+                    del multidata["datapackage"][game]
+                else:
+                    data = Utils.restricted_loads(GameDataPackage.get(checksum=game_data["checksum"]).data)
+                    game_data_packages[game] = data
+
+        return self._load(multidata, game_data_packages, True)
 
     @db_session
     def init_save(self, enabled: bool = True):

WebHostLib/models.py

@@ -56,3 +56,8 @@ class Generation(db.Entity):
     options = Required(buffer, lazy=True)
     meta = Required(LongStr, default=lambda: "{\"race\": false}")
     state = Required(int, default=0, index=True)
+
+
+class GameDataPackage(db.Entity):
+    checksum = PrimaryKey(str)
+    data = Required(bytes)

WebHostLib/tracker.py

@@ -11,10 +11,10 @@ from werkzeug.exceptions import abort
 from MultiServer import Context, get_saving_second
 from NetUtils import SlotType
 from Utils import restricted_loads
-from worlds import lookup_any_item_id_to_name, lookup_any_location_id_to_name
+from worlds import lookup_any_item_id_to_name, lookup_any_location_id_to_name, network_data_package
 from worlds.alttp import Items
 from . import app, cache
-from .models import Room
+from .models import GameDataPackage, Room
 
 alttp_icons = {
     "Blue Shield": r"https://www.zeldadungeon.net/wiki/images/8/85/Fighters-Shield.png",
@@ -229,14 +229,15 @@ def render_timedelta(delta: datetime.timedelta):
 
 @pass_context
 def get_location_name(context: runtime.Context, loc: int) -> str:
+    # once all rooms embed data package, the chain lookup can be dropped
     context_locations = context.get("custom_locations", {})
-    return collections.ChainMap(lookup_any_location_id_to_name, context_locations).get(loc, loc)
+    return collections.ChainMap(context_locations, lookup_any_location_id_to_name).get(loc, loc)
 
 
 @pass_context
 def get_item_name(context: runtime.Context, item: int) -> str:
     context_items = context.get("custom_items", {})
-    return collections.ChainMap(lookup_any_item_id_to_name, context_items).get(item, item)
+    return collections.ChainMap(context_items, lookup_any_item_id_to_name).get(item, item)
 
 
 app.jinja_env.filters["location_name"] = get_location_name
@@ -274,11 +275,21 @@ def get_static_room_data(room: Room):
                   if slot_info.type == SlotType.group}
 
         for game in games.values():
-            if game in multidata["datapackage"]:
-                custom_locations.update(
-                    {id: name for name, id in multidata["datapackage"][game]["location_name_to_id"].items()})
-                custom_items.update(
-                    {id: name for name, id in multidata["datapackage"][game]["item_name_to_id"].items()})
+            if game not in multidata["datapackage"]:
+                continue
+            game_data = multidata["datapackage"][game]
+            if "checksum" in game_data:
+                if network_data_package["games"].get(game, {}).get("checksum") == game_data["checksum"]:
+                    # non-custom. remove from multidata
+                    # network_data_package import could be skipped once all rooms embed data package
+                    del multidata["datapackage"][game]
+                    continue
+                else:
+                    game_data = restricted_loads(GameDataPackage.get(checksum=game_data["checksum"]).data)
+            custom_locations.update(
+                {id_: name for name, id_ in game_data["location_name_to_id"].items()})
+            custom_items.update(
+                {id_: name for name, id_ in game_data["item_name_to_id"].items()})
     elif "games" in multidata:
         games = multidata["games"]
     seed_checks_in_area = checks_in_area.copy()

WebHostLib/upload.py

@@ -1,19 +1,22 @@
 import base64
 import json
+import pickle
 import typing
 import uuid
 import zipfile
-from io import BytesIO
+import zlib
 
+from io import BytesIO
 from flask import request, flash, redirect, url_for, session, render_template, Markup
-from pony.orm import flush, select
+from pony.orm import commit, flush, select, rollback
+from pony.orm.core import TransactionIntegrityError
 
 import MultiServer
 from NetUtils import NetworkSlot, SlotType
 from Utils import VersionException, __version__
 from worlds.Files import AutoPatchRegister
 from . import app
-from .models import Seed, Room, Slot
+from .models import Seed, Room, Slot, GameDataPackage
 
 banned_zip_contents = (".sfc", ".z64", ".n64", ".sms", ".gb")
 
@@ -78,6 +81,27 @@ def upload_zip_to_db(zfile: zipfile.ZipFile, owner=None, meta={"race": False}, s
     # Load multi data.
     if multidata:
         decompressed_multidata = MultiServer.Context.decompress(multidata)
+        recompress = False
+
+        if "datapackage" in decompressed_multidata:
+            # strip datapackage from multidata, leaving only the checksums
+            game_data_packages: typing.List[GameDataPackage] = []
+            for game, game_data in decompressed_multidata["datapackage"].items():
+                if game_data.get("checksum"):
+                    game_data_package = GameDataPackage(checksum=game_data["checksum"],
+                                                        data=pickle.dumps(game_data))
+                    decompressed_multidata["datapackage"][game] = {
+                        "version": game_data.get("version", 0),
+                        "checksum": game_data["checksum"]
+                    }
+                    recompress = True
+                    try:
+                        commit()  # commit game data package
+                        game_data_packages.append(game_data_package)
+                    except TransactionIntegrityError:
+                        del game_data_package
+                        rollback()
+
         if "slot_info" in decompressed_multidata:
             for slot, slot_info in decompressed_multidata["slot_info"].items():
                 # Ignore Player Groups (e.g. item links)
@@ -90,6 +114,9 @@ def upload_zip_to_db(zfile: zipfile.ZipFile, owner=None, meta={"race": False}, s
 
             flush()  # commit slots
 
+        if recompress:
+            multidata = multidata[0:1] + zlib.compress(pickle.dumps(decompressed_multidata), 9)
+
         seed = Seed(multidata=multidata, spoiler=spoiler, slots=slots, owner=owner, meta=json.dumps(meta),
                     id=sid if sid else uuid.uuid4())
         flush()  # create seed