Core: APWorld manifest (#4516)

Adds support for a manifest file (archipelago.json) inside an .apworld file. It tells AP the game, minimum core version (optional field), maximum core version (optional field), its own version (used to determine which file to prefer to load only currently)
The file itself is marked as required starting with core 0.7.0, prior, just a warning is printed, with error trace.

Co-authored-by: Doug Hoskisson <beauxq@users.noreply.github.com>
Co-authored-by: qwint <qwint.42@gmail.com>
Co-authored-by: Exempt-Medic <60412657+Exempt-Medic@users.noreply.github.com>
Co-authored-by: NewSoupVi <57900059+NewSoupVi@users.noreply.github.com>

Utils.py

@@ -49,6 +49,7 @@ class Version(typing.NamedTuple):
 
 __version__ = "0.6.4"
 version_tuple = tuplize_version(__version__)
+version = Version(*version_tuple)
 
 is_linux = sys.platform.startswith("linux")
 is_macos = sys.platform == "darwin"

docs/apworld specification.md

@@ -19,7 +19,21 @@ the world's folder in `worlds/`. I.e. `worlds/ror2.apworld` containing `ror2/__i
 
 ## Metadata
 
-No metadata is specified yet.
+Metadata about the apworld is defined in an `archipelago.json` file inside the zip archive.
+The current format version has at minimum:
+```json
+{
+    "version": 6,
+    "compatible_version": 5,
+    "game": "Game Name"
+}
+```
+
+with the following optional version fields using the format `"1.0.0"` to represent major.minor.build:
+* `minimum_ap_version` and `maximum_ap_version` - which if present will each be compared against the current
+  Archipelago version respectively to filter those files from being loaded
+* `world_version` - an arbitrary version for that world in order to only load the newest valid world.
+  An apworld without a world_version is always treated as older than one with a version
 
 
 ## Extra Data

setup.py

@@ -371,6 +371,8 @@ class BuildExeCommand(cx_Freeze.command.build_exe.build_exe):
         os.makedirs(self.buildfolder / "Players" / "Templates", exist_ok=True)
         from Options import generate_yaml_templates
         from worlds.AutoWorld import AutoWorldRegister
+        from worlds.Files import APWorldContainer
+        from Utils import version
         assert not non_apworlds - set(AutoWorldRegister.world_types), \
             f"Unknown world {non_apworlds - set(AutoWorldRegister.world_types)} designated for .apworld"
         folders_to_remove: list[str] = []
@@ -379,13 +381,26 @@ class BuildExeCommand(cx_Freeze.command.build_exe.build_exe):
             if worldname not in non_apworlds:
                 file_name = os.path.split(os.path.dirname(worldtype.__file__))[1]
                 world_directory = self.libfolder / "worlds" / file_name
+                if os.path.isfile(world_directory / "archipelago.json"):
+                    manifest = json.load(open(world_directory / "archipelago.json"))
+                else:
+                    manifest = {}
                 # this method creates an apworld that cannot be moved to a different OS or minor python version,
                 # which should be ok
-                with zipfile.ZipFile(self.libfolder / "worlds" / (file_name + ".apworld"), "x", zipfile.ZIP_DEFLATED,
+                zip_path = self.libfolder / "worlds" / (file_name + ".apworld")
+                apworld = APWorldContainer(str(zip_path))
+                apworld.minimum_ap_version = version
+                apworld.maximum_ap_version = version
+                apworld.game = worldtype.game
+                manifest.update(apworld.get_manifest())
+                apworld.manifest_path = f"{file_name}/archipelago.json"
+                with zipfile.ZipFile(zip_path, "x", zipfile.ZIP_DEFLATED,
                                      compresslevel=9) as zf:
                     for path in world_directory.rglob("*.*"):
                         relative_path = os.path.join(*path.parts[path.parts.index("worlds")+1:])
+                        if not relative_path.endswith("archipelago.json"):
                             zf.write(path, relative_path)
+                    zf.writestr(apworld.manifest_path, json.dumps(manifest))
                     folders_to_remove.append(file_name)
                 shutil.rmtree(world_directory)
         shutil.copyfile("meta.yaml", self.buildfolder / "Players" / "Templates" / "meta.yaml")

worlds/Files.py

@@ -8,7 +8,8 @@ import os
 import threading
 from io import BytesIO
 
-from typing import ClassVar, Dict, List, Literal, Tuple, Any, Optional, Union, BinaryIO, overload, Sequence
+from typing import (ClassVar, Dict, List, Literal, Tuple, Any, Optional, Union, BinaryIO, overload, Sequence,
+                    TYPE_CHECKING)
 
 import bsdiff4
 
@@ -16,6 +17,9 @@ semaphore = threading.Semaphore(os.cpu_count() or 4)
 
 del threading
 
+if TYPE_CHECKING:
+    from Utils import Version
+
 
 class AutoPatchRegister(abc.ABCMeta):
     patch_types: ClassVar[Dict[str, AutoPatchRegister]] = {}
@@ -163,6 +167,32 @@ class APContainer:
         }
 
 
+class APWorldContainer(APContainer):
+    """A zipfile containing a world implementation."""
+    game: str | None = None
+    world_version: "Version | None" = None
+    minimum_ap_version: "Version | None" = None
+    maximum_ap_version: "Version | None" = None
+
+    def read_contents(self, opened_zipfile: zipfile.ZipFile) -> Dict[str, Any]:
+        from Utils import tuplize_version, Version
+        manifest = super().read_contents(opened_zipfile)
+        self.game = manifest["game"]
+        for version_key in ("world_version", "minimum_ap_version", "maximum_ap_version"):
+            if version_key in manifest:
+                setattr(self, version_key, Version(*tuplize_version(manifest[version_key])))
+        return manifest
+
+    def get_manifest(self) -> Dict[str, Any]:
+        manifest = super().get_manifest()
+        manifest["game"] = self.game
+        for version_key in ("world_version", "minimum_ap_version", "maximum_ap_version"):
+            version = getattr(self, version_key)
+            if version:
+                manifest[version_key] = version.as_simple_string()
+        return manifest
+
+
 class APPlayerContainer(APContainer):
     """A zipfile containing at least archipelago.json meant for a player"""
     game: ClassVar[Optional[str]] = None

worlds/__init__.py

@@ -10,7 +10,7 @@ import dataclasses
 from typing import List
 
 from NetUtils import DataPackage
-from Utils import local_path, user_path
+from Utils import local_path, user_path, Version, version_tuple
 
 local_folder = os.path.dirname(__file__)
 user_folder = user_path("worlds") if user_path() != local_path() else user_path("custom_worlds")
@@ -38,6 +38,7 @@ class WorldSource:
     is_zip: bool = False
     relative: bool = True  # relative to regular world import folder
     time_taken: float = -1.0
+    version: Version = Version(0, 0, 0)
 
     def __repr__(self) -> str:
         return f"{self.__class__.__name__}({self.path}, is_zip={self.is_zip}, relative={self.relative})"
@@ -102,12 +103,73 @@ for folder in (folder for folder in (user_folder, local_folder) if folder):
 
 # import all submodules to trigger AutoWorldRegister
 world_sources.sort()
+apworlds: list[WorldSource] = []
 for world_source in world_sources:
+    # load all loose files first:
+    if world_source.is_zip:
+        apworlds.append(world_source)
+    else:
         world_source.load()
 
-# Build the data package for each game.
 from .AutoWorld import AutoWorldRegister
 
+if apworlds:
+    # encapsulation for namespace / gc purposes
+    def load_apworlds() -> None:
+        global apworlds
+        from .Files import APWorldContainer, InvalidDataError
+        core_compatible: list[tuple[WorldSource, APWorldContainer]] = []
+
+        def fail_world(game_name: str, reason: str, add_as_failed_to_load: bool = True) -> None:
+            if add_as_failed_to_load:
+                failed_world_loads.append(game_name)
+            logging.warning(reason)
+
+        for apworld_source in apworlds:
+            apworld: APWorldContainer = APWorldContainer(apworld_source.resolved_path)
+            # populate metadata
+            try:
+                apworld.read()
+            except InvalidDataError as e:
+                if version_tuple < (0, 7, 0):
+                    logging.error(
+                        f"Invalid or missing manifest file for {apworld_source.resolved_path}. "
+                        "This apworld will stop working with Archipelago 0.7.0."
+                    )
+                    logging.error(e)
+                else:
+                    raise e
+
+            if apworld.minimum_ap_version and apworld.minimum_ap_version > version_tuple:
+                fail_world(apworld.game,
+                           f"Did not load {apworld_source.path} "
+                           f"as its minimum core version {apworld.minimum_ap_version} "
+                           f"is higher than current core version {version_tuple}.")
+            elif apworld.maximum_ap_version and apworld.maximum_ap_version < version_tuple:
+                fail_world(apworld.game,
+                           f"Did not load {apworld_source.path} "
+                           f"as its maximum core version {apworld.maximum_ap_version} "
+                           f"is lower than current core version {version_tuple}.")
+            else:
+                core_compatible.append((apworld_source, apworld))
+        # load highest version first
+        core_compatible.sort(
+            key=lambda element: element[1].world_version if element[1].world_version else Version(0, 0, 0),
+            reverse=True)
+        for apworld_source, apworld in core_compatible:
+            if apworld.game and apworld.game in AutoWorldRegister.world_types:
+                fail_world(apworld.game,
+                           f"Did not load {apworld_source.path} "
+                           f"as its game {apworld.game} is already loaded.",
+                           add_as_failed_to_load=False)
+            else:
+                apworld_source.load()
+    load_apworlds()
+    del load_apworlds
+
+del apworlds
+
+# Build the data package for each game.
 network_data_package: DataPackage = {
     "games": {world_name: world.get_data_package_data() for world_name, world in AutoWorldRegister.world_types.items()},
 }