MultiServer: speed up location commands (#1926)

* MultiServer: speed up location commands

Adds optimized pure python wrapper around locations dict
Adds optimized cython implementation of the wrapper, saving cpu time and 80% memory use

* Speedups: auto-build on import and build during setup

* Speedups: add requirements

* CI: don't break with build_ext

* Speedups: use C++ compiler for pyximport

* Speedups: cleanup and more validation

* Speedups: add tests for LocationStore

* Setup: delete temp in-place build modules

* Speedups: more tests and safer indices

The change has no security implications, but ensures that entries[IndexEntry.start] is always valid.

* Speedups: add cython3 compatibility

* Speedups: remove unused import

* Speedups: reformat

* Speedup: fix empty set in test

* Speedups: use regular dict in Locations.get_for_player

* CI: run unittests with beta cython

now with 2x nicer names

NetUtils.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import typing
 import enum
+import warnings
 from json import JSONEncoder, JSONDecoder
 
 import websockets
@@ -343,3 +344,65 @@ class Hint(typing.NamedTuple):
     @property
     def local(self):
         return self.receiving_player == self.finding_player
+
+
+class _LocationStore(dict, typing.MutableMapping[int, typing.Dict[int, typing.Tuple[int, int, int]]]):
+    def find_item(self, slots: typing.Set[int], seeked_item_id: int
+                  ) -> typing.Generator[typing.Tuple[int, int, int, int, int], None, None]:
+        for finding_player, check_data in self.items():
+            for location_id, (item_id, receiving_player, item_flags) in check_data.items():
+                if receiving_player in slots and item_id == seeked_item_id:
+                    yield finding_player, location_id, item_id, receiving_player, item_flags
+
+    def get_for_player(self, slot: int) -> typing.Dict[int, typing.Set[int]]:
+        import collections
+        all_locations: typing.Dict[int, typing.Set[int]] = collections.defaultdict(set)
+        for source_slot, location_data in self.items():
+            for location_id, values in location_data.items():
+                if values[1] == slot:
+                    all_locations[source_slot].add(location_id)
+        return all_locations
+
+    def get_checked(self, state: typing.Dict[typing.Tuple[int, int], typing.Set[int]], team: int, slot: int
+                    ) -> typing.List[int]:
+        checked = state[team, slot]
+        if not checked:
+            # This optimizes the case where everyone connects to a fresh game at the same time.
+            return []
+        return [location_id for
+                location_id in self[slot] if
+                location_id in checked]
+
+    def get_missing(self, state: typing.Dict[typing.Tuple[int, int], typing.Set[int]], team: int, slot: int
+                    ) -> typing.List[int]:
+        checked = state[team, slot]
+        if not checked:
+            # This optimizes the case where everyone connects to a fresh game at the same time.
+            return list(self)
+        return [location_id for
+                location_id in self[slot] if
+                location_id not in checked]
+
+    def get_remaining(self, state: typing.Dict[typing.Tuple[int, int], typing.Set[int]], team: int, slot: int
+                      ) -> typing.List[int]:
+        checked = state[team, slot]
+        player_locations = self[slot]
+        return sorted([player_locations[location_id][0] for
+                       location_id in player_locations if
+                       location_id not in checked])
+
+
+if typing.TYPE_CHECKING:  # type-check with pure python implementation until we have a typing stub
+    LocationStore = _LocationStore
+else:
+    try:
+        import pyximport
+        pyximport.install()
+    except ImportError:
+        pyximport = None
+    try:
+        from _speedups import LocationStore
+    except ImportError:
+        warnings.warn("_speedups not available. Falling back to pure python LocationStore. "
+                      "Install a matching C++ compiler for your platform to compile _speedups.")
+        LocationStore = _LocationStore