diff --git a/.gitattributes b/.gitattributes index 537a05f68b67..5ab537933405 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,2 @@ worlds/blasphemous/region_data.py linguist-generated=true +worlds/yachtdice/YachtWeights.py linguist-generated=true diff --git a/.github/pyright-config.json b/.github/pyright-config.json index 6ad7fa5f19b5..de7758a71566 100644 --- a/.github/pyright-config.json +++ b/.github/pyright-config.json @@ -1,8 +1,20 @@ { "include": [ - "type_check.py", + "../BizHawkClient.py", + "../Patch.py", + "../test/general/test_groups.py", + "../test/general/test_helpers.py", + "../test/general/test_memory.py", + "../test/general/test_names.py", + "../test/multiworld/__init__.py", + "../test/multiworld/test_multiworlds.py", + "../test/netutils/__init__.py", + "../test/programs/__init__.py", + "../test/programs/test_multi_server.py", + "../test/utils/__init__.py", + "../test/webhost/test_descriptions.py", "../worlds/AutoSNIClient.py", - "../Patch.py" + "type_check.py" ], "exclude": [ @@ -16,7 +28,7 @@ "reportMissingImports": true, "reportMissingTypeStubs": true, - "pythonVersion": "3.8", + "pythonVersion": "3.10", "pythonPlatform": "Windows", "executionEnvironments": [ diff --git a/.github/workflows/analyze-modified-files.yml b/.github/workflows/analyze-modified-files.yml index c9995fa2d043..b59336fafe9b 100644 --- a/.github/workflows/analyze-modified-files.yml +++ b/.github/workflows/analyze-modified-files.yml @@ -53,7 +53,7 @@ jobs: - uses: actions/setup-python@v5 if: env.diff != '' with: - python-version: 3.8 + python-version: '3.10' - name: "Install dependencies" if: env.diff != '' diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 23c463fb947a..27ca76e41f8f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,14 +24,15 @@ env: jobs: # build-release-macos: # LF volunteer - build-win-py38: # RCs will still be built and signed by hand + build-win: # RCs will still be built and signed by hand runs-on: windows-latest steps: - uses: actions/checkout@v4 - name: Install python uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '~3.12.7' + check-latest: true - name: Download run-time dependencies run: | Invoke-WebRequest -Uri https://github.com/Ijwu/Enemizer/releases/download/${Env:ENEMIZER_VERSION}/win-x64.zip -OutFile enemizer.zip @@ -111,10 +112,11 @@ jobs: - name: Get a recent python uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '~3.12.7' + check-latest: true - name: Install build-time dependencies run: | - echo "PYTHON=python3.11" >> $GITHUB_ENV + echo "PYTHON=python3.12" >> $GITHUB_ENV wget -nv https://github.com/AppImage/AppImageKit/releases/download/$APPIMAGETOOL_VERSION/appimagetool-x86_64.AppImage chmod a+rx appimagetool-x86_64.AppImage ./appimagetool-x86_64.AppImage --appimage-extract diff --git a/.github/workflows/ctest.yml b/.github/workflows/ctest.yml index 9492c83c9e53..a0ae2cb25206 100644 --- a/.github/workflows/ctest.yml +++ b/.github/workflows/ctest.yml @@ -11,7 +11,7 @@ on: - '**.hh?' - '**.hpp' - '**.hxx' - - '**.CMakeLists' + - '**/CMakeLists.txt' - '.github/workflows/ctest.yml' pull_request: paths: @@ -21,7 +21,7 @@ on: - '**.hh?' - '**.hpp' - '**.hxx' - - '**.CMakeLists' + - '**/CMakeLists.txt' - '.github/workflows/ctest.yml' jobs: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3f8651d408e7..aec4f90998cf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,10 +44,11 @@ jobs: - name: Get a recent python uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '~3.12.7' + check-latest: true - name: Install build-time dependencies run: | - echo "PYTHON=python3.11" >> $GITHUB_ENV + echo "PYTHON=python3.12" >> $GITHUB_ENV wget -nv https://github.com/AppImage/AppImageKit/releases/download/$APPIMAGETOOL_VERSION/appimagetool-x86_64.AppImage chmod a+rx appimagetool-x86_64.AppImage ./appimagetool-x86_64.AppImage --appimage-extract diff --git a/.github/workflows/scan-build.yml b/.github/workflows/scan-build.yml index 5234d862b4d3..ac842070625f 100644 --- a/.github/workflows/scan-build.yml +++ b/.github/workflows/scan-build.yml @@ -40,10 +40,10 @@ jobs: run: | wget https://apt.llvm.org/llvm.sh chmod +x ./llvm.sh - sudo ./llvm.sh 17 + sudo ./llvm.sh 19 - name: Install scan-build command run: | - sudo apt install clang-tools-17 + sudo apt install clang-tools-19 - name: Get a recent python uses: actions/setup-python@v5 with: @@ -56,7 +56,7 @@ jobs: - name: scan-build run: | source venv/bin/activate - scan-build-17 --status-bugs -o scan-build-reports -disable-checker deadcode.DeadStores python setup.py build -y + scan-build-19 --status-bugs -o scan-build-reports -disable-checker deadcode.DeadStores python setup.py build -y - name: Store report if: failure() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/strict-type-check.yml b/.github/workflows/strict-type-check.yml index bafd572a26ae..2ccdad8d11af 100644 --- a/.github/workflows/strict-type-check.yml +++ b/.github/workflows/strict-type-check.yml @@ -26,7 +26,7 @@ jobs: - name: "Install dependencies" run: | - python -m pip install --upgrade pip pyright==1.1.358 + python -m pip install --upgrade pip pyright==1.1.392.post0 python ModuleUpdate.py --append "WebHostLib/requirements.txt" --force --yes - name: "pyright: strict check on specific files" diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index a38fef8fda08..88b5d12987ad 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -33,13 +33,11 @@ jobs: matrix: os: [ubuntu-latest] python: - - {version: '3.8'} - - {version: '3.9'} - {version: '3.10'} - {version: '3.11'} - {version: '3.12'} include: - - python: {version: '3.8'} # win7 compat + - python: {version: '3.10'} # old compat os: windows-latest - python: {version: '3.12'} # current os: windows-latest diff --git a/BaseClasses.py b/BaseClasses.py index 46edeb5ea059..3d0004806cc5 100644 --- a/BaseClasses.py +++ b/BaseClasses.py @@ -1,18 +1,16 @@ from __future__ import annotations import collections -import itertools import functools import logging import random import secrets -import typing # this can go away when Python 3.8 support is dropped from argparse import Namespace from collections import Counter, deque from collections.abc import Collection, MutableSequence from enum import IntEnum, IntFlag from typing import (AbstractSet, Any, Callable, ClassVar, Dict, Iterable, Iterator, List, Mapping, NamedTuple, - Optional, Protocol, Set, Tuple, Union, Type) + Optional, Protocol, Set, Tuple, Union, TYPE_CHECKING) from typing_extensions import NotRequired, TypedDict @@ -20,7 +18,8 @@ import Options import Utils -if typing.TYPE_CHECKING: +if TYPE_CHECKING: + from entrance_rando import ERPlacementState from worlds import AutoWorld @@ -231,7 +230,7 @@ def set_options(self, args: Namespace) -> None: for player in self.player_ids: world_type = AutoWorld.AutoWorldRegister.world_types[self.game[player]] self.worlds[player] = world_type(self, player) - options_dataclass: typing.Type[Options.PerGameCommonOptions] = world_type.options_dataclass + options_dataclass: type[Options.PerGameCommonOptions] = world_type.options_dataclass self.worlds[player].options = options_dataclass(**{option_key: getattr(args, option_key)[player] for option_key in options_dataclass.type_hints}) @@ -428,12 +427,12 @@ def get_entrance(self, entrance_name: str, player: int) -> Entrance: def get_location(self, location_name: str, player: int) -> Location: return self.regions.location_cache[player][location_name] - def get_all_state(self, use_cache: bool) -> CollectionState: + def get_all_state(self, use_cache: bool, allow_partial_entrances: bool = False) -> CollectionState: cached = getattr(self, "_all_state", None) if use_cache and cached: return cached.copy() - ret = CollectionState(self) + ret = CollectionState(self, allow_partial_entrances) for item in self.itempool: self.worlds[item.player].collect(ret, item) @@ -606,6 +605,49 @@ def get_spheres(self) -> Iterator[Set[Location]]: state.collect(location.item, True, location) locations -= sphere + def get_sendable_spheres(self) -> Iterator[Set[Location]]: + """ + yields a set of multiserver sendable locations (location.item.code: int) for each logical sphere + + If there are unreachable locations, the last sphere of reachable locations is followed by an empty set, + and then a set of all of the unreachable locations. + """ + state = CollectionState(self) + locations: Set[Location] = set() + events: Set[Location] = set() + for location in self.get_filled_locations(): + if type(location.item.code) is int: + locations.add(location) + else: + events.add(location) + + while locations: + sphere: Set[Location] = set() + + # cull events out + done_events: Set[Union[Location, None]] = {None} + while done_events: + done_events = set() + for event in events: + if event.can_reach(state): + state.collect(event.item, True, event) + done_events.add(event) + events -= done_events + + for location in locations: + if location.can_reach(state): + sphere.add(location) + + yield sphere + if not sphere: + if locations: + yield locations # unreachable locations + break + + for location in sphere: + state.collect(location.item, True, location) + locations -= sphere + def fulfills_accessibility(self, state: Optional[CollectionState] = None): """Check if accessibility rules are fulfilled with current or supplied state.""" if not state: @@ -676,10 +718,11 @@ class CollectionState(): path: Dict[Union[Region, Entrance], PathValue] locations_checked: Set[Location] stale: Dict[int, bool] + allow_partial_entrances: bool additional_init_functions: List[Callable[[CollectionState, MultiWorld], None]] = [] additional_copy_functions: List[Callable[[CollectionState, CollectionState], CollectionState]] = [] - def __init__(self, parent: MultiWorld): + def __init__(self, parent: MultiWorld, allow_partial_entrances: bool = False): self.prog_items = {player: Counter() for player in parent.get_all_ids()} self.multiworld = parent self.reachable_regions = {player: set() for player in parent.get_all_ids()} @@ -688,6 +731,7 @@ def __init__(self, parent: MultiWorld): self.path = {} self.locations_checked = set() self.stale = {player: True for player in parent.get_all_ids()} + self.allow_partial_entrances = allow_partial_entrances for function in self.additional_init_functions: function(self, parent) for items in parent.precollected_items.values(): @@ -722,6 +766,8 @@ def _update_reachable_regions_explicit_indirect_conditions(self, player: int, qu if new_region in reachable_regions: blocked_connections.remove(connection) elif connection.can_reach(self): + if self.allow_partial_entrances and not new_region: + continue assert new_region, f"tried to search through an Entrance \"{connection}\" with no connected Region" reachable_regions.add(new_region) blocked_connections.remove(connection) @@ -747,7 +793,9 @@ def _update_reachable_regions_auto_indirect_conditions(self, player: int, queue: if new_region in reachable_regions: blocked_connections.remove(connection) elif connection.can_reach(self): - assert new_region, f"tried to search through an Entrance \"{connection}\" with no Region" + if self.allow_partial_entrances and not new_region: + continue + assert new_region, f"tried to search through an Entrance \"{connection}\" with no connected Region" reachable_regions.add(new_region) blocked_connections.remove(connection) blocked_connections.update(new_region.exits) @@ -767,6 +815,7 @@ def copy(self) -> CollectionState: ret.advancements = self.advancements.copy() ret.path = self.path.copy() ret.locations_checked = self.locations_checked.copy() + ret.allow_partial_entrances = self.allow_partial_entrances for function in self.additional_copy_functions: ret = function(self, ret) return ret @@ -820,21 +869,40 @@ def sweep_for_advancements(self, locations: Optional[Iterable[Location]] = None) def has(self, item: str, player: int, count: int = 1) -> bool: return self.prog_items[player][item] >= count + # for loops are specifically used in all/any/count methods, instead of all()/any()/sum(), to avoid the overhead of + # creating and iterating generator instances. In `return all(player_prog_items[item] for item in items)`, the + # argument to all() would be a new generator instance, for example. def has_all(self, items: Iterable[str], player: int) -> bool: """Returns True if each item name of items is in state at least once.""" - return all(self.prog_items[player][item] for item in items) + player_prog_items = self.prog_items[player] + for item in items: + if not player_prog_items[item]: + return False + return True def has_any(self, items: Iterable[str], player: int) -> bool: """Returns True if at least one item name of items is in state at least once.""" - return any(self.prog_items[player][item] for item in items) + player_prog_items = self.prog_items[player] + for item in items: + if player_prog_items[item]: + return True + return False def has_all_counts(self, item_counts: Mapping[str, int], player: int) -> bool: """Returns True if each item name is in the state at least as many times as specified.""" - return all(self.prog_items[player][item] >= count for item, count in item_counts.items()) + player_prog_items = self.prog_items[player] + for item, count in item_counts.items(): + if player_prog_items[item] < count: + return False + return True def has_any_count(self, item_counts: Mapping[str, int], player: int) -> bool: """Returns True if at least one item name is in the state at least as many times as specified.""" - return any(self.prog_items[player][item] >= count for item, count in item_counts.items()) + player_prog_items = self.prog_items[player] + for item, count in item_counts.items(): + if player_prog_items[item] >= count: + return True + return False def count(self, item: str, player: int) -> int: return self.prog_items[player][item] @@ -862,11 +930,20 @@ def has_from_list_unique(self, items: Iterable[str], player: int, count: int) -> def count_from_list(self, items: Iterable[str], player: int) -> int: """Returns the cumulative count of items from a list present in state.""" - return sum(self.prog_items[player][item_name] for item_name in items) + player_prog_items = self.prog_items[player] + total = 0 + for item_name in items: + total += player_prog_items[item_name] + return total def count_from_list_unique(self, items: Iterable[str], player: int) -> int: """Returns the cumulative count of items from a list present in state. Ignores duplicates of the same item.""" - return sum(self.prog_items[player][item_name] > 0 for item_name in items) + player_prog_items = self.prog_items[player] + total = 0 + for item_name in items: + if player_prog_items[item_name] > 0: + total += 1 + return total # item name group related def has_group(self, item_name_group: str, player: int, count: int = 1) -> bool: @@ -931,6 +1008,11 @@ def remove(self, item: Item): self.stale[item.player] = True +class EntranceType(IntEnum): + ONE_WAY = 1 + TWO_WAY = 2 + + class Entrance: access_rule: Callable[[CollectionState], bool] = staticmethod(lambda state: True) hide_path: bool = False @@ -938,19 +1020,24 @@ class Entrance: name: str parent_region: Optional[Region] connected_region: Optional[Region] = None + randomization_group: int + randomization_type: EntranceType # LttP specific, TODO: should make a LttPEntrance addresses = None target = None - def __init__(self, player: int, name: str = "", parent: Optional[Region] = None) -> None: + def __init__(self, player: int, name: str = "", parent: Optional[Region] = None, + randomization_group: int = 0, randomization_type: EntranceType = EntranceType.ONE_WAY) -> None: self.name = name self.parent_region = parent self.player = player + self.randomization_group = randomization_group + self.randomization_type = randomization_type def can_reach(self, state: CollectionState) -> bool: assert self.parent_region, f"called can_reach on an Entrance \"{self}\" with no parent_region" if self.parent_region.can_reach(state) and self.access_rule(state): - if not self.hide_path and not self in state.path: + if not self.hide_path and self not in state.path: state.path[self] = (self.name, state.path.get(self.parent_region, (self.parent_region.name, None))) return True @@ -962,6 +1049,32 @@ def connect(self, region: Region, addresses: Any = None, target: Any = None) -> self.addresses = addresses region.entrances.append(self) + def is_valid_source_transition(self, er_state: "ERPlacementState") -> bool: + """ + Determines whether this is a valid source transition, that is, whether the entrance + randomizer is allowed to pair it to place any other regions. By default, this is the + same as a reachability check, but can be modified by Entrance implementations to add + other restrictions based on the placement state. + + :param er_state: The current (partial) state of the ongoing entrance randomization + """ + return self.can_reach(er_state.collection_state) + + def can_connect_to(self, other: Entrance, dead_end: bool, er_state: "ERPlacementState") -> bool: + """ + Determines whether a given Entrance is a valid target transition, that is, whether + the entrance randomizer is allowed to pair this Entrance to that Entrance. By default, + only allows connection between entrances of the same type (one ways only go to one ways, + two ways always go to two ways) and prevents connecting an exit to itself in coupled mode. + + :param other: The proposed Entrance to connect to + :param dead_end: Whether the other entrance considered a dead end by Entrance randomization + :param er_state: The current (partial) state of the ongoing entrance randomization + """ + # the implementation of coupled causes issues for self-loops since the reverse entrance will be the + # same as the forward entrance. In uncoupled they are ok. + return self.randomization_type == other.randomization_type and (not er_state.coupled or self.name != other.name) + def __repr__(self): multiworld = self.parent_region.multiworld if self.parent_region else None return multiworld.get_name_string_for_object(self) if multiworld else f'{self.name} (Player {self.player})' @@ -975,7 +1088,7 @@ class Region: entrances: List[Entrance] exits: List[Entrance] locations: List[Location] - entrance_type: ClassVar[Type[Entrance]] = Entrance + entrance_type: ClassVar[type[Entrance]] = Entrance class Register(MutableSequence): region_manager: MultiWorld.RegionManager @@ -1075,7 +1188,7 @@ def get_connecting_entrance(self, is_main_entrance: Callable[[Entrance], bool]) return entrance.parent_region.get_connecting_entrance(is_main_entrance) def add_locations(self, locations: Dict[str, Optional[int]], - location_type: Optional[Type[Location]] = None) -> None: + location_type: Optional[type[Location]] = None) -> None: """ Adds locations to the Region object, where location_type is your Location class and locations is a dict of location names to address. @@ -1111,8 +1224,18 @@ def create_exit(self, name: str) -> Entrance: self.exits.append(exit_) return exit_ + def create_er_target(self, name: str) -> Entrance: + """ + Creates and returns an Entrance object as an entrance to this region + + :param name: name of the Entrance being created + """ + entrance = self.entrance_type(self.player, name) + entrance.connect(self) + return entrance + def add_exits(self, exits: Union[Iterable[str], Dict[str, Optional[str]]], - rules: Dict[str, Callable[[CollectionState], bool]] = None) -> None: + rules: Dict[str, Callable[[CollectionState], bool]] = None) -> List[Entrance]: """ Connects current region to regions in exit dictionary. Passed region names must exist first. @@ -1122,10 +1245,14 @@ def add_exits(self, exits: Union[Iterable[str], Dict[str, Optional[str]]], """ if not isinstance(exits, Dict): exits = dict.fromkeys(exits) - for connecting_region, name in exits.items(): - self.connect(self.multiworld.get_region(connecting_region, self.player), - name, - rules[connecting_region] if rules and connecting_region in rules else None) + return [ + self.connect( + self.multiworld.get_region(connecting_region, self.player), + name, + rules[connecting_region] if rules and connecting_region in rules else None, + ) + for connecting_region, name in exits.items() + ] def __repr__(self): return self.multiworld.get_name_string_for_object(self) if self.multiworld else f'{self.name} (Player {self.player})' @@ -1209,13 +1336,26 @@ def hint_text(self) -> str: class ItemClassification(IntFlag): - filler = 0b0000 # aka trash, as in filler items like ammo, currency etc, - progression = 0b0001 # Item that is logically relevant - useful = 0b0010 # Item that is generally quite useful, but not required for anything logical - trap = 0b0100 # detrimental item - skip_balancing = 0b1000 # should technically never occur on its own - # Item that is logically relevant, but progression balancing should not touch. - # Typically currency or other counted items. + filler = 0b0000 + """ aka trash, as in filler items like ammo, currency etc """ + + progression = 0b0001 + """ Item that is logically relevant. + Protects this item from being placed on excluded or unreachable locations. """ + + useful = 0b0010 + """ Item that is especially useful. + Protects this item from being placed on excluded or unreachable locations. + When combined with another flag like "progression", it means "an especially useful progression item". """ + + trap = 0b0100 + """ Item that is detrimental in some way. """ + + skip_balancing = 0b1000 + """ should technically never occur on its own + Item that is logically relevant, but progression balancing should not touch. + Typically currency or other counted items. """ + progression_skip_balancing = 0b1001 # only progression gets balanced def as_flag(self) -> int: @@ -1264,6 +1404,10 @@ def useful(self) -> bool: def trap(self) -> bool: return ItemClassification.trap in self.classification + @property + def filler(self) -> bool: + return not (self.advancement or self.useful or self.trap) + @property def excludable(self) -> bool: return not (self.advancement or self.useful) @@ -1386,14 +1530,21 @@ def create_playthrough(self, create_paths: bool = True) -> None: # second phase, sphere 0 removed_precollected: List[Item] = [] - for item in (i for i in chain.from_iterable(multiworld.precollected_items.values()) if i.advancement): - logging.debug('Checking if %s (Player %d) is required to beat the game.', item.name, item.player) - multiworld.precollected_items[item.player].remove(item) - multiworld.state.remove(item) - if not multiworld.can_beat_game(): - multiworld.push_precollected(item) - else: - removed_precollected.append(item) + + for precollected_items in multiworld.precollected_items.values(): + # The list of items is mutated by removing one item at a time to determine if each item is required to beat + # the game, and re-adding that item if it was required, so a copy needs to be made before iterating. + for item in precollected_items.copy(): + if not item.advancement: + continue + logging.debug('Checking if %s (Player %d) is required to beat the game.', item.name, item.player) + precollected_items.remove(item) + multiworld.state.remove(item) + if not multiworld.can_beat_game(): + # Add the item back into `precollected_items` and collect it into `multiworld.state`. + multiworld.push_precollected(item) + else: + removed_precollected.append(item) # we are now down to just the required progress items in collection_spheres. Unfortunately # the previous pruning stage could potentially have made certain items dependant on others @@ -1532,7 +1683,7 @@ def write_option(option_key: str, option_obj: Options.AssembleOptions) -> None: [f" {location}: {item}" for (location, item) in sphere.items()] if isinstance(sphere, dict) else [f" {item}" for item in sphere])) for (sphere_nr, sphere) in self.playthrough.items()])) if self.unreachables: - outfile.write('\n\nUnreachable Items:\n\n') + outfile.write('\n\nUnreachable Progression Items:\n\n') outfile.write( '\n'.join(['%s: %s' % (unreachable.item, unreachable) for unreachable in self.unreachables])) diff --git a/CommonClient.py b/CommonClient.py index 77ed85b5c652..33792f0ed28b 100644 --- a/CommonClient.py +++ b/CommonClient.py @@ -23,7 +23,7 @@ from MultiServer import CommandProcessor from NetUtils import (Endpoint, decode, NetworkItem, encode, JSONtoTextParser, ClientStatus, Permission, NetworkSlot, - RawJSONtoTextParser, add_json_text, add_json_location, add_json_item, JSONTypes, SlotType) + RawJSONtoTextParser, add_json_text, add_json_location, add_json_item, JSONTypes, HintStatus, SlotType) from Utils import Version, stream_input, async_start from worlds import network_data_package, AutoWorldRegister import os @@ -31,6 +31,7 @@ if typing.TYPE_CHECKING: import kvui + import argparse logger = logging.getLogger("Client") @@ -412,6 +413,7 @@ async def disconnect(self, allow_autoreconnect: bool = False): await self.server.socket.close() if self.server_task is not None: await self.server_task + self.ui.update_hints() async def send_msgs(self, msgs: typing.List[typing.Any]) -> None: """ `msgs` JSON serializable """ @@ -458,6 +460,13 @@ async def send_connect(self, **kwargs: typing.Any) -> None: await self.send_msgs([payload]) await self.send_msgs([{"cmd": "Get", "keys": ["_read_race_mode"]}]) + async def check_locations(self, locations: typing.Collection[int]) -> set[int]: + """Send new location checks to the server. Returns the set of actually new locations that were sent.""" + locations = set(locations) & self.missing_locations + if locations: + await self.send_msgs([{"cmd": 'LocationChecks', "locations": tuple(locations)}]) + return locations + async def console_input(self) -> str: if self.ui: self.ui.focus_textinput() @@ -551,7 +560,14 @@ async def shutdown(self): await self.ui_task if self.input_task: self.input_task.cancel() - + + # Hints + def update_hint(self, location: int, finding_player: int, status: typing.Optional[HintStatus]) -> None: + msg = {"cmd": "UpdateHint", "location": location, "player": finding_player} + if status is not None: + msg["status"] = status + async_start(self.send_msgs([msg]), name="update_hint") + # DataPackage async def prepare_data_package(self, relevant_games: typing.Set[str], remote_date_package_versions: typing.Dict[str, int], @@ -693,8 +709,16 @@ def handle_connection_loss(self, msg: str) -> None: logger.exception(msg, exc_info=exc_info, extra={'compact_gui': True}) self._messagebox_connection_loss = self.gui_error(msg, exc_info[1]) - def make_gui(self) -> typing.Type["kvui.GameManager"]: - """To return the Kivy App class needed for run_gui so it can be overridden before being built""" + def make_gui(self) -> "type[kvui.GameManager]": + """ + To return the Kivy `App` class needed for `run_gui` so it can be overridden before being built + + Common changes are changing `base_title` to update the window title of the client and + updating `logging_pairs` to automatically make new tabs that can be filled with their respective logger. + + ex. `logging_pairs.append(("Foo", "Bar"))` + will add a "Bar" tab which follows the logger returned from `logging.getLogger("Foo")` + """ from kvui import GameManager class TextManager(GameManager): @@ -710,6 +734,11 @@ def run_gui(self): def run_cli(self): if sys.stdin: + if sys.stdin.fileno() != 0: + from multiprocessing import parent_process + if parent_process(): + return # ignore MultiProcessing pipe + # steam overlay breaks when starting console_loop if 'gameoverlayrenderer' in os.environ.get('LD_PRELOAD', ''): logger.info("Skipping terminal input, due to conflicting Steam Overlay detected. Please use GUI only.") @@ -878,6 +907,7 @@ async def process_server_cmd(ctx: CommonContext, args: dict): ctx.disconnected_intentionally = True ctx.event_invalid_game() elif 'IncompatibleVersion' in errors: + ctx.disconnected_intentionally = True raise Exception('Server reported your client version as incompatible. ' 'This probably means you have to update.') elif 'InvalidItemsHandling' in errors: @@ -1028,6 +1058,32 @@ def get_base_parser(description: typing.Optional[str] = None): return parser +def handle_url_arg(args: "argparse.Namespace", + parser: "typing.Optional[argparse.ArgumentParser]" = None) -> "argparse.Namespace": + """ + Parse the url arg "archipelago://name:pass@host:port" from launcher into correct launch args for CommonClient + If alternate data is required the urlparse response is saved back to args.url if valid + """ + if not args.url: + return args + + url = urllib.parse.urlparse(args.url) + if url.scheme != "archipelago": + if not parser: + parser = get_base_parser() + parser.error(f"bad url, found {args.url}, expected url in form of archipelago://archipelago.gg:38281") + return args + + args.url = url + args.connect = url.netloc + if url.username: + args.name = urllib.parse.unquote(url.username) + if url.password: + args.password = urllib.parse.unquote(url.password) + + return args + + def run_as_textclient(*args): class TextContext(CommonContext): # Text Mode to use !hint and such with games that have no text entry @@ -1040,7 +1096,7 @@ async def server_auth(self, password_requested: bool = False): if password_requested and not self.password: await super(TextContext, self).server_auth(password_requested) await self.get_username() - await self.send_connect() + await self.send_connect(game="") def on_package(self, cmd: str, args: dict): if cmd == "Connected": @@ -1069,17 +1125,7 @@ async def main(args): parser.add_argument("url", nargs="?", help="Archipelago connection url") args = parser.parse_args(args) - # handle if text client is launched using the "archipelago://name:pass@host:port" url from webhost - if args.url: - url = urllib.parse.urlparse(args.url) - if url.scheme == "archipelago": - args.connect = url.netloc - if url.username: - args.name = urllib.parse.unquote(url.username) - if url.password: - args.password = urllib.parse.unquote(url.password) - else: - parser.error(f"bad url, found {args.url}, expected url in form of archipelago://archipelago.gg:38281") + args = handle_url_arg(args, parser=parser) # use colorama to display colored text highlighting on windows colorama.init() diff --git a/Fill.py b/Fill.py index 706cca657457..d1773c82139b 100644 --- a/Fill.py +++ b/Fill.py @@ -36,7 +36,8 @@ def sweep_from_pool(base_state: CollectionState, itempool: typing.Sequence[Item] def fill_restrictive(multiworld: MultiWorld, base_state: CollectionState, locations: typing.List[Location], item_pool: typing.List[Item], single_player_placement: bool = False, lock: bool = False, swap: bool = True, on_place: typing.Optional[typing.Callable[[Location], None]] = None, - allow_partial: bool = False, allow_excluded: bool = False, name: str = "Unknown") -> None: + allow_partial: bool = False, allow_excluded: bool = False, one_item_per_player: bool = True, + name: str = "Unknown") -> None: """ :param multiworld: Multiworld to be filled. :param base_state: State assumed before fill. @@ -63,14 +64,22 @@ def fill_restrictive(multiworld: MultiWorld, base_state: CollectionState, locati placed = 0 while any(reachable_items.values()) and locations: - # grab one item per player - items_to_place = [items.pop() - for items in reachable_items.values() if items] + if one_item_per_player: + # grab one item per player + items_to_place = [items.pop() + for items in reachable_items.values() if items] + else: + next_player = multiworld.random.choice([player for player, items in reachable_items.items() if items]) + items_to_place = [] + if item_pool: + items_to_place.append(reachable_items[next_player].pop()) + for item in items_to_place: for p, pool_item in enumerate(item_pool): if pool_item is item: item_pool.pop(p) break + maximum_exploration_state = sweep_from_pool( base_state, item_pool + unplaced_items, multiworld.get_filled_locations(item.player) if single_player_placement else None) @@ -226,18 +235,30 @@ def remaining_fill(multiworld: MultiWorld, locations: typing.List[Location], itempool: typing.List[Item], name: str = "Remaining", - move_unplaceable_to_start_inventory: bool = False) -> None: + move_unplaceable_to_start_inventory: bool = False, + check_location_can_fill: bool = False) -> None: unplaced_items: typing.List[Item] = [] placements: typing.List[Location] = [] swapped_items: typing.Counter[typing.Tuple[int, str]] = Counter() total = min(len(itempool), len(locations)) placed = 0 + + # Optimisation: Decide whether to do full location.can_fill check (respect excluded), or only check the item rule + if check_location_can_fill: + state = CollectionState(multiworld) + + def location_can_fill_item(location_to_fill: Location, item_to_fill: Item): + return location_to_fill.can_fill(state, item_to_fill, check_access=False) + else: + def location_can_fill_item(location_to_fill: Location, item_to_fill: Item): + return location_to_fill.item_rule(item_to_fill) + while locations and itempool: item_to_place = itempool.pop() spot_to_fill: typing.Optional[Location] = None for i, location in enumerate(locations): - if location.item_rule(item_to_place): + if location_can_fill_item(location, item_to_place): # popping by index is faster than removing by content, spot_to_fill = locations.pop(i) # skipping a scan for the element @@ -258,7 +279,7 @@ def remaining_fill(multiworld: MultiWorld, location.item = None placed_item.location = None - if location.item_rule(item_to_place): + if location_can_fill_item(location, item_to_place): # Add this item to the existing placement, and # add the old item to the back of the queue spot_to_fill = placements.pop(i) @@ -480,7 +501,14 @@ def mark_for_locking(location: Location): if prioritylocations: # "priority fill" fill_restrictive(multiworld, multiworld.state, prioritylocations, progitempool, - single_player_placement=single_player, swap=False, on_place=mark_for_locking, name="Priority") + single_player_placement=single_player, swap=False, on_place=mark_for_locking, + name="Priority", one_item_per_player=True, allow_partial=True) + + if prioritylocations: + # retry with one_item_per_player off because some priority fills can fail to fill with that optimization + fill_restrictive(multiworld, multiworld.state, prioritylocations, progitempool, + single_player_placement=single_player, swap=False, on_place=mark_for_locking, + name="Priority Retry", one_item_per_player=False) accessibility_corrections(multiworld, multiworld.state, prioritylocations, progitempool) defaultlocations = prioritylocations + defaultlocations @@ -509,7 +537,8 @@ def mark_for_locking(location: Location): if progitempool: raise FillError( f"Not enough locations for progression items. " - f"There are {len(progitempool)} more progression items than there are available locations.", + f"There are {len(progitempool)} more progression items than there are available locations.\n" + f"Unfilled locations:\n{multiworld.get_unfilled_locations()}.", multiworld=multiworld, ) accessibility_corrections(multiworld, multiworld.state, defaultlocations) @@ -527,7 +556,7 @@ def mark_for_locking(location: Location): if excludedlocations: raise FillError( f"Not enough filler items for excluded locations. " - f"There are {len(excludedlocations)} more excluded locations than filler or trap items.", + f"There are {len(excludedlocations)} more excluded locations than excludable items.", multiworld=multiworld, ) @@ -548,6 +577,26 @@ def mark_for_locking(location: Location): print_data = {"items": items_counter, "locations": locations_counter} logging.info(f"Per-Player counts: {print_data})") + more_locations = locations_counter - items_counter + more_items = items_counter - locations_counter + for player in multiworld.player_ids: + if more_locations[player]: + logging.error( + f"Player {multiworld.get_player_name(player)} had {more_locations[player]} more locations than items.") + elif more_items[player]: + logging.warning( + f"Player {multiworld.get_player_name(player)} had {more_items[player]} more items than locations.") + if unfilled: + raise FillError( + f"Unable to fill all locations.\n" + + f"Unfilled locations({len(unfilled)}): {unfilled}" + ) + else: + logging.warning( + f"Unable to place all items.\n" + + f"Unplaced items({len(unplaced)}): {unplaced}" + ) + def flood_items(multiworld: MultiWorld) -> None: # get items to distribute @@ -978,15 +1027,32 @@ def failed(warning: str, force: typing.Union[bool, str]) -> None: multiworld.random.shuffle(items) count = 0 err: typing.List[str] = [] - successful_pairs: typing.List[typing.Tuple[Item, Location]] = [] + successful_pairs: typing.List[typing.Tuple[int, Item, Location]] = [] + claimed_indices: typing.Set[typing.Optional[int]] = set() for item_name in items: - item = multiworld.worlds[player].create_item(item_name) + index_to_delete: typing.Optional[int] = None + if from_pool: + try: + # If from_pool, try to find an existing item with this name & player in the itempool and use it + index_to_delete, item = next( + (i, item) for i, item in enumerate(multiworld.itempool) + if item.player == player and item.name == item_name and i not in claimed_indices + ) + except StopIteration: + warn( + f"Could not remove {item_name} from pool for {multiworld.player_name[player]} as it's already missing from it.", + placement['force']) + item = multiworld.worlds[player].create_item(item_name) + else: + item = multiworld.worlds[player].create_item(item_name) + for location in reversed(candidates): if (location.address is None) == (item.code is None): # either both None or both not None if not location.item: if location.item_rule(item): if location.can_fill(multiworld.state, item, False): - successful_pairs.append((item, location)) + successful_pairs.append((index_to_delete, item, location)) + claimed_indices.add(index_to_delete) candidates.remove(location) count = count + 1 break @@ -998,6 +1064,7 @@ def failed(warning: str, force: typing.Union[bool, str]) -> None: err.append(f"Cannot place {item_name} into already filled location {location}.") else: err.append(f"Mismatch between {item_name} and {location}, only one is an event.") + if count == maxcount: break if count < placement['count']['min']: @@ -1005,17 +1072,16 @@ def failed(warning: str, force: typing.Union[bool, str]) -> None: failed( f"Plando block failed to place {m - count} of {m} item(s) for {multiworld.player_name[player]}, error(s): {' '.join(err)}", placement['force']) - for (item, location) in successful_pairs: + + # Sort indices in reverse so we can remove them one by one + successful_pairs = sorted(successful_pairs, key=lambda successful_pair: successful_pair[0] or 0, reverse=True) + + for (index, item, location) in successful_pairs: multiworld.push_item(location, item, collect=False) location.locked = True logging.debug(f"Plando placed {item} at {location}") - if from_pool: - try: - multiworld.itempool.remove(item) - except ValueError: - warn( - f"Could not remove {item} from pool for {multiworld.player_name[player]} as it's already missing from it.", - placement['force']) + if index is not None: # If this item is from_pool and was found in the pool, remove it. + multiworld.itempool.pop(index) except Exception as e: raise Exception( diff --git a/Generate.py b/Generate.py index bc359a203da7..b057db25a311 100644 --- a/Generate.py +++ b/Generate.py @@ -42,7 +42,9 @@ def mystery_argparse(): help="Path to output folder. Absolute or relative to cwd.") # absolute or relative to cwd parser.add_argument('--race', action='store_true', default=defaults.race) parser.add_argument('--meta_file_path', default=defaults.meta_file_path) - parser.add_argument('--log_level', default='info', help='Sets log level') + parser.add_argument('--log_level', default=defaults.loglevel, help='Sets log level') + parser.add_argument('--log_time', help="Add timestamps to STDOUT", + default=defaults.logtime, action='store_true') parser.add_argument("--csv_output", action="store_true", help="Output rolled player options to csv (made for async multiworld).") parser.add_argument("--plando", default=defaults.plando_options, @@ -75,7 +77,7 @@ def main(args=None) -> Tuple[argparse.Namespace, int]: seed = get_seed(args.seed) - Utils.init_logging(f"Generate_{seed}", loglevel=args.log_level) + Utils.init_logging(f"Generate_{seed}", loglevel=args.log_level, add_timestamp=args.log_time) random.seed(seed) seed_name = get_seed_name(random) @@ -114,7 +116,14 @@ def main(args=None) -> Tuple[argparse.Namespace, int]: os.path.join(args.player_files_path, fname) not in {args.meta_file_path, args.weights_file_path}: path = os.path.join(args.player_files_path, fname) try: - weights_cache[fname] = read_weights_yamls(path) + weights_for_file = [] + for doc_idx, yaml in enumerate(read_weights_yamls(path)): + if yaml is None: + logging.warning(f"Ignoring empty yaml document #{doc_idx + 1} in {fname}") + else: + weights_for_file.append(yaml) + weights_cache[fname] = tuple(weights_for_file) + except Exception as e: raise ValueError(f"File {fname} is invalid. Please fix your yaml.") from e @@ -431,7 +440,7 @@ def roll_settings(weights: dict, plando_options: PlandoOptions = PlandoOptions.b if "linked_options" in weights: weights = roll_linked_options(weights) - valid_keys = set() + valid_keys = {"triggers"} if "triggers" in weights: weights = roll_triggers(weights, weights["triggers"], valid_keys) @@ -453,6 +462,10 @@ def roll_settings(weights: dict, plando_options: PlandoOptions = PlandoOptions.b raise Exception(f"Option {option_key} has to be in a game's section, not on its own.") ret.game = get_choice("game", weights) + if not isinstance(ret.game, str): + if ret.game is None: + raise Exception('"game" not specified') + raise Exception(f"Invalid game: {ret.game}") if ret.game not in AutoWorldRegister.world_types: from worlds import failed_world_loads picks = Utils.get_fuzzy_results(ret.game, list(AutoWorldRegister.world_types) + failed_world_loads, limit=1)[0] @@ -486,15 +499,23 @@ def roll_settings(weights: dict, plando_options: PlandoOptions = PlandoOptions.b for option_key, option in world_type.options_dataclass.type_hints.items(): handle_option(ret, game_weights, option_key, option, plando_options) valid_keys.add(option_key) - for option_key in game_weights: - if option_key in {"triggers", *valid_keys}: - continue - logging.warning(f"{option_key} is not a valid option name for {ret.game} and is not present in triggers.") + + # TODO remove plando_items after moving it to the options system + valid_keys.add("plando_items") if PlandoOptions.items in plando_options: ret.plando_items = copy.deepcopy(game_weights.get("plando_items", [])) if ret.game == "A Link to the Past": + # TODO there are still more LTTP options not on the options system + valid_keys |= {"sprite_pool", "sprite", "random_sprite_on_event"} roll_alttp_settings(ret, game_weights) + # log a warning for options within a game section that aren't determined as valid + for option_key in game_weights: + if option_key in valid_keys: + continue + logging.warning(f"{option_key} is not a valid option name for {ret.game} and is not present in triggers " + f"for player {ret.name}.") + return ret diff --git a/LICENSE b/LICENSE index 40716cff4275..60d31b7b7de8 100644 --- a/LICENSE +++ b/LICENSE @@ -1,7 +1,7 @@ MIT License Copyright (c) 2017 LLCoolDave -Copyright (c) 2022 Berserker66 +Copyright (c) 2025 Berserker66 Copyright (c) 2022 CaitSith2 Copyright (c) 2021 LegendaryLinux diff --git a/Launcher.py b/Launcher.py index 2620f786a54b..22c0944ab1a4 100644 --- a/Launcher.py +++ b/Launcher.py @@ -22,16 +22,15 @@ from shutil import which from typing import Callable, Optional, Sequence, Tuple, Union -import Utils -import settings -from worlds.LauncherComponents import Component, components, Type, SuffixIdentifier, icon_paths - if __name__ == "__main__": import ModuleUpdate ModuleUpdate.update() -from Utils import is_frozen, user_path, local_path, init_logging, open_filename, messagebox, \ - is_windows, is_macos, is_linux +import settings +import Utils +from Utils import (init_logging, is_frozen, is_linux, is_macos, is_windows, local_path, messagebox, open_filename, + user_path) +from worlds.LauncherComponents import Component, components, icon_paths, SuffixIdentifier, Type def open_host_yaml(): @@ -127,12 +126,13 @@ def handle_uri(path: str, launch_args: Tuple[str, ...]) -> None: elif component.display_name == "Text Client": text_client_component = component - from kvui import App, Button, BoxLayout, Label, Clock, Window + if client_component is None: + run_component(text_client_component, *launch_args) + return - class Popup(App): - timer_label: Label - remaining_time: Optional[int] + from kvui import App, Button, BoxLayout, Label, Window + class Popup(App): def __init__(self): self.title = "Connect to Multiworld" self.icon = r"data/icon.png" @@ -140,47 +140,29 @@ def __init__(self): def build(self): layout = BoxLayout(orientation="vertical") + layout.add_widget(Label(text="Select client to open and connect with.")) + button_row = BoxLayout(orientation="horizontal", size_hint=(1, 0.4)) - if client_component is None: - self.remaining_time = 7 - label_text = (f"A game client able to parse URIs was not detected for {game}.\n" - f"Launching Text Client in 7 seconds...") - self.timer_label = Label(text=label_text) - layout.add_widget(self.timer_label) - Clock.schedule_interval(self.update_label, 1) - else: - layout.add_widget(Label(text="Select client to open and connect with.")) - button_row = BoxLayout(orientation="horizontal", size_hint=(1, 0.4)) - - text_client_button = Button( - text=text_client_component.display_name, - on_release=lambda *args: run_component(text_client_component, *launch_args) - ) - button_row.add_widget(text_client_button) + text_client_button = Button( + text=text_client_component.display_name, + on_release=lambda *args: run_component(text_client_component, *launch_args) + ) + button_row.add_widget(text_client_button) - game_client_button = Button( - text=client_component.display_name, - on_release=lambda *args: run_component(client_component, *launch_args) - ) - button_row.add_widget(game_client_button) + game_client_button = Button( + text=client_component.display_name, + on_release=lambda *args: run_component(client_component, *launch_args) + ) + button_row.add_widget(game_client_button) - layout.add_widget(button_row) + layout.add_widget(button_row) return layout - def update_label(self, dt): - if self.remaining_time > 1: - # countdown the timer and string replace the number - self.remaining_time -= 1 - self.timer_label.text = self.timer_label.text.replace( - str(self.remaining_time + 1), str(self.remaining_time) - ) - else: - # our timer is finished so launch text client and close down - run_component(text_client_component, *launch_args) - Clock.unschedule(self.update_label) - App.get_running_app().stop() - Window.close() + def _stop(self, *largs): + # see run_gui Launcher _stop comment for details + self.root_window.close() + super()._stop(*largs) Popup().run() @@ -242,9 +224,8 @@ def launch(exe, in_terminal=False): def run_gui(): - from kvui import App, ContainerLayout, GridLayout, Button, Label, ScrollBox, Widget + from kvui import App, ContainerLayout, GridLayout, Button, Label, ScrollBox, Widget, ApAsyncImage from kivy.core.window import Window - from kivy.uix.image import AsyncImage from kivy.uix.relativelayout import RelativeLayout class Launcher(App): @@ -277,8 +258,8 @@ def build_button(component: Component) -> Widget: button.component = component button.bind(on_release=self.component_action) if component.icon != "icon": - image = AsyncImage(source=icon_paths[component.icon], - size=(38, 38), size_hint=(None, 1), pos=(5, 0)) + image = ApAsyncImage(source=icon_paths[component.icon], + size=(38, 38), size_hint=(None, 1), pos=(5, 0)) box_layout = RelativeLayout(size_hint_y=None, height=40) box_layout.add_widget(button) box_layout.add_widget(image) diff --git a/LinksAwakeningClient.py b/LinksAwakeningClient.py index 298788098d9e..e2e16922fa95 100644 --- a/LinksAwakeningClient.py +++ b/LinksAwakeningClient.py @@ -235,7 +235,7 @@ async def async_read_memory_safe(self, address, size=1): def check_command_response(self, command: str, response: bytes): if command == "VERSION": - ok = re.match("\d+\.\d+\.\d+", response.decode('ascii')) is not None + ok = re.match(r"\d+\.\d+\.\d+", response.decode('ascii')) is not None else: ok = response.startswith(command.encode()) if not ok: @@ -560,6 +560,10 @@ async def server_auth(self, password_requested: bool = False): while self.client.auth == None: await asyncio.sleep(0.1) + + # Just return if we're closing + if self.exit_event.is_set(): + return self.auth = self.client.auth await self.send_connect() diff --git a/LttPAdjuster.py b/LttPAdjuster.py index 7e33a3d5efe8..963557e8da81 100644 --- a/LttPAdjuster.py +++ b/LttPAdjuster.py @@ -33,10 +33,15 @@ WINDOW_MIN_WIDTH = 425 class AdjusterWorld(object): + class AdjusterSubWorld(object): + def __init__(self, random): + self.random = random + def __init__(self, sprite_pool): import random self.sprite_pool = {1: sprite_pool} self.per_slot_randoms = {1: random} + self.worlds = {1: self.AdjusterSubWorld(random)} class ArgumentDefaultsHelpFormatter(argparse.RawTextHelpFormatter): diff --git a/Main.py b/Main.py index 4008ca5e9017..d0e7a7f8793d 100644 --- a/Main.py +++ b/Main.py @@ -148,50 +148,44 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No else: multiworld.worlds[1].options.non_local_items.value = set() multiworld.worlds[1].options.local_items.value = set() - + + AutoWorld.call_all(multiworld, "connect_entrances") AutoWorld.call_all(multiworld, "generate_basic") # remove starting inventory from pool items. # Because some worlds don't actually create items during create_items this has to be as late as possible. - if any(getattr(multiworld.worlds[player].options, "start_inventory_from_pool", None) for player in multiworld.player_ids): - new_items: List[Item] = [] - old_items: List[Item] = [] - depletion_pool: Dict[int, Dict[str, int]] = { - player: getattr(multiworld.worlds[player].options, - "start_inventory_from_pool", - StartInventoryPool({})).value.copy() - for player in multiworld.player_ids - } - for player, items in depletion_pool.items(): - player_world: AutoWorld.World = multiworld.worlds[player] - for count in items.values(): - for _ in range(count): - new_items.append(player_world.create_filler()) - target: int = sum(sum(items.values()) for items in depletion_pool.values()) - for i, item in enumerate(multiworld.itempool): + fallback_inventory = StartInventoryPool({}) + depletion_pool: Dict[int, Dict[str, int]] = { + player: getattr(multiworld.worlds[player].options, "start_inventory_from_pool", fallback_inventory).value.copy() + for player in multiworld.player_ids + } + target_per_player = { + player: sum(target_items.values()) for player, target_items in depletion_pool.items() if target_items + } + + if target_per_player: + new_itempool: List[Item] = [] + + # Make new itempool with start_inventory_from_pool items removed + for item in multiworld.itempool: if depletion_pool[item.player].get(item.name, 0): - target -= 1 depletion_pool[item.player][item.name] -= 1 - # quick abort if we have found all items - if not target: - old_items.extend(multiworld.itempool[i+1:]) - break else: - old_items.append(item) - - # leftovers? - if target: - for player, remaining_items in depletion_pool.items(): - remaining_items = {name: count for name, count in remaining_items.items() if count} - if remaining_items: - logger.warning(f"{multiworld.get_player_name(player)}" - f" is trying to remove items from their pool that don't exist: {remaining_items}") - # find all filler we generated for the current player and remove until it matches - removables = [item for item in new_items if item.player == player] - for _ in range(sum(remaining_items.values())): - new_items.remove(removables.pop()) - assert len(multiworld.itempool) == len(new_items + old_items), "Item Pool amounts should not change." - multiworld.itempool[:] = new_items + old_items + new_itempool.append(item) + + # Create filler in place of the removed items, warn if any items couldn't be found in the multiworld itempool + for player, target in target_per_player.items(): + unfound_items = {item: count for item, count in depletion_pool[player].items() if count} + + if unfound_items: + player_name = multiworld.get_player_name(player) + logger.warning(f"{player_name} tried to remove items from their pool that don't exist: {unfound_items}") + + needed_items = target_per_player[player] - sum(unfound_items.values()) + new_itempool += [multiworld.worlds[player].create_filler() for _ in range(needed_items)] + + assert len(multiworld.itempool) == len(new_itempool), "Item Pool amounts should not change." + multiworld.itempool[:] = new_itempool multiworld.link_items() @@ -249,6 +243,7 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No def write_multidata(): import NetUtils + from NetUtils import HintStatus slot_data = {} client_versions = {} games = {} @@ -273,10 +268,10 @@ def write_multidata(): for slot in multiworld.player_ids: slot_data[slot] = multiworld.worlds[slot].fill_slot_data() - def precollect_hint(location): + def precollect_hint(location: Location, auto_status: HintStatus): entrance = er_hint_data.get(location.player, {}).get(location.address, "") hint = NetUtils.Hint(location.item.player, location.player, location.address, - location.item.code, False, entrance, location.item.flags) + location.item.code, False, entrance, location.item.flags, auto_status) precollected_hints[location.player].add(hint) if location.item.player not in multiworld.groups: precollected_hints[location.item.player].add(hint) @@ -289,19 +284,22 @@ def precollect_hint(location): if type(location.address) == int: assert location.item.code is not None, "item code None should be event, " \ "location.address should then also be None. Location: " \ - f" {location}" + f" {location}, Item: {location.item}" assert location.address not in locations_data[location.player], ( f"Locations with duplicate address. {location} and " f"{locations_data[location.player][location.address]}") locations_data[location.player][location.address] = \ location.item.code, location.item.player, location.item.flags + auto_status = HintStatus.HINT_AVOID if location.item.trap else HintStatus.HINT_PRIORITY if location.name in multiworld.worlds[location.player].options.start_location_hints: - precollect_hint(location) + if not location.item.trap: # Unspecified status for location hints, except traps + auto_status = HintStatus.HINT_UNSPECIFIED + precollect_hint(location, auto_status) elif location.item.name in multiworld.worlds[location.item.player].options.start_hints: - precollect_hint(location) + precollect_hint(location, auto_status) elif any([location.item.name in multiworld.worlds[player].options.start_hints for player in multiworld.groups.get(location.item.player, {}).get("players", [])]): - precollect_hint(location) + precollect_hint(location, auto_status) # embedded data package data_package = { @@ -313,11 +311,10 @@ def precollect_hint(location): # get spheres -> filter address==None -> skip empty spheres: List[Dict[int, Set[int]]] = [] - for sphere in multiworld.get_spheres(): + for sphere in multiworld.get_sendable_spheres(): current_sphere: Dict[int, Set[int]] = collections.defaultdict(set) for sphere_location in sphere: - if type(sphere_location.address) is int: - current_sphere[sphere_location.player].add(sphere_location.address) + current_sphere[sphere_location.player].add(sphere_location.address) if current_sphere: spheres.append(dict(current_sphere)) diff --git a/ModuleUpdate.py b/ModuleUpdate.py index f49182bb7863..04cf25ea5594 100644 --- a/ModuleUpdate.py +++ b/ModuleUpdate.py @@ -5,8 +5,15 @@ import warnings -if sys.version_info < (3, 8, 6): - raise RuntimeError("Incompatible Python Version. 3.8.7+ is supported.") +if sys.platform in ("win32", "darwin") and sys.version_info < (3, 10, 11): + # Official micro version updates. This should match the number in docs/running from source.md. + raise RuntimeError(f"Incompatible Python Version found: {sys.version_info}. Official 3.10.15+ is supported.") +elif sys.platform in ("win32", "darwin") and sys.version_info < (3, 10, 15): + # There are known security issues, but no easy way to install fixed versions on Windows for testing. + warnings.warn(f"Python Version {sys.version_info} has security issues. Don't use in production.") +elif sys.version_info < (3, 10, 1): + # Other platforms may get security backports instead of micro updates, so the number is unreliable. + raise RuntimeError(f"Incompatible Python Version found: {sys.version_info}. 3.10.1+ is supported.") # don't run update if environment is frozen/compiled or if not the parent process (skip in subprocess) _skip_update = bool(getattr(sys, "frozen", False) or multiprocessing.parent_process()) diff --git a/MultiServer.py b/MultiServer.py index 764b56362ecc..a310808b3aec 100644 --- a/MultiServer.py +++ b/MultiServer.py @@ -28,9 +28,11 @@ if typing.TYPE_CHECKING: import ssl + from NetUtils import ServerConnection -import websockets import colorama +import websockets +from websockets.extensions.permessage_deflate import PerMessageDeflate try: # ponyorm is a requirement for webhost, not default server, so may not be importable from pony.orm.dbapiprovider import OperationalError @@ -41,7 +43,8 @@ import Utils from Utils import version_tuple, restricted_loads, Version, async_start, get_intended_text from NetUtils import Endpoint, ClientStatus, NetworkItem, decode, encode, NetworkPlayer, Permission, NetworkSlot, \ - SlotType, LocationStore + SlotType, LocationStore, Hint, HintStatus +from BaseClasses import ItemClassification min_client_version = Version(0, 1, 6) colorama.init() @@ -118,13 +121,14 @@ def get_saving_second(seed_name: str, interval: int = 60) -> int: class Client(Endpoint): version = Version(0, 0, 0) - tags: typing.List[str] = [] + tags: typing.List[str] remote_items: bool remote_start_inventory: bool no_items: bool no_locations: bool + no_text: bool - def __init__(self, socket: websockets.WebSocketServerProtocol, ctx: Context): + def __init__(self, socket: "ServerConnection", ctx: Context) -> None: super().__init__(socket) self.auth = False self.team = None @@ -174,6 +178,7 @@ class Context: "compatibility": int} # team -> slot id -> list of clients authenticated to slot. clients: typing.Dict[int, typing.Dict[int, typing.List[Client]]] + endpoints: list[Client] locations: LocationStore # typing.Dict[int, typing.Dict[int, typing.Tuple[int, int, int]]] location_checks: typing.Dict[typing.Tuple[int, int], typing.Set[int]] hints_used: typing.Dict[typing.Tuple[int, int], int] @@ -228,7 +233,7 @@ def __init__(self, host: str, port: int, server_password: str, password: str, lo self.hint_cost = hint_cost self.location_check_points = location_check_points self.hints_used = collections.defaultdict(int) - self.hints: typing.Dict[team_slot, typing.Set[NetUtils.Hint]] = collections.defaultdict(set) + self.hints: typing.Dict[team_slot, typing.Set[Hint]] = collections.defaultdict(set) self.release_mode: str = release_mode self.remaining_mode: str = remaining_mode self.collect_mode: str = collect_mode @@ -363,18 +368,28 @@ async def broadcast_send_encoded_msgs(self, endpoints: typing.Iterable[Endpoint] return True def broadcast_all(self, msgs: typing.List[dict]): - msgs = self.dumper(msgs) - endpoints = (endpoint for endpoint in self.endpoints if endpoint.auth) - async_start(self.broadcast_send_encoded_msgs(endpoints, msgs)) + msg_is_text = all(msg["cmd"] == "PrintJSON" for msg in msgs) + data = self.dumper(msgs) + endpoints = ( + endpoint + for endpoint in self.endpoints + if endpoint.auth and not (msg_is_text and endpoint.no_text) + ) + async_start(self.broadcast_send_encoded_msgs(endpoints, data)) def broadcast_text_all(self, text: str, additional_arguments: dict = {}): self.logger.info("Notice (all): %s" % text) self.broadcast_all([{**{"cmd": "PrintJSON", "data": [{ "text": text }]}, **additional_arguments}]) def broadcast_team(self, team: int, msgs: typing.List[dict]): - msgs = self.dumper(msgs) - endpoints = (endpoint for endpoint in itertools.chain.from_iterable(self.clients[team].values())) - async_start(self.broadcast_send_encoded_msgs(endpoints, msgs)) + msg_is_text = all(msg["cmd"] == "PrintJSON" for msg in msgs) + data = self.dumper(msgs) + endpoints = ( + endpoint + for endpoint in itertools.chain.from_iterable(self.clients[team].values()) + if not (msg_is_text and endpoint.no_text) + ) + async_start(self.broadcast_send_encoded_msgs(endpoints, data)) def broadcast(self, endpoints: typing.Iterable[Client], msgs: typing.List[dict]): msgs = self.dumper(msgs) @@ -388,13 +403,13 @@ async def disconnect(self, endpoint: Client): await on_client_disconnected(self, endpoint) def notify_client(self, client: Client, text: str, additional_arguments: dict = {}): - if not client.auth: + if not client.auth or client.no_text: return self.logger.info("Notice (Player %s in team %d): %s" % (client.name, client.team + 1, text)) async_start(self.send_msgs(client, [{"cmd": "PrintJSON", "data": [{ "text": text }], **additional_arguments}])) def notify_client_multiple(self, client: Client, texts: typing.List[str], additional_arguments: dict = {}): - if not client.auth: + if not client.auth or client.no_text: return async_start(self.send_msgs(client, [{"cmd": "PrintJSON", "data": [{ "text": text }], **additional_arguments} @@ -443,7 +458,7 @@ def _load(self, decoded_obj: dict, game_data_packages: typing.Dict[str, typing.A self.slot_info = decoded_obj["slot_info"] self.games = {slot: slot_info.game for slot, slot_info in self.slot_info.items()} - self.groups = {slot: slot_info.group_members for slot, slot_info in self.slot_info.items() + self.groups = {slot: set(slot_info.group_members) for slot, slot_info in self.slot_info.items() if slot_info.type == SlotType.group} self.clients = {0: {}} @@ -656,13 +671,29 @@ def get_hint_cost(self, slot): return max(1, int(self.hint_cost * 0.01 * len(self.locations[slot]))) return 0 - def recheck_hints(self, team: typing.Optional[int] = None, slot: typing.Optional[int] = None): + def recheck_hints(self, team: typing.Optional[int] = None, slot: typing.Optional[int] = None, + changed: typing.Optional[typing.Set[team_slot]] = None) -> None: + """Refreshes the hints for the specified team/slot. Providing 'None' for either team or slot + will refresh all teams or all slots respectively. If a set is passed for 'changed', each (team,slot) + pair that has at least one hint modified will be added to the set. + """ for hint_team, hint_slot in self.hints: - if (team is None or team == hint_team) and (slot is None or slot == hint_slot): - self.hints[hint_team, hint_slot] = { - hint.re_check(self, hint_team) for hint in - self.hints[hint_team, hint_slot] - } + if team != hint_team and team is not None: + continue # Check specified team only, all if team is None + if slot != hint_slot and slot is not None: + continue # Check specified slot only, all if slot is None + new_hints: typing.Set[Hint] = set() + for hint in self.hints[hint_team, hint_slot]: + new_hint = hint.re_check(self, hint_team) + new_hints.add(new_hint) + if hint == new_hint: + continue + for player in self.slot_set(hint.receiving_player) | {hint.finding_player}: + if changed is not None: + changed.add((hint_team,player)) + if slot is not None and slot != player: + self.replace_hint(hint_team, player, hint, new_hint) + self.hints[hint_team, hint_slot] = new_hints def get_rechecked_hints(self, team: int, slot: int): self.recheck_hints(team, slot) @@ -711,7 +742,7 @@ def get_aliased_name(self, team: int, slot: int): else: return self.player_names[team, slot] - def notify_hints(self, team: int, hints: typing.List[NetUtils.Hint], only_new: bool = False, + def notify_hints(self, team: int, hints: typing.List[Hint], only_new: bool = False, recipients: typing.Sequence[int] = None): """Send and remember hints.""" if only_new: @@ -726,7 +757,8 @@ def notify_hints(self, team: int, hints: typing.List[NetUtils.Hint], only_new: b concerns[player].append(data) if not hint.local and data not in concerns[hint.finding_player]: concerns[hint.finding_player].append(data) - # remember hints in all cases + + # only remember hints that were not already found at the time of creation if not hint.found: # since hints are bidirectional, finding player and receiving player, # we can check once if hint already exists @@ -742,13 +774,24 @@ def notify_hints(self, team: int, hints: typing.List[NetUtils.Hint], only_new: b self.on_new_hint(team, slot) for slot, hint_data in concerns.items(): if recipients is None or slot in recipients: - clients = self.clients[team].get(slot) + clients = filter(lambda c: not c.no_text, self.clients[team].get(slot, [])) if not clients: continue client_hints = [datum[1] for datum in sorted(hint_data, key=lambda x: x[0].finding_player != slot)] for client in clients: async_start(self.send_msgs(client, client_hints)) + def get_hint(self, team: int, finding_player: int, seeked_location: int) -> typing.Optional[Hint]: + for hint in self.hints[team, finding_player]: + if hint.location == seeked_location and hint.finding_player == finding_player: + return hint + return None + + def replace_hint(self, team: int, slot: int, old_hint: Hint, new_hint: Hint) -> None: + if old_hint in self.hints[team, slot]: + self.hints[team, slot].remove(old_hint) + self.hints[team, slot].add(new_hint) + # "events" def on_goal_achieved(self, client: Client): @@ -790,7 +833,7 @@ def update_aliases(ctx: Context, team: int): async_start(ctx.send_encoded_msgs(client, cmd)) -async def server(websocket, path: str = "/", ctx: Context = None): +async def server(websocket: "ServerConnection", path: str = "/", ctx: Context = None) -> None: client = Client(websocket, ctx) ctx.endpoints.append(client) @@ -881,6 +924,10 @@ async def on_client_joined(ctx: Context, client: Client): "If your client supports it, " "you may have additional local commands you can list with /help.", {"type": "Tutorial"}) + if not any(isinstance(extension, PerMessageDeflate) for extension in client.socket.extensions): + ctx.notify_client(client, "Warning: your client does not support compressed websocket connections! " + "It may stop working in the future. If you are a player, please report this to the " + "client's developer.") ctx.client_connection_timers[client.team, client.slot] = datetime.datetime.now(datetime.timezone.utc) @@ -947,9 +994,13 @@ def get_status_string(ctx: Context, team: int, tag: str): tagged = len([client for client in ctx.clients[team][slot] if tag in client.tags]) completion_text = f"({len(ctx.location_checks[team, slot])}/{len(ctx.locations[slot])})" tag_text = f" {tagged} of which are tagged {tag}" if connected and tag else "" - goal_text = " and has finished." if ctx.client_game_state[team, slot] == ClientStatus.CLIENT_GOAL else "." + status_text = ( + " and has finished." if ctx.client_game_state[team, slot] == ClientStatus.CLIENT_GOAL else + " and is ready." if ctx.client_game_state[team, slot] == ClientStatus.CLIENT_READY else + "." + ) text += f"\n{ctx.get_aliased_name(team, slot)} has {connected} connection{'' if connected == 1 else 's'}" \ - f"{tag_text}{goal_text} {completion_text}" + f"{tag_text}{status_text} {completion_text}" return text @@ -1027,21 +1078,37 @@ def send_items_to(ctx: Context, team: int, target_slot: int, *items: NetworkItem def register_location_checks(ctx: Context, team: int, slot: int, locations: typing.Iterable[int], count_activity: bool = True): + slot_locations = ctx.locations[slot] new_locations = set(locations) - ctx.location_checks[team, slot] - new_locations.intersection_update(ctx.locations[slot]) # ignore location IDs unknown to this multidata + new_locations.intersection_update(slot_locations) # ignore location IDs unknown to this multidata if new_locations: if count_activity: ctx.client_activity_timers[team, slot] = datetime.datetime.now(datetime.timezone.utc) + + sortable: list[tuple[int, int, int, int]] = [] for location in new_locations: - item_id, target_player, flags = ctx.locations[slot][location] + # extract all fields to avoid runtime overhead in LocationStore + item_id, target_player, flags = slot_locations[location] + # sort/group by receiver and item + sortable.append((target_player, item_id, location, flags)) + + info_texts: list[dict[str, typing.Any]] = [] + for target_player, item_id, location, flags in sorted(sortable): new_item = NetworkItem(item_id, location, slot, flags) send_items_to(ctx, team, target_player, new_item) ctx.logger.info('(Team #%d) %s sent %s to %s (%s)' % ( team + 1, ctx.player_names[(team, slot)], ctx.item_names[ctx.slot_info[target_player].game][item_id], ctx.player_names[(team, target_player)], ctx.location_names[ctx.slot_info[slot].game][location])) - info_text = json_format_send_event(new_item, target_player) - ctx.broadcast_team(team, [info_text]) + if len(info_texts) >= 140: + # split into chunks that are close to compression window of 64K but not too big on the wire + # (roughly 1300-2600 bytes after compression depending on repetitiveness) + ctx.broadcast_team(team, info_texts) + info_texts.clear() + info_texts.append(json_format_send_event(new_item, target_player)) + ctx.broadcast_team(team, info_texts) + del info_texts + del sortable ctx.location_checks[team, slot] |= new_locations send_new_items(ctx) @@ -1050,14 +1117,15 @@ def register_location_checks(ctx: Context, team: int, slot: int, locations: typi "hint_points": get_slot_points(ctx, team, slot), "checked_locations": new_locations, # send back new checks only }]) - old_hints = ctx.hints[team, slot].copy() - ctx.recheck_hints(team, slot) - if old_hints != ctx.hints[team, slot]: - ctx.on_changed_hints(team, slot) + updated_slots: typing.Set[tuple[int, int]] = set() + ctx.recheck_hints(team, slot, updated_slots) + for hint_team, hint_slot in updated_slots: + ctx.on_changed_hints(hint_team, hint_slot) ctx.save() -def collect_hints(ctx: Context, team: int, slot: int, item: typing.Union[int, str]) -> typing.List[NetUtils.Hint]: +def collect_hints(ctx: Context, team: int, slot: int, item: typing.Union[int, str], auto_status: HintStatus) \ + -> typing.List[Hint]: hints = [] slots: typing.Set[int] = {slot} for group_id, group in ctx.groups.items(): @@ -1067,31 +1135,58 @@ def collect_hints(ctx: Context, team: int, slot: int, item: typing.Union[int, st seeked_item_id = item if isinstance(item, int) else ctx.item_names_for_game(ctx.games[slot])[item] for finding_player, location_id, item_id, receiving_player, item_flags \ in ctx.locations.find_item(slots, seeked_item_id): - found = location_id in ctx.location_checks[team, finding_player] - entrance = ctx.er_hint_data.get(finding_player, {}).get(location_id, "") - hints.append(NetUtils.Hint(receiving_player, finding_player, location_id, item_id, found, entrance, - item_flags)) + prev_hint = ctx.get_hint(team, finding_player, location_id) + if prev_hint: + hints.append(prev_hint) + else: + found = location_id in ctx.location_checks[team, finding_player] + entrance = ctx.er_hint_data.get(finding_player, {}).get(location_id, "") + new_status = auto_status + if found: + new_status = HintStatus.HINT_FOUND + elif item_flags & ItemClassification.trap: + new_status = HintStatus.HINT_AVOID + hints.append(Hint(receiving_player, finding_player, location_id, item_id, found, entrance, + item_flags, new_status)) return hints -def collect_hint_location_name(ctx: Context, team: int, slot: int, location: str) -> typing.List[NetUtils.Hint]: +def collect_hint_location_name(ctx: Context, team: int, slot: int, location: str, auto_status: HintStatus) \ + -> typing.List[Hint]: seeked_location: int = ctx.location_names_for_game(ctx.games[slot])[location] - return collect_hint_location_id(ctx, team, slot, seeked_location) + return collect_hint_location_id(ctx, team, slot, seeked_location, auto_status) -def collect_hint_location_id(ctx: Context, team: int, slot: int, seeked_location: int) -> typing.List[NetUtils.Hint]: +def collect_hint_location_id(ctx: Context, team: int, slot: int, seeked_location: int, auto_status: HintStatus) \ + -> typing.List[Hint]: + prev_hint = ctx.get_hint(team, slot, seeked_location) + if prev_hint: + return [prev_hint] result = ctx.locations[slot].get(seeked_location, (None, None, None)) if any(result): item_id, receiving_player, item_flags = result found = seeked_location in ctx.location_checks[team, slot] entrance = ctx.er_hint_data.get(slot, {}).get(seeked_location, "") - return [NetUtils.Hint(receiving_player, slot, seeked_location, item_id, found, entrance, item_flags)] + new_status = auto_status + if found: + new_status = HintStatus.HINT_FOUND + elif item_flags & ItemClassification.trap: + new_status = HintStatus.HINT_AVOID + return [Hint(receiving_player, slot, seeked_location, item_id, found, entrance, item_flags, + new_status)] return [] -def format_hint(ctx: Context, team: int, hint: NetUtils.Hint) -> str: +status_names: typing.Dict[HintStatus, str] = { + HintStatus.HINT_FOUND: "(found)", + HintStatus.HINT_UNSPECIFIED: "(unspecified)", + HintStatus.HINT_NO_PRIORITY: "(no priority)", + HintStatus.HINT_AVOID: "(avoid)", + HintStatus.HINT_PRIORITY: "(priority)", +} +def format_hint(ctx: Context, team: int, hint: Hint) -> str: text = f"[Hint]: {ctx.player_names[team, hint.receiving_player]}'s " \ f"{ctx.item_names[ctx.slot_info[hint.receiving_player].game][hint.item]} is " \ f"at {ctx.location_names[ctx.slot_info[hint.finding_player].game][hint.location]} " \ @@ -1099,7 +1194,8 @@ def format_hint(ctx: Context, team: int, hint: NetUtils.Hint) -> str: if hint.entrance: text += f" at {hint.entrance}" - return text + (". (found)" if hint.found else ".") + + return text + ". " + status_names.get(hint.status, "(unknown)") def json_format_send_event(net_item: NetworkItem, receiving_player: int): @@ -1503,7 +1599,7 @@ def _cmd_getitem(self, item_name: str) -> bool: def get_hints(self, input_text: str, for_location: bool = False) -> bool: points_available = get_client_points(self.ctx, self.client) cost = self.ctx.get_hint_cost(self.client.slot) - + auto_status = HintStatus.HINT_UNSPECIFIED if for_location else HintStatus.HINT_PRIORITY if not input_text: hints = {hint.re_check(self.ctx, self.client.team) for hint in self.ctx.hints[self.client.team, self.client.slot]} @@ -1529,9 +1625,9 @@ def get_hints(self, input_text: str, for_location: bool = False) -> bool: self.output(f"Sorry, \"{hint_name}\" is marked as non-hintable.") hints = [] elif not for_location: - hints = collect_hints(self.ctx, self.client.team, self.client.slot, hint_id) + hints = collect_hints(self.ctx, self.client.team, self.client.slot, hint_id, auto_status) else: - hints = collect_hint_location_id(self.ctx, self.client.team, self.client.slot, hint_id) + hints = collect_hint_location_id(self.ctx, self.client.team, self.client.slot, hint_id, auto_status) else: game = self.ctx.games[self.client.slot] @@ -1551,16 +1647,16 @@ def get_hints(self, input_text: str, for_location: bool = False) -> bool: hints = [] for item_name in self.ctx.item_name_groups[game][hint_name]: if item_name in self.ctx.item_names_for_game(game): # ensure item has an ID - hints.extend(collect_hints(self.ctx, self.client.team, self.client.slot, item_name)) + hints.extend(collect_hints(self.ctx, self.client.team, self.client.slot, item_name, auto_status)) elif not for_location and hint_name in self.ctx.item_names_for_game(game): # item name - hints = collect_hints(self.ctx, self.client.team, self.client.slot, hint_name) + hints = collect_hints(self.ctx, self.client.team, self.client.slot, hint_name, auto_status) elif hint_name in self.ctx.location_name_groups[game]: # location group name hints = [] for loc_name in self.ctx.location_name_groups[game][hint_name]: if loc_name in self.ctx.location_names_for_game(game): - hints.extend(collect_hint_location_name(self.ctx, self.client.team, self.client.slot, loc_name)) + hints.extend(collect_hint_location_name(self.ctx, self.client.team, self.client.slot, loc_name, auto_status)) else: # location name - hints = collect_hint_location_name(self.ctx, self.client.team, self.client.slot, hint_name) + hints = collect_hint_location_name(self.ctx, self.client.team, self.client.slot, hint_name, auto_status) else: self.output(response) @@ -1725,7 +1821,9 @@ async def process_client_cmd(ctx: Context, client: Client, args: dict): ctx.clients[team][slot].append(client) client.version = args['version'] client.tags = args['tags'] - client.no_locations = 'TextOnly' in client.tags or 'Tracker' in client.tags + client.no_locations = "TextOnly" in client.tags or "Tracker" in client.tags + # set NoText for old PopTracker clients that predate the tag to save traffic + client.no_text = "NoText" in client.tags or ("PopTracker" in client.tags and client.version < (0, 5, 1)) connected_packet = { "cmd": "Connected", "team": client.team, "slot": client.slot, @@ -1798,6 +1896,9 @@ async def process_client_cmd(ctx: Context, client: Client, args: dict): client.tags = args["tags"] if set(old_tags) != set(client.tags): client.no_locations = 'TextOnly' in client.tags or 'Tracker' in client.tags + client.no_text = "NoText" in client.tags or ( + "PopTracker" in client.tags and client.version < (0, 5, 1) + ) ctx.broadcast_text_all( f"{ctx.get_aliased_name(client.team, client.slot)} (Team #{client.team + 1}) has changed tags " f"from {old_tags} to {client.tags}.", @@ -1826,19 +1927,63 @@ async def process_client_cmd(ctx: Context, client: Client, args: dict): for location in args["locations"]: if type(location) is not int: await ctx.send_msgs(client, - [{'cmd': 'InvalidPacket', "type": "arguments", "text": 'LocationScouts', + [{'cmd': 'InvalidPacket', "type": "arguments", + "text": 'Locations has to be a list of integers', "original_cmd": cmd}]) return target_item, target_player, flags = ctx.locations[client.slot][location] if create_as_hint: - hints.extend(collect_hint_location_id(ctx, client.team, client.slot, location)) + hints.extend(collect_hint_location_id(ctx, client.team, client.slot, location, + HintStatus.HINT_UNSPECIFIED)) locs.append(NetworkItem(target_item, location, target_player, flags)) ctx.notify_hints(client.team, hints, only_new=create_as_hint == 2) if locs and create_as_hint: ctx.save() await ctx.send_msgs(client, [{'cmd': 'LocationInfo', 'locations': locs}]) - + + elif cmd == 'UpdateHint': + location = args["location"] + player = args["player"] + status = args["status"] + if not isinstance(player, int) or not isinstance(location, int) \ + or (status is not None and not isinstance(status, int)): + await ctx.send_msgs(client, + [{'cmd': 'InvalidPacket', "type": "arguments", "text": 'UpdateHint', + "original_cmd": cmd}]) + return + hint = ctx.get_hint(client.team, player, location) + if not hint: + return # Ignored safely + if client.slot not in ctx.slot_set(hint.receiving_player): + await ctx.send_msgs(client, + [{'cmd': 'InvalidPacket', "type": "arguments", "text": 'UpdateHint: No Permission', + "original_cmd": cmd}]) + return + new_hint = hint + if status is None: + return + try: + status = HintStatus(status) + except ValueError: + await ctx.send_msgs(client, + [{'cmd': 'InvalidPacket', "type": "arguments", + "text": 'UpdateHint: Invalid Status', "original_cmd": cmd}]) + return + if status == HintStatus.HINT_FOUND: + await ctx.send_msgs(client, + [{'cmd': 'InvalidPacket', "type": "arguments", + "text": 'UpdateHint: Cannot manually update status to "HINT_FOUND"', "original_cmd": cmd}]) + return + new_hint = new_hint.re_prioritize(ctx, status) + if hint == new_hint: + return + ctx.replace_hint(client.team, hint.finding_player, hint, new_hint) + ctx.replace_hint(client.team, hint.receiving_player, hint, new_hint) + ctx.save() + ctx.on_changed_hints(client.team, hint.finding_player) + ctx.on_changed_hints(client.team, hint.receiving_player) + elif cmd == 'StatusUpdate': update_client_status(ctx, client, args["status"]) @@ -1886,6 +2031,7 @@ async def process_client_cmd(ctx: Context, client: Client, args: dict): args["cmd"] = "SetReply" value = ctx.stored_data.get(args["key"], args.get("default", 0)) args["original_value"] = copy.copy(value) + args["slot"] = client.slot for operation in args["operations"]: func = modify_functions[operation["operation"]] value = func(value, operation["value"]) @@ -2143,9 +2289,9 @@ def _cmd_hint(self, player_name: str, *item_name: str) -> bool: hints = [] for item_name_from_group in self.ctx.item_name_groups[game][item]: if item_name_from_group in self.ctx.item_names_for_game(game): # ensure item has an ID - hints.extend(collect_hints(self.ctx, team, slot, item_name_from_group)) + hints.extend(collect_hints(self.ctx, team, slot, item_name_from_group, HintStatus.HINT_PRIORITY)) else: # item name or id - hints = collect_hints(self.ctx, team, slot, item) + hints = collect_hints(self.ctx, team, slot, item, HintStatus.HINT_PRIORITY) if hints: self.ctx.notify_hints(team, hints) @@ -2179,14 +2325,17 @@ def _cmd_hint_location(self, player_name: str, *location_name: str) -> bool: if usable: if isinstance(location, int): - hints = collect_hint_location_id(self.ctx, team, slot, location) + hints = collect_hint_location_id(self.ctx, team, slot, location, + HintStatus.HINT_UNSPECIFIED) elif game in self.ctx.location_name_groups and location in self.ctx.location_name_groups[game]: hints = [] for loc_name_from_group in self.ctx.location_name_groups[game][location]: if loc_name_from_group in self.ctx.location_names_for_game(game): - hints.extend(collect_hint_location_name(self.ctx, team, slot, loc_name_from_group)) + hints.extend(collect_hint_location_name(self.ctx, team, slot, loc_name_from_group, + HintStatus.HINT_UNSPECIFIED)) else: - hints = collect_hint_location_name(self.ctx, team, slot, location) + hints = collect_hint_location_name(self.ctx, team, slot, location, + HintStatus.HINT_UNSPECIFIED) if hints: self.ctx.notify_hints(team, hints) else: @@ -2276,6 +2425,8 @@ def parse_args() -> argparse.Namespace: parser.add_argument('--cert_key', help="Path to SSL Certificate Key file") parser.add_argument('--loglevel', default=defaults["loglevel"], choices=['debug', 'info', 'warning', 'error', 'critical']) + parser.add_argument('--logtime', help="Add timestamps to STDOUT", + default=defaults["logtime"], action='store_true') parser.add_argument('--location_check_points', default=defaults["location_check_points"], type=int) parser.add_argument('--hint_cost', default=defaults["hint_cost"], type=int) parser.add_argument('--disable_item_cheat', default=defaults["disable_item_cheat"], action='store_true') @@ -2356,7 +2507,9 @@ def load_server_cert(path: str, cert_key: typing.Optional[str]) -> "ssl.SSLConte async def main(args: argparse.Namespace): - Utils.init_logging("Server", loglevel=args.loglevel.lower()) + Utils.init_logging(name="Server", + loglevel=args.loglevel.lower(), + add_timestamp=args.logtime) ctx = Context(args.host, args.port, args.server_password, args.password, args.location_check_points, args.hint_cost, not args.disable_item_cheat, args.release_mode, args.collect_mode, diff --git a/NetUtils.py b/NetUtils.py index 4776b228db17..f2ae2a63a056 100644 --- a/NetUtils.py +++ b/NetUtils.py @@ -5,11 +5,20 @@ import warnings from json import JSONEncoder, JSONDecoder -import websockets +if typing.TYPE_CHECKING: + from websockets import WebSocketServerProtocol as ServerConnection from Utils import ByValue, Version +class HintStatus(ByValue, enum.IntEnum): + HINT_UNSPECIFIED = 0 + HINT_NO_PRIORITY = 10 + HINT_AVOID = 20 + HINT_PRIORITY = 30 + HINT_FOUND = 40 + + class JSONMessagePart(typing.TypedDict, total=False): text: str # optional @@ -19,6 +28,8 @@ class JSONMessagePart(typing.TypedDict, total=False): player: int # if type == item indicates item flags flags: int + # if type == hint_status + hint_status: HintStatus class ClientStatus(ByValue, enum.IntEnum): @@ -141,7 +152,7 @@ def _object_hook(o: typing.Any) -> typing.Any: class Endpoint: - socket: websockets.WebSocketServerProtocol + socket: "ServerConnection" def __init__(self, socket): self.socket = socket @@ -184,6 +195,7 @@ class JSONTypes(str, enum.Enum): location_name = "location_name" location_id = "location_id" entrance_name = "entrance_name" + hint_status = "hint_status" class JSONtoTextParser(metaclass=HandlerMeta): @@ -224,7 +236,7 @@ def _handle_text(self, node: JSONMessagePart): def _handle_player_id(self, node: JSONMessagePart): player = int(node["text"]) - node["color"] = 'magenta' if player == self.ctx.slot else 'yellow' + node["color"] = 'magenta' if self.ctx.slot_concerns_self(player) else 'yellow' node["text"] = self.ctx.player_names[player] return self._handle_color(node) @@ -265,6 +277,10 @@ def _handle_entrance_name(self, node: JSONMessagePart): node["color"] = 'blue' return self._handle_color(node) + def _handle_hint_status(self, node: JSONMessagePart): + node["color"] = status_colors.get(node["hint_status"], "red") + return self._handle_color(node) + class RawJSONtoTextParser(JSONtoTextParser): def _handle_color(self, node: JSONMessagePart): @@ -297,6 +313,27 @@ def add_json_location(parts: list, location_id: int, player: int = 0, **kwargs) parts.append({"text": str(location_id), "player": player, "type": JSONTypes.location_id, **kwargs}) +status_names: typing.Dict[HintStatus, str] = { + HintStatus.HINT_FOUND: "(found)", + HintStatus.HINT_UNSPECIFIED: "(unspecified)", + HintStatus.HINT_NO_PRIORITY: "(no priority)", + HintStatus.HINT_AVOID: "(avoid)", + HintStatus.HINT_PRIORITY: "(priority)", +} +status_colors: typing.Dict[HintStatus, str] = { + HintStatus.HINT_FOUND: "green", + HintStatus.HINT_UNSPECIFIED: "white", + HintStatus.HINT_NO_PRIORITY: "slateblue", + HintStatus.HINT_AVOID: "salmon", + HintStatus.HINT_PRIORITY: "plum", +} + + +def add_json_hint_status(parts: list, hint_status: HintStatus, text: typing.Optional[str] = None, **kwargs): + parts.append({"text": text if text != None else status_names.get(hint_status, "(unknown)"), + "hint_status": hint_status, "type": JSONTypes.hint_status, **kwargs}) + + class Hint(typing.NamedTuple): receiving_player: int finding_player: int @@ -305,14 +342,21 @@ class Hint(typing.NamedTuple): found: bool entrance: str = "" item_flags: int = 0 + status: HintStatus = HintStatus.HINT_UNSPECIFIED def re_check(self, ctx, team) -> Hint: - if self.found: + if self.found and self.status == HintStatus.HINT_FOUND: return self found = self.location in ctx.location_checks[team, self.finding_player] if found: - return Hint(self.receiving_player, self.finding_player, self.location, self.item, found, self.entrance, - self.item_flags) + return self._replace(found=found, status=HintStatus.HINT_FOUND) + return self + + def re_prioritize(self, ctx, status: HintStatus) -> Hint: + if self.found and status != HintStatus.HINT_FOUND: + status = HintStatus.HINT_FOUND + if status != self.status: + return self._replace(status=status) return self def __hash__(self): @@ -334,10 +378,7 @@ def as_network_message(self) -> dict: else: add_json_text(parts, "'s World") add_json_text(parts, ". ") - if self.found: - add_json_text(parts, "(found)", type="color", color="green") - else: - add_json_text(parts, "(not found)", type="color", color="red") + add_json_hint_status(parts, self.status) return {"cmd": "PrintJSON", "data": parts, "type": "Hint", "receiving": self.receiving_player, @@ -383,6 +424,8 @@ def get_checked(self, state: typing.Dict[typing.Tuple[int, int], typing.Set[int] checked = state[team, slot] if not checked: # This optimizes the case where everyone connects to a fresh game at the same time. + if slot not in self: + raise KeyError(slot) return [] return [location_id for location_id in self[slot] if diff --git a/OoTAdjuster.py b/OoTAdjuster.py index 9519b191e704..1581d6539825 100644 --- a/OoTAdjuster.py +++ b/OoTAdjuster.py @@ -1,7 +1,6 @@ import tkinter as tk import argparse import logging -import random import os import zipfile from itertools import chain @@ -197,7 +196,6 @@ def set_icon(window): def adjust(args): # Create a fake multiworld and OOTWorld to use as a base multiworld = MultiWorld(1) - multiworld.per_slot_randoms = {1: random} ootworld = OOTWorld(multiworld, 1) # Set options in the fake OOTWorld for name, option in chain(cosmetic_options.items(), sfx_options.items()): diff --git a/Options.py b/Options.py index aa6f175fa58d..49e82069ee8d 100644 --- a/Options.py +++ b/Options.py @@ -15,7 +15,7 @@ from schema import And, Optional, Or, Schema from typing_extensions import Self -from Utils import get_fuzzy_results, is_iterable_except_str, output_path +from Utils import get_file_safe_name, get_fuzzy_results, is_iterable_except_str, output_path if typing.TYPE_CHECKING: from BaseClasses import MultiWorld, PlandoOptions @@ -137,7 +137,7 @@ class Option(typing.Generic[T], metaclass=AssembleOptions): If this is False, the docstring is instead interpreted as plain text, and displayed as-is on the WebHost with whitespace preserved. - If this is None, it inherits the value of `World.rich_text_options_doc`. For + If this is None, it inherits the value of `WebWorld.rich_text_options_doc`. For backwards compatibility, this defaults to False, but worlds are encouraged to set it to True and use reStructuredText for their Option documentation. @@ -496,7 +496,7 @@ class TextChoice(Choice): def __init__(self, value: typing.Union[str, int]): assert isinstance(value, str) or isinstance(value, int), \ - f"{value} is not a valid option for {self.__class__.__name__}" + f"'{value}' is not a valid option for '{self.__class__.__name__}'" self.value = value @property @@ -617,17 +617,17 @@ def validate_plando_bosses(cls, options: typing.List[str]) -> None: used_locations.append(location) used_bosses.append(boss) if not cls.valid_boss_name(boss): - raise ValueError(f"{boss.title()} is not a valid boss name.") + raise ValueError(f"'{boss.title()}' is not a valid boss name.") if not cls.valid_location_name(location): - raise ValueError(f"{location.title()} is not a valid boss location name.") + raise ValueError(f"'{location.title()}' is not a valid boss location name.") if not cls.can_place_boss(boss, location): - raise ValueError(f"{location.title()} is not a valid location for {boss.title()} to be placed.") + raise ValueError(f"'{location.title()}' is not a valid location for {boss.title()} to be placed.") else: if cls.duplicate_bosses: if not cls.valid_boss_name(option): - raise ValueError(f"{option} is not a valid boss name.") + raise ValueError(f"'{option}' is not a valid boss name.") else: - raise ValueError(f"{option.title()} is not formatted correctly.") + raise ValueError(f"'{option.title()}' is not formatted correctly.") @classmethod def can_place_boss(cls, boss: str, location: str) -> bool: @@ -689,9 +689,9 @@ def from_text(cls, text: str) -> Range: @classmethod def weighted_range(cls, text) -> Range: if text == "random-low": - return cls(cls.triangular(cls.range_start, cls.range_end, cls.range_start)) + return cls(cls.triangular(cls.range_start, cls.range_end, 0.0)) elif text == "random-high": - return cls(cls.triangular(cls.range_start, cls.range_end, cls.range_end)) + return cls(cls.triangular(cls.range_start, cls.range_end, 1.0)) elif text == "random-middle": return cls(cls.triangular(cls.range_start, cls.range_end)) elif text.startswith("random-range-"): @@ -717,11 +717,11 @@ def custom_range(cls, text) -> Range: f"{random_range[0]}-{random_range[1]} is outside allowed range " f"{cls.range_start}-{cls.range_end} for option {cls.__name__}") if text.startswith("random-range-low"): - return cls(cls.triangular(random_range[0], random_range[1], random_range[0])) + return cls(cls.triangular(random_range[0], random_range[1], 0.0)) elif text.startswith("random-range-middle"): return cls(cls.triangular(random_range[0], random_range[1])) elif text.startswith("random-range-high"): - return cls(cls.triangular(random_range[0], random_range[1], random_range[1])) + return cls(cls.triangular(random_range[0], random_range[1], 1.0)) else: return cls(random.randint(random_range[0], random_range[1])) @@ -739,8 +739,16 @@ def __str__(self) -> str: return str(self.value) @staticmethod - def triangular(lower: int, end: int, tri: typing.Optional[int] = None) -> int: - return int(round(random.triangular(lower, end, tri), 0)) + def triangular(lower: int, end: int, tri: float = 0.5) -> int: + """ + Integer triangular distribution for `lower` inclusive to `end` inclusive. + + Expects `lower <= end` and `0.0 <= tri <= 1.0`. The result of other inputs is undefined. + """ + # Use the continuous range [lower, end + 1) to produce an integer result in [lower, end]. + # random.triangular is actually [a, b] and not [a, b), so there is a very small chance of getting exactly b even + # when a != b, so ensure the result is never more than `end`. + return min(end, math.floor(random.triangular(0.0, 1.0, tri) * (end - lower + 1) + lower)) class NamedRange(Range): @@ -754,7 +762,7 @@ def __init__(self, value: int) -> None: elif value > self.range_end and value not in self.special_range_names.values(): raise Exception(f"{value} is higher than maximum {self.range_end} for option {self.__class__.__name__} " + f"and is also not one of the supported named special values: {self.special_range_names}") - + # See docstring for key in self.special_range_names: if key != key.lower(): @@ -817,18 +825,21 @@ def verify(self, world: typing.Type[World], player_name: str, plando_options: "P for item_name in self.value: if item_name not in world.item_names: picks = get_fuzzy_results(item_name, world.item_names, limit=1) - raise Exception(f"Item {item_name} from option {self} " - f"is not a valid item name from {world.game}. " + raise Exception(f"Item '{item_name}' from option '{self}' " + f"is not a valid item name from '{world.game}'. " f"Did you mean '{picks[0][0]}' ({picks[0][1]}% sure)") elif self.verify_location_name: for location_name in self.value: if location_name not in world.location_names: picks = get_fuzzy_results(location_name, world.location_names, limit=1) - raise Exception(f"Location {location_name} from option {self} " - f"is not a valid location name from {world.game}. " + raise Exception(f"Location '{location_name}' from option '{self}' " + f"is not a valid location name from '{world.game}'. " f"Did you mean '{picks[0][0]}' ({picks[0][1]}% sure)") + def __iter__(self) -> typing.Iterator[typing.Any]: + return self.value.__iter__() + class OptionDict(Option[typing.Dict[str, typing.Any]], VerifyKeys, typing.Mapping[str, typing.Any]): default = {} supports_weighting = False @@ -860,6 +871,8 @@ class ItemDict(OptionDict): verify_item_name = True def __init__(self, value: typing.Dict[str, int]): + if any(item_count is None for item_count in value.values()): + raise Exception("Items must have counts associated with them. Please provide positive integer values in the format \"item\": count .") if any(item_count < 1 for item_count in value.values()): raise Exception("Cannot have non-positive item counts.") super(ItemDict, self).__init__(value) @@ -1106,11 +1119,11 @@ def validate_plando_connections(cls, connections: typing.Iterable[PlandoConnecti used_entrances.append(entrance) used_exits.append(exit) if not cls.validate_entrance_name(entrance): - raise ValueError(f"{entrance.title()} is not a valid entrance.") + raise ValueError(f"'{entrance.title()}' is not a valid entrance.") if not cls.validate_exit_name(exit): - raise ValueError(f"{exit.title()} is not a valid exit.") + raise ValueError(f"'{exit.title()}' is not a valid exit.") if not cls.can_connect(entrance, exit): - raise ValueError(f"Connection between {entrance.title()} and {exit.title()} is invalid.") + raise ValueError(f"Connection between '{entrance.title()}' and '{exit.title()}' is invalid.") @classmethod def from_any(cls, data: PlandoConFromAnyType) -> Self: @@ -1175,7 +1188,7 @@ def __len__(self) -> int: class Accessibility(Choice): """ Set rules for reachability of your items/locations. - + **Full:** ensure everything can be reached and acquired. **Minimal:** ensure what is needed to reach your goal can be acquired. @@ -1193,7 +1206,7 @@ class Accessibility(Choice): class ItemsAccessibility(Accessibility): """ Set rules for reachability of your items/locations. - + **Full:** ensure everything can be reached and acquired. **Minimal:** ensure what is needed to reach your goal can be acquired. @@ -1244,12 +1257,16 @@ class CommonOptions(metaclass=OptionsMetaProperty): progression_balancing: ProgressionBalancing accessibility: Accessibility - def as_dict(self, *option_names: str, casing: str = "snake") -> typing.Dict[str, typing.Any]: + def as_dict(self, + *option_names: str, + casing: typing.Literal["snake", "camel", "pascal", "kebab"] = "snake", + toggles_as_bools: bool = False) -> typing.Dict[str, typing.Any]: """ Returns a dictionary of [str, Option.value] :param option_names: names of the options to return :param casing: case of the keys to return. Supports `snake`, `camel`, `pascal`, `kebab` + :param toggles_as_bools: whether toggle options should be output as bools instead of strings """ assert option_names, "options.as_dict() was used without any option names." option_results = {} @@ -1271,6 +1288,8 @@ def as_dict(self, *option_names: str, casing: str = "snake") -> typing.Dict[str, value = getattr(self, option_name).value if isinstance(value, set): value = sorted(value) + elif toggles_as_bools and issubclass(type(self).type_hints[option_name], Toggle): + value = bool(value) option_results[display_name] = value else: raise ValueError(f"{option_name} not found in {tuple(type(self).type_hints)}") @@ -1368,8 +1387,8 @@ def verify_items(items: typing.List[str], item_link: str, pool_name: str, world, picks_group = get_fuzzy_results(item_name, world.item_name_groups.keys(), limit=1) picks_group = f" or '{picks_group[0][0]}' ({picks_group[0][1]}% sure)" if allow_item_groups else "" - raise Exception(f"Item {item_name} from item link {item_link} " - f"is not a valid item from {world.game} for {pool_name}. " + raise Exception(f"Item '{item_name}' from item link '{item_link}' " + f"is not a valid item from '{world.game}' for '{pool_name}'. " f"Did you mean '{picks[0][0]}' ({picks[0][1]}% sure){picks_group}") if allow_item_groups: pool |= world.item_name_groups.get(item_name, {item_name}) @@ -1460,22 +1479,26 @@ class OptionGroup(typing.NamedTuple): def get_option_groups(world: typing.Type[World], visibility_level: Visibility = Visibility.template) -> typing.Dict[ str, typing.Dict[str, typing.Type[Option[typing.Any]]]]: """Generates and returns a dictionary for the option groups of a specified world.""" - option_groups = {option: option_group.name - for option_group in world.web.option_groups - for option in option_group.options} - # add a default option group for uncategorized options to get thrown into - ordered_groups = ["Game Options"] - [ordered_groups.append(group) for group in option_groups.values() if group not in ordered_groups] - grouped_options = {group: {} for group in ordered_groups} - for option_name, option in world.options_dataclass.type_hints.items(): - if visibility_level & option.visibility: - grouped_options[option_groups.get(option, "Game Options")][option_name] = option + option_to_name = {option: option_name for option_name, option in world.options_dataclass.type_hints.items()} - # if the world doesn't have any ungrouped options, this group will be empty so just remove it - if not grouped_options["Game Options"]: - del grouped_options["Game Options"] + ordered_groups = {group.name: group.options for group in world.web.option_groups} - return grouped_options + # add a default option group for uncategorized options to get thrown into + if "Game Options" not in ordered_groups: + grouped_options = set(option for group in ordered_groups.values() for option in group) + ungrouped_options = [option for option in option_to_name if option not in grouped_options] + # only add the game options group if we have ungrouped options + if ungrouped_options: + ordered_groups = {**{"Game Options": ungrouped_options}, **ordered_groups} + + return { + group: { + option_to_name[option]: option + for option in group_options + if (visibility_level in option.visibility and option in option_to_name) + } + for group, group_options in ordered_groups.items() + } def generate_yaml_templates(target_folder: typing.Union[str, "pathlib.Path"], generate_hidden: bool = True) -> None: @@ -1531,7 +1554,7 @@ def yaml_dump_scalar(scalar) -> str: del file_data - with open(os.path.join(target_folder, game_name + ".yaml"), "w", encoding="utf-8-sig") as f: + with open(os.path.join(target_folder, get_file_safe_name(game_name) + ".yaml"), "w", encoding="utf-8-sig") as f: f.write(res) @@ -1559,7 +1582,7 @@ def dump_player_options(multiworld: MultiWorld) -> None: } output.append(player_output) for option_key, option in world.options_dataclass.type_hints.items(): - if issubclass(Removed, option): + if option.visibility == Visibility.none: continue display_name = getattr(option, "display_name", option_key) player_output[display_name] = getattr(world.options, option_key).current_option_name diff --git a/README.md b/README.md index 0e57bce53b51..d60f1b96651f 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,10 @@ Currently, the following games are supported: * Kingdom Hearts 1 * Mega Man 2 * Yacht Dice +* Faxanadu +* Saving Princess +* Castlevania: Circle of the Moon +* Inscryption For setup and instructions check out our [tutorials page](https://archipelago.gg/tutorial/). Downloads can be found at [Releases](https://github.com/ArchipelagoMW/Archipelago/releases), including compiled diff --git a/SNIClient.py b/SNIClient.py index 19440e1dc5be..9140c73c14e2 100644 --- a/SNIClient.py +++ b/SNIClient.py @@ -243,6 +243,9 @@ def on_package(self, cmd: str, args: typing.Dict[str, typing.Any]) -> None: # Once the games handled by SNIClient gets made to be remote items, # this will no longer be needed. async_start(self.send_msgs([{"cmd": "LocationScouts", "locations": list(new_locations)}])) + + if self.client_handler is not None: + self.client_handler.on_package(self, cmd, args) def run_gui(self) -> None: from kvui import GameManager diff --git a/Utils.py b/Utils.py index 412011200f8a..0aa81af1502e 100644 --- a/Utils.py +++ b/Utils.py @@ -18,8 +18,8 @@ from argparse import Namespace from settings import Settings, get_settings -from typing import BinaryIO, Coroutine, Optional, Set, Dict, Any, Union -from typing_extensions import TypeGuard +from time import sleep +from typing import BinaryIO, Coroutine, Optional, Set, Dict, Any, Union, TypeGuard from yaml import load, load_all, dump try: @@ -47,7 +47,7 @@ def as_simple_string(self) -> str: return ".".join(str(item) for item in self) -__version__ = "0.5.1" +__version__ = "0.6.0" version_tuple = tuplize_version(__version__) is_linux = sys.platform.startswith("linux") @@ -152,8 +152,15 @@ def home_path(*path: str) -> str: if hasattr(home_path, 'cached_path'): pass elif sys.platform.startswith('linux'): - home_path.cached_path = os.path.expanduser('~/Archipelago') - os.makedirs(home_path.cached_path, 0o700, exist_ok=True) + xdg_data_home = os.getenv('XDG_DATA_HOME', os.path.expanduser('~/.local/share')) + home_path.cached_path = xdg_data_home + '/Archipelago' + if not os.path.isdir(home_path.cached_path): + legacy_home_path = os.path.expanduser('~/Archipelago') + if os.path.isdir(legacy_home_path): + os.renames(legacy_home_path, home_path.cached_path) + os.symlink(home_path.cached_path, legacy_home_path) + else: + os.makedirs(home_path.cached_path, 0o700, exist_ok=True) else: # not implemented home_path.cached_path = local_path() # this will generate the same exceptions we got previously @@ -421,7 +428,8 @@ def find_class(self, module: str, name: str) -> type: if module == "builtins" and name in safe_builtins: return getattr(builtins, name) # used by MultiServer -> savegame/multidata - if module == "NetUtils" and name in {"NetworkItem", "ClientStatus", "Hint", "SlotType", "NetworkSlot"}: + if module == "NetUtils" and name in {"NetworkItem", "ClientStatus", "Hint", + "SlotType", "NetworkSlot", "HintStatus"}: return getattr(self.net_utils_module, name) # Options and Plando are unpickled by WebHost -> Generate if module == "worlds.generic" and name == "PlandoItem": @@ -484,9 +492,9 @@ def get_text_after(text: str, start: str) -> str: loglevel_mapping = {'error': logging.ERROR, 'info': logging.INFO, 'warning': logging.WARNING, 'debug': logging.DEBUG} -def init_logging(name: str, loglevel: typing.Union[str, int] = logging.INFO, write_mode: str = "w", - log_format: str = "[%(name)s at %(asctime)s]: %(message)s", - exception_logger: typing.Optional[str] = None): +def init_logging(name: str, loglevel: typing.Union[str, int] = logging.INFO, + write_mode: str = "w", log_format: str = "[%(name)s at %(asctime)s]: %(message)s", + add_timestamp: bool = False, exception_logger: typing.Optional[str] = None): import datetime loglevel: int = loglevel_mapping.get(loglevel, loglevel) log_folder = user_path("logs") @@ -513,11 +521,15 @@ def __init__(self, filter_name: str, condition: typing.Callable[[logging.LogReco def filter(self, record: logging.LogRecord) -> bool: return self.condition(record) - file_handler.addFilter(Filter("NoStream", lambda record: not getattr(record, "NoFile", False))) + file_handler.addFilter(Filter("NoStream", lambda record: not getattr(record, "NoFile", False))) + file_handler.addFilter(Filter("NoCarriageReturn", lambda record: '\r' not in record.getMessage())) root_logger.addHandler(file_handler) if sys.stdout: + formatter = logging.Formatter(fmt='[%(asctime)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S') stream_handler = logging.StreamHandler(sys.stdout) stream_handler.addFilter(Filter("NoFile", lambda record: not getattr(record, "NoStream", False))) + if add_timestamp: + stream_handler.setFormatter(formatter) root_logger.addHandler(stream_handler) # Relay unhandled exceptions to logger. @@ -529,7 +541,8 @@ def handle_exception(exc_type, exc_value, exc_traceback): sys.__excepthook__(exc_type, exc_value, exc_traceback) return logging.getLogger(exception_logger).exception("Uncaught exception", - exc_info=(exc_type, exc_value, exc_traceback)) + exc_info=(exc_type, exc_value, exc_traceback), + extra={"NoStream": exception_logger is None}) return orig_hook(exc_type, exc_value, exc_traceback) handle_exception._wrapped = True @@ -552,7 +565,7 @@ def _cleanup(): import platform logging.info( f"Archipelago ({__version__}) logging initialized" - f" on {platform.platform()}" + f" on {platform.platform()} process {os.getpid()}" f" running Python {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" f"{' (frozen)' if is_frozen() else ''}" ) @@ -568,6 +581,8 @@ def queuer(): else: if text: queue.put_nowait(text) + else: + sleep(0.01) # non-blocking stream from threading import Thread thread = Thread(target=queuer, name=f"Stream handler for {stream.name}", daemon=True) @@ -852,11 +867,10 @@ def async_start(co: Coroutine[None, None, typing.Any], name: Optional[str] = Non task.add_done_callback(_faf_tasks.discard) -def deprecate(message: str): +def deprecate(message: str, add_stacklevels: int = 0): if __debug__: raise Exception(message) - import warnings - warnings.warn(message) + warnings.warn(message, stacklevel=2 + add_stacklevels) class DeprecateDict(dict): @@ -870,10 +884,9 @@ def __init__(self, message: str, error: bool = False) -> None: def __getitem__(self, item: Any) -> Any: if self.should_error: - deprecate(self.log_message) + deprecate(self.log_message, add_stacklevels=1) elif __debug__: - import warnings - warnings.warn(self.log_message) + warnings.warn(self.log_message, stacklevel=2) return super().__getitem__(item) @@ -927,7 +940,7 @@ def freeze_support() -> None: def visualize_regions(root_region: Region, file_name: str, *, show_entrance_names: bool = False, show_locations: bool = True, show_other_regions: bool = True, - linetype_ortho: bool = True) -> None: + linetype_ortho: bool = True, regions_to_highlight: set[Region] | None = None) -> None: """Visualize the layout of a world as a PlantUML diagram. :param root_region: The region from which to start the diagram from. (Usually the "Menu" region of your world.) @@ -943,16 +956,22 @@ def visualize_regions(root_region: Region, file_name: str, *, Items without ID will be shown in italics. :param show_other_regions: (default True) If enabled, regions that can't be reached by traversing exits are shown. :param linetype_ortho: (default True) If enabled, orthogonal straight line parts will be used; otherwise polylines. + :param regions_to_highlight: Regions that will be highlighted in green if they are reachable. Example usage in World code: from Utils import visualize_regions - visualize_regions(self.multiworld.get_region("Menu", self.player), "my_world.puml") + state = self.multiworld.get_all_state(False) + state.update_reachable_regions(self.player) + visualize_regions(self.get_region("Menu"), "my_world.puml", show_entrance_names=True, + regions_to_highlight=state.reachable_regions[self.player]) Example usage in Main code: from Utils import visualize_regions for player in multiworld.player_ids: visualize_regions(multiworld.get_region("Menu", player), f"{multiworld.get_out_file_name_base(player)}.puml") """ + if regions_to_highlight is None: + regions_to_highlight = set() assert root_region.multiworld, "The multiworld attribute of root_region has to be filled" from BaseClasses import Entrance, Item, Location, LocationProgressType, MultiWorld, Region from collections import deque @@ -1005,7 +1024,7 @@ def visualize_locations(region: Region) -> None: uml.append(f"\"{fmt(region)}\" : {{field}} {lock}{fmt(location)}") def visualize_region(region: Region) -> None: - uml.append(f"class \"{fmt(region)}\"") + uml.append(f"class \"{fmt(region)}\" {'#00FF00' if region in regions_to_highlight else ''}") if show_locations: visualize_locations(region) visualize_exits(region) diff --git a/WebHost.py b/WebHost.py index e597de24763d..768eeb512289 100644 --- a/WebHost.py +++ b/WebHost.py @@ -12,11 +12,12 @@ # in case app gets imported by something like gunicorn import Utils import settings +from Utils import get_file_safe_name if typing.TYPE_CHECKING: from flask import Flask -Utils.local_path.cached_path = os.path.dirname(__file__) or "." # py3.8 is not abs. remove "." when dropping 3.8 +Utils.local_path.cached_path = os.path.dirname(__file__) settings.no_gui = True configpath = os.path.abspath("config.yaml") if not os.path.exists(configpath): # fall back to config.yaml in home @@ -33,7 +34,7 @@ def get_app() -> "Flask": app.config.from_file(configpath, yaml.safe_load) logging.info(f"Updated config from {configpath}") # inside get_app() so it's usable in systems like gunicorn, which do not run WebHost.py, but import it. - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(allow_abbrev=False) parser.add_argument('--config_override', default=None, help="Path to yaml config file that overrules config.yaml.") args = parser.parse_known_args()[0] @@ -71,7 +72,7 @@ def create_ordered_tutorials_file() -> typing.List[typing.Dict[str, typing.Any]] shutil.rmtree(base_target_path, ignore_errors=True) for game, world in worlds.items(): # copy files from world's docs folder to the generated folder - target_path = os.path.join(base_target_path, game) + target_path = os.path.join(base_target_path, get_file_safe_name(game)) os.makedirs(target_path, exist_ok=True) if world.zip_path: diff --git a/WebHostLib/__init__.py b/WebHostLib/__init__.py index fdf3037fe015..9c713419c986 100644 --- a/WebHostLib/__init__.py +++ b/WebHostLib/__init__.py @@ -9,7 +9,7 @@ from pony.flask import Pony from werkzeug.routing import BaseConverter -from Utils import title_sorted +from Utils import title_sorted, get_file_safe_name UPLOAD_FOLDER = os.path.relpath('uploads') LOGS_FOLDER = os.path.relpath('logs') @@ -20,6 +20,7 @@ app.jinja_env.filters['any'] = any app.jinja_env.filters['all'] = all +app.jinja_env.filters['get_file_safe_name'] = get_file_safe_name app.config["SELFHOST"] = True # application process is in charge of running the websites app.config["GENERATORS"] = 8 # maximum concurrent world gens @@ -38,6 +39,8 @@ app.config["JOB_THRESHOLD"] = 1 # after what time in seconds should generation be aborted, freeing the queue slot. Can be set to None to disable. app.config["JOB_TIME"] = 600 +# memory limit for generator processes in bytes +app.config["GENERATOR_MEMORY_LIMIT"] = 4294967296 app.config['SESSION_PERMANENT'] = True # waitress uses one thread for I/O, these are for processing of views that then get sent @@ -84,6 +87,6 @@ def register(): from WebHostLib.customserver import run_server_process # to trigger app routing picking up on it - from . import tracker, upload, landing, check, generate, downloads, api, stats, misc, robots, options + from . import tracker, upload, landing, check, generate, downloads, api, stats, misc, robots, options, session app.register_blueprint(api.api_endpoints) diff --git a/WebHostLib/api/__init__.py b/WebHostLib/api/__init__.py index cf05e87374ab..d0b9d05c16b8 100644 --- a/WebHostLib/api/__init__.py +++ b/WebHostLib/api/__init__.py @@ -3,13 +3,13 @@ from flask import Blueprint -from ..models import Seed +from ..models import Seed, Slot api_endpoints = Blueprint('api', __name__, url_prefix="/api") def get_players(seed: Seed) -> List[Tuple[str, str]]: - return [(slot.player_name, slot.game) for slot in seed.slots] + return [(slot.player_name, slot.game) for slot in seed.slots.order_by(Slot.player_id)] from . import datapackage, generate, room, user # trigger registration diff --git a/WebHostLib/api/user.py b/WebHostLib/api/user.py index 116d3afa2288..0ddb6fe83ed8 100644 --- a/WebHostLib/api/user.py +++ b/WebHostLib/api/user.py @@ -30,4 +30,4 @@ def get_seeds(): "creation_time": seed.creation_time, "players": get_players(seed.slots), }) - return jsonify(response) \ No newline at end of file + return jsonify(response) diff --git a/WebHostLib/autolauncher.py b/WebHostLib/autolauncher.py index 08a1309ebc73..8ba093e014c5 100644 --- a/WebHostLib/autolauncher.py +++ b/WebHostLib/autolauncher.py @@ -6,6 +6,7 @@ import typing from datetime import timedelta, datetime from threading import Event, Thread +from typing import Any from uuid import UUID from pony.orm import db_session, select, commit @@ -53,7 +54,21 @@ def launch_generator(pool: multiprocessing.pool.Pool, generation: Generation): generation.state = STATE_STARTED -def init_db(pony_config: dict): +def init_generator(config: dict[str, Any]) -> None: + try: + import resource + except ModuleNotFoundError: + pass # unix only module + else: + # set soft limit for memory to from config (default 4GiB) + soft_limit = config["GENERATOR_MEMORY_LIMIT"] + old_limit, hard_limit = resource.getrlimit(resource.RLIMIT_AS) + if soft_limit != old_limit: + resource.setrlimit(resource.RLIMIT_AS, (soft_limit, hard_limit)) + logging.debug(f"Changed AS mem limit {old_limit} -> {soft_limit}") + del resource, soft_limit, hard_limit + + pony_config = config["PONY"] db.bind(**pony_config) db.generate_mapping() @@ -105,8 +120,8 @@ def keep_running(): try: with Locker("autogen"): - with multiprocessing.Pool(config["GENERATORS"], initializer=init_db, - initargs=(config["PONY"],), maxtasksperchild=10) as generator_pool: + with multiprocessing.Pool(config["GENERATORS"], initializer=init_generator, + initargs=(config,), maxtasksperchild=10) as generator_pool: with db_session: to_start = select(generation for generation in Generation if generation.state == STATE_STARTED) diff --git a/WebHostLib/check.py b/WebHostLib/check.py index 97cb797f7a56..4e0cf1178f4b 100644 --- a/WebHostLib/check.py +++ b/WebHostLib/check.py @@ -105,8 +105,9 @@ def roll_options(options: Dict[str, Union[dict, str]], plando_options=plando_options) else: for i, yaml_data in enumerate(yaml_datas): - rolled_results[f"{filename}/{i + 1}"] = roll_settings(yaml_data, - plando_options=plando_options) + if yaml_data is not None: + rolled_results[f"{filename}/{i + 1}"] = roll_settings(yaml_data, + plando_options=plando_options) except Exception as e: if e.__cause__: results[filename] = f"Failed to generate options in {filename}: {e} - {e.__cause__}" diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index a2eef108b0a1..76a2b8a4dc15 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -117,6 +117,7 @@ def load(self, room_id: int): self.gamespackage = {"Archipelago": static_gamespackage.get("Archipelago", {})} # this may be modified by _load self.item_name_groups = {"Archipelago": static_item_name_groups.get("Archipelago", {})} self.location_name_groups = {"Archipelago": static_location_name_groups.get("Archipelago", {})} + missing_checksum = False for game in list(multidata.get("datapackage", {})): game_data = multidata["datapackage"][game] @@ -132,11 +133,13 @@ def load(self, room_id: int): continue else: self.logger.warning(f"Did not find game_data_package for {game}: {game_data['checksum']}") + else: + missing_checksum = True # Game rolled on old AP and will load data package from multidata self.gamespackage[game] = static_gamespackage.get(game, {}) self.item_name_groups[game] = static_item_name_groups.get(game, {}) self.location_name_groups[game] = static_location_name_groups.get(game, {}) - if not game_data_packages: + if not game_data_packages and not missing_checksum: # all static -> use the static dicts directly self.gamespackage = static_gamespackage self.item_name_groups = static_item_name_groups diff --git a/WebHostLib/generate.py b/WebHostLib/generate.py index b19f3d483515..0bd9f7e5e066 100644 --- a/WebHostLib/generate.py +++ b/WebHostLib/generate.py @@ -31,11 +31,11 @@ def get_meta(options_source: dict, race: bool = False) -> Dict[str, Union[List[s server_options = { "hint_cost": int(options_source.get("hint_cost", ServerOptions.hint_cost)), - "release_mode": options_source.get("release_mode", ServerOptions.release_mode), - "remaining_mode": options_source.get("remaining_mode", ServerOptions.remaining_mode), - "collect_mode": options_source.get("collect_mode", ServerOptions.collect_mode), + "release_mode": str(options_source.get("release_mode", ServerOptions.release_mode)), + "remaining_mode": str(options_source.get("remaining_mode", ServerOptions.remaining_mode)), + "collect_mode": str(options_source.get("collect_mode", ServerOptions.collect_mode)), "item_cheat": bool(int(options_source.get("item_cheat", not ServerOptions.disable_item_cheat))), - "server_password": options_source.get("server_password", None), + "server_password": str(options_source.get("server_password", None)), } generator_options = { "spoiler": int(options_source.get("spoiler", GeneratorOptions.spoiler)), diff --git a/WebHostLib/misc.py b/WebHostLib/misc.py index c49b1ae17801..6be0e470b3b4 100644 --- a/WebHostLib/misc.py +++ b/WebHostLib/misc.py @@ -18,13 +18,6 @@ def get_world_theme(game_name: str): return 'grass' -@app.before_request -def register_session(): - session.permanent = True # technically 31 days after the last visit - if not session.get("_id", None): - session["_id"] = uuid4() # uniquely identify each session without needing a login - - @app.errorhandler(404) @app.errorhandler(jinja2.exceptions.TemplateNotFound) def page_not_found(err): diff --git a/WebHostLib/requirements.txt b/WebHostLib/requirements.txt index 5c79415312d4..b7b14dea1e6f 100644 --- a/WebHostLib/requirements.txt +++ b/WebHostLib/requirements.txt @@ -5,9 +5,7 @@ waitress>=3.0.0 Flask-Caching>=2.3.0 Flask-Compress>=1.15 Flask-Limiter>=3.8.0 -bokeh>=3.1.1; python_version <= '3.8' -bokeh>=3.4.3; python_version == '3.9' -bokeh>=3.5.2; python_version >= '3.10' +bokeh>=3.5.2 markupsafe>=2.1.5 Markdown>=3.7 mdx-breakless-lists>=1.0.1 diff --git a/WebHostLib/session.py b/WebHostLib/session.py new file mode 100644 index 000000000000..d5dab7d6e6e6 --- /dev/null +++ b/WebHostLib/session.py @@ -0,0 +1,31 @@ +from uuid import uuid4, UUID + +from flask import session, render_template + +from WebHostLib import app + + +@app.before_request +def register_session(): + session.permanent = True # technically 31 days after the last visit + if not session.get("_id", None): + session["_id"] = uuid4() # uniquely identify each session without needing a login + + +@app.route('/session') +def show_session(): + return render_template( + "session.html", + ) + + +@app.route('/session/') +def set_session(_id: str): + new_id: UUID = UUID(_id, version=4) + old_id: UUID = session["_id"] + if old_id != new_id: + session["_id"] = new_id + return render_template( + "session.html", + old_id=old_id, + ) diff --git a/WebHostLib/static/assets/faq/en.md b/WebHostLib/static/assets/faq/en.md index e64535b42d03..96e526612be6 100644 --- a/WebHostLib/static/assets/faq/en.md +++ b/WebHostLib/static/assets/faq/en.md @@ -22,7 +22,7 @@ players to rely upon each other to complete their game. While a multiworld game traditionally requires all players to be playing the same game, a multi-game multiworld allows players to randomize any of the supported games, and send items between them. This allows players of different -games to interact with one another in a single multiplayer environment. Archipelago supports multi-game multiworld. +games to interact with one another in a single multiplayer environment. Archipelago supports multi-game multiworlds. Here is a list of our [Supported Games](https://archipelago.gg/games). ## Can I generate a single-player game with Archipelago? diff --git a/WebHostLib/templates/gameInfo.html b/WebHostLib/templates/gameInfo.html index c5ebba82848d..3b908004b1be 100644 --- a/WebHostLib/templates/gameInfo.html +++ b/WebHostLib/templates/gameInfo.html @@ -11,7 +11,7 @@ {% block body %} {% include 'header/'+theme+'Header.html' %} -
+
{% endblock %} diff --git a/WebHostLib/templates/genericTracker.html b/WebHostLib/templates/genericTracker.html index 947cf2837278..b92097ceea08 100644 --- a/WebHostLib/templates/genericTracker.html +++ b/WebHostLib/templates/genericTracker.html @@ -98,6 +98,8 @@ {% if hint.finding_player == player %} {{ player_names_with_alias[(team, hint.finding_player)] }} + {% elif get_slot_info(team, hint.finding_player).type == 2 %} + {{ player_names_with_alias[(team, hint.finding_player)] }} {% else %} {{ player_names_with_alias[(team, hint.finding_player)] }} @@ -107,6 +109,8 @@ {% if hint.receiving_player == player %} {{ player_names_with_alias[(team, hint.receiving_player)] }} + {% elif get_slot_info(team, hint.receiving_player).type == 2 %} + {{ player_names_with_alias[(team, hint.receiving_player)] }} {% else %} {{ player_names_with_alias[(team, hint.receiving_player)] }} diff --git a/WebHostLib/templates/hostRoom.html b/WebHostLib/templates/hostRoom.html index 8e76dafc12fa..c5996d181ee0 100644 --- a/WebHostLib/templates/hostRoom.html +++ b/WebHostLib/templates/hostRoom.html @@ -178,8 +178,15 @@ }) .then(text => new DOMParser().parseFromString(text, 'text/html')) .then(newDocument => { - let el = newDocument.getElementById("host-room-info"); - document.getElementById("host-room-info").innerHTML = el.innerHTML; + ["host-room-info", "slots-table"].forEach(function(id) { + const newEl = newDocument.getElementById(id); + const oldEl = document.getElementById(id); + if (oldEl && newEl) { + oldEl.innerHTML = newEl.innerHTML; + } else if (newEl) { + console.warn(`Did not find element to replace for ${id}`) + } + }); }); } diff --git a/WebHostLib/templates/islandFooter.html b/WebHostLib/templates/islandFooter.html index 08cf227990b8..7de14f0d827c 100644 --- a/WebHostLib/templates/islandFooter.html +++ b/WebHostLib/templates/islandFooter.html @@ -1,6 +1,6 @@ {% block footer %}