Add generated map support to ladder_service (#732)

* Add generated map support to ladder_service * resolve formatting * resolve sql formatting * add unknown type log statement * format sql * formatting * formatting * quotes * change logging * Fix game timeout * isort * increase test timeout * increase test timeout * formatting Co-authored-by: Askaholic <askaholic@protonmail.com> * increase timeout * formatting * add timestamps to map_pool_map_version Co-authored-by: Askaholic <askaholic@protonmail.com>
FAForever · Mar 16, 2021 · 6e4362a · 6e4362a
1 parent f0ffd64
commit 6e4362a
Show file tree

Hide file tree

Showing 14 changed files with 274 additions and 62 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -4,7 +4,8 @@ name: Test
 on: [push, pull_request]
 
 env:
-  FAF_DB_VERSION: v106
+  FAF_DB_VERSION: v112
+  FLYWAY_VERSION: 7.5.4
 
 jobs:
   # Static Analysis
@@ -58,8 +59,8 @@ jobs:
           FLYWAY_LOCATIONS: filesystem:db/migrations
         run: |
           git clone --depth 1 --branch ${FAF_DB_VERSION} https://github.com/FAForever/db
-          wget -qO- https://repo1.maven.org/maven2/org/flywaydb/flyway-commandline/6.5.3/flyway-commandline-6.5.3-linux-x64.tar.gz | tar xz
-          flyway-6.5.3/flyway migrate
+          wget -qO- https://repo1.maven.org/maven2/org/flywaydb/flyway-commandline/${FLYWAY_VERSION}/flyway-commandline-${FLYWAY_VERSION}-linux-x64.tar.gz | tar xz
+          flyway-${FLYWAY_VERSION}/flyway migrate
 
       - name: Install dependencies with pipenv
         run: |

diff --git a/server/db/models.py b/server/db/models.py
@@ -256,8 +256,13 @@
 
 map_pool_map_version = Table(
     "map_pool_map_version", metadata,
-    Column("map_pool_id",       Integer, ForeignKey("map_pool.id"),     nullable=False),
-    Column("map_version_id",    Integer, ForeignKey("map_version.id"),  nullable=False),
+    Column("id",             Integer,       primary_key=True),
+    Column("map_pool_id",    Integer,       ForeignKey("map_pool.id"),      nullable=False),
+    Column("map_version_id", Integer,       ForeignKey("map_version.id")),
+    Column("weight",         Integer,       nullable=False),
+    Column("map_params",     Text),
+    Column("create_time",    TIMESTAMP,     nullable=False),
+    Column("update_time",    TIMESTAMP,     nullable=False)
 )
 
 map_version = Table(

diff --git a/server/games/coop.py b/server/games/coop.py
@@ -1,3 +1,5 @@
+import asyncio
+
 from server.abc.base_game import InitMode
 
 from .game import Game
@@ -21,6 +23,7 @@ def __init__(self, *args, **kwargs):
             "Difficulty": 3,
             "Expansion": "true"
         })
+        asyncio.get_event_loop().create_task(self.timeout_game(60))
 
     async def validate_game_mode_settings(self):
         """

diff --git a/server/games/custom_game.py b/server/games/custom_game.py
@@ -1,3 +1,4 @@
+import asyncio
 import time
 
 from server.abc.base_game import InitMode
@@ -18,6 +19,7 @@ def __init__(self, id_, *args, **kwargs):
         }
         new_kwargs.update(kwargs)
         super().__init__(id_, *args, **new_kwargs)
+        asyncio.get_event_loop().create_task(self.timeout_game())
 
     async def _run_pre_rate_validity_checks(self):
         limit = len(self.players) * 60

diff --git a/server/games/game.py b/server/games/game.py
@@ -118,12 +118,9 @@ def __init__(
         self._launch_fut = asyncio.Future()
 
         self._logger.debug("%s created", self)
-        asyncio.get_event_loop().create_task(self.timeout_game())
 
-    async def timeout_game(self):
-        # coop takes longer to set up
-        tm = 30 if self.game_mode != FeaturedModType.COOP else 60
-        await asyncio.sleep(tm)
+    async def timeout_game(self, timeout: int = 30):
+        await asyncio.sleep(timeout)
         if self.state is GameState.INITIALIZING:
             self._is_hosted.set_exception(
                 asyncio.TimeoutError("Game setup timed out")

diff --git a/server/ladder_service.py b/server/ladder_service.py
@@ -1,5 +1,5 @@
 import asyncio
-import itertools
+import json
 import random
 import re
 from collections import defaultdict
@@ -32,8 +32,7 @@
 from .games import LadderGame
 from .matchmaker import MapPool, MatchmakerQueue, OnMatchedCallback, Search
 from .players import Player, PlayerState
-from .rating import RatingType
-from .types import GameLaunchOptions, Map
+from .types import GameLaunchOptions, Map, NeroxisGeneratedMap
 
 
 @with_logger
@@ -107,6 +106,8 @@ async def fetch_map_pools(self, conn) -> Dict[int, Tuple[str, List[Map]]]:
             select([
                 map_pool.c.id,
                 map_pool.c.name,
+                map_pool_map_version.c.weight,
+                map_pool_map_version.c.map_params,
                 map_version.c.map_id,
                 map_version.c.filename,
                 t_map.c.display_name
@@ -125,8 +126,29 @@ async def fetch_map_pools(self, conn) -> Dict[int, Tuple[str, List[Map]]]:
             _, map_list = map_pool_maps[id_]
             if row.map_id is not None:
                 map_list.append(
-                    Map(row.map_id, row.display_name, row.filename)
+                    Map(row.map_id, row.display_name, row.filename, row.weight)
                 )
+            elif row.map_params is not None:
+                try:
+                    params = json.loads(row.map_params)
+                    map_type = params["type"]
+                    if map_type == "neroxis":
+                        map_list.append(NeroxisGeneratedMap.of(params, row.weight))
+                    else:
+                        self._logger.warning(
+                            "Unsupported map type %s in pool %s",
+                             map_type,
+                             row.id
+                        )
+
+                except Exception:
+                    self._logger.warning(
+                        "Failed to load map in map pool %d "
+                        "parameters specified as %s",
+                        row.id,
+                        row.map_params,
+                        exc_info=True
+                    )
 
         return map_pool_maps
 
@@ -367,7 +389,7 @@ async def start_game(
             pool = queue.get_map_pool_for_rating(rating)
             if not pool:
                 raise RuntimeError(f"No map pool available for rating {rating}!")
-            map_id, map_name, map_path = pool.choose_map(played_map_ids)
+            _, _, map_path, _ = pool.choose_map(played_map_ids)
 
             game = self.game_service.create_game(
                 game_class=LadderGame,
@@ -410,6 +432,7 @@ def get_player_mean(player):
             mapname = re.match("maps/(.+).zip", map_path).group(1)
             # FIXME: Database filenames contain the maps/ prefix and .zip suffix.
             # Really in the future, just send a better description
+
             self._logger.debug("Starting ladder game: %s", game)
             # Options shared by all players
             options = GameLaunchOptions(
@@ -428,7 +451,7 @@ def game_options(player: Player) -> GameLaunchOptions:
                 game, is_host=True, options=game_options(host)
             )
             try:
-                await game.wait_hosted(30)
+                await game.wait_hosted(60)
             finally:
                 # TODO: Once the client supports `match_cancelled`, don't
                 # send `launch_game` to the client if the host timed out. Until
@@ -443,7 +466,7 @@ def game_options(player: Player) -> GameLaunchOptions:
                     for guest in all_guests
                     if guest.lobby_connection is not None
                 ])
-            await game.wait_launched(30 + 10 * len(all_guests))
+            await game.wait_launched(60 + 10 * len(all_guests))
             self._logger.debug("Ladder game launched successfully")
         except Exception:
             if game:

diff --git a/server/matchmaker/map_pool.py b/server/matchmaker/map_pool.py
@@ -1,9 +1,9 @@
 import random
 from collections import Counter
-from typing import Iterable
+from typing import Iterable, Union
 
 from ..decorators import with_logger
-from ..types import Map
+from ..types import Map, NeroxisGeneratedMap
 
 
 @with_logger
@@ -12,7 +12,7 @@ def __init__(
         self,
         map_pool_id: int,
         name: str,
-        maps: Iterable[Map] = ()
+        maps: Iterable[Union[Map, NeroxisGeneratedMap]] = ()
     ):
         self.id = map_pool_id
         self.name = name
@@ -39,14 +39,21 @@ def choose_map(self, played_map_ids: Iterable[int] = ()) -> Map:
         counter.update(id_ for id_ in played_map_ids if id_ in self.maps)
 
         least_common = counter.most_common()[::-1]
-        least_count = least_common[0][1]
+        least_count = 1
+        for map_count in least_common:
+            if isinstance(self.maps[map_count[0]], Map):
+                least_count = map_count[1]
+                break
+
         # Trim off the maps with higher play counts
         for i, (_, count) in enumerate(least_common):
-            if count != least_count:
+            if count > least_count:
                 least_common = least_common[:i]
                 break
 
-        return self.maps[random.choice(least_common)[0]]
+        weights = [self.maps[id_].weight for id_, _ in least_common]
+
+        return self.maps[random.choices(least_common, weights=weights, k=1)[0][0]].get_map()
 
     def __repr__(self):
         return f"MapPool({self.id}, {self.name}, {list(self.maps.values())})"
diff --git a/server/types.py b/server/types.py
@@ -1,3 +1,5 @@
+import base64
+import random
 from typing import NamedTuple, Optional
 
 
@@ -27,3 +29,56 @@ class Map(NamedTuple):
     id: int
     name: str
     path: str
+    weight: int = 1
+
+    def get_map(self) -> "Map":
+        return self
+
+
+# FIXME: Use typing.Protocol in python3.8 to subtype with MAP
+class NeroxisGeneratedMap(NamedTuple):
+    id: int
+    version: str
+    spawns: int
+    map_size_pixels: int
+    weight: int = 1
+
+    @classmethod
+    def of(cls, params: dict, weight: int = 1):
+        assert params["type"] == "neroxis"
+
+        map_size_pixels = int(params["size"])
+
+        if map_size_pixels <= 0:
+            raise Exception("Map size is zero or negative")
+
+        if map_size_pixels & (map_size_pixels - 1) != 0:
+            raise Exception("Map size is not a power of 2")
+
+        spawns = int(params["spawns"])
+        if spawns % 2 != 0:
+            raise Exception("spawns is not a multiple of 2")
+
+        version = params["version"]
+        return NeroxisGeneratedMap(
+            -int.from_bytes(bytes(f"{version}_{spawns}_{map_size_pixels}", encoding="ascii"), "big"),
+            version,
+            spawns,
+            map_size_pixels,
+            weight
+        )
+
+    def get_map(self) -> Map:
+        """
+            Generate a map name based on the version and parameters. If invalid parameters are specified
+            hand back None
+        """
+        seed_bytes = random.getrandbits(64).to_bytes(8, "big")
+        size_byte = (self.map_size_pixels // 64).to_bytes(1, "big")
+        spawn_byte = self.spawns.to_bytes(1, "big")
+        option_bytes = spawn_byte + size_byte
+        seed_str = base64.b32encode(seed_bytes).decode("ascii").replace("=", "").lower()
+        option_str = base64.b32encode(option_bytes).decode("ascii").replace("=", "").lower()
+        map_name = f"neroxis_map_generator_{self.version}_{seed_str}_{option_str}"
+        map_path = f"maps/{map_name}.zip"
+        return Map(self.id, map_name, map_path, self.weight)
diff --git a/tests/data/test-data.sql b/tests/data/test-data.sql
@@ -53,6 +53,8 @@ DELETE FROM matchmaker_queue_game;
 DELETE FROM matchmaker_queue_map_pool;
 DELETE FROM map_pool;
 DELETE FROM map_pool_map_version;
+DELETE FROM user_group;
+DELETE FROM group_permission_assignment;
 
 SET FOREIGN_KEY_CHECKS=1;
 
@@ -94,11 +96,21 @@ insert into name_history (id, change_time, user_id, previous_name) values
   (1, date_sub(now(), interval 12 month), 1, 'test_maniac'),
   (2, date_sub(now(), interval 1 month), 2, 'YoungDostya');
 
+insert into user_group (id, technical_name, public, name_key) values
+  (1, 'faf_server_administrators', true, 'admins'),
+  (2, 'faf_moderators_global', true, 'mods');
+
 -- Permissions
-insert into lobby_admin (user_id, `group`) values (1,2);
-insert into user_group_assignment(user_id, group_id)  values (1, (SELECT id from user_group WHERE technical_name = 'faf_server_administrators'));
-insert into user_group_assignment(user_id, group_id)  values (2, (SELECT id from user_group WHERE technical_name = 'faf_moderators_global'));
-insert into user_group_assignment(user_id, group_id)  values (20, (SELECT id from user_group WHERE technical_name = 'faf_moderators_global'));
+insert into group_permission_assignment (id, group_id, permission_id) values
+  (1, (SELECT id from user_group WHERE technical_name = 'faf_server_administrators'), (SELECT id from group_permission WHERE technical_name = 'ADMIN_BROADCAST_MESSAGE')),
+  (2, (SELECT id from user_group WHERE technical_name = 'faf_server_administrators'), (SELECT id from group_permission WHERE technical_name = 'ADMIN_KICK_SERVER')),
+  (3, (SELECT id from user_group WHERE technical_name = 'faf_server_administrators'), (SELECT id from group_permission WHERE technical_name = 'ADMIN_JOIN_CHANNEL')),
+  (4, (SELECT id from user_group WHERE technical_name = 'faf_moderators_global'), (SELECT id from group_permission WHERE technical_name = 'ADMIN_KICK_SERVER'));
+
+insert into lobby_admin (user_id, `group`) values (1, 2);
+insert into user_group_assignment(user_id, group_id) values (1, (SELECT id from user_group WHERE technical_name = 'faf_server_administrators'));
+insert into user_group_assignment(user_id, group_id) values (2, (SELECT id from user_group WHERE technical_name = 'faf_moderators_global'));
+insert into user_group_assignment(user_id, group_id) values (20, (SELECT id from user_group WHERE technical_name = 'faf_moderators_global'));
 
 insert into leaderboard (id, technical_name, name_key, description_key) values
   (1, "global", "leaderboard.global.name", "leaderboard.global.desc"),
@@ -200,7 +212,7 @@ insert into map_version (id, description, max_players, width, height, version, f
   (15, 'SCMP 015', 8, 512, 512, 1, 'maps/scmp_015.zip', 0, 1, 15),
   (16, 'SCMP 015', 8, 512, 512, 2, 'maps/scmp_015.v0002.zip', 0, 1, 15),
   (17, 'SCMP 015', 8, 512, 512, 3, 'maps/scmp_015.v0003.zip', 0, 1, 15),
-  (18, 'Sneaky_Map', 8, 512, 512, 1, "maps/neroxis_map_generator_sneaky_map.zip", 0, 0, 16);
+  (18, 'Sneaky_Map', 8, 512, 512, 1, 'maps/neroxis_map_generator_sneaky_map.zip', 0, 0, 16);
 
 insert into ladder_map (id, idmap) values
   (1,1),
@@ -272,7 +284,8 @@ insert into game_player_stats (gameId, playerId, AI, faction, color, team, place
 insert into matchmaker_queue (id, technical_name, featured_mod_id, leaderboard_id, name_key, team_size, enabled) values
   (1, "ladder1v1", 6, 2, "matchmaker.ladder1v1", 1, true),
   (2, "tmm2v2", 1, 3, "matchmaker.tmm2v2", 2, true),
-  (3, "disabled", 1, 1, "matchmaker.disabled", 4, false);
+  (3, "disabled", 1, 1, "matchmaker.disabled", 4, false),
+  (4, "neroxis1v1", 1, 2, "matchmaker.neroxis", 1, true);
 
 insert into matchmaker_queue_game (matchmaker_queue_id, game_stats_id) values
   (1, 1),
@@ -301,18 +314,26 @@ insert into matchmaker_queue_game (matchmaker_queue_id, game_stats_id) values
 insert into map_pool (id, name) values
   (1, "Ladder1v1 season 1: 5-10k"),
   (2, "Ladder1v1 season 1: all"),
-  (3, "Large maps");
-
-insert into map_pool_map_version (map_pool_id, map_version_id) values
-  (1, 15), (1, 16), (1, 17),
-  (2, 11), (2, 14), (2, 15), (2, 16), (2, 17),
-  (3, 1),  (3, 2),  (3, 3);
+  (3, "Large maps"),
+  (4, "Generated Maps with Errors");
+
+insert into map_pool_map_version (map_pool_id, map_version_id, weight, map_params) values
+  (1, 15, 1, NULL), (1, 16, 1, NULL), (1, 17, 1, NULL),
+  (2, 11, 1, NULL), (2, 14, 1, NULL), (2, 15, 1, NULL), (2, 16, 1, NULL), (2, 17, 1, NULL),
+  (3, 1, 1, NULL), (3, 2, 1, NULL), (3, 3, 1, NULL),
+  (4, NULL, 1, '{"type": "neroxis", "size": 512, "spawns": 2, "version": "0.0.0"}'),
+  -- Bad Generated Map Parameters should not be included in pool
+  (4, NULL, 1, '{"type": "neroxis", "size": 513, "spawns": 2, "version": "0.0.0"}'),
+  (4, NULL, 1, '{"type": "neroxis", "size": 0, "spawns": 2, "version": "0.0.0"}'),
+  (4, NULL, 1, '{"type": "neroxis", "size": 512, "spawns": 3, "version": "0.0.0"}'),
+  (4, NULL, 1, '{"type": "beroxis", "size": 512, "spawns": 2, "version": "0.0.0"}');
 
 insert into matchmaker_queue_map_pool (matchmaker_queue_id, map_pool_id, min_rating, max_rating) values
   (1, 1, NULL, 800),
   (1, 2, 800, NULL),
   (1, 3, 1000, NULL),
-  (2, 3, NULL, NULL);
+  (2, 3, NULL, NULL),
+  (4, 4, NULL, NULL);
 
 insert into friends_and_foes (user_id, subject_id, `status`) values
   (1, 3, 'FOE'),