diff --git a/Makefile b/Makefile index 5655a86a..10e6ebf0 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,6 @@ VERSION = SOFT_VERSION-$(shell git rev-parse --short HEAD) REPO_FILES = $(shell git ls-tree -r --name-only HEAD) TESTS := $(patsubst %.cpp,%.exe,$(subst tests/,bin/,$(shell find tests/ -iname "test_*.cpp"))) -PYTHON := python3 export CMAKE_TOOLCHAIN_FILE ?= toolchains/mingw-w64-i686.cmake BASE_DIR := $(BUILDDIR)/$(notdir $(basename $(CMAKE_TOOLCHAIN_FILE)))-$(CMAKE_BUILD_TYPE) @@ -14,12 +13,9 @@ BUILD_DIR = $(BASE_DIR)/build export DEST_DIR = $(BASE_DIR)/pkg BUILDER ?= builder -export CPPCHECK ?= cppcheck export CM_FILES = $(filter %CMakeLists.txt, $(REPO_FILES)) export CPP_SOURCES = $(filter %.cpp %.hpp %.c %.h, $(REPO_FILES)) export W32_FILES := process.cpp state_parser.cpp dll_inject.cpp network.cpp addscn/addscn.cpp -GAMEMD_PATCHED := $(DEST_DIR)/bin/gamemd-spawn-patched.exe -EXTRA_PATCHES ?= export UID := $(shell id -u) export GID := $(shell id -g) @@ -28,16 +24,9 @@ export CMAKE_TARGET ?= all export CMAKE_EXTRA_ARGS ?= export CXXFLAGS ?= -Wall -Wextra -DLL_LOADER_UNIX = $(BUILD_DIR)/load_dll.bin -DLL_LOADER = $(subst \,\\,$(shell winepath -w $(DLL_LOADER_UNIX))) - -INTEGRATION_TEST ?= docker-compose.integration.yml -INTEGRATION_TEST_TARGET ?= ./pyra2yr/test_sell_mcv.py -COMPOSE_ARGS ?= --abort-on-container-exit pyra2yr tunnel wm vnc novnc game-0 game-1 -compose_cmd := docker-compose -f docker-compose.yml -f $(INTEGRATION_TEST) # need -T flag for this to work properly in shell scripts, but this causes ctrl+c not to work. # TODO: find a workaround -compose_build = docker-compose run -e BUILDDIR -e CMAKE_TOOLCHAIN_FILE -e CMAKE_TARGET -e NPROC -e CMAKE_BUILD_TYPE -e EXTRA_PATCHES -e CMAKE_EXTRA_ARGS -e CXXFLAGS -e TAG_NAME -T --rm $(BUILDER) +compose_build = docker-compose run -e BUILDDIR -e CMAKE_TOOLCHAIN_FILE -e CMAKE_TARGET -e NPROC -e CMAKE_BUILD_TYPE -e CMAKE_EXTRA_ARGS -e CXXFLAGS -e TAG_NAME -T --rm $(BUILDER) doc: @@ -77,33 +66,7 @@ build_cpp: $(BUILD_DIR) cmake --build $(BUILD_DIR) --config $(CMAKE_BUILD_TYPE) --target $(CMAKE_TARGET) -j $(NPROC) cmake --build $(BUILD_DIR) --config $(CMAKE_BUILD_TYPE) --target install/fast -build: build_cpp $(GAMEMD_PATCHED) - -$(BUILD_DIR)/.gamemd-spawn.exe: gamemd-spawn.exe - cp $< $@ - -# FIXME: if building just core lib, ra2yrcppcli.exe is unavailable -$(BUILD_DIR)/p_text2.txt: $(BUILD_DIR)/.gamemd-spawn.exe - $(DEST_DIR)/bin/ra2yrcppcli.exe \ - --address-GetProcAddr=0x7e1250 \ - --address-LoadLibraryA=0x7e1220 \ - --generate-dll-loader=$(DLL_LOADER) - $(DEST_DIR)/bin/addscn.exe $< .p_text2 0x1000 0x60000020 > $@ - wineserver -w - -# FIXME: autodetect detour address -$(GAMEMD_PATCHED): $(BUILD_DIR)/p_text2.txt - $(eval s_ptext2 := $(shell cat $(<))) - $(eval s_ptext2_addr := $(shell cat $(<) | cut -f 3 -d":")) - $(PYTHON) ./scripts/patch_gamemd.py \ - -p "d0x7cd80f:$(BUILD_DIR)/load_dll.bin" \ - $(EXTRA_PATCHES) \ - -d $(s_ptext2_addr) \ - -s ".p_text:0x00004d66:0x00b7a000:0x0047e000" \ - -s ".text:0x003df38d:0x00401000:0x00001000" \ - -s "$(s_ptext2)" \ - -i $(BUILD_DIR)/.gamemd-spawn.exe > $(BUILD_DIR)/.gamemd-spawn-patched.exe - install -D $(BUILD_DIR)/.gamemd-spawn-patched.exe $@ +build: build_cpp build_protobuf: mkdir -p $(BUILD_DIR) @@ -129,11 +92,8 @@ test: wineboot -s; \ wine $(DEST_DIR)/$$f; done -test_integration: $(GAMEMD_PATCHED) - BUILDDIR=$(BUILDDIR) COMMAND='./scripts/run_gamemd.sh' COMMAND_PYRA2YR='python3 $(INTEGRATION_TEST_TARGET)' $(compose_cmd) up $(COMPOSE_ARGS) - -docker_base: - docker-compose build --build-arg USER_ID=$(UID) +docker-base: + docker-compose build builder pyra2yr tunnel vnc docker_test: set -e; for f in $(TESTS); do \ @@ -141,8 +101,7 @@ docker_test: COMMAND="sh -c 'UID=$(UID) BUILDDIR=$(BUILDDIR) make BUILDDIR=$(BUILDDIR) DEST_DIR=$(DEST_DIR) TESTS=$$f test'" docker-compose up --abort-on-container-exit builder; done # NB. using "run" the env. vars need to be specified with -e flag -# actually we dont wanna pass TC in env var, because it overrides the --toolchain flag, which we use to transform relative path -docker_build: +docker-build: $(compose_build) make build cppcheck: @@ -151,15 +110,12 @@ cppcheck: check: cmake_format lint format ./scripts/check.sh -clean-gamemd: - rm -f $(GAMEMD_PATCHED) $(BUILD_DIR)/p_text2.txt $(BUILD_DIR)/.gamemd-spawn.exe - clean: rm -rf $(BUILDDIR); mkdir -p $(BUILDDIR) - rm -f test_data/*.status $(GAMEMD_PATCHED) + rm -f test_data/*.status # TODO: check out the special $(MAKE) variable that presumably passes flags docker-release: $(compose_build) ./scripts/create-release.sh -.PHONY: build build_cpp doc lint format test docker docker_build check cppcheck clean clean-gamemd +.PHONY: build build_cpp doc lint format test docker docker_build check cppcheck clean diff --git a/data/patches.txt b/data/patches.txt new file mode 100644 index 00000000..8d93df86 --- /dev/null +++ b/data/patches.txt @@ -0,0 +1,7 @@ +d0x7cd80f:609c5589e568646c6c00686370702e6861327972686c6962728d45f050a120127e00ffd089ec506a656872766963685f69736568696e69748d45ec508b45fc50a150127e00ffd06a0068b93800006a10ffd089ec5d9d61:s10 +d0x55de4f:s7 +d0x72dfb0:s6 +d0x7b3d6f:s6 +d0x7b3f15:s6 +d0x643c62:s6 +d0x4068e0:s6 \ No newline at end of file diff --git a/docker-compose.debug.yml b/docker-compose.debug.yml deleted file mode 100644 index 1e494354..00000000 --- a/docker-compose.debug.yml +++ /dev/null @@ -1,45 +0,0 @@ -version: "3.9" -services: - game-0: - cap_add: - - SYS_PTRACE - stop_signal: SIGKILL - image: shmocz/ra2yrcpp:latest - depends_on: - - wm - user: "${UID}:${GID}" - working_dir: /home/user/project - volumes: - - gamedata:/home/user/RA2 - - ./wine-dir:/home/user/.wine - - .:/home/user/project - command: "${COMMAND}" - env_file: - - integration.env - environment: - PLAYER_ID: player_0 - RA2YRCPP_PORT: 14521 - RA2YRCPP_WS_PORT: 14525 - WINE_CMD: wine Z:/usr/share/win32/gdbserver.exe localhost:12340 - network_mode: service:vnc - game-1: - stop_signal: SIGKILL - cap_add: - - SYS_PTRACE - image: shmocz/ra2yrcpp:latest - depends_on: - - wm - user: "${UID}:${GID}" - working_dir: /home/user/project - volumes: - - gamedata:/home/user/RA2 - - ./wine-dir:/home/user/.wine - - .:/home/user/project - command: "${COMMAND}" - env_file: - - integration.env - environment: - PLAYER_ID: player_1 - RA2YRCPP_PORT: 14522 - RA2YRCPP_WS_PORT: 14526 - network_mode: service:vnc diff --git a/docker-compose.integration.yml b/docker-compose.integration.yml index 8c6bdf5b..9c7d6f37 100644 --- a/docker-compose.integration.yml +++ b/docker-compose.integration.yml @@ -1,42 +1,31 @@ version: "3.9" + +x-common: &common + cap_add: + - SYS_PTRACE + command: "${COMMAND}" + depends_on: + - wm + env_file: + - integration.env + image: shmocz/ra2yrcpp:latest + network_mode: service:vnc + stop_signal: SIGKILL + user: "${UID}:${GID}" + volumes: + - .:/home/user/project + - gamedata:/home/user/RA2 + - ./.wine-dir:/home/user/.wine + working_dir: /home/user/project + services: game-0: - cap_add: - - SYS_PTRACE - stop_signal: SIGKILL - image: shmocz/ra2yrcpp:latest - depends_on: - - wm - user: "${UID}:${GID}" - working_dir: /home/user/project - volumes: - - gamedata:/home/user/RA2 - - ./wine-dir:/home/user/.wine - - .:/home/user/project - command: "${COMMAND}" - env_file: - - integration.env + <<: *common environment: PLAYER_ID: player_0 RA2YRCPP_PORT: 14521 - RA2YRCPP_WS_PORT: 14525 - network_mode: service:vnc game-1: - stop_signal: SIGKILL - image: shmocz/ra2yrcpp:latest - depends_on: - - wm - user: "${UID}:${GID}" - working_dir: /home/user/project - volumes: - - gamedata:/home/user/RA2 - - ./wine-dir:/home/user/.wine - - .:/home/user/project - command: "${COMMAND}" - env_file: - - integration.env + <<: *common environment: PLAYER_ID: player_1 RA2YRCPP_PORT: 14522 - RA2YRCPP_WS_PORT: 14526 - network_mode: service:vnc diff --git a/docker-compose.yml b/docker-compose.yml index ebcf513f..c81cb59e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,5 @@ version: "3.9" + services: vnc: build: @@ -13,7 +14,6 @@ services: - "12001:5901" # vnc - "50000:50000" # tunnel - "14521:14521" # ra2yrcpp - - "14525:14525" # ws proxy - "12340:12340" # debugger novnc: image: shmocz/vnc:latest diff --git a/docker/tunnel.Dockerfile b/docker/tunnel.Dockerfile index 6314665b..50b3b709 100644 --- a/docker/tunnel.Dockerfile +++ b/docker/tunnel.Dockerfile @@ -1,11 +1,10 @@ FROM alpine:3.16.2 -ARG USER_ID RUN apk add --update --no-cache python3 git RUN python3 -m ensurepip RUN pip3 install --no-cache --upgrade pip setuptools RUN python3 -m pip install 'pycncnettunnel @ git+https://github.com/shmocz/pycncnettunnel.git' -RUN adduser -D -u $USER_ID user +RUN adduser -D user USER user CMD pycncnettunnel diff --git a/docker/vnc.Dockerfile b/docker/vnc.Dockerfile index 51d03f6f..655e5d3f 100644 --- a/docker/vnc.Dockerfile +++ b/docker/vnc.Dockerfile @@ -7,10 +7,9 @@ COPY --from=src /noVNC / RUN apk add --no-cache bash git nodejs python3 FROM novnc -ARG USER_ID RUN apk add --no-cache tigervnc openbox xterm terminus-font bash -RUN adduser -D -u $USER_ID user +RUN adduser -D user USER user WORKDIR /home/user diff --git a/integration.env b/integration.env index 9ea3151a..8a6b2740 100644 --- a/integration.env +++ b/integration.env @@ -1,19 +1,6 @@ DISPLAY=:1 HOME=/home/user -PLAYERS_CONFIG=test_data/envs.tsv RA2YRCPP_GAME_DIR=/home/user/RA2 -RA2YRCPP_TEST_INSTANCES_DIR=/home/user/project/${BUILDDIR}/test_instances WINEARCH=win32 -# TODO: find a workaround for this -DEST_DIR=${DEST_DIR} USER=${UID} GID=${GID} -MAP_PATH=test_data/dry_heat.map -INI_OVERRIDE=test_data/cheap_items.ini -# set to 1 for 60FPS in single player -GAME_SPEED=0 -FRAME_SEND_RATE=1 -RA2_MODE=False -PROTOCOL_VERSION=0 -TUNNEL_ADDRESS="0.0.0.0" -TUNNEL_PORT=50000 \ No newline at end of file diff --git a/scripts/debug.sh b/scripts/debug.sh index cfec2a88..784b3286 100755 --- a/scripts/debug.sh +++ b/scripts/debug.sh @@ -7,11 +7,12 @@ HOMEDIR="/home/user/project" : ${BUILDDIR:="cbuild_docker"} TOOLCHAIN="$(echo $CMAKE_TOOLCHAIN_FILE | sed -E 's/.+\/(.+)\.cmake/\1/g')-${CMAKE_BUILD_TYPE}" : ${TARGET:="localhost:12340"} +: ${INTEGRATION_TEST_TARGET:="./pyra2yr/test_sell_mcv.py"} # Executable to be passed to wine and it's args, example: # EXE="$HOMEDIR/$BUILDDIR/$TOOLCHAIN/pkg/bin/test_dll_inject.exe --gtest_repeat=-1 --gtest_filter=*IServiceDLL*" : ${EXE:="$HOMEDIR/$BUILDDIR/$TOOLCHAIN/pkg/bin/test_dll_inject.exe --gtest_repeat=-1 --gtest_filter=*IServiceDLL*"} -GDB_COMMAND='x86_64-w64-mingw32-gdb -iex "set pagination off" -ex "target extended-remote '"$TARGET"'" -ex "set pagination off" -ex "set logging overwrite on" -ex "set logging on" -ex "set disassembly-flavor intel"' +GDB_COMMAND='gdb' : ${GDB_SCRIPT:="$HOMEDIR/scripts/debug.gdb"} function dcmd_generic() { @@ -23,18 +24,21 @@ function dcmd_generic() { function dcmd_integration() { : ${user:="root"} - cmd="$(printf 'WINEPREFIX=${HOME}/project/%s/test_instances/${PLAYER_ID}/.wine %s' "$BUILDDIR" "$1")" - docker-compose -f docker-compose.yml -f docker-compose.integration.yml exec --user "$user" \ - -w "$HOMEDIR"/$BUILDDIR/test_instances/player_${PLAYER_ID} \ - game-$PLAYER_ID bash -c "$cmd" + : ${it=""} + docker exec $it --user "$user" -w "$HOMEDIR"/$BUILDDIR/test_instances/player_${PLAYER_ID} \ + game-0-$PLAYER_ID bash -c "$1" } function gdb_connect() { - dcmd_integration "$(printf '%s %s' "$GDB_COMMAND" "$1")" + it="-it" dcmd_integration "$(printf '%s %s' "$GDB_COMMAND" "$1")" } function debug_integration_test() { - a="$(printf '%s "source "%s""' "-ex" "$GDB_SCRIPT")" + # get target PID + game_pid="$(dcmd_integration "pgrep -f gamemd-spawn-ra2yrcpp.exe")" + a="$(printf -- '-p %s %s "source "%s""' "$game_pid" "-ex" "$GDB_SCRIPT")" + + # attach (and load symbols) gdb_connect "$a" } @@ -48,8 +52,13 @@ function tmux_send_keys() { } function debug_integration() { - make INTEGRATION_TEST_TARGET="$INTEGRATION_TEST_TARGET" test_integration & - sleep 3 + python ./scripts/run-gamemd.py \ + -b $BUILDDIR/test_instances \ + -B $BUILDDIR/mingw-w64-i686-Release/pkg/bin \ + -S maingame/gamemd-spawn-n.exe \ + run-docker-instance \ + -e pyra2yr/test_sell_mcv.py & + sleep 5 tmux_send_keys C-c tmux_send_keys "CMAKE_TOOLCHAIN_FILE=$CMAKE_TOOLCHAIN_FILE CMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE DEBUG_FN=debug_integration_test ./scripts/debug.sh" C-m wait diff --git a/scripts/generate-spawnini.py b/scripts/generate-spawnini.py deleted file mode 100755 index 95432c05..00000000 --- a/scripts/generate-spawnini.py +++ /dev/null @@ -1,224 +0,0 @@ -#!/usr/bin/env python3 -import csv -import argparse -from typing import List -from dataclasses import dataclass, fields -from collections import OrderedDict - - -def parse_args(): - a = argparse.ArgumentParser( - description="generate spawn.ini", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - a.add_argument("-i", "--input-path", help="player config path", type=str) - a.add_argument( - "-f", - "--frame-send-rate", - help="FrameSendRate param", - type=int, - default=1, - ) - a.add_argument( - "-pr", - "--protocol-version", - help="protocol version", - type=int, - default=0, - ) - a.add_argument( - "-g", - "--game-mode", - help="game mode", - choices=["yr", "ra2"], - default="yr", - type=str, - ) - a.add_argument("-s", "--speed", help="Game speed", type=int, default=0) - a.add_argument("-p", "--player-id", help="Current player id", type=int) - a.add_argument( - "-t", "--tunnel-address", help="Tunnel address", default="0.0.0.0" - ) - a.add_argument("-tp", "--tunnel-port", help="Tunnel port", default=50000) - return a.parse_args() - - -@dataclass -class PlayerSetting: - name: str - color: int - side: int - is_host: str - is_observer: str - ai_difficulty: int - port: int - - -@dataclass -class GlobalSettings: - Credits: int - FrameSendRate: int - GameMode: int - GameSpeed: int - MCVRedeploy: bool - Protocol: int - Ra2Mode: bool - Scenario: str - Seed: int - ShortGame: bool - SidebarHack: bool - SuperWeapons: bool - GameID: int - Bases: str - UnitCount: int - Host: bool - Port: int - UIGameMode: str - - -@dataclass -class TunnelSettings: - Ip: str - Port: int - - -def parse_players(input_path) -> List[PlayerSetting]: - fld = {f.name: f.type for f in fields(PlayerSetting)} - res = [] - with open(input_path, "r") as f: - R = csv.DictReader(f, delimiter="\t") - for e in R: - entries = {k: fld[k](v) for k, v in e.items()} - res.append(PlayerSetting(**entries)) - return res - - -def get_main_settings( - s: GlobalSettings, p: PlayerSetting, p_all: List[PlayerSetting] -): - ai_players = len([x for x in p_all if x.ai_difficulty != -1]) - player_count = len(p_all) - ai_players - entries = OrderedDict( - [ - ("AIPlayers", ai_players), - ("PlayerCount", player_count), - ("IsSpectator", p.is_observer), - ("Name", p.name), - ("Color", p.color), - ("Side", p.side), - ] - ) - for f in fields(s): - entries[f.name] = getattr(s, f.name) - entries["Host"] = p.is_host - entries["Port"] = p.port - return "\n".join(["[Settings]"] + [f"{k}={v}" for k, v in entries.items()]) - - -def main(): - args = parse_args() - entries = None - with open(args.input_path, "r") as f: - R = csv.DictReader(f, delimiter="\t") - entries = [x for x in R] - - ai_section = [] - others_section = [] - for i, e in enumerate(x for x in entries if x["ai_difficulty"] != "-1"): - ai_section.append((i, e)) - - for i, e in ( - (j, x) - for j, x in enumerate(entries) - if x["ai_difficulty"] == "-1" and j != args.player_id - ): - others_section.append((i, e)) - - all_players = parse_players(args.input_path) - G = GlobalSettings( - Credits=10000, - FrameSendRate=args.frame_send_rate, - GameMode=1, - GameSpeed=args.speed, - MCVRedeploy=True, - Protocol=args.protocol_version, - Ra2Mode=args.game_mode == "ra2", - Scenario="spawnmap.ini", - Seed=123, - ShortGame=True, - SidebarHack="Yes", - SuperWeapons=True, - GameID=12850, - Bases="Yes", - UnitCount=0, - Host=False, - Port=-1, - UIGameMode="Battle", - ) - T = TunnelSettings(Ip=args.tunnel_address, Port=args.tunnel_port) - - sections = [] - # emit main settings - p = all_players[args.player_id] - sections.append(get_main_settings(G, p, all_players)) - - # emit ai entries - if ai_section: - sections.append( - "\n".join( - ["[HouseHandicaps]"] - + [ - "Multi{}={}".format(i + 1, e["ai_difficulty"]) - for i, e in ai_section - ] - ) - ) - sections.append( - "\n".join( - ["[HouseCountries]"] - + ["Multi{}={}".format(i + 1, e["side"]) for i, e in ai_section] - ) - ) - sections.append( - "\n".join( - ["[HouseColors]"] - + [ - "Multi{}={}".format(i + 1, e["color"]) - for i, e in ai_section - ] - ) - ) - - # emit other players - keymap = OrderedDict( - [ - ("Name", "name"), - ("Side", "side"), - ("IsSpectator", "is_observer"), - ("Port", "port"), - ("Color", "color"), - ] - ) - if others_section: - for i, e in others_section: - ee = [f"{k}={e[v]}" for k, v in keymap.items()] + [f"Ip={T.Ip}"] - entries = "\n".join(ee) - sections.append("\n".join([f"[Other{i+1}]", entries])) - - sections.append( - "\n".join( - ["[SpawnLocations]"] - + ["Multi{}={}".format(i + 1, i) for i, _ in enumerate(all_players)] - ) - ) - - if others_section: - entries = "\n".join(f"{f.name}={getattr(T, f.name)}" for f in fields(T)) - # for f in fields(T): - # entries[f.name] = getattr(s, f.name) - sections.append("\n".join(["[Tunnel]"] + [entries])) - print("\n\n".join(sections)) - - -if __name__ == "__main__": - main() diff --git a/scripts/patch_gamemd.py b/scripts/patch_gamemd.py index c82a2e28..ceb81ac0 100644 --- a/scripts/patch_gamemd.py +++ b/scripts/patch_gamemd.py @@ -3,11 +3,10 @@ import re import os import struct -from typing import List, Tuple +from typing import List, Any, Iterable from dataclasses import dataclass import argparse import logging as lg -from iced_x86 import * @dataclass @@ -60,8 +59,8 @@ def map_paddr(sections: List[Section], paddr: int) -> int: return r -def read_file(p): - with open(p, "r") as f: +def read_file(p, mode="r"): + with open(p, mode) as f: return f.read() @@ -76,6 +75,8 @@ def pushret(address): def decode_bytes(b: bytes, ip=0, count=0): + from iced_x86 import Decoder + d = Decoder(32, b, ip=ip) for inst, _ in zip(d, range(count)): yield inst @@ -87,95 +88,65 @@ def decode_until(b: bytes, ip=0, max_ins=10): return inst.ip - ip -def create_detour( +@dataclass +class Hook: + vaddr: int + paddr: int + size: int + + +@dataclass +class Patch: + vaddr: int + paddr: int + size: int + code: bytearray = None + + @classmethod + def from_address(cls, b: bytearray, sections, vaddr: int): + paddr = map_vaddr(sections, vaddr) + c = decode_until(b[paddr : (paddr + 256)], ip=paddr) + return Patch(vaddr, paddr, c) + + def to_patch_string(self): + if not self.code: + return f"d0x{self.vaddr:02x}:s{self.size}" + return f"d0x{self.vaddr:02x}:{self.code.hex()}:s{self.size}" + + +@dataclass +class Handle: + binary: bytearray + args: Any + patches: List[Patch] = None + sections: List[Section] = None + + +def make_detour( binary: bytes, - address: int, + h: Patch, addr_detours: int, - code: bytes, sections: List[Section] = None, -) -> int: - """ - Parameters - ---------- - binary : bytes - PE binary - address : int - target virtual address - addr_detours : int - detours virtual address - code : bytes - code to write - sections : List[Section], optional - - Returns - ------- - int - Size of newly created detour - """ - paddr = map_vaddr(sections, address) + code=None, +): + code = code or bytearray() section_detours = get_section(sections, addr_detours) offset_detours = addr_detours - section_detours.vaddr - # determine numbers of instructions to disassemble - c = decode_until(binary[paddr : (paddr + 256)], ip=address) - - bb = decode_bytes(binary[paddr : (paddr + 256)], ip=address, count=c) - # FIXME: how to handle RET? - to_copy = bytearray() - if not all(list(str(x) in ["ret", "nop"] for x in bb)): - to_copy = bytearray(binary[paddr : (paddr + c)]) - else: # put another return val - binary[(paddr + c) : (paddr + c + 1)] = b"\xc3" - to_copy = bytearray(b"\x90" * 6) - - # make detour - detour = to_copy + code + pushret(address + c) - # copy bytes to trampoline area + detour = binary[h.paddr : (h.paddr + h.size)] + code + pushret(h.vaddr + h.size) patch(binary, detour, section_detours.paddr + offset_detours) - lg.info( - "patch: target,detour,len(detour)=%x,%x,%d", - address, - addr_detours, - len(detour), - ) pr_1 = pushret(addr_detours) - assert c - len(pr_1) >= 0 - b = pr_1 + bytearray([0x90] * (c - len(pr_1))) - # patch original binary - patch(binary, b, paddr) + pr_2 = pr_1 + bytearray([0x90] * (h.size - len(pr_1))) + patch(binary, pr_2, h.paddr) return len(detour) -def create_detour_trampolines( - binary: bytes, - addresses: List[int], - patches: List[Tuple[str, int, bytes]] = None, - sections: List[Section] = None, - detour_address: int = 0xB7E6AC, -): - patches = patches or [] - addr_detours = detour_address - # Create simple detours to target addresses - for addr in addresses: - addr_detours = addr_detours + create_detour( - binary, addr, addr_detours, b"", sections - ) - - # Create custom detours and raw patches - for ptype, addr, code in patches: - if ptype == "d": - addr_detours = addr_detours + create_detour( - binary, addr, addr_detours, code, sections - ) - elif ptype == "r": - patch(binary, code, map_vaddr(sections, addr)) - else: - raise RuntimeError(f"Invalid patch type {ptype}") - - -def write_out(b): +def write_out(b, path=None): fp = sys.stdout.buffer + if path: + fp = open(path, "wb") fp.write(b) fp.flush() + fp.close() def auto_int(x): @@ -204,8 +175,12 @@ def parse_args(): "-p", "--patches", action="append", + default=[], help="Extra patches to apply in format
:. can be 'd' (detour) or 'r' (raw) Address is in hex.", ) + a.add_argument( + "-r", "--raw", type=str, help="Apply raw patches specified in given file" + ) a.add_argument( "-f", "--source-file", @@ -213,42 +188,87 @@ def parse_args(): help="C++ source from which to parse hook addresses", ) a.add_argument( - "-i", "--input", default="gamemd-spawn.exe", help="gamemd path" + "-D", + "--dump-patches", + action="store_true", + help="Don't output patched binary. Rather all patches to be applied in a format suitable for -p parameter", + ) + a.add_argument("-i", "--input", default="gamemd-spawn.exe", help="gamemd path") + a.add_argument( + "-o", + "--output", + default=None, + help="Output file. If unspecified, write to stdout", ) return a.parse_args() -def get_patches(patches: List[str]): +def parse_patches(h: Handle, patches) -> Iterable[Patch]: for ptype, s_addr, path in [ re.match(r"(d|r)0x([a-fA-F0-9]+):(.+)", p).groups() for p in patches ]: addr = int(s_addr, base=16) if os.path.exists(path): - with open(path, "rb") as f: - yield (ptype, addr, f.read()) + p = Patch.from_address(h.binary, h.sections, addr) + p.code = bytearray(read_file(path, "rb")) + yield p + elif m := re.match(r"s(\d+)", path): + yield Patch(addr, map_vaddr(h.sections, addr), int(m[1])) + elif m := re.match(r"([a-fA-F0-9]+):s(\d+)", path): + code = bytearray.fromhex(m[1]) + yield Patch(addr, map_vaddr(h.sections, addr), int(m[2]), code=code) + elif ptype == "r": + code = bytearray.fromhex(path) + yield Patch(addr, map_vaddr(h.sections, addr), len(code), code=code) else: - yield (ptype, addr, bytearray.fromhex(path)) + raise RuntimeError(f"invalid patch type: {ptype} {path}") def get_sections(sections: List[str]) -> List[Section]: res = [] - for s in sections: - z = s.split(":") + for z in (s.split(":") for s in sections): res.append(Section(z[0], *[int(x, 0) for x in z[1:]])) return res +def do_patching(h: Handle) -> bytearray: + addr_detours = h.args.detour_address + binary_patched = h.binary.copy() + for p in h.patches: + addr_detours += make_detour(binary_patched, p, addr_detours, h.sections, p.code) + return binary_patched + + +def get_handle(a) -> Handle: + H = Handle(binary=bytearray(read_file(a.input, "rb")), args=a) + H.sections = get_sections(a.sections) + H.patches = list(parse_patches(H, H.args.patches)) + if a.raw: + raw_patches = re.split(r"\s+", read_file(H.args.raw)) + H.patches.extend(list(parse_patches(H, raw_patches))) + else: + addr_hooks = get_hook_addresses(read_file(a.source_file)) + H.patches.extend( + Patch.from_address(H.binary, H.sections, x) for x in addr_hooks + ) + return H + + def main(): lg.basicConfig(level=lg.INFO) a = parse_args() - addrs = get_hook_addresses(read_file(a.source_file)) - - with open(a.input, "rb") as f: - b = bytearray(f.read()) - patches = list(get_patches(a.patches)) - sections = get_sections(a.sections) - create_detour_trampolines(b, addrs, patches, sections, a.detour_address) - write_out(b) + + H = get_handle(a) + if H.args.dump_patches: + return write_out( + bytes( + "\n".join(p.to_patch_string() for p in H.patches), + encoding="utf8", + ), + a.output, + ) + binary_patched = do_patching(H) + return write_out(binary_patched, a.output) if __name__ == "__main__": diff --git a/scripts/prep_instance_dirs.sh b/scripts/prep_instance_dirs.sh deleted file mode 100755 index 53999fab..00000000 --- a/scripts/prep_instance_dirs.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env bash - -INSTANCES="$1" -PLAYER_ID=${2:-""} - -set -o nounset - -# Path to directory containing RA2 game data files -p_main="$RA2YRCPP_GAME_DIR" -# ra2yrcpp library files (FXIME) -p_libs="$RA2YRCPP_PKG_DIR" -p_map_path="$MAP_PATH" -p_ini_override="$INI_OVERRIDE" -: ${PLAYERS_CONFIG:=test_data/envs.tsv} - -[ -d "$p_libs" ] || { - echo "$p_libs not found" - exit 1 -} - -paths=( - BINKW32.DLL - spawner.xdp - ra2.mix - ra2md.mix - theme.mix - thememd.mix - langmd.mix - language.mix - expandmd01.mix - mapsmd03.mix - maps02.mix - maps01.mix - Ra2.tlb - INI - Maps - RA2.INI - RA2MD.ini - ddraw.ini - spawner2.xdp - Blowfish.dll - ddraw.dll) - -IFS=$'\n' read -d '' -r -a cfgs <"$PLAYERS_CONFIG" -cfgs=("${cfgs[@]:1}") - -function mklink() { - it=("$@") - params=("${@:1:$(($# - 1))}") - target="${it[-1]}" - for ii in "${params[@]}"; do - [ ! -e "$ii" ] && { - echo "$ii not found" && exit 1 - } - ln -vfrns "$ii" "$target" - done -} - -for i in ${!cfgs[@]}; do - if [ ! -z "$PLAYER_ID" ] && [[ "$PLAYER_ID" != "player_${i}" ]]; then - continue - fi - cfg="${cfgs[$i]}" - read -r name color side is_host is_observer port ai_difficulty <<<$(echo "$cfg") - if [ "$ai_difficulty" != "-1" ]; then - continue - fi - ifolder="$INSTANCES/player_${i}" - mkdir -p "$ifolder" - for p in ${paths[@]}; do - mklink "$p_main/$p" "$ifolder/$p" - done - - # link files - cd "$p_libs" - mklink "gamemd-spawn-patched.exe" "$ifolder"/gamemd-spawn.exe - mklink ra2yrcppcli.exe *.dll "$ifolder" - cd - - - cp -f "$p_map_path" "$ifolder/spawnmap.ini" - if [ -f "$p_ini_override" ]; then - cat "$p_ini_override" >>"$ifolder/spawnmap.ini" - fi - # write spawn.ini - # get_spawn_ini "$i" >"$ifolder/spawn.ini" - game_mode="yr" - if [ "$RA2_MODE" == "True" ]; then - game_mode="ra2" - fi - - ./scripts/generate-spawnini.py \ - -i "$PLAYERS_CONFIG" \ - -f "$FRAME_SEND_RATE" \ - -pr "$PROTOCOL_VERSION" \ - -g "$game_mode" \ - -p "$i" \ - -s "$GAME_SPEED" \ - -t "$TUNNEL_ADDRESS" \ - -tp "$TUNNEL_PORT" \ - >"$ifolder/spawn.ini" -done diff --git a/scripts/run-gamemd.py b/scripts/run-gamemd.py new file mode 100755 index 00000000..b8172ad5 --- /dev/null +++ b/scripts/run-gamemd.py @@ -0,0 +1,890 @@ +#!/usr/bin/env python3 + +import json +import functools +import tempfile +import traceback +import datetime +from enum import Enum +import re +import time +import atexit +import logging +import subprocess +from pathlib import Path +import shutil +import argparse +import sys +import os +from dataclasses import dataclass, fields, field + +NAME_SPAWNER = "gamemd-spawn.exe" +NAME_SPAWNER_PATCHED = "gamemd-spawn-ra2yrcpp.exe" +NAME_SPAWNER_SYRINGE = "gamemd-spawn-syr.exe" + +FILE_PATHS = f"""\ +BINKW32.DLL +spawner.xdp +ra2.mix +ra2md.mix +theme.mix +thememd.mix +langmd.mix +language.mix +expandmd01.mix +mapsmd03.mix +maps02.mix +maps01.mix +Ra2.tlb +INI +Maps +RA2.INI +RA2MD.ini +ddraw.ini +spawner2.xdp +Blowfish.dll +ddraw.dll +Syringe.exe +cncnet5.dll +{NAME_SPAWNER}\ +""" + + +LIB_PATHS = f"""\ +{NAME_SPAWNER_PATCHED} +libgcc_s_dw2-1.dll +libra2yrcpp.dll +libstdc++-6.dll +libwinpthread-1.dll +zlib1.dll\ +""" + + +def setup_logging(level=logging.INFO): + FORMAT = ( + "[%(levelname)s] %(asctime)s %(module)s.%(filename)s:%(lineno)d: %(message)s" + ) + logging.basicConfig(level=level, format=FORMAT) + start_time = datetime.datetime.now().isoformat() + level_name = logging.getLevelName(level) + logging.info("Logging started at: %s, level=%s", start_time, level_name) + + +def mklink(dst, src): + if os.path.islink(dst): + os.unlink(dst) + os.symlink(src, dst) + + +def read_file(p): + with open(p, "r") as f: + return f.read() + + +class ProtocolVersion(Enum): + zero = 0 + compress = 2 + + +# TODO: put to pyra2yr +class Color(Enum): + Yellow = 0 + Red = 1 + Blue = 2 + Green = 3 + Orange = 4 + Teal = 5 + Purple = 6 + Pink = 7 + + +@dataclass +class PlayerEntry: + name: str + color: Color + side: int + location: int + index: int + is_host: bool = False + is_observer: bool = False + ai_difficulty: int = -1 + port: int = 0 + + def __post_init__(self): + if self.port <= 0: + self.port = 13360 + self.index + if not isinstance(self.color, Color): + self.color = Color[self.color] + + +@dataclass +class ConfigFile: + """Class that maps directly to the JSON configuration.""" + + map_path: str + unit_count: int + start_credits: int + seed: int + ra2_mode: bool + short_game: bool + superweapons: bool + protocol: ProtocolVersion + tunnel: str + port: int + game_speed: int = 0 + frame_send_rate: int = 1 + crates: bool = False + mcv_redeploy: bool = True + allies_allowed: bool = True + multi_engineer: bool = False + bridges_destroyable: bool = True + build_off_ally: bool = True + players: list[PlayerEntry] = field(default_factory=list) + + def __post_init__(self): + for k in ["color", "location", "name"]: + if len(set(getattr(p, k) for p in self.players)) != len(self.players): + raise RuntimeError(f'Duplicate player attribute: "{k}"') + + @classmethod + def load(cls, path): + fld = {f.name: f.type for f in fields(cls)} + args = {} + d = json.loads(read_file(path)) + for k, v in d.items(): + if k == "players": + args[k] = [PlayerEntry(index=i, **x) for (i, x) in enumerate(v)] + elif k == "protocol": + args[k] = ProtocolVersion[v] + else: + args[k] = fld[k](v) + return cls(**args) + + def kv_to_string(self, name, x): + return f"[{name}]\n" + "\n".join(f"{k}={v}" for k, v in x) + + def player_values(self, p: PlayerEntry): + kmap = [ + ("Name", "name"), + ("Side", "side"), + ("IsSpectator", "is_observer"), + ("Port", "port"), + ] + return [(k, getattr(p, v)) for k, v in kmap] + [("Color", p.color.value)] + + def others_sections(self, player_index: int): + oo = [ + p for p in self.players if p.ai_difficulty < 0 and p.index != player_index + ] + res = [] + for o in oo: + pv = self.player_values(o) + [("Ip", self.tunnel)] + res.append(self.kv_to_string(f"Other{o.index + 1}", pv)) + return "\n\n".join(res) + + def to_ini(self, player_index: int): + player = next(p for p in self.players if p.index == player_index) + ai_players = [p for p in self.players if p.ai_difficulty > -1] + main_section_values = [ + ("Credits", self.start_credits), + ("FrameSendRate", self.frame_send_rate), + ("GameMode", 1), + ("GameSpeed", self.game_speed), + ("MCVRedeploy", self.mcv_redeploy), + ("Protocol", self.protocol.value), + ("Ra2Mode", self.ra2_mode), + ("ShortGame", self.short_game), + ("SidebarHack", "Yes"), + ("SuperWeapons", self.superweapons), + ("GameID", 12850), + ("Bases", "Yes"), + ("UnitCount", self.unit_count), + ("UIGameMode", "Battle"), + ("Host", player.is_host), + ("Seed", self.seed), + ("Scenario", "spawnmap.ini"), + ("PlayerCount", len(self.players) - len(ai_players)), + ("AIPlayers", len(ai_players)), + ("Crates", self.crates), + ("AlliesAllowed", self.allies_allowed), + ("MultiEngineer", self.multi_engineer), + ("BridgeDestory", self.bridges_destroyable), + ("BuildOffAlly", self.build_off_ally), + ] + + main_section_values.extend(self.player_values(player)) + + res = [] + res.append(self.kv_to_string("Settings", main_section_values)) + res.append(self.others_sections(player_index)) + res.append( + self.kv_to_string( + "SpawnLocations", + [(f"Multi{p.index + 1}", p.location) for p in self.players], + ) + ) + res.append( + self.kv_to_string("Tunnel", [("Ip", self.tunnel), ("Port", self.port)]) + ) + if ai_players: + res.append( + self.kv_to_string( + "HouseHandicaps", + [(f"Multi{x.index + 1}", f"{x.ai_difficulty}") for x in ai_players], + ) + ) + res.append( + self.kv_to_string( + "HouseCountries", + [(f"Multi{x.index + 1}", f"{x.side}") for x in ai_players], + ) + ) + res.append( + self.kv_to_string( + "HouseColors", + [(f"Multi{x.index + 1}", f"{x.color.value}") for x in ai_players], + ) + ) + return "\n\n".join(res) + "\n" + + +def prun(args, **kwargs): + cmdline = [str(x) for x in args] + logging.info("exec: %s", cmdline) + return subprocess.run(cmdline, **kwargs) + + +def popen(args, **kwargs): + return subprocess.Popen([str(x) for x in args], **kwargs) + + +def docker_exit(): + subprocess.run("docker-compose down --remove-orphans -t 1", shell=True, check=True) + + +def read_players_config(path) -> list[PlayerEntry]: + D = json.loads(read_file(path)) + return [ + PlayerEntry(**{k: p.get(k, -1) for k in ["name", "ai_difficulty"]}) + for p in D["players"] + ] + + +def try_fn(fn, retry_interval=2.0, tries=5): + for _ in range(tries): + try: + r = fn() + return r + except KeyboardInterrupt: + raise + except: + time.sleep(retry_interval) + raise RuntimeError("Timeout") + + +class Docker: + @classmethod + def _common(cls, compose_files=None): + cf = compose_files or ["docker-compose.yml"] + r = ["docker-compose"] + for c in cf: + r.extend(["-f", c]) + return r + + @classmethod + def run( + cls, + cmd, + service, + compose_files=None, + uid=None, + name=None, + env=None, + ): + r = cls._common(compose_files=compose_files) + r.extend(["run", "--rm", "-T"]) + if uid: + r.extend(["-u", f"{uid}:{uid}"]) + if env: + for k, v in env: + r.extend(["-e", f"{k}={v}"]) + if name: + r.append(f"--name={name}") + r.append(service) + r.extend(cmd) + return r + + @classmethod + def exec(cls, cmd, service, compose_files=None, uid=None, env=None): + r = cls._common(compose_files=compose_files) + r.extend(["exec", "-T"]) + if uid: + r.extend(["-u", f"{uid}:{uid}"]) + if env: + for k, v in env: + r.extend(["-e", f"{k}={v}"]) + r.append(service) + r.extend(cmd) + return r + + @classmethod + def up(cls, services, compose_files=None): + r = cls._common(compose_files) + r.append("up") + r.extend(services) + return r + + +def get_game_uid(): + return int( + prun( + Docker.run( + [ + "python3", + "-c", + 'import os; print(os.stat("/home/user/project").st_uid)', + ], + "builder", + ), + check=True, + capture_output=True, + ).stdout.strip() + ) + + +class MainCommand: + def __init__(self, args): + self.args = args + + @functools.cached_property + def uid(self): + return get_game_uid() + + +def cmd_gamemd_patch(): + return [ + "python3", + "./scripts/patch_gamemd.py", + "-s", + ".p_text:0x00004d66:0x00b7a000:0x0047e000", + "-s", + ".text:0x003df38d:0x00401000:0x00001000", + ] + + +def cmd_addscn(addscn_path, dst): + return [addscn_path, dst, ".p_text2", "0x1000", "0x60000020"] + + +class PatchGameMD(MainCommand): + def main(self): + args = self.args + bdir = Path(args.build_dir) + + dst = args.output + + def _run(c): + return prun( + c, + check=True, + capture_output=True, + ) + + td = tempfile.TemporaryDirectory() + + s_temp = Path(td.name) / "tmp.exe" + # Copy original spawner to temp directory + shutil.copy(args.spawner_path, s_temp) + + # Append new section to copied spawner and write PE section info to text file + o = _run(cmd_addscn(bdir / "addscn.exe", s_temp)) + pe_info_d = str(o.stdout, encoding="utf-8").strip() + p_text_2_addr = pe_info_d.split(":")[2] + + # wait wine to exit + if sys.platform != "win32": + _run(["wineserver", "-w"]) + + # Patch the binary + o = _run( + cmd_gamemd_patch() + + [ + "-s", + pe_info_d, + "-d", + p_text_2_addr, + "-r", + Path("data") / "patches.txt", + "-i", + f"{s_temp}", + "-o", + f"{dst}", + ] + ) + logging.info("patch successful: %s", dst.absolute()) + + +class RunDockerInstance(MainCommand): + def prepare_patched_spawner(self): + args = self.args + bdir = Path(args.build_dir) + + dst = bdir / NAME_SPAWNER_PATCHED + if dst.exists(): + logging.info("not overwriting %s", str(dst)) + return + + dll_loader = bdir / "load_dll.bin" + pe_info = bdir / "p_text2.txt" + patch_info = bdir / "patches.txt" + + def _run(c): + return prun( + Docker.run(c, "builder", uid=self.uid), + check=True, + capture_output=True, + ) + + _run( + [ + bdir / "ra2yrcppcli.exe", + "--address-GetProcAddr=0x7e1250", + "--address-LoadLibraryA=0x7e1220", + f"--generate-dll-loader={dll_loader}", + ] + ) + + # Copy original spawner to build directory + shutil.copy(args.spawner_path, dst) + + # Append new section to copied spawner and write PE section info to text file + o = _run([bdir / "addscn.exe", dst, ".p_text2", "0x1000", "0x60000020"]) + pe_info_d = str(o.stdout, encoding="utf-8").strip() + p_text_2_addr = pe_info_d.split(":")[2] + with open(pe_info, "w") as f: + f.write(pe_info_d) + + # wait wine to eexit + _run(["wineserver", "-w"]) + + cmdline_patch = cmd_gamemd_patch() + [ + "-d", + p_text_2_addr, + "-s", + pe_info_d, + "-i", + dst, + ] + + # Get raw patch + o = _run( + cmdline_patch + + [ + "-D", + "-p", + f"d0x7cd80f:{dll_loader}", + "-o", + patch_info, + ] + ) + + # Patch the binary + o = _run( + cmdline_patch + + [ + "-r", + patch_info, + "-o", + f"{dst}", + ] + ) + + def create_game_instance( + self, i, player_name, port + ) -> tuple[str, subprocess.Popen]: + """Create Docker game instance + + Parameters + ---------- + i : int + Index of the player in the player list + name : str + Player name + port : int + ra2yrcpp server port + + Returns + ------- + (str, subprocess.Popen) + Tuple of container name, and docker process object. + + Raises + ------ + subprocess.CalledProcessError: If the process fails. + """ + args = self.args + args_ini_overrides = list(sum([("-i", x) for x in args.ini_overrides], ())) + cname = f"game-0-{i}" + cmdline = Docker.run( + ["./scripts/run-gamemd.py"] + + args_ini_overrides + + [ + "-p", + args.players_config, + "--base-dir", + args.base_dir, + "-B", + args.build_dir, + "-n", + player_name, + "-r", + args.registry, + "-t", + args.type, + "-g", + args.game_mode, + "-s", + args.game_speed, + "-S", + args.spawner_path, + "-Sy", + args.syringe_spawner_path, + "run-gamemd", + ], + "game-0", + compose_files=["docker-compose.yml", "docker-compose.integration.yml"], + uid=self.uid, + name=cname, + env=[ + ("RA2YRCPP_RECORD_PATH", str(int(time.time())) + ".pb.gz"), + ("RA2YRCPP_PORT", f"{port}"), + ], + ) + return (cname, popen(cmdline)) + + def run_docker_instance(self): + pyra2yr_script = self.args.script.as_posix() + os.environ["COMMAND_PYRA2YR"] = f"python3 {pyra2yr_script}" + + uid = self.uid + main_service = popen(Docker.up("tunnel wm vnc novnc pyra2yr".split(" "))) + + # hack to wait until pyra2yr service has started + try_fn(lambda: prun(Docker.exec(["ls", "-l"], "pyra2yr", uid=uid), check=True)) + + procs = [] + + def _stop_instances(): + main_service.terminate() + for cname, p in procs: + prun(["docker", "stop", cname], check=True) + p.wait() + + atexit.register(_stop_instances) + + # Create game instances + for i, c in enumerate( + p + for p in ConfigFile.load(self.args.players_config).players + if p.ai_difficulty < 0 + ): + cname, proc = self.create_game_instance(i, c.name, 14520 + i + 1) + procs.append((cname, proc)) + + while True: + time.sleep(1) + + +class RunGameMD: + def __init__(self, args): + self.args = args + self.cfg = ConfigFile.load(self.args.players_config) + self.players_config = self.cfg.players + self.player_index, self.player_config = next( + (i, p) + for i, p in enumerate(self.players_config) + if p.name == self.args.player_name + ) + self.base_dir = Path(self.args.base_dir) + self.instance_dir = self.base_dir / self.player_config.name + self.build_dir = Path(self.args.build_dir) + self.map_path = self.instance_dir / "spawnmap.ini" + self.spawn_path = self.instance_dir / "spawn.ini" + self.wineprefix_dir = self.instance_dir / ".wine" + self.ra2yrcpp_spawner_path = self.instance_dir / NAME_SPAWNER_PATCHED + + def create_symlinks(self): + """Symlink relevant data files for this test instance.""" + # Clear old symlinks + for p in os.listdir(self.instance_dir): + if os.path.islink(p): + os.unlink(p) + + for p in re.split(r"\s+", FILE_PATHS): + mklink(self.instance_dir / p, self.args.game_data_dir / p) + + for p in re.split(r"\s+", LIB_PATHS): + mklink(self.instance_dir / p, (self.build_dir / p).absolute()) + + for p in self.args.syringe_dlls: + mklink(self.instance_dir / p, (self.build_dir / p).absolute()) + + if self.args.syringe_spawner_path.exists(): + mklink( + self.instance_dir / NAME_SPAWNER_SYRINGE, + self.args.syringe_spawner_path.absolute(), + ) + + def create_ini_files(self): + shutil.copy(self.cfg.map_path, self.map_path) + + def apply_ini_overrides(self): + m = read_file(self.map_path) + with open(self.map_path, "w") as f: + f.write("\n\n".join([m] + [read_file(p) for p in self.args.ini_overrides])) + + def generate_spawnini(self): + with open(self.spawn_path, "w") as f: + f.write(self.cfg.to_ini(self.player_index)) + + def prepare_wine_prefix(self): + os.environ["WINEPREFIX"] = str(self.wineprefix_dir.absolute()) + if self.wineprefix_dir.exists(): + return + self.wineprefix_dir.mkdir(parents=True) + try: + for c in ( + ["wineboot", "-ik"], + ["wine", "regedit", self.args.registry], + ["wineboot", "-s"], + ["wineserver", "-w"], + ): + r = prun(c, check=True) + except: + logging.error("%s", traceback.format_exc()) + shutil.rmtree(self.wineprefix_dir) + + def prepare_instance_directory(self): + self.create_symlinks() + self.create_ini_files() + self.apply_ini_overrides() + self.generate_spawnini() + + def run(self): + cmd = { + "syringe": ["Syringe.exe", f" {NAME_SPAWNER_SYRINGE}", "-SPAWN", "-CD"], + "static": [NAME_SPAWNER_PATCHED, "-SPAWN"], + }[self.args.type] + so = open(self.instance_dir / "out.log", "w") + se = open(self.instance_dir / "err.log", "w") + prun(["wine"] + cmd, cwd=self.instance_dir, stdout=so, stderr=se) + + +def run_docker_instance(args): + M = RunDockerInstance(args) + atexit.register(docker_exit) + try: + if args.type == "static": + M.prepare_patched_spawner() + M.run_docker_instance() + except KeyboardInterrupt: + logging.info("Stop program") + except subprocess.CalledProcessError as e: + logging.error("%s %s", traceback.format_exc(), e.stderr) + except Exception: + logging.error("%s", traceback.format_exc()) + + +def run_gamemd(args): + M = RunGameMD(args) + M.prepare_wine_prefix() + M.prepare_instance_directory() + M.run() + + +def patch_gamemd(args): + M = PatchGameMD(args) + try: + M.main() + except subprocess.CalledProcessError as e: + logging.error("%s: %s", e.cmd, e.stderr) + + +def build(args): + uid = get_game_uid() + + def _run(c): + return prun( + Docker.run("builder", c, uid=uid), + check=True, + ) + + _run(["make", "build_cpp"]) + + +def parse_args(): + a = argparse.ArgumentParser( + description="RA2YR game launcher helper", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + sp = a.add_subparsers(title="Subcommands", dest="subcommand") + a1 = sp.add_parser( + "run-docker-instance", help="Execute one or more game instances with Docker." + ) + a1.add_argument( + "-e", + "--script", + help="pyra2yr script to run (relative to pyra2yr directory)", + type=Path, + default=Path("pyra2yr") / "test_sell_mcv.py", + ) + a1.set_defaults(func=run_docker_instance) + + a2 = sp.add_parser("run-gamemd") + a2.set_defaults(func=run_gamemd) + + a4 = sp.add_parser( + "patch-gamemd", help="Patch original spawner for ra2yrcpp library." + ) + a4.add_argument( + "-o", + "--output", + help="Output for patched spawner", + type=Path, + default=Path(".") / NAME_SPAWNER_PATCHED, + ) + a4.set_defaults(func=patch_gamemd) + + a3 = sp.add_parser("build") + a3.add_argument( + "-c", + "--config", + default="Release", + choices=("Release", "RelWithDebInfo", "Debug"), + help="CMake config type", + ) + a3.add_argument( + "-tc", + "--toolchain", + type=Path, + default=Path("toolchains") / "mingw-w64-i686-docker.cmake", + help="CMake toolchain file path", + ) + a3.set_defaults(func=build) + + a.add_argument( + "-b", + "--base-dir", + type=str, + default=os.getenv("RA2YRCPP_TEST_INSTANCES_DIR"), + help="Base directory for test instance directories.", + ) + a.add_argument( + "-t", + "--type", + type=str, + choices=("static", "syringe"), + default="static", + help="The type of spawner to use. 'static' means statically patched spawner, 'syringe' uses unmodified spawner and loads provided DLL's using Syringe", + ) + a.add_argument( + "-T", + "--tunnel-address", + type=str, + default="0.0.0.0", + help="Tunnel server IP address.", + ) + a.add_argument( + "-tp", "--tunnel-port", type=int, default=50000, help="Tunnel server port." + ) + a.add_argument( + "-d", + "--syringe-dlls", + type=str, + action="append", + default=[], + help="DLL file (in the CnCNet main folder) to use for Syringe. This option can be specified multiple times.", + ) + a.add_argument( + "-p", + "--players-config", + type=str, + default=os.path.join("test_data", "envs.json"), + help="Path to Docker test instance configuration file.", + ) + a.add_argument( + "-i", + "--ini-overrides", + action="append", + type=str, + default=[], + help="INI files to concatenate into the generated spawnmap ini. This option can be specified multiple times.", + ) + a.add_argument( + "-n", + "--player-name", + type=str, + help="Current player name.", + ) + a.add_argument( + "-r", + "--registry", + type=Path, + default=Path("test_data") / "env.reg", + help="Windows registry setup for the test environment.", + ) + a.add_argument("-f", "--frame-send-rate", type=int, default=1) + a.add_argument("-m", "--map-path", type=str, default=os.getenv("MAP_PATH")) + a.add_argument( + "-g", + "--game-mode", + type=str, + choices=("ra2", "yr"), + default="yr", + help="Game mode", + ) + a.add_argument( + "-s", + "--game-speed", + type=int, + choices=tuple(range(6)), + default=0, + help="Game speed. 0 = Fastest, 5 = Slowest.", + ) + a.add_argument( + "-G", + "--game-data-dir", + type=Path, + default=os.getenv("RA2YRCPP_GAME_DIR"), + help="Game data directory. You probably shouldn't modify this as it points to the default path specified in Docker compose and related files.", + ) + a.add_argument( + "-B", + "--build-dir", + type=Path, + help="Path to folder containing ra2yrcpp library and related binaries.", + ) + a.add_argument( + "-S", + "--spawner-path", + required=True, + type=Path, + help="Path to original spawner binary.", + ) + a.add_argument( + "-Sy", + "--syringe-spawner-path", + type=Path, + help="Path to syringe spawner (basically vanilla gamemd).", + ) + return a.parse_args() + + +def main(): + setup_logging() + a = parse_args() + a.func(a) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_gamemd.sh b/scripts/run_gamemd.sh deleted file mode 100755 index 3e9445c3..00000000 --- a/scripts/run_gamemd.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -set -o nounset - -export WINEPREFIX="$RA2YRCPP_TEST_INSTANCES_DIR/$PLAYER_ID/.wine" -# Initialize wine env if not done -if [ ! -d "$WINEPREFIX" ]; then - mkdir -p "$WINEPREFIX" - wineboot -ik - wine regedit test_data/env.reg - wineboot -s - # need to wait to avoid losing inserted registry entries - wineserver -w -fi - -: ${WINE_CMD:="wine"} -idir="$RA2YRCPP_TEST_INSTANCES_DIR/$PLAYER_ID" -mkdir -p "$idir" - -# Prepare instance directory -{ - RA2YRCPP_PKG_DIR="$DEST_DIR/bin" ./scripts/prep_instance_dirs.sh "$RA2YRCPP_TEST_INSTANCES_DIR" "$PLAYER_ID" - cd "$idir" - RA2YRCPP_RECORD_PATH="$(date '+%s.pb.gz')" WINEDEBUG="+loaddll" $WINE_CMD gamemd-spawn.exe -SPAWN 2>err.log -} >"$idir/out.log" diff --git a/test_data/envs.tsv b/test_data/envs.tsv deleted file mode 100644 index d8c35e6c..00000000 --- a/test_data/envs.tsv +++ /dev/null @@ -1,3 +0,0 @@ -name color side is_host is_observer port ai_difficulty -player_0 1 6 True False 13367 -1 -player_1 2 6 False False 13368 -1 diff --git a/test_data/test_env.json b/test_data/test_env.json new file mode 100644 index 00000000..9cc7e3ae --- /dev/null +++ b/test_data/test_env.json @@ -0,0 +1,28 @@ +{ + "game_speed": 0, + "map_path": "test_data/dry_heat.map", + "port": 50000, + "protocol": "zero", + "ra2_mode": false, + "seed": 123, + "short_game": true, + "start_credits": 10000, + "superweapons": true, + "tunnel": "0.0.0.0", + "unit_count": 0, + "players": [ + { + "name": "player_0", + "color": "Red", + "side": "6", + "location": 0, + "is_host": true + }, + { + "name": "player_1", + "color": "Yellow", + "side": "6", + "location": 1 + } + ] +}