diff --git a/algobattle/battle.py b/algobattle/battle.py index e16a02ba..1f28caac 100644 --- a/algobattle/battle.py +++ b/algobattle/battle.py @@ -4,6 +4,7 @@ some basic battle types, and related classed. """ from dataclasses import dataclass +from enum import StrEnum from functools import wraps from importlib.metadata import entry_points from abc import abstractmethod @@ -30,6 +31,7 @@ ConfigDict, Field, GetCoreSchemaHandler, + SerializeAsAny, ValidationError, ValidationInfo, ValidatorFunctionWrapHandler, @@ -39,12 +41,20 @@ from algobattle.program import ( Generator, - ProgramRunInfo, + GeneratorResult, + ProgramResult, ProgramUi, + RunConfigOverride, Solver, + SolverResult, +) +from algobattle.problem import InstanceModel, Problem, SolutionModel +from algobattle.util import ( + Encodable, + EncodableModel, + ExceptionInfo, + BaseModel, ) -from algobattle.problem import Problem -from algobattle.util import Encodable, ExceptionInfo, BaseModel _BattleConfig: TypeAlias = Any @@ -61,6 +71,54 @@ Type = type +class ProgramLogConfigTime(StrEnum): + """When to log a programs i/o.""" + + never = "never" + error = "error" + always = "always" + + +class ProgramLogConfigLocation(StrEnum): + """Where to log a programs i/o.""" + + disabled = "disabled" + inline = "inline" + + +class ProgramLogConfigView(Protocol): # noqa: D101 + when: ProgramLogConfigTime = ProgramLogConfigTime.error + output: ProgramLogConfigLocation = ProgramLogConfigLocation.inline + + +class ProgramRunInfo(BaseModel): + """Data about a program's execution.""" + + runtime: float = 0 + overriden: RunConfigOverride = Field(default_factory=dict) + error: ExceptionInfo | None = None + battle_data: SerializeAsAny[EncodableModel] | None = None + instance: SerializeAsAny[InstanceModel] | None = None + solution: SerializeAsAny[SolutionModel[InstanceModel]] | None = None + + @classmethod + def from_result(cls, result: ProgramResult, *, inline_output: bool) -> Self: + """Converts the program run info into a jsonable model.""" + info = cls( + runtime=result.runtime, + overriden=result.overriden, + error=result.error, + ) + if inline_output: + if isinstance(result.battle_data, EncodableModel): + info.battle_data = result.battle_data + if isinstance(result.solution, SolutionModel): + info.solution = result.solution + if isinstance(result, GeneratorResult) and isinstance(result.instance, InstanceModel): + info.instance = result.instance + return info + + class Fight(BaseModel): """The result of one fight between the participating teams. @@ -79,6 +137,28 @@ class Fight(BaseModel): solver: ProgramRunInfo | None """Data about the solver's execution.""" + @classmethod + def from_results( + cls, + max_size: int, + score: float, + generator: GeneratorResult, + solver: SolverResult | None, + *, + config: ProgramLogConfigView, + ) -> Self: + """Turns the involved result objects into a jsonable model.""" + inline_output = config.when == "always" or ( + config.when == "error" + and (generator.error is not None or (solver is not None and solver.error is not None)) + ) + return cls( + max_size=max_size, + score=score, + generator=ProgramRunInfo.from_result(generator, inline_output=inline_output), + solver=ProgramRunInfo.from_result(solver, inline_output=inline_output) if solver is not None else None, + ) + class FightUi(ProgramUi, Protocol): """Provides an interface for :class:`Fight` to update the ui.""" @@ -113,6 +193,7 @@ class FightHandler: battle: "Battle" ui: FightUi set_cpus: str | None + log_config: ProgramLogConfigView @_save_result async def run( @@ -175,8 +256,14 @@ async def run( set_cpus=self.set_cpus, ui=ui, ) - if gen_result.info.error is not None: - return Fight(score=1, max_size=max_size, generator=gen_result.info, solver=None) + if gen_result.error is not None: + return Fight.from_results( + score=1, + max_size=max_size, + generator=gen_result, + solver=None, + config=self.log_config, + ) assert gen_result.instance is not None sol_result = await self.solver.run( @@ -190,8 +277,10 @@ async def run( set_cpus=self.set_cpus, ui=ui, ) - if sol_result.info.error is not None: - return Fight(score=0, max_size=max_size, generator=gen_result.info, solver=sol_result.info) + if sol_result.error is not None: + return Fight.from_results( + score=0, max_size=max_size, generator=gen_result, solver=sol_result, config=self.log_config + ) assert sol_result.solution is not None if self.problem.with_solution: @@ -202,7 +291,13 @@ async def run( else: score = self.problem.score(gen_result.instance, solution=sol_result.solution) score = max(0, min(1, float(score))) - return Fight(score=score, max_size=max_size, generator=gen_result.info, solver=sol_result.info) + return Fight.from_results( + score=score, + max_size=max_size, + generator=gen_result, + solver=sol_result, + config=self.log_config, + ) # We need this to be here to prevent an import cycle between match.py and battle.py diff --git a/algobattle/cli.py b/algobattle/cli.py index cf20da51..4e0ea1af 100644 --- a/algobattle/cli.py +++ b/algobattle/cli.py @@ -183,9 +183,8 @@ def run_match( console.print(Padding(leaderboard, (1, 0, 0, 0))) if save: - res_string = result.model_dump_json(exclude_defaults=True) out_path = config.project.results.joinpath(f"match-{timestamp()}.json") - out_path.write_text(res_string) + out_path.write_text(result.format(error_detail=config.project.error_detail)) console.print("Saved match result to ", out_path) return result except KeyboardInterrupt: diff --git a/algobattle/match.py b/algobattle/match.py index d4a7fa68..5cc980de 100644 --- a/algobattle/match.py +++ b/algobattle/match.py @@ -5,7 +5,7 @@ from itertools import combinations from pathlib import Path import tomllib -from typing import Annotated, Any, Iterable, Protocol, ClassVar, Self, TypeAlias, TypeVar, cast +from typing import Annotated, Any, Iterable, Literal, Protocol, ClassVar, Self, TypeAlias, TypeVar, cast from typing_extensions import override from typing_extensions import TypedDict @@ -28,7 +28,15 @@ from anyio.to_thread import current_default_thread_limiter from docker.types import LogConfig, Ulimit -from algobattle.battle import Battle, FightHandler, FightUi, BattleUi, Iterated +from algobattle.battle import ( + Battle, + FightHandler, + FightUi, + BattleUi, + Iterated, + ProgramLogConfigLocation, + ProgramLogConfigTime, +) from algobattle.program import ProgramConfigView, ProgramUi, Matchup, TeamHandler, BuildUi from algobattle.problem import Problem from algobattle.util import ( @@ -91,6 +99,7 @@ async def _run_battle( battle=battle, ui=battle_ui, set_cpus=set_cpus, + log_config=config.project.log_program_io, ) try: await battle.run_battle( @@ -184,6 +193,30 @@ def calculate_points(self, total_points_per_team: int) -> dict[str, float]: return points + def format(self, *, indent: int | None = 2, error_detail: Literal["high", "low"] = "low") -> str: + """Nicely formats the match result into a json string.""" + match error_detail: + case "high": + exclude = None + case "low": + detail = {"detail"} + program = {"error": detail} + exclude = { + "excluded_teams": {"__all__": detail}, + "battles": { + "__all__": { + "runtime_error": detail, + "fights": { + "__all__": { + "generator": program, + "solver": program, + } + }, + } + }, + } + return self.model_dump_json(exclude_defaults=True, indent=indent, exclude=exclude) + class Ui(BuildUi, Protocol): """Base class for a UI that observes a Match and displays its data. @@ -581,6 +614,13 @@ class DynamicProblemConfig(BaseModel): class ProjectConfig(BaseModel): """Various project settings.""" + class ProgramOutputConfig(BaseModel): + """How to log program output.""" + + # a bit janky atm, allows for future expansion + when: ProgramLogConfigTime = ProgramLogConfigTime.error + output: ProgramLogConfigLocation = ProgramLogConfigLocation.inline + parallel_battles: int = 1 """Number of battles exectuted in parallel.""" name_images: bool = True @@ -589,6 +629,12 @@ class ProjectConfig(BaseModel): """Whether to clean up the images after we use them.""" set_cpus: str | list[str] | None = None """Wich cpus to run programs on, if it is a list each battle will use a different cpu specification for it.""" + error_detail: Literal["low", "high"] = "high" + """How detailed error messages should be. + Higher settings help in debugging, but may leak information from other teams. + """ + log_program_io: ProgramOutputConfig = ProgramOutputConfig() + """How to log program output.""" points: int = 100 """Highest number of points each team can achieve.""" results: RelativePath = Field(default=Path("./results"), validate_default=True) diff --git a/algobattle/program.py b/algobattle/program.py index f62f5e90..e836cee8 100644 --- a/algobattle/program.py +++ b/algobattle/program.py @@ -21,7 +21,6 @@ from docker.models.containers import Container as DockerContainer from docker.types import Mount from requests import Timeout, ConnectionError -from pydantic import Field from anyio import run as run_async from anyio.to_thread import run_sync from urllib3.exceptions import ReadTimeoutError @@ -37,7 +36,6 @@ TempDir, ValidationError, Role, - BaseModel, ) from algobattle.problem import Problem, Instance, Solution @@ -162,35 +160,25 @@ def __exit__(self, exc: Any, val: Any, tb: Any): self._output.__exit__(exc, val, tb) -class ProgramRunInfo(BaseModel): - """Data about a program's execution.""" +@dataclass(frozen=True) +class SolverResult: + """The result of a solver execution.""" runtime: float = 0 - overriden: RunConfigOverride = Field(default_factory=dict) + overriden: RunConfigOverride = field(default_factory=RunConfigOverride) error: ExceptionInfo | None = None - - -@dataclass -class ProgramResult: - """The result of a program execution.""" - - info: ProgramRunInfo battle_data: Encodable | None = None + solution: Solution[Instance] | None = None -@dataclass -class GeneratorResult(ProgramResult): - """Result of a single generator execution.""" +@dataclass(frozen=True) +class GeneratorResult(SolverResult): + """The result of a generator execution.""" instance: Instance | None = None - solution: Solution[Instance] | None = None - -@dataclass -class SolverResult(ProgramResult): - """Result of a single solver execution.""" - solution: Solution[Instance] | None = None +ProgramResult = GeneratorResult | SolverResult @dataclass @@ -573,7 +561,9 @@ async def run( except Exception as e: exception_info = ExceptionInfo.from_exception(e) return GeneratorResult( - info=ProgramRunInfo(runtime=runtime, overriden=specs.overriden, error=exception_info), + runtime=runtime, + overriden=specs.overriden, + error=exception_info, battle_data=battle_data, instance=instance, solution=solution, @@ -582,8 +572,8 @@ async def run( def test(self, max_size: int | None = None) -> Instance | ExceptionInfo: """Tests whether the generator runs without issues and creates a syntactically valid instance.""" res = run_async(self.run, max_size or self.problem.min_size) - if res.info.error: - return res.info.error + if res.error: + return res.error else: assert res.instance is not None return res.instance @@ -658,7 +648,9 @@ async def run( except Exception as e: exception_info = ExceptionInfo.from_exception(e) return SolverResult( - info=ProgramRunInfo(runtime=runtime, overriden=specs.overriden, error=exception_info), + runtime=runtime, + overriden=specs.overriden, + error=exception_info, battle_data=battle_data, solution=solution, ) @@ -666,8 +658,8 @@ async def run( def test(self, instance: Instance) -> ExceptionInfo | None: """Tests whether the solver runs without issues and creates a syntactically valid solution.""" res = run_async(self.run, instance, instance.size) - if res.info.error: - return res.info.error + if res.error: + return res.error else: return None @@ -828,8 +820,6 @@ async def build( except Exception as e: handler.excluded[name] = ExceptionInfo.from_exception(e) ui.finish_build(name, False) - except BaseException: - raise else: ui.finish_build(name, True) return handler diff --git a/algobattle/util.py b/algobattle/util.py index 1c87cb07..f7691eaa 100644 --- a/algobattle/util.py +++ b/algobattle/util.py @@ -35,7 +35,7 @@ class Role(StrEnum): class BaseModel(PydandticBaseModel): """Base class for all pydantic models.""" - model_config = ConfigDict(extra="forbid", from_attributes=True) + model_config = ConfigDict(extra="forbid", from_attributes=True, hide_input_in_errors=True) class Encodable(ABC): @@ -198,10 +198,16 @@ def from_exception(cls, error: Exception) -> Self: message=error.message, detail=error.detail, ) - else: + elif isinstance(error, PydanticValidationError): return cls( type=error.__class__.__name__, message=str(error), + detail=str(error.errors(include_input=True, include_url=False)), + ) + else: + return cls( + type=error.__class__.__name__, + message="Unknown exception occurred.", detail=format_exception(error), ) diff --git a/docs/advanced/config.md b/docs/advanced/config.md index 274bf37e..cbd19336 100644 --- a/docs/advanced/config.md +++ b/docs/advanced/config.md @@ -173,6 +173,23 @@ structure with both keys being mandatory: : Path to the folder where result files are saved. Each result file will be a json file with a name containing the command that created it and the current timestamp. Defaults to `results` + `error_detail` + : Used to specify how detailed error messages included in the log files should be. Can be set to `high`, which + includes full details and stack traces for any exceptions that occur, or `low` to hide sensitive data that may leak + other team's strategic information. + + `log_program_io` + : A table that specifies how each program's output should be logged. + + `when` + : When to save the data. Can be either `never`, `error`, or `always`. When set to `never` or `always` it has the + expected behaviour, when set to `error` it will save the data only if an error occurred during the fight. + Defaults to `error`. + + `output` + : Where to store each program's output data. Currently only supports `disabled` to turn of logging program output + or `inline` to store jsonable data in the match result json file. Defaults to `inline.` + ### `docker` : Contains various advanced Docker settings that are passed through to the Docker daemon without influencing Algobattle itself. You generally should not need to use these settings. If you are running into a problem you cannot solve without diff --git a/pyproject.toml b/pyproject.toml index de5c3042..cdd5477a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "pdm.backend" [project] name = "algobattle-base" -version = "4.0.2" +version = "4.1.0" description = "The Algobattle lab course package." readme = "README.md" requires-python = ">=3.11" diff --git a/tests/test_battles.py b/tests/test_battles.py index c0459548..83ce7c26 100644 --- a/tests/test_battles.py +++ b/tests/test_battles.py @@ -5,9 +5,9 @@ from typing import Iterable, TypeVar from unittest import IsolatedAsyncioTestCase, main -from algobattle.battle import Battle, Fight, FightHandler, Iterated +from algobattle.battle import Battle, Fight, FightHandler, Iterated, ProgramRunInfo from algobattle.match import BattleObserver, EmptyUi -from algobattle.program import Matchup, ProgramRunInfo, Team +from algobattle.program import Matchup, Team from algobattle.util import Encodable, ExceptionInfo diff --git a/tests/test_docker.py b/tests/test_docker.py index 0f6cc658..2880c0dd 100644 --- a/tests/test_docker.py +++ b/tests/test_docker.py @@ -35,7 +35,7 @@ async def test_gen_lax_timeout(self): path=self.problem_path / "generator_timeout", problem=TestProblem, config=self.config_short ) as gen: res = await gen.run(5) - self.assertIsNone(res.info.error) + self.assertIsNone(res.error) async def test_gen_strict_timeout(self): """The generator times out.""" @@ -45,8 +45,8 @@ async def test_gen_strict_timeout(self): config=self.config_strict, ) as gen: res = await gen.run(5) - assert res.info.error is not None - self.assertEqual(res.info.error.type, "ExecutionTimeout") + assert res.error is not None + self.assertEqual(res.error.type, "ExecutionTimeout") async def test_gen_exec_err(self): """The generator doesn't execute properly.""" @@ -54,8 +54,8 @@ async def test_gen_exec_err(self): path=self.problem_path / "generator_execution_error", problem=TestProblem, config=self.config ) as gen: res = await gen.run(5) - assert res.info.error is not None - self.assertEqual(res.info.error.type, "ExecutionError") + assert res.error is not None + self.assertEqual(res.error.type, "ExecutionError") async def test_gen_syn_err(self): """The generator outputs a syntactically incorrect solution.""" @@ -63,8 +63,8 @@ async def test_gen_syn_err(self): path=self.problem_path / "generator_syntax_error", problem=TestProblem, config=self.config ) as gen: res = await gen.run(5) - assert res.info.error is not None - self.assertEqual(res.info.error.type, "EncodingError") + assert res.error is not None + self.assertEqual(res.error.type, "EncodingError") async def test_gen_sem_err(self): """The generator outputs a semantically incorrect solution.""" @@ -72,8 +72,8 @@ async def test_gen_sem_err(self): path=self.problem_path / "generator_semantics_error", problem=TestProblem, config=self.config ) as gen: res = await gen.run(5) - assert res.info.error is not None - self.assertEqual(res.info.error.type, "ValidationError") + assert res.error is not None + self.assertEqual(res.error.type, "ValidationError") async def test_gen_succ(self): """The generator returns the fixed instance.""" @@ -90,8 +90,8 @@ async def test_sol_strict_timeout(self): path=self.problem_path / "solver_timeout", problem=TestProblem, config=self.config_strict ) as sol: res = await sol.run(self.instance, 5) - assert res.info.error is not None - self.assertEqual(res.info.error.type, "ExecutionTimeout") + assert res.error is not None + self.assertEqual(res.error.type, "ExecutionTimeout") async def test_sol_lax_timeout(self): """The solver times out but still outputs a correct solution.""" @@ -99,7 +99,7 @@ async def test_sol_lax_timeout(self): path=self.problem_path / "solver_timeout", problem=TestProblem, config=self.config_short ) as sol: res = await sol.run(self.instance, 5) - self.assertIsNone(res.info.error) + self.assertIsNone(res.error) async def test_sol_exec_err(self): """The solver doesn't execute properly.""" @@ -107,8 +107,8 @@ async def test_sol_exec_err(self): path=self.problem_path / "solver_execution_error", problem=TestProblem, config=self.config ) as sol: res = await sol.run(self.instance, 5) - assert res.info.error is not None - self.assertEqual(res.info.error.type, "ExecutionError") + assert res.error is not None + self.assertEqual(res.error.type, "ExecutionError") async def test_sol_syn_err(self): """The solver outputs a syntactically incorrect solution.""" @@ -116,8 +116,8 @@ async def test_sol_syn_err(self): path=self.problem_path / "solver_syntax_error", problem=TestProblem, config=self.config ) as sol: res = await sol.run(self.instance, 5) - assert res.info.error is not None - self.assertEqual(res.info.error.type, "EncodingError") + assert res.error is not None + self.assertEqual(res.error.type, "EncodingError") async def test_sol_sem_err(self): """The solver outputs a semantically incorrect solution.""" @@ -125,8 +125,8 @@ async def test_sol_sem_err(self): path=self.problem_path / "solver_semantics_error", problem=TestProblem, config=self.config ) as sol: res = await sol.run(self.instance, 5) - assert res.info.error is not None - self.assertEqual(res.info.error.type, "ValidationError") + assert res.error is not None + self.assertEqual(res.error.type, "ValidationError") async def test_sol_succ(self): """The solver outputs a solution with a low quality.""" diff --git a/tests/test_match.py b/tests/test_match.py index df90b184..0e998a84 100644 --- a/tests/test_match.py +++ b/tests/test_match.py @@ -6,7 +6,7 @@ from pydantic import ByteSize, ValidationError -from algobattle.battle import Fight, Iterated, Averaged +from algobattle.battle import Fight, Iterated, Averaged, ProgramRunInfo from algobattle.match import ( DynamicProblemConfig, MatchupStr, @@ -17,7 +17,7 @@ RunConfig, TeamInfo, ) -from algobattle.program import ProgramRunInfo, Team, Matchup, TeamHandler +from algobattle.program import Team, Matchup, TeamHandler from .testsproblem.problem import TestProblem