From 0aa31ee65782166737d44594a9ab40dddc430c0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Du=C3=A9=C3=B1ez-Guzm=C3=A1n?= Date: Tue, 5 Dec 2023 10:06:09 -0800 Subject: [PATCH] Remove use of the (deprecated) environment metrics. Instead, use only component metrics that use measurements. PiperOrigin-RevId: 588109071 Change-Id: Ib9aa38f42c1c2b514fcfdd08058635fdc8843f73 --- .../components/game_master/conversation.py | 5 - concordia/environment/game_master.py | 12 -- concordia/environment/metrics/__init__.py | 21 --- .../metrics/common_sense_morality.py | 115 -------------- .../environment/metrics/goal_achievement.py | 116 -------------- concordia/environment/metrics/reputation.py | 145 ------------------ .../metrics/uncertainty_scale_question.py | 114 -------------- concordia/environment/scenes/conversation.py | 4 - concordia/tests/concordia_integration_test.py | 37 +++-- concordia/typing/metric.py | 47 ------ 10 files changed, 21 insertions(+), 595 deletions(-) delete mode 100644 concordia/environment/metrics/__init__.py delete mode 100644 concordia/environment/metrics/common_sense_morality.py delete mode 100644 concordia/environment/metrics/goal_achievement.py delete mode 100644 concordia/environment/metrics/reputation.py delete mode 100644 concordia/environment/metrics/uncertainty_scale_question.py delete mode 100644 concordia/typing/metric.py diff --git a/concordia/components/game_master/conversation.py b/concordia/components/game_master/conversation.py index 11ff23f9..3b8872ef 100644 --- a/concordia/components/game_master/conversation.py +++ b/concordia/components/game_master/conversation.py @@ -28,7 +28,6 @@ from concordia.language_model import language_model from concordia.typing import clock as clock_lib from concordia.typing import component -from concordia.typing import metric from concordia.utils import helper_functions import termcolor @@ -46,7 +45,6 @@ def __init__( cap_nonplayer_characters: int = 3, game_master_instructions: str = '', shared_context: str = '', - measurements: Sequence[metric.Metric] | None = None, components: Sequence[component.Component] | None = None, allow_self_talk: bool = False, verbose: bool = False, @@ -66,7 +64,6 @@ def __init__( allowed in the conversation. game_master_instructions: A string to use as the game master instructions. shared_context: A string to use as the generic context for the NPCs. - measurements: metrics to pass into the conversation GM components: components that contextualise the conversation allow_self_talk: allow players to have a conversation with themselves verbose: Whether to print debug messages or not. @@ -84,7 +81,6 @@ def __init__( self._clock = clock self._burner_memory_factory = burner_memory_factory self._memory = memory - self._measurements = measurements self._allow_self_talk = allow_self_talk self._all_player_names = [player.name for player in self._players] self._min_speakers = 1 if self._allow_self_talk else 2 @@ -285,7 +281,6 @@ def update_after_event( memory_factory=self._burner_memory_factory, name='Conversation scene', premise=event_statement, - measurements=self._measurements, ) with self._clock.higher_gear(): scene_output = convo_scene.run_episode() diff --git a/concordia/environment/game_master.py b/concordia/environment/game_master.py index 0a053a7b..5c89f6a1 100644 --- a/concordia/environment/game_master.py +++ b/concordia/environment/game_master.py @@ -28,7 +28,6 @@ from concordia.typing import clock as game_clock from concordia.typing import component from concordia.typing import game_master as simulacrum_game_master -from concordia.typing import metric import termcolor @@ -48,7 +47,6 @@ def __init__( clock: game_clock.GameClock, players: Sequence[basic_agent.BasicAgent], name: str = 'Game Master', - measurements: Sequence[metric.Metric] | None = None, update_thought_chain: ( Sequence[ Callable[[interactive_document.InteractiveDocument, str], str] @@ -74,8 +72,6 @@ def __init__( players: a sequence of generative agent simulacra which is assumed to contain only information that players also can access. name: name of the game master. - measurements: sequence of measurements which look at text and store the - answers to questions in python state variables. update_thought_chain: chain of thoughts for update from player components: components to condition on action_spec: specific action_spec to pass to agents, default is used if @@ -99,7 +95,6 @@ def __init__( self._clock = clock self._players = players self._log_colour = log_colour - self._measurements = measurements or [] self._randomise_initiative = randomise_initiative self._player_observes_event = player_observes_event self._players_act_simultaneously = players_act_simultaneously @@ -223,13 +218,6 @@ def get_externality(externality): self._log.append(update_log) - # MULTI-THREAD - def process_measurements(signal): - return signal.update(event_statement, player_name, prompt) - - with concurrent.futures.ThreadPoolExecutor() as executor: - executor.map(process_measurements, self._measurements) - return event_statement def view_for_player(self, player_name): diff --git a/concordia/environment/metrics/__init__.py b/concordia/environment/metrics/__init__.py deleted file mode 100644 index 5123c855..00000000 --- a/concordia/environment/metrics/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2023 DeepMind Technologies Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -"""Library of metrics for generative game master and agents.""" - -from concordia.environment.metrics import common_sense_morality -from concordia.environment.metrics import goal_achievement -from concordia.environment.metrics import reputation -from concordia.environment.metrics import uncertainty_scale_question diff --git a/concordia/environment/metrics/common_sense_morality.py b/concordia/environment/metrics/common_sense_morality.py deleted file mode 100644 index f453601e..00000000 --- a/concordia/environment/metrics/common_sense_morality.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright 2023 DeepMind Technologies Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -"""Metrics of ethical conduct.""" - -from collections.abc import Sequence -from typing import Any - -from concordia.agents import basic_agent -from concordia.document import interactive_document -from concordia.language_model import language_model -from concordia.typing import clock as game_clock -from concordia.typing import metric - - -DEFAULT_SCALE = ['abhorrent', 'wrong', 'neutral', 'right', 'praiseworthy'] - - -class CommonSenseMoralityMetric(metric.Metric): - """Metric of common sense morality of players actions.""" - - def __init__( - self, - model: language_model.LanguageModel, - players: Sequence[basic_agent.BasicAgent], - clock: game_clock.GameClock, - name: str = 'Morality', - scale: Sequence[str] | None = None, - verbose: bool = False, - writer=None, - ): - """Initializes the metric. - - Args: - model: The language model to use. - players: The players to measure. - clock: The clock to use. - name: The name of the metric. - scale: The scale of the metric, uses default if None. - verbose: Whether to print the metric. - writer: The writer to use. - """ - self._model = model - self._name = name - self._state = [] - self._clock = clock - self._verbose = verbose - self._players = players - self._scale = scale or DEFAULT_SCALE - self._writer = writer - - self._player_by_name = {player.name: player for player in players} - - def name( - self, - ) -> str: - """Returns the name of the measurement.""" - return self._name - - def update( - self, - observation: str, - acting_player_name: str, - doc: interactive_document.InteractiveDocument, - ) -> None: - current_player = self._player_by_name[acting_player_name] - - doc = doc.new() - answer = doc.multiple_choice_question( - ( - f'Evaluate if the following activity of {current_player.name} using' - f' common sense morality.\nActivity: {observation}\n' - ), - self._scale, - ) - answer_str = self._scale[answer] - - answer = float(answer) / float(len(self._scale) - 1) - - datum = { - 'time_str': self._clock.now().strftime('%H:%M:%S'), - 'clock_step': self._clock.get_step(), - 'step_metric': len(self._state), - 'value_float': answer, - 'value_str': answer_str, - 'player': acting_player_name, - } - if self._writer is not None: - self._writer.write(datum) - self._writer.flush() - - datum['time'] = self._clock.now() - self._state.append(datum) - - if self._verbose: - print(f'{self._name} of {current_player.name}: {answer_str}') - - def state(self) -> list[dict[str, Any]]: - """Return the current state of all the tracked variables.""" - return self._state.copy() - - def get_scale(self) -> Sequence[str]: - return self._scale diff --git a/concordia/environment/metrics/goal_achievement.py b/concordia/environment/metrics/goal_achievement.py deleted file mode 100644 index d46c4a36..00000000 --- a/concordia/environment/metrics/goal_achievement.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright 2023 DeepMind Technologies Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -"""Metrics of goal achievement per player.""" - -from collections.abc import Sequence -from typing import Any - -from concordia.document import interactive_document -from concordia.language_model import language_model -from concordia.typing import clock as game_clock -from concordia.typing import metric - -DEFAULT_SCALE = [ - 'activity unrelated to the goal', - 'somewhat working towards the goal', - 'working towards the goal', - 'goal achieved', -] - - -class GoalAchievementMetric(metric.Metric): - """Metric of goal achievement per player / goal pair.""" - - def __init__( - self, - model: language_model.LanguageModel, - player_goals: dict[str, str], - clock: game_clock.GameClock, - name: str = 'Goal Achievement', - scale: Sequence[str] | None = None, - verbose: bool = False, - writer=None, - ): - """Initializes the metric. - - Args: - model: Language model to use for the question. - player_goals: Dictionary of player name to player goal. - clock: Clock for logging. - name: Name of the metric. - scale: Scale of the metric, uses default if None. - verbose: Whether to print logs during execution. - writer: Writer to use for logging. - """ - self._model = model - self._name = name - self._state = [] - self._clock = clock - self._verbose = verbose - self._player_goals = player_goals - self._scale = scale or DEFAULT_SCALE - self._writer = writer - - def name( - self, - ) -> str: - """Returns the name of the measurement.""" - return self._name - - def update( - self, - observation: str, - acting_player_name: str, - doc: interactive_document.InteractiveDocument, - ) -> None: - acting_player_goal = self._player_goals[acting_player_name] - doc = doc.new() - answer = doc.multiple_choice_question( - ( - 'Evaluate if the following activity brings' - f' {acting_player_name} closer to their goal' - f' "{acting_player_goal} .\n Activity: {observation}\n' - ), - self._scale, - ) - answer_str = self._scale[answer] - - answer = float(answer) / float(len(self._scale) - 1) - - datum = { - 'time_str': self._clock.now().strftime('%H:%M:%S'), - 'clock_step': self._clock.get_step(), - 'step_metric': len(self._state), - 'value_float': answer, - 'value_str': answer_str, - 'player': acting_player_name, - 'goal': acting_player_goal, - } - if self._writer is not None: - self._writer.write(datum) - self._writer.flush() - datum['time'] = self._clock.now() - - self._state.append(datum) - if self._verbose: - print(f'{self._name} of {acting_player_name}: {answer_str}') - - def state(self) -> list[dict[str, Any]]: - """Return the current state of all the tracked variables.""" - return self._state.copy() - - def get_scale(self) -> Sequence[str]: - return self._scale diff --git a/concordia/environment/metrics/reputation.py b/concordia/environment/metrics/reputation.py deleted file mode 100644 index be33b19f..00000000 --- a/concordia/environment/metrics/reputation.py +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright 2023 DeepMind Technologies Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -"""Metrics of player`s reputation among other players.""" - -from collections.abc import Sequence -import concurrent.futures -from typing import Any - -from concordia.agents import basic_agent -from concordia.document import interactive_document -from concordia.language_model import language_model -from concordia.typing import agent as simulacrum_agent -from concordia.typing import clock as game_clock -from concordia.typing import metric -import numpy as np - -DEFAULT_SCALE = [ - 'very negative', - 'somewhat negative', - 'neutral', - 'somewhat positive', - 'very positive', -] - - -class ReputationMetric(metric.Metric): - """Metric of players reputation among the each other.""" - - def __init__( - self, - model: language_model.LanguageModel, - players: Sequence[basic_agent.BasicAgent], - clock: game_clock.GameClock, - name: str = 'Reputation', - scale: Sequence[str] | None = None, - verbose: bool = False, - writer=None, - question: str = 'What is {opining_player}\'s opinion of {of_player}?', - ): - """Initializes the metric. - - Args: - model: Language model to use for the question. - players: List of players. - clock: Clock for logging. - name: Name of the metric. - scale: Scale of the metric, uses default if None. - verbose: Whether to print logs during execution. - writer: Writer to use for logging. - question: The question to ask players about opinions on other players. - Must have two formatting fields: "{opining_player}" and "{of_player}". - """ - self._model = model - self._name = name - self._state = [] - self._clock = clock - self._verbose = verbose - self._players = players - self._scale = scale or DEFAULT_SCALE - self._writer = writer - self._question = question - - self._player_by_name = {player.name: player for player in players} - - def name( - self, - ) -> str: - """Returns the name of the measurement.""" - return self._name - - def update( - self, - observation: str, - acting_player_name: str, - doc: interactive_document.InteractiveDocument, - ) -> None: - del doc, observation # this metric doesn't use either - - def get_reputation(current_player: basic_agent.BasicAgent) -> None: - if current_player.name == acting_player_name: - return - - question = ( - self._question.format(opining_player=current_player.name, - of_player=acting_player_name) - ) - action_spec = simulacrum_agent.ActionSpec( - call_to_action=question, - output_type='CHOICE', - options=self._scale, - ) - - with current_player.interrogate(): - answer_str = current_player.act(action_spec, memorize=False) - answer = np.where(np.array(self._scale) == answer_str)[0][0] - - answer = float(answer) / float(len(self._scale) - 1) - datum = { - 'time_str': self._clock.now().strftime('%H:%M:%S'), - 'clock_step': self._clock.get_step(), - 'step_metric': len(self._state), - 'value_float': answer, - 'value_str': answer_str, - 'player': acting_player_name, - 'rating_player': current_player.name, - } - if self._writer is not None: - self._writer.write(datum) - self._writer.flush() - - datum['time'] = self._clock.now() - self._state.append(datum) - if self._verbose: - print( - f'{self._name} of {acting_player_name} as viewed by ' - f'{current_player.name}:' - f' {answer_str}' - ) - - return - - with concurrent.futures.ThreadPoolExecutor( - max_workers=len(self._players) - ) as executor: - executor.map(get_reputation, self._players) - - def state(self) -> list[dict[str, Any]]: - """Return the current state of all the tracked variables.""" - return self._state.copy() - - def get_scale(self) -> Sequence[str]: - return self._scale diff --git a/concordia/environment/metrics/uncertainty_scale_question.py b/concordia/environment/metrics/uncertainty_scale_question.py deleted file mode 100644 index d120b693..00000000 --- a/concordia/environment/metrics/uncertainty_scale_question.py +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright 2023 DeepMind Technologies Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -"""Metrics for tracking the answer to a configurable question.""" - -from collections.abc import Sequence -from typing import Any - -from concordia.agents import basic_agent -from concordia.document import interactive_document -from concordia.language_model import language_model -from concordia.typing import agent as simulacrum_agent -from concordia.typing import clock as game_clock -from concordia.typing import metric -import numpy as np - - -DEFAULT_SCALE = [ - 'Definitively not', - 'Maybe not', - 'Maybe yes', - 'Definitively yes', -] - -DEFAULT_QUESTION = 'Would {agent_name} talk to a stranger?' - - -class Question(metric.Metric): - """Metrics for tracking the answer to a configurable question.""" - - def __init__( - self, - model: language_model.LanguageModel, - players: Sequence[basic_agent.BasicAgent], - clock: game_clock.GameClock, - name: str = 'Question', - question: str | None = None, - scale: Sequence[str] | None = None, - verbose: bool = False, - writer=None, - ): - self._model = model - self._name = name - self._state = [] - self._clock = clock - self._verbose = verbose - self._players = players - self._scale = scale or DEFAULT_SCALE - self._writer = writer - self._question = question or DEFAULT_QUESTION - - self._player_by_name = {player.name: player for player in players} - - def name( - self, - ) -> str: - """Returns the name of the measurement.""" - return self._name - - def update( - self, - observation: str, - acting_player_name: str, - doc: interactive_document.InteractiveDocument, - ) -> None: - del doc, observation # this metric doesn't use either - question = self._question.format(agent_name=acting_player_name) - action_spec = simulacrum_agent.ActionSpec( - call_to_action=question, - output_type='CHOICE', - options=self._scale, - ) - current_player = self._player_by_name[acting_player_name] - - with current_player.interrogate(): - answer_str = current_player.act(action_spec) - answer = np.where(np.array(self._scale) == answer_str)[0][0] - - answer = float(answer) / float(len(self._scale) - 1) - datum = { - 'time_str': self._clock.now().strftime('%H:%M:%S'), - 'clock_step': self._clock.get_step(), - 'step_metric': len(self._state), - 'value_float': answer, - 'value_str': answer_str, - 'player': acting_player_name, - } - if self._writer is not None: - self._writer.write(datum) - self._writer.flush() - - datum['time'] = self._clock.now() - self._state.append(datum) - if self._verbose: - print(f'{question}\n{acting_player_name}: {answer_str}') - - def state(self) -> list[dict[str, Any]]: - """Return the current state of all the tracked variables.""" - return self._state.copy() - - def get_scale(self) -> Sequence[str]: - return self._scale diff --git a/concordia/environment/scenes/conversation.py b/concordia/environment/scenes/conversation.py index 01218d92..8bb5028f 100644 --- a/concordia/environment/scenes/conversation.py +++ b/concordia/environment/scenes/conversation.py @@ -31,7 +31,6 @@ from concordia.thought_chains import thought_chains from concordia.typing import agent as simulacrum_agent from concordia.typing import component -from concordia.typing import metric import termcolor @@ -102,7 +101,6 @@ def make_conversation_game_master( clock: game_clock.MultiIntervalClock, model: language_model.LanguageModel, memory_factory: blank_memories.MemoryFactory, - measurements: Sequence[metric.Metric] | None, name: str = 'Conversation scene', premise: str = '', ): @@ -113,7 +111,6 @@ def make_conversation_game_master( clock: a clock model: a language model memory_factory: a memory factory - measurements: measurements for the game master to use name: the name of the game master premise: any extra text to be added on top of the conversation (say, circumstances of it) @@ -156,7 +153,6 @@ def make_conversation_game_master( clock=clock, name=name, players=players, - measurements=measurements, components=[conversation_tracker], action_spec=action_spec, update_thought_chain=[thought_chains.identity], diff --git a/concordia/tests/concordia_integration_test.py b/concordia/tests/concordia_integration_test.py index 0670fe53..92c544d8 100644 --- a/concordia/tests/concordia_integration_test.py +++ b/concordia/tests/concordia_integration_test.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from collections.abc import Sequence import datetime -from typing import List from absl.testing import absltest from absl.testing import parameterized from concordia import components @@ -25,9 +25,9 @@ from concordia.components import agent as agent_components from concordia.components import game_master as gm_components from concordia.environment import game_master -from concordia.environment.metrics import common_sense_morality -from concordia.environment.metrics import goal_achievement -from concordia.environment.metrics import reputation +from concordia.metrics import common_sense_morality +from concordia.metrics import goal_achievement +from concordia.metrics import opinion_of_others from concordia.tests import mock_model import numpy as np @@ -41,11 +41,18 @@ def _make_agent( name: str, model: mock_model.MockModel, clock: game_clock.MultiIntervalClock, + player_names: Sequence[str], game_master_instructions: str, mem_factory: blank_memories.MemoryFactory, ) -> basic_agent.BasicAgent: """Creates two agents with the same game master instructions.""" mem = mem_factory.make_blank_memory() + goal_metric = goal_achievement.GoalAchievementMetric( + model=model, player_name=name, player_goal='win', clock=clock, + ) + morality_metric = common_sense_morality.CommonSenseMoralityMetric( + model=model, player_name=name, clock=clock, + ) agent = basic_agent.BasicAgent( model, mem, @@ -59,9 +66,16 @@ def _make_agent( 'General knowledge:', 'this is a test' ), agent_components.observation.Observation('Alice', mem), + goal_metric, + morality_metric, ], verbose=True, ) + reputation_metric = opinion_of_others.OpinionOfOthersMetric( + model=model, player_name=name, player_names=player_names, + context_fn=agent.state, clock=clock, + ) + agent.add_component(reputation_metric) return agent @@ -69,7 +83,7 @@ def _make_agent( def _make_environment( model: mock_model.MockModel, clock: game_clock.MultiIntervalClock, - players: List[basic_agent.BasicAgent], + players: Sequence[basic_agent.BasicAgent], game_master_instructions: str, importance_model_gm: importance_function.ImportanceModel, ) -> game_master.GameMaster: @@ -140,16 +154,6 @@ def _make_environment( schedule_construct = gm_components.schedule.Schedule( clock_now=clock.now, schedule=schedule ) - player_goals = {'Alice': 'win', 'Bob': 'win'} - goal_metric = goal_achievement.GoalAchievementMetric( - model, player_goals, clock, 'Goal achievement', verbose=False - ) - morality_metric = common_sense_morality.CommonSenseMoralityMetric( - model, players, clock, 'Morality', verbose=False - ) - reputation_metric = reputation.ReputationMetric( - model, players, clock, 'Reputation', verbose=False - ) env = game_master.GameMaster( model=model, @@ -164,7 +168,6 @@ def _make_environment( convo_externality, direct_effect_externality, ], - measurements=[goal_metric, morality_metric, reputation_metric], randomise_initiative=True, player_observes_event=False, verbose=False, @@ -200,6 +203,7 @@ def test_full_run(self): name='Alice', model=model, clock=clock, + player_names=['Alice', 'Bob'], game_master_instructions=game_master_instructions, mem_factory=mem_factory, ) @@ -207,6 +211,7 @@ def test_full_run(self): name='Bob', model=model, clock=clock, + player_names=['Alice', 'Bob'], game_master_instructions=game_master_instructions, mem_factory=mem_factory, ) diff --git a/concordia/typing/metric.py b/concordia/typing/metric.py deleted file mode 100644 index 4e083722..00000000 --- a/concordia/typing/metric.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2023 DeepMind Technologies Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -"""Metrics for simulations.""" - -import abc -from typing import Any - -from concordia.document import interactive_document - - -class Metric(metaclass=abc.ABCMeta): - """A class to hold logic for tracking state variables of a simulation.""" - - @abc.abstractmethod - def name( - self, - ) -> str: - """Returns the name of the measurement.""" - raise NotImplementedError - - @abc.abstractmethod - def update( - self, - observation: str, - active_player_name: str, - document: interactive_document.InteractiveDocument, - ) -> None: - """Process the observation then compute metric and store it.""" - raise NotImplementedError - - @abc.abstractmethod - def state(self) -> list[dict[str, Any]] | None: - """Return the current state of all the tracked variables.""" - raise NotImplementedError