Skip to content

Commit

Permalink
Update LMEvalJob resource (#2219)
Browse files Browse the repository at this point in the history
  • Loading branch information
adolfo-ab authored Dec 4, 2024
1 parent 1046423 commit 345a820
Showing 1 changed file with 17 additions and 1 deletion.
18 changes: 17 additions & 1 deletion ocp_resources/lm_eval_job.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ class LMEvalJob(NamespacedResource):

def __init__(
self,
allow_code_execution: Optional[bool] = None,
allow_online: Optional[bool] = None,
batch_size: Optional[str] = "",
gen_args: Optional[List[Any]] = None,
limit: Optional[str] = "",
Expand All @@ -29,6 +31,12 @@ def __init__(
) -> None:
"""
Args:
allow_code_execution (bool): AllowCodeExecution specifies whether the LMEvalJob can execute remote
code. Default is false.
allow_online (bool): AllowOnly specifies whether the LMEvalJob can directly download remote
code, datasets and metrics. Default is false.
batch_size (str): Batch size for the evaluation. This is used by the models that run and
are loaded locally and not apply for the commercial APIs.
Expand All @@ -47,7 +55,7 @@ def __init__(
num_few_shot (int): Sets the number of few-shot examples to place in context
offline (Dict[str, Any]): Offline specifies settings for running LMEvalJobs in a offline mode
offline (Dict[str, Any]): Offline specifies settings for running LMEvalJobs in an offline mode
outputs (Dict[str, Any]): Outputs specifies storage for evaluation results
Expand All @@ -61,6 +69,8 @@ def __init__(
"""
super().__init__(**kwargs)

self.allow_code_execution = allow_code_execution
self.allow_online = allow_online
self.batch_size = batch_size
self.gen_args = gen_args
self.limit = limit
Expand Down Expand Up @@ -90,6 +100,12 @@ def to_dict(self) -> None:
_spec["model"] = self.model
_spec["taskList"] = self.task_list

if self.allow_code_execution is not None:
_spec["allowCodeExecution"] = self.allow_code_execution

if self.allow_online is not None:
_spec["allowOnline"] = self.allow_online

if self.batch_size:
_spec["batchSize"] = self.batch_size

Expand Down

0 comments on commit 345a820

Please sign in to comment.