update metric calculation and requirements

code/run_eval.py

@@ -123,6 +123,11 @@ def main():
         with open(os.path.join(output_dir, f'{config}_correct.jsonl'), 'w') as f:
             for e in correct_data:
                 f.write(json.dumps(e) + '\n')
+
+        acc1 = np.mean([e['match'] for e in error_data]) * 100
+        acc2 = np.mean([e['match'] for e in correct_data]) * 100
+        f1 = 2 * acc1 * acc2 / (acc1 + acc2)
+        print(f'{config} error acc: {acc1:.1f}, correct acc: {acc2:.1f}, f1: {f1:.1f}')
 
 
 if __name__ == '__main__':

code/run_eval_prm_rlhflow.py

@@ -17,6 +17,7 @@
 """
 
 import os
+import numpy as np
 import json
 from tqdm import tqdm
 from multiprocessing import Pool
@@ -74,6 +75,11 @@ def single_process(d):
         with open(f'outputs/Llama3.1-8B-PRM-Mistral-Data/{config}_correct.jsonl', 'w') as f:
             for e in data2:
                 f.write(json.dumps(e) + '\n')
+
+        acc1 = np.mean([e['match'] for e in data1]) * 100
+        acc2 = np.mean([e['match'] for e in data2]) * 100
+        f1 = 2 * acc1 * acc2 / (acc1 + acc2)
+        print(f'{config} error acc: {acc1:.1f}, correct acc: {acc2:.1f}, f1: {f1:.1f}')
 
 if __name__ == '__main__':
     main()

requirements.txt

@@ -0,0 +1,3 @@
+torch==2.4.0
+transformers==4.46.1
+vllm==0.6.3.post1