Merge pull request #98 from uiuc-focal-lab/json_eval3

Fix JSON evaluation for gemma
uiuc-focal-lab · Aug 7, 2024 · f4f9cb4 · f4f9cb4
2 parents 21593dd + eab96b7
commit f4f9cb4
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 2 deletions.
diff --git a/syncode/dataset.py b/syncode/dataset.py
@@ -50,6 +50,8 @@ def __init__(self, dataset, language=None, num_few_shot=0):
             ds = load_dataset("NousResearch/json-mode-eval", split = "train")
             self.problems = []
             for problem in ds:
+                prompt = [{'content': problem['prompt'][0]['content'] + problem['prompt'][1]['content'], 'role': 'user'}]
+                problem['prompt'] = prompt
                 self.problems.append({**problem, 'prompt': problem['prompt'], 'ground_truth': problem['completion'], 'schema': problem['schema']})
         elif dataset == "folio":
             self.dataset_name = "folio"

diff --git a/syncode/evaluation/json_eval.py b/syncode/evaluation/json_eval.py
@@ -29,13 +29,12 @@ def run_json_eval(
         pbar = tqdm(total=len(problems) * syncode.num_samples)
         results = defaultdict(list)
 
-        for task_id, problem in enumerate(problems[:10]):
+        for task_id, problem in enumerate(problems):
             output = JSONEval.run_eval_for_task(syncode, syncode.num_samples, problem, samples, pbar, task_id)
             if debug_task_id is not None:
                 return output
             outputs.append(outputs) 
 
-
         schema_result = validate_json_data(syncode, samples, results)
 
         # exact match evaluation doesn't make sense