hot fix for llama4

end-to-end-use-cases/benchmarks/llm_eval_harness/meta_eval/meta_template/gpqa_cot/utils.py

@@ -4,9 +4,16 @@ import re
 import datasets
 
 
-
+token_convert = {
+    "<|start_header_id|>":"<|header_start|>",
+    "<|end_header_id|>":"<|header_end|>",
+    "<|eot_id|>":"<|eot|>",
+}
 def doc_to_text(doc: dict) -> str:
-    return doc["input_final_prompts"][0]
+    prompt = doc["input_final_prompts"][0]
+    for k,v in token_convert.items():
+        prompt = prompt.replace(k,v)
+    return prompt
 def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:
     def _process_doc(doc: dict) -> dict:
         out_doc = {

end-to-end-use-cases/benchmarks/llm_eval_harness/meta_eval/meta_template/math_hard/utils.py

@@ -19,8 +19,16 @@ please install sympy via pip install lm-eval[math] or pip install -e .[math]",
 
 # taken from
 # https://github.com/wellecks/lm-evaluation-harness/blob/master/lm_eval/tasks/minerva_math.py
+token_convert = {
+    "<|start_header_id|>":"<|header_start|>",
+    "<|end_header_id|>":"<|header_end|>",
+    "<|eot_id|>":"<|eot|>",
+}
 def doc_to_text(doc: dict) -> str:
-    return doc["input_final_prompts"][0]
+    prompt = doc["input_final_prompts"][0]
+    for k,v in token_convert.items():
+        prompt = prompt.replace(k,v)
+    return prompt
 
 def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:
     def _process_doc(doc: dict) -> dict:

end-to-end-use-cases/benchmarks/llm_eval_harness/meta_eval/meta_template/mmlu_pro/utils.py

@@ -3,8 +3,16 @@ import string
 import datasets
 
 
+token_convert = {
+    "<|start_header_id|>":"<|header_start|>",
+    "<|end_header_id|>":"<|header_end|>",
+    "<|eot_id|>":"<|eot|>",
+}
 def doc_to_text(doc: dict) -> str:
-    return doc["input_final_prompts"][0]
+    prompt = doc["input_final_prompts"][0]
+    for k,v in token_convert.items():
+        prompt = prompt.replace(k,v)
+    return prompt
 
 
 def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: