Upload folder using huggingface_hub

check.py

@@ -0,0 +1,74 @@
+import json
+import os
+from collections import Counter
+
+base_dir = "generate_result/zero_shot/bd_math/generation/llama3.1/1"
+
+
+def has_repetition(text, threshold=3):
+    """
+    Check if the given text contains repetitive substrings and return the repetitive phrases.
+
+    :param text: The text to check for repetition
+    :param threshold: The number of repetitions to consider as repetitive
+    :return: A list of repetitive phrases if found, otherwise an empty list
+    """
+    words = text.split()
+    repetitive_phrases = []
+    for n in range(15, 20):
+        phrases = [" ".join(words[i : i + n]) for i in range(len(words) - n + 1)]
+        phrase_counts = Counter(phrases)
+        repetitive_phrases.extend(
+            [phrase for phrase, count in phrase_counts.items() if count >= threshold]
+        )
+        break
+    return repetitive_phrases
+
+
+total_items = 0
+items_with_repetition = 0
+repetition_data = []
+
+for i in range(8):
+    file_path = os.path.join(base_dir, f"{i}.json")
+
+    if not os.path.exists(file_path):
+        print(f"File {file_path} does not exist. Skipping.")
+        continue
+
+    with open(file_path, "r") as file:
+        for line_number, line in enumerate(file, 1):
+            try:
+                data = json.loads(line)
+                model_output = data.get("total output", "")[0]
+                total_items += 1
+
+                repetitive_phrases = has_repetition(model_output)
+                if len(repetitive_phrases):
+                    items_with_repetition += 1
+                    repetition_data.append(
+                        {
+                            "file": f"{i}.json",
+                            "line": line_number,
+                            "prompt": data.get("prompt", ""),
+                            "repetitive_phrases": repetitive_phrases,
+                        }
+                    )
+                    # print(repetitive_phrases[0])
+            except json.JSONDecodeError:
+                print(f"Error decoding JSON in file {i}.json, line {line_number}")
+            except Exception as e:
+                print(f"Error processing file {i}.json, line {line_number}: {str(e)}")
+
+# Calculate the ratio
+ratio = items_with_repetition / total_items if total_items > 0 else 0
+print(
+    f"Ratio of items with repetition: {ratio:.2f} ({items_with_repetition}/{total_items})"
+)
+
+# Save repetition data to a file
+output_file = "repetition_analysis.json"
+with open(os.path.join(base_dir, output_file), "w") as f:
+    json.dump(repetition_data, f, indent=2)
+
+print(f"Repetition analysis completed. Results saved to {output_file}")

vllm_generate.py

@@ -11,7 +11,7 @@ import sys
 import os
 import numpy as np
 
-few_shot_string = '''Question: Find the domain of the expression $\frac{\sqrt{x-2}}{\sqrt{5-x}}$.}
+few_shot_string = """Question: Find the domain of the expression $\frac{\sqrt{x-2}}{\sqrt{5-x}}$.}
 Let's think step by step. The expressions inside each square root must be non-negative. Therefore, $x-2 \ge 0$, so $x\ge2$, and $5 - x \ge 0$, so $x \le 5$. Also, the denominator cannot be equal to zero, so $5-x>0$, which gives $x<5$. Therefore, the domain of the expression is $[2,5)$. Final Answer: The answer is $[2,5)$. I hope it is correct.
 
 Question: If $\det \mathbf{A} = 2$ and $\det \mathbf{B} = 12,$ then find $\det (\mathbf{A} \mathbf{B}).$
@@ -35,7 +35,7 @@ Let's think step by step. If we multiply the first equation by $-\frac{3}{2}$, w
 $$-\frac{3}{2}a=b\Rightarrow\frac{a}{b}=-\frac{2}{3}.$$
 Final Answer: The answer is $-\frac{2}{3}$. I hope it is correct.
 
-'''
+"""
 
 PROMPT_DICT = {
     "lean4": (
@@ -49,9 +49,9 @@ PROMPT_DICT = {
         "Write a response that appropriately completes the request.\n\n"
         "### Instruction:\n{instruction}\n\n### Response:"
     ),
-    'old_prompt_bd': '''Question: {question}
-Let's think step by step.''',
-    'vallina':'''{question}''',
+    "old_prompt_bd": """Question: {question}
+Let's think step by step.""",
+    "vallina": """{question}""",
 }