Fix duplicates

app.py

@@ -67,46 +67,42 @@ async def clone_yara_repo():
         return None
 
 def preprocess_yara_rules(repo_path: Path) -> Path:
-    """Preprocess YARA rules to fix syntax issues"""
+    """Preprocess YARA rules to fix syntax issues and ensure unique rule names"""
     processed_dir = Path("processed_yara_rules")
     if processed_dir.exists():
         shutil.rmtree(processed_dir)
     processed_dir.mkdir()
 
+    rule_pattern = re.compile(r"rule\s+(\w+)")  # Match YARA rule names
+    seen_rules = set()  # Track rule names to prevent duplicates
+    rule_counter = 1  # Counter for renaming duplicate rules
+
     for yara_file in repo_path.glob("**/*.yar"):
         try:
             content = yara_file.read_text(encoding='utf-8', errors='replace')
-            
-            # Fix invalid rule names and string identifiers
             processed_content = []
-            rule_counter = 1
-            string_counter = 1
             current_rule = None
-            
+
             for line in content.split('\n'):
-                # Normalize rule names
                 if line.strip().startswith('rule '):
                     rule_name = line.split('{')[0].split('rule ')[1].strip()
-                    # Replace invalid characters and ensure unique names
                     clean_name = rule_name.replace('#', '').replace(' ', '_').replace('-', '_')
-                    clean_name = f"Rule_{rule_counter}_{clean_name}"[:128]
+
+                    # If the rule name is already seen, rename it
+                    if clean_name in seen_rules:
+                        clean_name = f"{clean_name}_{rule_counter}"
+                        rule_counter += 1
+
+                    seen_rules.add(clean_name)
                     processed_content.append(f"rule {clean_name} {{")
                     current_rule = clean_name
-                    rule_counter += 1
-                    string_counter = 1
-                # Make string identifiers unique per rule
-                elif '$a_01_' in line and current_rule:
-                    new_id = f"${current_rule}_str_{string_counter}"
-                    line = line.replace('$a_01_', new_id, 1)
-                    string_counter += 1
-                    processed_content.append(line)
                 else:
                     processed_content.append(line)
 
             # Save processed file
             processed_file = processed_dir / yara_file.name
             processed_file.write_text('\n'.join(processed_content))
-            
+
         except Exception as e:
             logger.warning(f"Error processing {yara_file}: {str(e)}")
             continue
@@ -116,8 +112,8 @@ def preprocess_yara_rules(repo_path: Path) -> Path:
 def compile_yara_rules(repo_path: Path) -> Optional[yara.Rules]:
     """Compile YARA rules from repository with error handling"""
     try:
-        # processed_dir = preprocess_yara_rules(repo_path)
-        yara_files = list(repo_path.glob("**/*.yara"))
+        processed_dir = preprocess_yara_rules(repo_path)
+        yara_files = list(processed_dir.glob("**/*.yara"))
         
         if not yara_files:
             logger.warning("No YARA files found in repository")
@@ -126,9 +122,9 @@ def compile_yara_rules(repo_path: Path) -> Optional[yara.Rules]:
         logger.info(f"Found {len(yara_files)} YARA files, compiling rules")
         rules = {}
         
-        for yara_file in yara_files:
+        for i, yara_file in enumerate(yara_files):
             try:
-                rules[str(yara_file)] = str(yara_file)
+                rules[str(f"{yara_file}_{i}")] = str(yara_file)
             except Exception as e:
                 logger.warning(f"Error processing {yara_file}: {str(e)}")