File size: 4,560 Bytes
eca5751 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 | """Code Formatter - Format code samples cho training."""
from __future__ import annotations
import re
from typing import Dict, Any, List, Optional
class CodeFormatter:
"""Format code samples cho training.
Features:
- Strip excessive blank lines
- Normalize indentation
- Add language tags to code blocks
- Wrap code in markdown fences if needed
- Detect language automatically
"""
LANG_BY_EXT = {
".py": "python", ".js": "javascript", ".ts": "typescript",
".go": "go", ".rs": "rust", ".java": "java",
".c": "c", ".cpp": "cpp", ".h": "c", ".hpp": "cpp",
".cs": "csharp", ".rb": "ruby", ".php": "php",
".swift": "swift", ".kt": "kotlin", ".scala": "scala",
".sql": "sql", ".sh": "bash", ".bash": "bash",
".html": "html", ".css": "css", ".json": "json",
".yaml": "yaml", ".yml": "yaml", ".toml": "toml",
".xml": "xml", ".md": "markdown",
}
# Language detection patterns
LANG_PATTERNS = {
"python": [r"^\s*def\s+\w+", r"^\s*class\s+\w+", r"^\s*import\s+\w+", r"^\s*from\s+\w+\s+import"],
"javascript": [r"^\s*function\s+\w+", r"^\s*const\s+\w+\s*=", r"^\s*let\s+\w+\s*=", r"=>\s*\{?"],
"typescript": [r":\s*(string|number|boolean|void|any)\b", r"interface\s+\w+", r"type\s+\w+\s*="],
"go": [r"^\s*func\s+\w+", r"^\s*package\s+\w+", r"^\s*import\s+\("],
"rust": [r"^\s*fn\s+\w+", r"^\s*impl\s+\w+", r"^\s*use\s+\w+", r"^\s*let\s+mut\s+"],
"java": [r"^\s*public\s+class\s+\w+", r"^\s*private\s+\w+\s+\w+", r"^\s*import\s+java\."],
"c": [r"^\s*#include\s*<", r"^\s*int\s+main\s*\("],
"cpp": [r"^\s*#include\s*<", r"^\s*std::", r"^\s*template\s*<"],
}
def detect_language(self, code: str, filename: Optional[str] = None) -> Optional[str]:
"""Detect programming language of code."""
if filename:
import os
ext = os.path.splitext(filename)[1].lower()
if ext in self.LANG_BY_EXT:
return self.LANG_BY_EXT[ext]
# Pattern matching
for lang, patterns in self.LANG_PATTERNS.items():
for pattern in patterns:
if re.search(pattern, code, re.MULTILINE):
return lang
return None
def format(self, code: str, language: Optional[str] = None) -> str:
"""Format code sample."""
# Detect language if not provided
if not language:
language = self.detect_language(code) or ""
# Strip trailing whitespace on each line
lines = [line.rstrip() for line in code.splitlines()]
# Remove excessive blank lines (max 2 consecutive)
formatted_lines = []
blank_count = 0
for line in lines:
if line.strip() == "":
blank_count += 1
if blank_count <= 2:
formatted_lines.append("")
else:
blank_count = 0
formatted_lines.append(line)
# Remove leading/trailing blank lines
while formatted_lines and formatted_lines[0] == "":
formatted_lines.pop(0)
while formatted_lines and formatted_lines[-1] == "":
formatted_lines.pop()
code_clean = "\n".join(formatted_lines)
return code_clean
def wrap_in_markdown(self, code: str, language: Optional[str] = None) -> str:
"""Wrap code in markdown fence."""
if not language:
language = self.detect_language(code) or ""
return f"```{language}\n{code}\n```"
def process(self, sample: Dict[str, Any]) -> Dict[str, Any]:
"""Process a code sample."""
sample = dict(sample)
text = sample.get("text", "")
language = sample.get("language") or sample.get("metadata", {}).get("language")
# Check if it's code
is_code = (
sample.get("language") or
sample.get("metadata", {}).get("language") or
self.detect_language(text) is not None
)
if is_code:
formatted = self.format(text, language)
sample["text"] = formatted
sample["metadata"] = sample.get("metadata", {})
sample["metadata"]["formatted"] = True
if not language:
language = self.detect_language(text)
sample["metadata"]["detected_language"] = language
return sample
|