Spaces:
Sleeping
Sleeping
File size: 3,050 Bytes
5ce9fab | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 | from __future__ import annotations
import json
import re
import zipfile
from pathlib import Path
from xml.etree import ElementTree
import pymupdf4llm
RUBRIC_EXTENSIONS = {".pdf", ".docx", ".txt", ".md"}
STUDENT_EXTENSIONS = RUBRIC_EXTENSIONS | {".ipynb"}
def supported_extensions(for_student: bool = False) -> set[str]:
return STUDENT_EXTENSIONS if for_student else RUBRIC_EXTENSIONS
def is_supported(filename: str, *, for_student: bool = False) -> bool:
return Path(filename).suffix.lower() in supported_extensions(for_student)
def supported_extensions_label(*, for_student: bool = False) -> str:
return ", ".join(sorted(supported_extensions(for_student)))
def clean_text(text: str) -> str:
text = text.replace("\x00", " ")
text = re.sub(r"[ \t]+", " ", text)
text = re.sub(r"\n{3,}", "\n\n", text)
return text.strip()
def _read_plain_text(path: Path) -> str:
return path.read_text("utf-8", errors="replace")
def _xml_text(xml_bytes: bytes) -> str:
root = ElementTree.fromstring(xml_bytes)
ns = {"w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main"}
blocks: list[str] = []
for paragraph in root.findall(".//w:p", ns):
text = "".join(node.text or "" for node in paragraph.findall(".//w:t", ns))
if text.strip():
blocks.append(text)
return "\n".join(blocks)
def _read_docx(path: Path) -> str:
parts = [
"word/document.xml",
"word/comments.xml",
"word/footnotes.xml",
"word/endnotes.xml",
]
with zipfile.ZipFile(path) as archive:
text_parts = []
for part in parts:
try:
text = _xml_text(archive.read(part))
except KeyError:
continue
if text.strip():
text_parts.append(text)
return "\n\n".join(text_parts)
def _read_ipynb(path: Path) -> str:
try:
import nbformat
from nbconvert import MarkdownExporter
notebook = nbformat.read(path, as_version=4)
exporter = MarkdownExporter()
body, _ = exporter.from_notebook_node(notebook)
return body
except Exception:
data = json.loads(path.read_text("utf-8", errors="replace"))
blocks = []
for cell in data.get("cells", []):
source = cell.get("source", "")
if isinstance(source, list):
source = "".join(source)
if source:
blocks.append(source)
return "\n\n".join(blocks)
def parse_file_text(path: Path, filename: str | None = None) -> str:
name = filename or path.name
suffix = Path(name).suffix.lower()
if suffix == ".pdf":
text = pymupdf4llm.to_markdown(str(path))
elif suffix == ".docx":
text = _read_docx(path)
elif suffix in {".txt", ".md"}:
text = _read_plain_text(path)
elif suffix == ".ipynb":
text = _read_ipynb(path)
else:
raise ValueError(f"Unsupported file type: {suffix or name}")
return clean_text(text)
|