Download app/utils/map_builder.py from rk147/secforge: direct link, hf CLI and curl.
- Browser
- Download file 12.9 kB
-
https://huggingface.co/spaces/rk147/secforge/resolve/main/app/utils/map_builder.py
- Command line
-
hf download hf://spaces/rk147/secforge/app/utils/map_builder.py
-
curl -L -o map_builder.py https://huggingface.co/spaces/rk147/secforge/resolve/main/app/utils/map_builder.py
12.9 kB
| """ | |
| Map builder — clean call graph with full NL explanations per node and edge. | |
| Python: ast module (accurate). | |
| Other languages: LLM fallback. | |
| """ | |
| import ast | |
| import json | |
| from dataclasses import dataclass, field | |
| from typing import List, Dict, Optional | |
| from groq import Groq | |
| from app.config import GROQ_API_KEY, MODEL_NAME, MAX_TOKENS | |
| NOISE_BUILTINS = { | |
| "print", "len", "range", "append", "extend", "str", "int", "float", | |
| "list", "dict", "set", "tuple", "isinstance", "hasattr", "getattr", | |
| "setattr", "enumerate", "zip", "map", "filter", "sorted", "reversed", | |
| "open", "format", "repr", "type", "super", "staticmethod", "classmethod", | |
| "property", "encode", "decode", "strip", "split", "join", "lower", | |
| "upper", "replace", "get", "keys", "values", "items", "update", "pop", | |
| "insert", "remove", "index", "count", "copy", "clear", "write", "read", | |
| "close", "next", "iter", "vars", "dir", "id", "hash", "abs", "round", | |
| } | |
| class GraphNode: | |
| id: str | |
| label: str | |
| kind: str | |
| file: str | |
| line: int | |
| description: str = "" | |
| explanation: str = "" | |
| class GraphEdge: | |
| source: str | |
| target: str | |
| kind: str | |
| relationship: str = "" | |
| class CallGraph: | |
| nodes: List[GraphNode] = field(default_factory=list) | |
| edges: List[GraphEdge] = field(default_factory=list) | |
| narrative: str = "" | |
| errors: List[str] = field(default_factory=list) | |
| def to_dict(self) -> dict: | |
| return { | |
| "nodes": [n.__dict__ for n in self.nodes], | |
| "edges": [e.__dict__ for e in self.edges], | |
| "narrative": self.narrative, | |
| "errors": self.errors, | |
| } | |
| class _PythonVisitor(ast.NodeVisitor): | |
| def __init__(self, filename: str, code: str): | |
| self.filename = filename | |
| self.code = code | |
| self.nodes: List[GraphNode] = [] | |
| self.edges: List[GraphEdge] = [] | |
| self._scope_stack: List[str] = [] | |
| self._defined: set = set() | |
| self._called_targets: set = set() | |
| def _id(self, name: str) -> str: | |
| return f"{self.filename}::{name}" | |
| def _scope(self) -> Optional[str]: | |
| return self._scope_stack[-1] if self._scope_stack else None | |
| def visit_FunctionDef(self, node): | |
| name = node.name | |
| nid = self._id(name) | |
| self._defined.add(name) | |
| doc = ast.get_docstring(node) or "" | |
| self.nodes.append(GraphNode( | |
| id=nid, label=name, kind="function", | |
| file=self.filename, line=node.lineno, | |
| description=doc[:120], | |
| )) | |
| parent = self._scope() | |
| if parent: | |
| edge = GraphEdge(source=self._id(parent), target=nid, kind="calls") | |
| if not any(e.source == edge.source and e.target == edge.target for e in self.edges): | |
| self.edges.append(edge) | |
| self._called_targets.add(nid) | |
| self._scope_stack.append(name) | |
| self.generic_visit(node) | |
| self._scope_stack.pop() | |
| visit_AsyncFunctionDef = visit_FunctionDef | |
| def visit_ClassDef(self, node): | |
| name = node.name | |
| nid = self._id(name) | |
| self._defined.add(name) | |
| doc = ast.get_docstring(node) or "" | |
| self.nodes.append(GraphNode( | |
| id=nid, label=name, kind="class", | |
| file=self.filename, line=node.lineno, | |
| description=doc[:120], | |
| )) | |
| self._scope_stack.append(name) | |
| self.generic_visit(node) | |
| self._scope_stack.pop() | |
| def visit_Call(self, node): | |
| scope = self._scope() | |
| if not scope: | |
| self.generic_visit(node) | |
| return | |
| if isinstance(node.func, ast.Name): | |
| callee = node.func.id | |
| elif isinstance(node.func, ast.Attribute): | |
| callee = node.func.attr | |
| else: | |
| self.generic_visit(node) | |
| return | |
| if callee in NOISE_BUILTINS or callee.startswith("__"): | |
| self.generic_visit(node) | |
| return | |
| target_id = self._id(callee) | |
| self._called_targets.add(target_id) | |
| if callee not in self._defined: | |
| if not any(n.id == target_id for n in self.nodes): | |
| self.nodes.append(GraphNode( | |
| id=target_id, label=callee, kind="external", | |
| file=self.filename, line=getattr(node, "lineno", 0), | |
| )) | |
| edge = GraphEdge(source=self._id(scope), target=target_id, kind="calls") | |
| if not any(e.source == edge.source and e.target == edge.target for e in self.edges): | |
| self.edges.append(edge) | |
| self.generic_visit(node) | |
| def resolve_entry_points(self): | |
| all_targets = {e.target for e in self.edges} | |
| for node in self.nodes: | |
| if node.kind == "function" and node.id not in all_targets: | |
| node.kind = "entry" | |
| def _analyze_python(filename: str, code: str) -> CallGraph: | |
| graph = CallGraph() | |
| try: | |
| tree = ast.parse(code) | |
| visitor = _PythonVisitor(filename, code) | |
| visitor.visit(tree) | |
| visitor.resolve_entry_points() | |
| graph.nodes = visitor.nodes | |
| graph.edges = visitor.edges | |
| except SyntaxError as e: | |
| graph.errors.append(f"Syntax error in {filename}: {e}") | |
| return graph | |
| def _analyze_llm_fallback(filename: str, code: str, language: str) -> CallGraph: | |
| graph = CallGraph() | |
| try: | |
| client = Groq(api_key=GROQ_API_KEY) | |
| prompt = f"""Analyze this {language} code. Return ONLY valid JSON: | |
| {{"nodes":[{{"id":"uid","label":"name","kind":"function|class|entry|external","line":1,"description":"one sentence"}}],"edges":[{{"source":"id1","target":"id2","kind":"calls"}}]}} | |
| Only functions/classes. No branches/loops/exceptions. Skip trivial calls. | |
| File: {filename} | |
| Code: | |
| {code[:3000]} | |
| JSON only:""" | |
| resp = client.chat.completions.create( | |
| model=MODEL_NAME, | |
| messages=[{"role": "user", "content": prompt}], | |
| max_tokens=MAX_TOKENS, temperature=0.1, | |
| ) | |
| text = resp.choices[0].message.content.strip() | |
| s, e = text.find("{"), text.rfind("}") + 1 | |
| if s >= 0 and e > s: | |
| data = json.loads(text[s:e]) | |
| for n in data.get("nodes", []): | |
| graph.nodes.append(GraphNode( | |
| id=f"{filename}::{n.get('id','')}", | |
| label=n.get("label", ""), | |
| kind=n.get("kind", "function"), | |
| file=filename, line=n.get("line", 0), | |
| description=n.get("description", ""), | |
| )) | |
| for e_ in data.get("edges", []): | |
| graph.edges.append(GraphEdge( | |
| source=f"{filename}::{e_.get('source','')}", | |
| target=f"{filename}::{e_.get('target','')}", | |
| kind=e_.get("kind", "calls"), | |
| )) | |
| except Exception as ex: | |
| graph.errors.append(f"LLM fallback error for {filename}: {ex}") | |
| return graph | |
| def _generate_node_explanations(nodes: List[GraphNode], files: dict) -> List[GraphNode]: | |
| targets = [n for n in nodes if n.kind in ("function", "class", "entry") and not n.explanation] | |
| if not targets: | |
| return nodes | |
| snippets = [] | |
| for n in targets[:25]: | |
| code_text, _ = files.get(n.file, ("", "")) | |
| lines = code_text.split("\n") | |
| start = max(0, n.line - 1) | |
| snippet = "\n".join(lines[start:start + 12]) | |
| snippets.append(f"### {n.label} ({n.kind})\n```\n{snippet}\n```") | |
| try: | |
| client = Groq(api_key=GROQ_API_KEY) | |
| prompt = f"""For each function/class below, return a JSON array (same order) where each object has: | |
| - "short": one sentence max 10 words, what it does | |
| - "full": 2-3 sentences — what it does in detail, what inputs it takes, what it returns, what breaks if it fails | |
| Return ONLY a valid JSON array. No markdown. | |
| {chr(10).join(f'{i+1}. {s}' for i, s in enumerate(snippets))} | |
| JSON array:""" | |
| resp = client.chat.completions.create( | |
| model=MODEL_NAME, | |
| messages=[{"role": "user", "content": prompt}], | |
| max_tokens=1500, temperature=0.2, | |
| ) | |
| text = resp.choices[0].message.content.strip() | |
| s = text.find("[") | |
| e = text.rfind("]") + 1 | |
| if s >= 0 and e > s: | |
| results = json.loads(text[s:e]) | |
| for i, n in enumerate(targets): | |
| if i < len(results): | |
| n.description = results[i].get("short", n.description) | |
| n.explanation = results[i].get("full", "") | |
| except Exception: | |
| pass | |
| return nodes | |
| def _generate_edge_relationships(edges: List[GraphEdge], nodes: List[GraphNode], files: dict) -> List[GraphEdge]: | |
| node_map = {n.id: n for n in nodes} | |
| meaningful = [ | |
| e for e in edges | |
| if e.kind == "calls" | |
| and node_map.get(e.source) and node_map.get(e.source).kind != "external" | |
| and node_map.get(e.target) and node_map.get(e.target).kind != "external" | |
| and not e.relationship | |
| ][:30] | |
| if not meaningful: | |
| return edges | |
| pairs = [] | |
| for e in meaningful: | |
| src = node_map.get(e.source) | |
| tgt = node_map.get(e.target) | |
| src_desc = (src.description or src.label) if src else e.source.split("::")[-1] | |
| tgt_desc = (tgt.description or tgt.label) if tgt else e.target.split("::")[-1] | |
| pairs.append(f'"{src_desc}" calls "{tgt_desc}"') | |
| try: | |
| client = Groq(api_key=GROQ_API_KEY) | |
| prompt = f"""For each function call relationship, write a SHORT phrase (5-8 words) describing what they do together. | |
| Focus on data/purpose flow. Examples: "passes user input for validation", "delegates file parsing to", "triggers cleanup after error". | |
| Return ONLY a JSON array of strings in same order. No markdown. | |
| {chr(10).join(f'{i+1}. {p}' for i, p in enumerate(pairs))} | |
| JSON array:""" | |
| resp = client.chat.completions.create( | |
| model=MODEL_NAME, | |
| messages=[{"role": "user", "content": prompt}], | |
| max_tokens=800, temperature=0.3, | |
| ) | |
| text = resp.choices[0].message.content.strip() | |
| s = text.find("[") | |
| e_idx = text.rfind("]") + 1 | |
| if s >= 0 and e_idx > s: | |
| results = json.loads(text[s:e_idx]) | |
| for i, edge in enumerate(meaningful): | |
| if i < len(results): | |
| edge.relationship = results[i] | |
| except Exception: | |
| pass | |
| return edges | |
| def _generate_narrative(graph: CallGraph) -> str: | |
| try: | |
| client = Groq(api_key=GROQ_API_KEY) | |
| entries = [n for n in graph.nodes if n.kind == "entry"] | |
| funcs = [n for n in graph.nodes if n.kind in ("function", "class")] | |
| edge_summary = "\n".join( | |
| f" {e.source.split('::')[-1]} → {e.target.split('::')[-1]}" | |
| + (f" ({e.relationship})" if e.relationship else "") | |
| for e in graph.edges[:40] if e.kind == "calls" | |
| ) | |
| prompt = f"""Write a plain-English narrative (4-6 sentences) for this codebase: | |
| - What starts execution and why | |
| - Main flow through the code | |
| - Key dependencies and handoffs | |
| - What the system achieves overall | |
| Entry points: {', '.join(n.label + (f' ({n.description})' if n.description else '') for n in entries[:8])} | |
| Functions: {', '.join(n.label + (f' ({n.description})' if n.description else '') for n in funcs[:20])} | |
| Flow: | |
| {edge_summary} | |
| Narrative:""" | |
| resp = client.chat.completions.create( | |
| model=MODEL_NAME, | |
| messages=[{"role": "user", "content": prompt}], | |
| max_tokens=400, temperature=0.3, | |
| ) | |
| return resp.choices[0].message.content.strip() | |
| except Exception as e: | |
| return f"Could not generate narrative: {e}" | |
| def build_graph(files: dict) -> CallGraph: | |
| from app.modules.language_detector import detect_language | |
| combined = CallGraph() | |
| for filename, (code, language) in files.items(): | |
| if language == "python": | |
| g = _analyze_python(filename, code) | |
| else: | |
| g = _analyze_llm_fallback(filename, code, language) | |
| combined.nodes.extend(g.nodes) | |
| combined.edges.extend(g.edges) | |
| combined.errors.extend(g.errors) | |
| seen = set() | |
| deduped = [] | |
| for n in combined.nodes: | |
| if n.id not in seen: | |
| seen.add(n.id) | |
| deduped.append(n) | |
| combined.nodes = deduped | |
| defined_ids = {n.id for n in combined.nodes if n.kind != "external"} | |
| has_incoming = set() | |
| for e in combined.edges: | |
| if e.source in defined_ids: | |
| has_incoming.add(e.target) | |
| combined.nodes = [ | |
| n for n in combined.nodes | |
| if n.kind != "external" or n.id in has_incoming | |
| ] | |
| combined.nodes = _generate_node_explanations(combined.nodes, files) | |
| combined.edges = _generate_edge_relationships(combined.edges, combined.nodes, files) | |
| combined.narrative = _generate_narrative(combined) | |
| return combined |