secforge / app /utils /map_builder.py
rk147's picture
Update app/utils/map_builder.py
22c45fc verified
Raw History Blame Contribute Delete
12.9 kB
"""
Map builder — clean call graph with full NL explanations per node and edge.
Python: ast module (accurate).
Other languages: LLM fallback.
"""
import ast
import json
from dataclasses import dataclass, field
from typing import List, Dict, Optional
from groq import Groq
from app.config import GROQ_API_KEY, MODEL_NAME, MAX_TOKENS
NOISE_BUILTINS = {
"print", "len", "range", "append", "extend", "str", "int", "float",
"list", "dict", "set", "tuple", "isinstance", "hasattr", "getattr",
"setattr", "enumerate", "zip", "map", "filter", "sorted", "reversed",
"open", "format", "repr", "type", "super", "staticmethod", "classmethod",
"property", "encode", "decode", "strip", "split", "join", "lower",
"upper", "replace", "get", "keys", "values", "items", "update", "pop",
"insert", "remove", "index", "count", "copy", "clear", "write", "read",
"close", "next", "iter", "vars", "dir", "id", "hash", "abs", "round",
}
@dataclass
class GraphNode:
id: str
label: str
kind: str
file: str
line: int
description: str = ""
explanation: str = ""
@dataclass
class GraphEdge:
source: str
target: str
kind: str
relationship: str = ""
@dataclass
class CallGraph:
nodes: List[GraphNode] = field(default_factory=list)
edges: List[GraphEdge] = field(default_factory=list)
narrative: str = ""
errors: List[str] = field(default_factory=list)
def to_dict(self) -> dict:
return {
"nodes": [n.__dict__ for n in self.nodes],
"edges": [e.__dict__ for e in self.edges],
"narrative": self.narrative,
"errors": self.errors,
}
class _PythonVisitor(ast.NodeVisitor):
def __init__(self, filename: str, code: str):
self.filename = filename
self.code = code
self.nodes: List[GraphNode] = []
self.edges: List[GraphEdge] = []
self._scope_stack: List[str] = []
self._defined: set = set()
self._called_targets: set = set()
def _id(self, name: str) -> str:
return f"{self.filename}::{name}"
def _scope(self) -> Optional[str]:
return self._scope_stack[-1] if self._scope_stack else None
def visit_FunctionDef(self, node):
name = node.name
nid = self._id(name)
self._defined.add(name)
doc = ast.get_docstring(node) or ""
self.nodes.append(GraphNode(
id=nid, label=name, kind="function",
file=self.filename, line=node.lineno,
description=doc[:120],
))
parent = self._scope()
if parent:
edge = GraphEdge(source=self._id(parent), target=nid, kind="calls")
if not any(e.source == edge.source and e.target == edge.target for e in self.edges):
self.edges.append(edge)
self._called_targets.add(nid)
self._scope_stack.append(name)
self.generic_visit(node)
self._scope_stack.pop()
visit_AsyncFunctionDef = visit_FunctionDef
def visit_ClassDef(self, node):
name = node.name
nid = self._id(name)
self._defined.add(name)
doc = ast.get_docstring(node) or ""
self.nodes.append(GraphNode(
id=nid, label=name, kind="class",
file=self.filename, line=node.lineno,
description=doc[:120],
))
self._scope_stack.append(name)
self.generic_visit(node)
self._scope_stack.pop()
def visit_Call(self, node):
scope = self._scope()
if not scope:
self.generic_visit(node)
return
if isinstance(node.func, ast.Name):
callee = node.func.id
elif isinstance(node.func, ast.Attribute):
callee = node.func.attr
else:
self.generic_visit(node)
return
if callee in NOISE_BUILTINS or callee.startswith("__"):
self.generic_visit(node)
return
target_id = self._id(callee)
self._called_targets.add(target_id)
if callee not in self._defined:
if not any(n.id == target_id for n in self.nodes):
self.nodes.append(GraphNode(
id=target_id, label=callee, kind="external",
file=self.filename, line=getattr(node, "lineno", 0),
))
edge = GraphEdge(source=self._id(scope), target=target_id, kind="calls")
if not any(e.source == edge.source and e.target == edge.target for e in self.edges):
self.edges.append(edge)
self.generic_visit(node)
def resolve_entry_points(self):
all_targets = {e.target for e in self.edges}
for node in self.nodes:
if node.kind == "function" and node.id not in all_targets:
node.kind = "entry"
def _analyze_python(filename: str, code: str) -> CallGraph:
graph = CallGraph()
try:
tree = ast.parse(code)
visitor = _PythonVisitor(filename, code)
visitor.visit(tree)
visitor.resolve_entry_points()
graph.nodes = visitor.nodes
graph.edges = visitor.edges
except SyntaxError as e:
graph.errors.append(f"Syntax error in {filename}: {e}")
return graph
def _analyze_llm_fallback(filename: str, code: str, language: str) -> CallGraph:
graph = CallGraph()
try:
client = Groq(api_key=GROQ_API_KEY)
prompt = f"""Analyze this {language} code. Return ONLY valid JSON:
{{"nodes":[{{"id":"uid","label":"name","kind":"function|class|entry|external","line":1,"description":"one sentence"}}],"edges":[{{"source":"id1","target":"id2","kind":"calls"}}]}}
Only functions/classes. No branches/loops/exceptions. Skip trivial calls.
File: {filename}
Code:
{code[:3000]}
JSON only:"""
resp = client.chat.completions.create(
model=MODEL_NAME,
messages=[{"role": "user", "content": prompt}],
max_tokens=MAX_TOKENS, temperature=0.1,
)
text = resp.choices[0].message.content.strip()
s, e = text.find("{"), text.rfind("}") + 1
if s >= 0 and e > s:
data = json.loads(text[s:e])
for n in data.get("nodes", []):
graph.nodes.append(GraphNode(
id=f"{filename}::{n.get('id','')}",
label=n.get("label", ""),
kind=n.get("kind", "function"),
file=filename, line=n.get("line", 0),
description=n.get("description", ""),
))
for e_ in data.get("edges", []):
graph.edges.append(GraphEdge(
source=f"{filename}::{e_.get('source','')}",
target=f"{filename}::{e_.get('target','')}",
kind=e_.get("kind", "calls"),
))
except Exception as ex:
graph.errors.append(f"LLM fallback error for {filename}: {ex}")
return graph
def _generate_node_explanations(nodes: List[GraphNode], files: dict) -> List[GraphNode]:
targets = [n for n in nodes if n.kind in ("function", "class", "entry") and not n.explanation]
if not targets:
return nodes
snippets = []
for n in targets[:25]:
code_text, _ = files.get(n.file, ("", ""))
lines = code_text.split("\n")
start = max(0, n.line - 1)
snippet = "\n".join(lines[start:start + 12])
snippets.append(f"### {n.label} ({n.kind})\n```\n{snippet}\n```")
try:
client = Groq(api_key=GROQ_API_KEY)
prompt = f"""For each function/class below, return a JSON array (same order) where each object has:
- "short": one sentence max 10 words, what it does
- "full": 2-3 sentences — what it does in detail, what inputs it takes, what it returns, what breaks if it fails
Return ONLY a valid JSON array. No markdown.
{chr(10).join(f'{i+1}. {s}' for i, s in enumerate(snippets))}
JSON array:"""
resp = client.chat.completions.create(
model=MODEL_NAME,
messages=[{"role": "user", "content": prompt}],
max_tokens=1500, temperature=0.2,
)
text = resp.choices[0].message.content.strip()
s = text.find("[")
e = text.rfind("]") + 1
if s >= 0 and e > s:
results = json.loads(text[s:e])
for i, n in enumerate(targets):
if i < len(results):
n.description = results[i].get("short", n.description)
n.explanation = results[i].get("full", "")
except Exception:
pass
return nodes
def _generate_edge_relationships(edges: List[GraphEdge], nodes: List[GraphNode], files: dict) -> List[GraphEdge]:
node_map = {n.id: n for n in nodes}
meaningful = [
e for e in edges
if e.kind == "calls"
and node_map.get(e.source) and node_map.get(e.source).kind != "external"
and node_map.get(e.target) and node_map.get(e.target).kind != "external"
and not e.relationship
][:30]
if not meaningful:
return edges
pairs = []
for e in meaningful:
src = node_map.get(e.source)
tgt = node_map.get(e.target)
src_desc = (src.description or src.label) if src else e.source.split("::")[-1]
tgt_desc = (tgt.description or tgt.label) if tgt else e.target.split("::")[-1]
pairs.append(f'"{src_desc}" calls "{tgt_desc}"')
try:
client = Groq(api_key=GROQ_API_KEY)
prompt = f"""For each function call relationship, write a SHORT phrase (5-8 words) describing what they do together.
Focus on data/purpose flow. Examples: "passes user input for validation", "delegates file parsing to", "triggers cleanup after error".
Return ONLY a JSON array of strings in same order. No markdown.
{chr(10).join(f'{i+1}. {p}' for i, p in enumerate(pairs))}
JSON array:"""
resp = client.chat.completions.create(
model=MODEL_NAME,
messages=[{"role": "user", "content": prompt}],
max_tokens=800, temperature=0.3,
)
text = resp.choices[0].message.content.strip()
s = text.find("[")
e_idx = text.rfind("]") + 1
if s >= 0 and e_idx > s:
results = json.loads(text[s:e_idx])
for i, edge in enumerate(meaningful):
if i < len(results):
edge.relationship = results[i]
except Exception:
pass
return edges
def _generate_narrative(graph: CallGraph) -> str:
try:
client = Groq(api_key=GROQ_API_KEY)
entries = [n for n in graph.nodes if n.kind == "entry"]
funcs = [n for n in graph.nodes if n.kind in ("function", "class")]
edge_summary = "\n".join(
f" {e.source.split('::')[-1]} → {e.target.split('::')[-1]}"
+ (f" ({e.relationship})" if e.relationship else "")
for e in graph.edges[:40] if e.kind == "calls"
)
prompt = f"""Write a plain-English narrative (4-6 sentences) for this codebase:
- What starts execution and why
- Main flow through the code
- Key dependencies and handoffs
- What the system achieves overall
Entry points: {', '.join(n.label + (f' ({n.description})' if n.description else '') for n in entries[:8])}
Functions: {', '.join(n.label + (f' ({n.description})' if n.description else '') for n in funcs[:20])}
Flow:
{edge_summary}
Narrative:"""
resp = client.chat.completions.create(
model=MODEL_NAME,
messages=[{"role": "user", "content": prompt}],
max_tokens=400, temperature=0.3,
)
return resp.choices[0].message.content.strip()
except Exception as e:
return f"Could not generate narrative: {e}"
def build_graph(files: dict) -> CallGraph:
from app.modules.language_detector import detect_language
combined = CallGraph()
for filename, (code, language) in files.items():
if language == "python":
g = _analyze_python(filename, code)
else:
g = _analyze_llm_fallback(filename, code, language)
combined.nodes.extend(g.nodes)
combined.edges.extend(g.edges)
combined.errors.extend(g.errors)
seen = set()
deduped = []
for n in combined.nodes:
if n.id not in seen:
seen.add(n.id)
deduped.append(n)
combined.nodes = deduped
defined_ids = {n.id for n in combined.nodes if n.kind != "external"}
has_incoming = set()
for e in combined.edges:
if e.source in defined_ids:
has_incoming.add(e.target)
combined.nodes = [
n for n in combined.nodes
if n.kind != "external" or n.id in has_incoming
]
combined.nodes = _generate_node_explanations(combined.nodes, files)
combined.edges = _generate_edge_relationships(combined.edges, combined.nodes, files)
combined.narrative = _generate_narrative(combined)
return combined