update doc

gradio_mcp_space.py

@@ -108,16 +108,42 @@ def initialize_knowledge_graph(
 @observe(as_type="tool")
 def get_node_info(node_id: str) -> str:
     """
-    Get detailed information about a node in the knowledge graph.
-
-    Returns information including the node's type, name, description,
-    declared/called entities, and type-specific details.
+    Retrieve comprehensive details about any node in the Transformers library knowledge graph.
+
+    PURPOSE:
+    Use this tool to inspect the full metadata and content of a specific node when you need
+    to understand what a particular code element contains, what entities it declares or calls,
+    and how it fits into the codebase structure.
+
+    WHEN TO USE:
+    - After finding a node ID from search_nodes, list_nodes_by_type, or get_neighbors
+    - To see the actual code content of a chunk node
+    - To understand what entities (classes, functions, variables) are declared in a file or chunk
+    - To examine entity metadata including aliases, declaration locations, and usage locations
+    - To get file metadata like language and path information
+
+    NODE TYPES SUPPORTED:
+    - 'chunk': Code segments with content, declared/called entities, and file position
+    - 'file': Source files with path, language, and entity summaries
+    - 'directory': Folder nodes with path information
+    - 'entity': Programming constructs (classes, functions, methods, variables) with declaration/usage tracking
+    - 'repo': Repository root node
+
+    TYPICAL WORKFLOW:
+    1. search_nodes("attention mechanism") -> get node IDs
+    2. get_node_info(node_id) -> see full content and metadata
+    3. get_neighbors(node_id) or find_usages(entity_name) -> explore relationships
 
     Args:
-        node_id: The ID of the node to retrieve information for
+        node_id: The unique identifier of the node (e.g., 'src/transformers/models/bert/modeling_bert.py::chunk_3' for chunks, or 'BertModel' for entities)
 
     Returns:
-        str: A formatted string with node information
+        str: Formatted details including node type, name, description, content (for chunks), declared entities, called entities, and type-specific metadata
+
+    Example node_ids:
+        - Chunk: 'src/transformers/models/bert/modeling_bert.py::chunk_5'
+        - File: 'src/transformers/models/bert/modeling_bert.py'
+        - Entity: 'BertModel', 'forward', 'attention_mask'
     """
     if knowledge_graph is None:
         return "Error: Knowledge graph not initialized"
@@ -195,15 +221,37 @@ def get_node_info(node_id: str) -> str:
 @observe(as_type="tool")
 def get_node_edges(node_id: str) -> str:
     """
-    List all incoming and outgoing edges for a node.
+    List all graph edges (relationships) connected to a specific node in the knowledge graph.
+
+    PURPOSE:
+    Use this tool to understand how a node is connected to other parts of the codebase.
+    Reveals the dependency structure and relationships that link code elements together.
+
+    WHEN TO USE:
+    - To discover what code calls or depends on a specific function/class
+    - To find parent-child relationships (e.g., which file contains a chunk)
+    - To trace declaration and usage patterns through the codebase
+    - To understand the connectivity of an entity in the dependency graph
+    - When you need a raw view of all relationships without filtering
+
+    EDGE TYPES YOU'LL SEE:
+    - 'contains': Parent-child (file→chunk, directory→file, repo→directory)
+    - 'calls': Entity usage relationships (chunk→entity it calls)
+    - 'declares': Entity declaration relationships (chunk→entity it defines)
+
+    DIRECTION MEANINGS:
+    - Incoming edges (←): Other nodes pointing TO this node (e.g., "who calls me?")
+    - Outgoing edges (→): This node pointing TO others (e.g., "what do I call?")
 
-    Shows relationships to other nodes in the knowledge graph.
+    COMPARISON WITH get_neighbors:
+    - get_node_edges: Shows edge metadata and direction, raw relationship view
+    - get_neighbors: Shows neighboring node details, easier for exploration
 
     Args:
-        node_id: The ID of the node whose edges to list
+        node_id: The unique identifier of the node to inspect edges for
 
     Returns:
-        str: A formatted string showing all edges
+        str: List of incoming and outgoing edges with source/target node IDs and relationship types
     """
     if knowledge_graph is None:
         return "Error: Knowledge graph not initialized"
@@ -247,17 +295,39 @@ Incoming Edges ({len(incoming)}):
 @observe(as_type="tool")
 def search_nodes(query: str, limit: int = 10, page: int = 1) -> str:
     """
-    Search for chunk nodes in the knowledge graph by query string.
-
-    Uses keyword search via the code index.
+    Search the Transformers codebase using keyword matching against code content and metadata.
+
+    PURPOSE:
+    This is your PRIMARY SEARCH TOOL for exploring the codebase. Use it to find relevant
+    code chunks based on natural language queries, function names, class names, comments,
+    or any text that might appear in the source code.
+
+    WHEN TO USE:
+    - FIRST STEP when investigating any topic in the Transformers library
+    - To find implementations of specific features (e.g., "rotary embeddings", "flash attention")
+    - To locate code by function/class name when you don't have the exact node ID
+    - To discover code related to a concept (e.g., "gradient checkpointing", "tokenization")
+    - When you don't know where something is implemented
+
+    SEARCH TIPS:
+    - Use specific technical terms: "rope embedding" rather than just "embedding"
+    - Include class/function names if known: "BertSelfAttention forward"
+    - Try multiple related queries if first results aren't satisfactory
+    - Results are ranked by relevance to your query
+
+    TYPICAL WORKFLOW:
+    1. search_nodes("attention mask handling") -> find relevant chunks
+    2. get_node_info(chunk_id) -> examine the code content
+    3. get_chunk_context(chunk_id) -> see surrounding code for fuller picture
+    4. go_to_definition(entity_name) -> find where an entity is defined
 
     Args:
-        query: The search string to match against code index
-        limit: Maximum number of results to return per page (default: 10)
-        page: Page number for pagination, 1-indexed (default: 1)
+        query: Search terms to match against code content. Can be natural language, function names, class names, or code snippets. More specific queries yield better results.
+        limit: Results per page (default: 10, max recommended: 50). Use smaller limits for faster responses.
+        page: Page number starting from 1. Use pagination to browse through many results.
 
     Returns:
-        str: A formatted string with search results
+        str: Ranked list of matching code chunks with IDs and content previews. Use the returned IDs with get_node_info or get_chunk_context for full details.
     """
     if knowledge_graph is None:
         return "Error: Knowledge graph not initialized"
@@ -321,18 +391,32 @@ def search_nodes(query: str, limit: int = 10, page: int = 1) -> str:
 @observe(as_type="tool")
 def get_graph_stats() -> str:
     """
-    Get comprehensive statistics about the knowledge graph.
-
-    Returns detailed information about the repository structure including:
-    - Chunks: Code segments that represent portions of files (functions, classes, etc.)
-    - Entities: Programming constructs like classes, functions, methods, variables
-    - Files and directories in the repository
-    - Relationships between different components
-
-    For entity nodes, provides a breakdown by entity type (class, function, method, etc.).
+    Get a comprehensive statistical overview of the Transformers library knowledge graph.
+
+    PURPOSE:
+    Use this tool to understand the scope and structure of the knowledge graph.
+    Provides counts and breakdowns of all node types, entity types, and relationship types.
+
+    WHEN TO USE:
+    - At the START of an exploration session to understand the codebase scope
+    - To learn what types of entities and relationships are available for querying
+    - To understand the terminology used in this knowledge graph (chunks, entities, edges)
+    - When you need to report on the overall structure of the Transformers library
+
+    WHAT YOU'LL LEARN:
+    - Total number of nodes and edges in the graph
+    - Breakdown of node types (chunks, files, directories, entities)
+    - Entity type distribution (classes, functions, methods, variables, etc.)
+    - Edge relationship types (contains, calls, declares)
+    - Definitions of key concepts used throughout the tools
+
+    GRAPH TERMINOLOGY:
+    - Chunks: Logical code segments (a function body, a class definition, etc.)
+    - Entities: Named programming constructs tracked across the codebase
+    - Edges: Relationships connecting nodes (contains, calls, declares)
 
     Returns:
-        str: A formatted string with comprehensive graph statistics
+        str: Detailed statistics including node counts by type, entity breakdown, edge relation counts, and concept definitions to help you use other tools effectively.
     """
     if knowledge_graph is None:
         return "Error: Knowledge graph not initialized"
@@ -436,17 +520,44 @@ Edge Relations:
 @observe(as_type="tool")
 def list_nodes_by_type(node_type: str, limit: int = 20, page: int = 1) -> str:
     """
-    List nodes of a specific type in the knowledge graph.
-    
-    For entities, use entity_type (e.g., 'class', 'function', 'method').
-    For other nodes, use node_type (e.g., 'file', 'chunk', 'directory').
-    
+    List all nodes of a specific type in the Transformers knowledge graph with pagination.
+
+    PURPOSE:
+    Use this tool to browse and discover nodes by their type. Helpful when you want to
+    see what classes, functions, files, or other constructs exist in the codebase.
+
+    WHEN TO USE:
+    - To get a list of all classes in the Transformers library: node_type='class'
+    - To see all Python files: node_type='file'
+    - To list all functions: node_type='function'
+    - To browse all methods: node_type='method'
+    - When you need to find node IDs for further exploration
+
+    VALID node_type VALUES:
+    For entities (programming constructs):
+    - 'class': Class definitions (e.g., BertModel, GPT2LMHeadModel)
+    - 'function': Standalone function definitions
+    - 'method': Class method definitions
+    - 'variable': Variable declarations
+    - 'parameter': Function/method parameters
+
+    For structural nodes:
+    - 'file': Source code files
+    - 'chunk': Code segments within files
+    - 'directory': Folder structure nodes
+    - 'repo': Repository root (typically one)
+
+    COMPARISON WITH search_by_type_and_name:
+    - list_nodes_by_type: Browse ALL nodes of a type (no name filter)
+    - search_by_type_and_name: Filter by type AND search by name substring
+
     Args:
-        node_type: The type of nodes to list (e.g., 'function', 'class', 'file')
-        limit: Maximum number of nodes to return (default: 20)
-        page: Page number for pagination (default: 1)
+        node_type: The type to filter by. Use lowercase: 'class', 'function', 'method', 'file', 'chunk', 'directory'
+        limit: Maximum results per page (default: 20). Increase for broader browsing.
+        page: Page number starting from 1 for pagination through large result sets.
+
     Returns:
-        str: A formatted string with matching nodes
+        str: Alphabetically sorted list of matching nodes with their IDs and types. Use IDs with get_node_info for details.
     """
     if knowledge_graph is None:
         return "Error: Knowledge graph not initialized"
@@ -537,15 +648,42 @@ def list_nodes_by_type(node_type: str, limit: int = 20, page: int = 1) -> str:
 @observe(as_type="tool")
 def get_neighbors(node_id: str, limit: int = 20, page: int = 1) -> str:
     """
-    Retrieves all nodes directly connected to a given node.
-
-    Retrieves neighboring nodes with their relationship types.
+    Get all nodes directly connected to a given node with their relationship information.
+
+    PURPOSE:
+    Use this tool to explore the local neighborhood of any node in the knowledge graph.
+    Shows what's connected to a node and how, making it easy to navigate the codebase structure.
+
+    WHEN TO USE:
+    - To explore what a node is connected to (files, chunks, entities)
+    - To navigate from one code element to related elements
+    - To understand the local structure around a specific node
+    - After using get_node_info when you want to explore connected nodes
+    - To discover related code without knowing exact names
+
+    WHAT YOU'LL SEE:
+    - Neighbor node IDs and names
+    - Node types (chunk, file, entity, etc.)
+    - Relationship direction (→ outgoing, ← incoming)
+    - Relationship type (contains, calls, declares)
+
+    TYPICAL NAVIGATION PATTERNS:
+    - From a file: see its chunks and declared entities
+    - From a chunk: see entities it declares/calls and its parent file
+    - From an entity: see chunks that declare or call it
+    - From a directory: see contained files and subdirectories
+
+    COMPARISON WITH get_node_edges:
+    - get_neighbors: Shows neighboring NODE details (name, type) - better for exploration
+    - get_node_edges: Shows raw EDGE information - better for understanding relationships
 
     Args:
-        node_id: The ID of the node whose neighbors to retrieve
+        node_id: The ID of the node to explore neighbors for
+        limit: Maximum neighbors to return per page (default: 20)
+        page: Page number for pagination when node has many connections
 
     Returns:
-        str: A formatted string showing all neighbors
+        str: List of connected nodes with their IDs, names, types, and the relationships connecting them
     """
     if knowledge_graph is None:
         return "Error: Knowledge graph not initialized"
@@ -614,15 +752,39 @@ def get_neighbors(node_id: str, limit: int = 20, page: int = 1) -> str:
 @observe(as_type="tool")
 def go_to_definition(entity_name: str) -> str:
     """
-    Retrieve chunk node(s) where entity is declared or defined in the codebase.
-
-    Locates and retrieves the declaration point for functions, classes, variables, etc.
+    Jump to the source code location(s) where an entity is defined/declared.
+
+    PURPOSE:
+    Use this tool to find WHERE in the codebase a class, function, method, or variable
+    is defined. Returns the actual code content of the definition along with file location.
+
+    WHEN TO USE:
+    - To see the implementation of a class like 'BertModel' or 'GPT2Attention'
+    - To find where a function is defined when you know its name
+    - To examine the source code of any entity found through search or listing
+    - When you need to understand HOW something is implemented (not just WHERE it's used)
+    - To get the actual code definition for analysis or explanation
+
+    WHAT YOU'LL GET:
+    - Entity type (class, function, method, variable)
+    - Data type if available
+    - List of all locations where the entity is declared (some entities may be defined in multiple places)
+    - For each location: file path, chunk order, and FULL CODE CONTENT
+
+    TYPICAL WORKFLOW:
+    1. search_nodes("attention") -> find entity names
+    2. go_to_definition("BertSelfAttention") -> see the class implementation
+    3. find_usages("BertSelfAttention") -> see where it's used
+
+    COMPARISON WITH find_usages:
+    - go_to_definition: Shows WHERE entity is DEFINED (the implementation)
+    - find_usages: Shows WHERE entity is USED/CALLED (the consumers)
 
     Args:
-        entity_name: The name of the entity to retrieve the definition for
+        entity_name: Exact name of the entity (case-sensitive). Examples: 'BertModel', 'forward', 'attention_mask', 'get_extended_attention_mask'
 
     Returns:
-        str: A formatted string with definition locations
+        str: Entity type, file location(s), and complete source code of the definition(s). Returns error if entity not found.
     """
     if knowledge_graph is None:
         return "Error: Knowledge graph not initialized"
@@ -663,17 +825,46 @@ def go_to_definition(entity_name: str) -> str:
 @observe(as_type="tool")
 def find_usages(entity_name: str, limit: int = 20, page: int = 1) -> str:
     """
-    Retrieve all usages or calls of an entity in the codebase.
+    Find all locations in the codebase where an entity is used or called.
+
+    PURPOSE:
+    Use this tool to understand the impact and usage patterns of any entity.
+    Shows every place where a class is instantiated, a function is called,
+    or a variable is referenced throughout the Transformers library.
+
+    WHEN TO USE:
+    - To understand how widely used a class or function is
+    - To see usage examples of a particular API or function
+    - To assess the impact of changing an entity (who depends on it?)
+    - To learn how to use a class/function by seeing real examples
+    - To trace data flow through the codebase
+
+    WHAT YOU'LL GET:
+    - Total count of usage locations
+    - For each usage: file path, chunk position, and full code context showing the usage
+    - Paginated results for entities with many usages
+
+    TYPICAL WORKFLOWS:
+
+    Impact Analysis:
+    1. go_to_definition("deprecated_function") -> understand what it does
+    2. find_usages("deprecated_function") -> see all code that needs updating
+
+    Learning by Example:
+    1. list_nodes_by_type('class') -> find interesting classes
+    2. find_usages("BertModel") -> see how it's instantiated and used
 
-    Shows where functions, classes, variables, etc. are used.
+    COMPARISON WITH go_to_definition:
+    - find_usages: WHERE is this entity CALLED/USED (consumers)
+    - go_to_definition: WHERE is this entity DEFINED (implementation)
 
     Args:
-        entity_name: The name of the entity to retrieve usages for
-        limit: Maximum number of usages to return per page (default: 20)
-        page: Page number for pagination, 1-indexed (default: 1)
+        entity_name: Exact name of the entity to find usages for (case-sensitive)
+        limit: Usages per page (default: 20). Many popular classes have 100+ usages.
+        page: Page number for pagination (starts at 1)
 
     Returns:
-        str: A formatted string with usage locations
+        str: List of code chunks that use this entity, with file paths and full code content showing the usage in context
     """
     if knowledge_graph is None:
         return "Error: Knowledge graph not initialized"
@@ -738,15 +929,40 @@ def find_usages(entity_name: str, limit: int = 20, page: int = 1) -> str:
 @observe(as_type="tool")
 def get_file_structure(file_path: str) -> str:
     """
-    Get an overview of the structure of a file.
-
-    Shows chunks and declared entities within a specific file.
+    Get a structural overview of a source file showing its chunks and declared entities.
+
+    PURPOSE:
+    Use this tool to understand the organization of a specific file. Shows what classes,
+    functions, and other entities are defined in the file, plus how the file is divided into chunks.
+
+    WHEN TO USE:
+    - To get a table of contents for a file before diving into specifics
+    - To see what classes and functions a file defines
+    - To understand how code is organized within a file
+    - To find chunk IDs for further exploration with get_node_info or get_chunk_context
+    - When you know the file path but need to understand its contents
+
+    WHAT YOU'LL SEE:
+    - File path and detected programming language
+    - Total number of code chunks in the file
+    - List of declared entities (classes, functions, methods, variables) with their types
+    - Ordered list of chunks with their IDs and descriptions
+
+    HOW TO GET FILE PATHS:
+    - Use list_files_in_directory() to browse files
+    - Use search_nodes() and look at file paths in results
+    - Use list_nodes_by_type('file') to get file node IDs (which are the paths)
+
+    TYPICAL WORKFLOW:
+    1. list_files_in_directory('src/transformers/models/bert') -> find files
+    2. get_file_structure('src/transformers/models/bert/modeling_bert.py') -> see structure
+    3. get_node_info(chunk_id) -> examine specific code chunks
 
     Args:
-        file_path: The path of the file to get the structure for
+        file_path: The full path to the file (e.g., 'src/transformers/models/bert/modeling_bert.py'). Must match exactly as stored in the knowledge graph.
 
     Returns:
-        str: A formatted string with file structure
+        str: File overview including language, chunk count, declared entities list, and chunk descriptions
     """
     if knowledge_graph is None:
         return "Error: Knowledge graph not initialized"
@@ -792,18 +1008,42 @@ def get_file_structure(file_path: str) -> str:
 @observe(as_type="tool")
 def get_related_chunks(chunk_id: str, relation_type: str = "calls", limit: int = 20, page: int = 1) -> str:
     """
-    Retrieve chunks related to a given chunk by a specific relationship.
-
-    Retrieve chunks connected via relationships like 'calls', 'contains', etc.
+    Find code chunks connected to a given chunk through a specific relationship type.
+
+    PURPOSE:
+    Use this tool to trace code dependencies by following relationship edges from a chunk.
+    Helps understand what code a chunk depends on or what depends on it.
+
+    WHEN TO USE:
+    - To find what entities/code a chunk calls or uses (relation_type='calls')
+    - To trace dependencies from a specific piece of code
+    - To explore the call graph emanating from a chunk
+    - When you have a chunk ID and want to see connected code
+
+    RELATIONSHIP TYPES:
+    - 'calls': Entities/chunks that this chunk calls or references (most common)
+    - 'contains': Child nodes contained by this node (for files/directories)
+    - 'declares': Entities declared by this chunk
+    - 'all' or '': Get all outgoing relationships regardless of type
+
+    TYPICAL WORKFLOW:
+    1. search_nodes("BertAttention forward") -> find a chunk
+    2. get_related_chunks(chunk_id, 'calls') -> see what it calls
+    3. get_node_info(related_chunk_id) -> examine called code
+
+    COMPARISON WITH OTHER TOOLS:
+    - get_neighbors: All connected nodes (any direction, any type)
+    - get_related_chunks: Outgoing edges only, filtered by relationship type
+    - entity_relationships: Focused on entity nodes and their relationships
 
     Args:
-        chunk_id: The ID of the chunk to retrieve related chunks for
-        relation_type: The type of relationship to filter by (default: 'calls')
-        limit: Maximum number of results per page (default: 20)
-        page: Page number for pagination, 1-indexed (default: 1)
+        chunk_id: The ID of the chunk to explore from (e.g., 'src/transformers/models/bert/modeling_bert.py::chunk_5')
+        relation_type: Filter by relationship type: 'calls', 'contains', 'declares', or 'all' for everything (default: 'calls')
+        limit: Maximum results per page (default: 20)
+        page: Page number for pagination
 
     Returns:
-        str: A formatted string with related chunks
+        str: List of related chunks with their IDs, file paths, and entity names involved in the relationship
     """
     if knowledge_graph is None:
         return "Error: Knowledge graph not initialized"
@@ -891,19 +1131,54 @@ def list_all_entities(
     called_in_repo: Optional[bool] = None
 ) -> str:
     """
-    List all entities tracked in the knowledge graph with filtering and pagination options.
-
-    Shows entity types, declaration counts, and usage counts.
+    Browse all programming entities (classes, functions, methods, variables) tracked in the knowledge graph.
+
+    PURPOSE:
+    Use this tool to explore the full inventory of code entities in the Transformers library.
+    Supports filtering by type and usage patterns, making it powerful for targeted exploration.
+
+    WHEN TO USE:
+    - To browse all classes, functions, or methods in the codebase
+    - To find entities that are defined but never used (dead code analysis)
+    - To find external entities that are called but not defined in the repo
+    - To get an overview of entity distribution in the codebase
+    - When you need entity names for use with go_to_definition or find_usages
+
+    FILTERING OPTIONS:
+
+    By entity_type:
+    - 'class': Class definitions (BertModel, GPT2Config, etc.)
+    - 'function': Standalone functions
+    - 'method': Class methods
+    - 'variable': Variable declarations
+    - 'parameter': Function/method parameters
+    - None: All entity types
+
+    By declaration status (declared_in_repo):
+    - True: Only entities DEFINED in this repo (has source code)
+    - False: Only external entities (imported from other packages)
+    - None: All entities
+
+    By usage status (called_in_repo):
+    - True: Only entities that ARE USED somewhere in the code
+    - False: Only entities that are NEVER USED (potential dead code)
+    - None: All entities
+
+    USEFUL FILTER COMBINATIONS:
+    - All classes: entity_type='class'
+    - Defined classes: entity_type='class', declared_in_repo=True
+    - Unused functions: entity_type='function', called_in_repo=False
+    - External dependencies: declared_in_repo=False, called_in_repo=True
 
     Args:
-        limit: Maximum number of entities to return per page (default: 50)
-        page: Page number for pagination, 1-indexed (default: 1)
-        entity_type: Filter by entity type ('class', 'function', 'method', 'variable', 'parameter', 'function_call', 'method_call')
-        declared_in_repo: If True, only return entities with declarations. If False, only entities without declarations. If None, return all.
-        called_in_repo: If True, only return entities that have usages/calls in the repo. If False, only entities without usages. If None, return all.
+        limit: Entities per page (default: 50). Use larger values for comprehensive listings.
+        page: Page number starting from 1 for pagination
+        entity_type: Filter by type: 'class', 'function', 'method', 'variable', 'parameter', or None for all
+        declared_in_repo: True=defined in repo, False=external only, None=all
+        called_in_repo: True=has usages, False=never used, None=all
 
     Returns:
-        str: A formatted string with all entities for the requested page
+        str: List of entities with their types, declaration count, and usage count. Use entity names with go_to_definition or find_usages.
     """
     if knowledge_graph is None:
         return "Error: Knowledge graph not initialized"
@@ -1037,16 +1312,41 @@ def list_all_entities(
 @observe(as_type="tool")
 def diff_chunks(node_id_1: str, node_id_2: str) -> str:
     """
-    Show the diff between two code chunks or nodes.
-
-    Compares the content of two nodes and shows differences.
+    Compare two code chunks and show their differences in unified diff format.
+
+    PURPOSE:
+    Use this tool to compare two pieces of code side-by-side. Shows exactly what's
+    different between them using standard unified diff format (like git diff).
+
+    WHEN TO USE:
+    - To compare similar implementations (e.g., two attention mechanisms)
+    - To understand differences between related classes or functions
+    - To analyze variations in code patterns across the codebase
+    - To compare two versions or implementations of similar functionality
+    - When you suspect code duplication and want to see exact differences
+
+    DIFF FORMAT:
+    - Lines starting with '-' are only in the first chunk
+    - Lines starting with '+' are only in the second chunk
+    - Lines without prefix are common to both
+    - @@ markers show line number context
+
+    TYPICAL WORKFLOW:
+    1. search_nodes("attention") -> find attention implementations
+    2. Get chunk IDs from two different attention classes
+    3. diff_chunks(chunk_id_1, chunk_id_2) -> compare implementations
+
+    COMPARISON IDEAS:
+    - BertAttention vs GPT2Attention
+    - Different forward() implementations
+    - Similar utility functions in different modules
 
     Args:
-        node_id_1: The ID of the first node/chunk
-        node_id_2: The ID of the second node/chunk
+        node_id_1: ID of the first chunk/node to compare
+        node_id_2: ID of the second chunk/node to compare
 
     Returns:
-        str: A formatted string with the diff
+        str: Unified diff output showing line-by-line differences. Returns 'No differences found' if chunks are identical.
     """
     if knowledge_graph is None:
         return "Error: Knowledge graph not initialized"
@@ -1080,16 +1380,41 @@ def diff_chunks(node_id_1: str, node_id_2: str) -> str:
 @observe(as_type="tool")
 def print_tree(root_id: str = "root", max_depth: int = 3) -> str:
     """
-    Show a tree view of the repository structure.
-
-    Displays a hierarchical tree starting from a given node.
+    Display a hierarchical tree view of the repository structure starting from any node.
+
+    PURPOSE:
+    Use this tool to visualize the structure of the codebase. Shows parent-child relationships
+    in a familiar tree format, helping you understand how files and directories are organized.
+
+    WHEN TO USE:
+    - To explore the directory structure of the Transformers repository
+    - To see what's inside a specific directory (use directory as root_id)
+    - To understand the file organization for a component
+    - To get an overview of the codebase hierarchy
+    - When you need to understand where files are located
+
+    TREE VISUALIZATION:
+    - Each level shows node name and type (repo, directory, file, chunk)
+    - Indentation represents depth in the hierarchy
+    - Children are limited to prevent overwhelming output
+
+    TIPS:
+    - Start with max_depth=2 for a high-level overview
+    - Increase max_depth to see more detail (but output gets larger)
+    - Use a directory path as root_id to focus on a specific area
+    - Use list_files_in_directory for more detailed file listings
+
+    TYPICAL USAGE:
+    - print_tree('root', max_depth=2) -> see top-level structure
+    - print_tree('src/transformers/models', max_depth=2) -> see model organization
+    - print_tree('src/transformers/models/bert', max_depth=3) -> see bert module structure
 
     Args:
-        root_id: The node ID to start the tree from (default: 'root')
-        max_depth: Maximum depth to show (default: 3)
+        root_id: Starting node ID. Use 'root' for repository root, or a directory/file path to start from a specific location.
+        max_depth: How many levels deep to show (default: 3). Higher values show more detail but larger output.
 
     Returns:
-        str: A formatted string with the tree structure
+        str: ASCII tree visualization showing the hierarchical structure with node names and types
     """
     if knowledge_graph is None:
         return "Error: Knowledge graph not initialized"
@@ -1144,15 +1469,40 @@ def print_tree(root_id: str = "root", max_depth: int = 3) -> str:
 @observe(as_type="tool")
 def entity_relationships(node_id: str) -> str:
     """
-    Show all relationships for a given entity or node.
-
-    Displays incoming and outgoing relationships with their types.
+    Display all incoming and outgoing relationships for any node, with relationship types.
+
+    PURPOSE:
+    Use this tool to get a complete picture of how a node connects to the rest of the
+    knowledge graph. Shows both what points TO this node and what this node points TO.
+
+    WHEN TO USE:
+    - To understand all dependencies of an entity
+    - To see what declares or calls a specific entity
+    - To trace the full relationship network around any node
+    - When you need more detail than get_neighbors provides about relationship types
+    - For entity-centric analysis (understanding a class or function's connections)
+
+    WHAT YOU'LL SEE:
+    - Incoming relationships: Other nodes that have edges pointing TO this node
+      (e.g., chunks that CALL this function, files that CONTAIN this chunk)
+    - Outgoing relationships: This node's edges pointing TO other nodes
+      (e.g., entities this chunk CALLS, chunks this file CONTAINS)
+    - Relationship types for each edge (calls, declares, contains)
+
+    COMPARISON WITH SIMILAR TOOLS:
+    - get_node_edges: Same information but different formatting
+    - get_neighbors: Shows neighbor node details, not edge details
+    - get_related_chunks: Filtered by relationship type, chunks only
+
+    TYPICAL WORKFLOW:
+    1. go_to_definition("BertModel") -> find entity
+    2. entity_relationships("BertModel") -> see what calls/uses BertModel
 
     Args:
-        node_id: The node/entity ID to explore relationships for
+        node_id: The ID of any node (entity, chunk, file, directory)
 
     Returns:
-        str: A formatted string with all relationships
+        str: Complete list of incoming and outgoing relationships with source/target IDs and relationship types
     """
     if knowledge_graph is None:
         return "Error: Knowledge graph not initialized"
@@ -1195,22 +1545,50 @@ def entity_relationships(node_id: str) -> str:
 @observe(as_type="tool")
 def search_by_type_and_name(node_type: str, name_query: str, limit: int = 10, page: int = 1, partial_allowed: bool = True) -> str:
     """
-    Search for nodes/entities by type and name substring with partial matching support.
-
-    Filters nodes by type and searches for matching names. Supports partial matching
-    so searching for 'Embedding' will find 'BertEmbeddings', 'LlamaRotaryEmbedding', etc.
-    
-    For entities, searches by entity_type (e.g., 'class', 'function', 'method').
-    For other nodes, searches by node_type (e.g., 'file', 'chunk', 'directory').
+    Search for nodes by combining type filtering with name pattern matching.
+
+    PURPOSE:
+    Use this tool for precise, targeted searches when you know the type of node you're looking
+    for and have a partial name. More efficient than list_nodes_by_type when you have name hints.
+
+    WHEN TO USE:
+    - To find all classes containing 'Attention': search_by_type_and_name('class', 'Attention')
+    - To find functions with 'forward' in name: search_by_type_and_name('function', 'forward')
+    - To find files named 'config': search_by_type_and_name('file', 'config')
+    - When you know the type AND have a partial name to search for
+    - For pattern-based discovery of related components
+
+    SEARCH BEHAVIOR:
+    - Case-insensitive matching
+    - partial_allowed=True (default): Fuzzy matching, finds 'BertEmbeddings' when searching 'Embed'
+    - partial_allowed=False: Requires exact substring match
+    - Results sorted by match quality (exact matches first, then substring, then fuzzy)
+
+    VALID node_type VALUES:
+    For entities: 'class', 'function', 'method', 'variable', 'parameter'
+    For structural: 'file', 'chunk', 'directory'
+
+    SEARCH EXAMPLES:
+    - All Attention classes: search_by_type_and_name('class', 'Attention')
+    - All Embedding classes: search_by_type_and_name('class', 'Embedding')
+    - Config files: search_by_type_and_name('file', 'config')
+    - Forward methods: search_by_type_and_name('method', 'forward')
+    - Test files: search_by_type_and_name('file', 'test_')
+
+    COMPARISON WITH SIMILAR TOOLS:
+    - search_nodes: Full-text search in code content (doesn't filter by type)
+    - list_nodes_by_type: Lists all of a type (no name filter)
+    - search_by_type_and_name: Combines type filter + name search (best of both)
 
     Args:
-        node_type: Type of node/entity (e.g., 'function', 'class', 'file', 'chunk', 'directory')
-        name_query: Substring to match in the name (case-insensitive, supports partial matches)
-        limit: Maximum results to return (default: 10)
-        partial_allowed: Enable partial matching (default: True). If False, requires exact substring match.
+        node_type: Type to filter by: 'class', 'function', 'method', 'file', 'chunk', 'directory', etc.
+        name_query: Name pattern to search for (case-insensitive). Can be partial.
+        limit: Results per page (default: 10)
+        page: Page number for pagination
+        partial_allowed: Enable fuzzy matching (default: True). Set False for stricter matching.
 
     Returns:
-        str: A formatted string with matching nodes
+        str: Matching nodes sorted by relevance, with IDs and types. Use IDs with get_node_info for details.
     """
     if knowledge_graph is None:
         return "Error: Knowledge graph not initialized"
@@ -1338,16 +1716,45 @@ def search_by_type_and_name(node_type: str, name_query: str, limit: int = 10, pa
 @observe(as_type="tool")
 def get_chunk_context(node_id: str) -> str:
     """
-    Get the full content of a code chunk along with its surrounding chunks.
-
-    Returns the full content of the previous, current, and next chunks,
-    organized by file and joined together.
+    Get expanded code context by retrieving a chunk along with its previous and next chunks.
+
+    PURPOSE:
+    Use this tool when you need to see MORE CODE CONTEXT around a specific chunk.
+    Chunks are logical code segments, but sometimes you need to see surrounding code
+    to fully understand the implementation.
+
+    WHEN TO USE:
+    - After search_nodes or get_node_info when you need more surrounding context
+    - When a chunk shows a partial function/class and you need the complete picture
+    - To understand code flow across chunk boundaries
+    - To see imports or setup code that precedes a chunk
+    - To see what code follows after a chunk
+
+    WHAT YOU'LL GET:
+    - The previous chunk's content (if it exists)
+    - The target chunk's content
+    - The next chunk's content (if it exists)
+    - All organized by file and joined together seamlessly
+
+    CONTEXT EXPANSION:
+    - Shows up to 3 consecutive chunks (prev + current + next)
+    - Useful for understanding function bodies that span chunks
+    - Helps see class context when looking at individual methods
+
+    TYPICAL WORKFLOW:
+    1. search_nodes("attention forward") -> find relevant chunk
+    2. get_node_info(chunk_id) -> see chunk content
+    3. get_chunk_context(chunk_id) -> see surrounding code for fuller understanding
+
+    COMPARISON WITH get_node_info:
+    - get_node_info: Single chunk content + full metadata
+    - get_chunk_context: Expanded code view (prev + current + next chunks), less metadata
 
     Args:
-        node_id: The node/chunk ID to get context for
+        node_id: The chunk ID to get context for (e.g., 'src/transformers/models/bert/modeling_bert.py::chunk_5')
 
     Returns:
-        str: The full content of surrounding code chunks
+        str: Combined content of previous, current, and next chunks organized by file. Provides seamless code view.
     """
 
     
@@ -1383,15 +1790,38 @@ def get_chunk_context(node_id: str) -> str:
 @observe(as_type="tool")
 def get_file_stats(path: str) -> str:
     """
-    Get statistics for a file or directory.
+    Get detailed statistics and metrics for a specific file or directory.
+
+    PURPOSE:
+    Use this tool to get quantitative metrics about a file including line counts,
+    entity counts, and chunk counts. Useful for understanding file complexity.
 
-    Shows number of entities, lines, chunks, etc.
+    WHEN TO USE:
+    - To assess the size and complexity of a file
+    - To see summary counts of entities declared and called
+    - To understand how a file is chunked
+    - For code metrics and analysis tasks
+    - When deciding which files to explore further
+
+    METRICS PROVIDED:
+    - Line count (total lines in the file)
+    - Declared entities count with a sample list
+    - Called entities count with a sample list
+    - Number of chunks the file is divided into
+
+    COMPARISON WITH get_file_structure:
+    - get_file_stats: Quantitative metrics (counts, numbers)
+    - get_file_structure: Qualitative overview (entity names, chunk IDs)
+
+    TYPICAL USAGE:
+    - get_file_stats('src/transformers/models/bert/modeling_bert.py') -> see metrics
+    - Use this to identify large/complex files before diving in
 
     Args:
-        path: The file or directory path to get statistics for
+        path: The file path to analyze. Must match the path as stored in the knowledge graph.
 
     Returns:
-        str: A formatted string with file statistics
+        str: Statistics including line count, declared entities, called entities, and chunk count
     """
     if knowledge_graph is None:
         return "Error: Knowledge graph not initialized"
@@ -1444,17 +1874,46 @@ def get_file_stats(path: str) -> str:
 @observe(as_type="tool")
 def find_path(source_id: str, target_id: str, max_depth: int = 5) -> str:
     """
-    Retrieve the shortest path between two nodes in the knowledge graph.
-
-    Uses graph traversal to retrieve connections between nodes.
+    Find the shortest path between two nodes in the knowledge graph.
+
+    PURPOSE:
+    Use this tool to discover how two code elements are connected through the graph.
+    Reveals the chain of relationships linking two seemingly unrelated pieces of code.
+
+    WHEN TO USE:
+    - To understand how two classes/functions are related
+    - To trace dependency chains between components
+    - To discover indirect connections between code elements
+    - To verify if two nodes are connected at all
+    - For understanding code architecture and coupling
+
+    WHAT YOU'LL GET:
+    - Path length (number of hops)
+    - Ordered list of nodes from source to target
+    - Visual representation of the path
+
+    LIMITATIONS:
+    - max_depth limits search to avoid long computations
+    - If no path found within max_depth, nodes may still be connected via longer path
+    - Very distant nodes may require increasing max_depth
+
+    EXAMPLE QUERIES:
+    - How is BertModel connected to GPT2Model?
+    - What's the path from a utility function to a model class?
+    - How many hops between two files?
+
+    TYPICAL WORKFLOW:
+    1. Identify two node IDs of interest
+    2. find_path(source, target) -> discover connection
+    3. get_node_info for nodes in the path to understand the relationship
 
     Args:
-        source_id: The ID of the source node
-        target_id: The ID of the target node
-        max_depth: Maximum depth to search for a path (default: 5)
+        source_id: Starting node ID (any node type)
+        target_id: Destination node ID (any node type)
+        max_depth: Maximum path length to search (default: 5). Increase for distant nodes.
 
     Returns:
-        str: A formatted string showing the path
+        str: Path from source to target showing each node in sequence, or message if no path found
     """
     if knowledge_graph is None:
         return "Error: Knowledge graph not initialized"
@@ -1493,17 +1952,50 @@ def find_path(source_id: str, target_id: str, max_depth: int = 5) -> str:
 @observe(as_type="tool")
 def get_subgraph(node_id: str, depth: int = 2, edge_types: Optional[str] = None) -> str:
     """
-    Retrieve a subgraph around a node up to a specified depth.
-
-    Optionally filters by edge types (comma-separated).
+    Extract a local subgraph around a node up to a specified depth.
+
+    PURPOSE:
+    Use this tool to get a bounded view of the graph neighborhood around any node.
+    Shows all nodes reachable within a certain number of hops, optionally filtered by edge type.
+
+    WHEN TO USE:
+    - To understand the local network around a class or function
+    - To extract a bounded region of the knowledge graph for analysis
+    - To see all nodes within N hops of a target node
+    - To analyze the dependency neighborhood of a component
+    - When get_neighbors isn't enough and you need multi-hop exploration
+
+    DEPTH EXPLANATION:
+    - depth=1: Only immediate neighbors (same as get_neighbors)
+    - depth=2: Neighbors and their neighbors (2 hops)
+    - depth=3+: Larger neighborhood (exponentially more nodes)
+
+    EDGE TYPE FILTERING:
+    - Pass comma-separated edge types to filter: 'calls,declares'
+    - Common types: 'calls', 'contains', 'declares'
+    - Leave empty or None for all edge types
+
+    OUTPUT:
+    - Node count and edge count in the subgraph
+    - List of all node IDs in the extracted subgraph
+    - Filtered by edge types if specified
+
+    TYPICAL WORKFLOW:
+    1. Find a central node of interest
+    2. get_subgraph(node_id, depth=2) -> see local neighborhood
+    3. Use node IDs from result with get_node_info for details
+
+    COMPARISON WITH get_neighbors:
+    - get_neighbors: Single hop, shows node details
+    - get_subgraph: Multi-hop, shows subgraph structure and counts
 
     Args:
-        node_id: The ID of the central node
-        depth: The depth/radius of the subgraph to Retrieve (default: 2)
-        edge_types: Optional comma-separated list of edge types (e.g., 'calls,contains')
+        node_id: Central node to build subgraph around
+        depth: Radius in hops from central node (default: 2). Higher = larger subgraph.
+        edge_types: Optional comma-separated filter: 'calls,contains,declares' or None for all
 
     Returns:
-        str: A formatted string describing the subgraph
+        str: Subgraph summary with node/edge counts and list of included node IDs
     """
     if knowledge_graph is None:
         return "Error: Knowledge graph not initialized"
@@ -1545,20 +2037,57 @@ def get_subgraph(node_id: str, depth: int = 2, edge_types: Optional[str] = None)
 @observe(as_type="tool")
 def list_files_in_directory(directory_path: str = "", pattern: str = "*", recursive: bool = True, limit: int = 50, page: int = 1) -> str:
     """
-    List files in a directory with optional glob pattern matching.
-
-    This provides hierarchical file listing, showing files within directories
-    rather than just top-level files. Supports glob patterns for filtering.
+    Browse and list files in the repository with flexible filtering options.
+
+    PURPOSE:
+    Use this tool to explore the file structure of the Transformers library.
+    Supports directory scoping, glob patterns, and recursive/non-recursive modes.
+
+    WHEN TO USE:
+    - To see what files exist in a directory
+    - To find files by pattern (e.g., all Python files, all test files)
+    - To explore the repository structure directory by directory
+    - To find specific file types in specific locations
+    - When you need file paths for use with other tools
+
+    FILTERING OPTIONS:
+
+    directory_path:
+    - Empty string '': Search all files in the repository
+    - 'src/transformers/models': Only files under this directory
+    - 'src/transformers/models/bert': Focus on a specific model
+
+    pattern (glob patterns):
+    - '*': All files (default)
+    - '*.py': Python files only
+    - 'test_*.py': Test files
+    - '*config*': Files with 'config' in name
+    - 'modeling_*.py': Modeling files
+
+    recursive:
+    - True (default): Include files in subdirectories
+    - False: Only files directly in the specified directory
+
+    COMMON USE CASES:
+    - All files: list_files_in_directory()
+    - Bert model files: list_files_in_directory('src/transformers/models/bert')
+    - All Python files: list_files_in_directory(pattern='*.py')
+    - Test files only: list_files_in_directory(pattern='test_*.py')
+    - Config files: list_files_in_directory(pattern='*config*')
+
+    COMPARISON WITH print_tree:
+    - print_tree: Visual hierarchy, includes directories
+    - list_files_in_directory: Flat file list with details, better for finding specific files
 
     Args:
-        directory_path: Path to the directory to list (empty string for root/all files)
-        pattern: Glob pattern to filter files (e.g., '*.py', 'test_*.py', '**/*.js')
-        recursive: Whether to search recursively in subdirectories (default: True)
-        limit: Maximum number of files to return per page (default: 50)
-        page: Page number for pagination, 1-indexed (default: 1)
+        directory_path: Directory to search in. Empty string for entire repository.
+        pattern: Glob pattern for filename filtering (default: '*' matches all)
+        recursive: Search subdirectories (default: True)
+        limit: Files per page (default: 50)
+        page: Page number for pagination
 
     Returns:
-        str: A formatted string with matching files
+        str: List of matching files with paths, languages, and entity counts
     """
     if knowledge_graph is None:
         return "Error: Knowledge graph not initialized"
@@ -1673,17 +2202,48 @@ def list_files_in_directory(directory_path: str = "", pattern: str = "*", recurs
 @observe(as_type="tool")
 def find_files_importing(module_or_entity: str, limit: int = 30, page: int = 1) -> str:
     """
-    Retrieve all files that import a specific module or entity.
-
-    Searches for import statements and usage patterns across the codebase.
+    Find all files that import or use a specific module, class, or function.
+
+    PURPOSE:
+    Use this tool to trace import dependencies and understand which parts of the
+    codebase depend on a particular module or entity.
+
+    WHEN TO USE:
+    - To find all files that import a specific module (e.g., 'torch', 'numpy')
+    - To trace dependencies on a class or function
+    - To understand the impact scope of a module
+    - To find usage patterns of external libraries
+    - For dependency analysis and impact assessment
+
+    SEARCH BEHAVIOR:
+    - Searches through 'called_entities' metadata
+    - Also scans code chunks for import statement patterns
+    - Matches import, from...import, require, use patterns
+    - Case-insensitive matching
+
+    WHAT YOU'LL GET:
+    - List of files that import/use the specified module or entity
+    - Match type (called_entity or import_statement)
+    - Matched entity names when applicable
+
+    EXAMPLE QUERIES:
+    - find_files_importing('torch') -> files using PyTorch
+    - find_files_importing('numpy') -> files using NumPy
+    - find_files_importing('BertModel') -> files using BertModel
+    - find_files_importing('attention') -> files related to attention
+
+    LIMITATIONS:
+    - May not catch all dynamic imports
+    - Pattern matching may have false positives/negatives
+    - For comprehensive search, combine with search_nodes
 
     Args:
-        module_or_entity: The name of the module or entity to retrieve imports of
-        limit: Maximum number of results to return per page (default: 30)
-        page: Page number for pagination, 1-indexed (default: 1)
+        module_or_entity: Name of the module, class, or function to search for (case-insensitive)
+        limit: Maximum results per page (default: 30)
+        page: Page number for pagination
 
     Returns:
-        str: A formatted string with files that import the specified module/entity
+        str: List of files that import or use the specified module/entity, with match details
     """
     if knowledge_graph is None:
         return "Error: Knowledge graph not initialized"
@@ -1801,17 +2361,54 @@ def find_files_importing(module_or_entity: str, limit: int = 30, page: int = 1)
 @observe(as_type="tool")
 def get_concept_overview(concept: str, limit: int = 15) -> str:
     """
-    Get a high-level overview of a concept across the codebase.
-
-    Combines multiple search strategies to provide a comprehensive view of how
-    a concept (like 'embeddings', 'authentication', 'caching') is implemented.
+    Get a high-level overview of how a concept is implemented across the Transformers codebase.
+
+    PURPOSE:
+    Use this tool for broad exploration of a concept or feature. Aggregates related
+    classes, functions, files, and code snippets into a single comprehensive view.
+    Ideal for initial investigation of a topic.
+
+    WHEN TO USE:
+    - FIRST STEP when exploring a new concept (before detailed searches)
+    - To understand how a feature is implemented across the codebase
+    - To discover all components related to a concept
+    - To get a bird's-eye view before diving into specifics
+    - When you're not sure where to start investigating
+
+    SEARCH STRATEGY:
+    This tool combines multiple search approaches:
+    - Searches entity names (classes, functions, methods) containing the concept
+    - Searches file names and paths
+    - Searches chunk content and descriptions
+    - Aggregates results into categorized sections
+
+    CONCEPT EXAMPLES:
+    - 'attention' -> attention mechanisms across all models
+    - 'embedding' -> embedding layers and utilities
+    - 'tokenizer' -> tokenization components
+    - 'generation' -> text generation utilities
+    - 'config' -> configuration classes
+    - 'cache' -> caching mechanisms
+    - 'rope' -> rotary position embeddings
+    - 'flash' -> flash attention implementations
+
+    OUTPUT STRUCTURE:
+    - Related Classes: Class definitions matching the concept
+    - Related Functions/Methods: Functions matching the concept
+    - Related Files: Files with concept in path/name
+    - Code Snippets: Relevant code chunks
+
+    TYPICAL WORKFLOW:
+    1. get_concept_overview('attention') -> see all attention-related components
+    2. Identify specific classes/functions of interest
+    3. go_to_definition or search_nodes for detailed exploration
 
     Args:
-        concept: The concept to search for (e.g., 'embedding', 'authentication', 'cache')
-        limit: Maximum number of results per category (default: 15)
+        concept: The concept to explore (e.g., 'attention', 'embedding', 'generation', 'tokenizer')
+        limit: Maximum items per category (default: 15)
 
     Returns:
-        str: A formatted overview of the concept across the codebase
+        str: Categorized overview with related classes, functions, files, and code snippets
     """
     if knowledge_graph is None:
         return "Error: Knowledge graph not initialized"
@@ -1984,8 +2581,8 @@ def create_gradio_app():
                 with gr.Column():
                     node_output = gr.Textbox(label="Node Information", lines=20, max_lines=30)
             node_info_btn.click(fn=get_node_info, inputs=node_id_input, outputs=node_output)
-            node_edges_btn.click(fn=get_node_edges, inputs=node_id_input, outputs=node_output)
             gr.Markdown("#Get Node Info:" + _tool_doc_md(get_node_info))
+            node_edges_btn.click(fn=get_node_edges, inputs=node_id_input, outputs=node_output)
             gr.Markdown("#Get Node Edges:" + _tool_doc_md(get_node_edges))
 
         with gr.Tab("🏗️ Structure"):