WIP before syncing with hf-https/main

APIarchitecture.mmd

@@ -0,0 +1,46 @@
+graph TB
+    subgraph "Client Applications"
+        APP1[Python App]
+        APP2[Node.js App]
+        APP3[Claude Desktop]
+        APP4[Web App]
+    end
+    
+    subgraph "API Endpoints"
+        REST[REST API<br/>HTTP/JSON]
+        SSE[SSE Endpoint<br/>MCP Protocol]
+        GRAD[Gradio API<br/>WebSocket]
+    end
+    
+    subgraph "Integration Methods"
+        M1[Direct HTTP Calls]
+        M2[MCP Client SDK]
+        M3[Gradio Client]
+    end
+    
+    subgraph "API Functions"
+        direction TB
+        FUNC[Available Functions]
+        F1[validate_rdf_tool<br/>POST /api/validate]
+        F2[get_ai_suggestions<br/>POST /api/suggest]
+        F3[get_ai_correction<br/>POST /api/correct]
+        F4[get_rdf_examples<br/>GET /api/examples]
+        
+        FUNC --> F1
+        FUNC --> F2
+        FUNC --> F3
+        FUNC --> F4
+    end
+    
+    APP1 --> M1
+    APP2 --> M1
+    APP3 --> M2
+    APP4 --> M3
+    
+    M1 --> REST
+    M2 --> SSE
+    M3 --> GRAD
+    
+    REST --> FUNC
+    SSE --> FUNC
+    GRAD --> FUNC

RDFandLLMcorrectionflow.mmd

@@ -0,0 +1,29 @@
+sequenceDiagram
+    participant User
+    participant App as MCP4RDF Service
+    participant Val as Validator
+    participant SHACL as SHACL Engine
+    participant LLM as LLM (Llama-3.3)
+    
+    User->>App: Submit RDF/XML
+    App->>Val: validate_rdf_tool()
+    Val->>SHACL: Load template & validate
+    
+    alt Valid RDF
+        SHACL-->>Val: Conforms
+        Val-->>App: {conforms: true, status: "Valid"}
+        App-->>User: Display success
+    else Invalid RDF
+        SHACL-->>Val: Violations found
+        Val-->>App: {conforms: false, results: "errors..."}
+        
+        App->>LLM: get_ai_suggestions(errors, rdf)
+        Note over LLM: Analyze errors<br/>Generate fix steps
+        LLM-->>App: Structured suggestions
+        
+        App->>LLM: get_ai_correction(errors, rdf)
+        Note over LLM: Generate corrected<br/>RDF/XML
+        LLM-->>App: Fixed RDF/XML
+        
+        App-->>User: Display errors +<br/>suggestions +<br/>corrected RDF
+    end

app.py

@@ -606,13 +606,17 @@ Be direct and solution-focused. No lengthy explanations."""
         chat_completion = client.chat.completions.create(
             model=HF_MODEL,
             messages=[
+                {
+                    "role": "system",
+                    "content": "You are a friendly librarian helping fix catalog records. Never use technical RDF or SHACL terminology."
+                },
                 {
                     "role": "user",
                     "content": prompt
                 }
             ],
             max_tokens=1500,
-            temperature=0.7,
+            temperature=0.6,
             top_p=0.9
         )
         
@@ -800,48 +804,136 @@ Please provide the corrected RDF/XML that addresses all validation issues.
 {generate_manual_correction_hints(validation_results, rdf_content)}"""
 
 def generate_manual_suggestions(validation_results: str) -> str:
-    """Generate rule-based suggestions when AI is not available"""
-    suggestions = []
-    
-    if "Constraint Violation" in validation_results:
-        suggestions.append("• Fix SHACL constraint violations by ensuring required properties are present")
-    
-    if "Missing property" in validation_results or "missing" in validation_results.lower():
-        suggestions.append("• Add missing required properties (check template requirements)")
-    
-    if "datatype" in validation_results.lower():
-        suggestions.append("• Correct data type mismatches (ensure proper literal types)")
-    
-    if "namespace" in validation_results.lower() or "prefix" in validation_results.lower():
-        suggestions.append("• Add missing namespace declarations at the top of your RDF")
-    
-    if "XML" in validation_results or "syntax" in validation_results.lower():
-        suggestions.append("• Fix XML syntax errors (check for unclosed tags, invalid characters)")
-    
+    """Generate generic, pattern-based suggestions when AI is not available.
+
+    Note: Avoid hardcoding SHACL rules or specific property requirements; rely only on
+    patterns present in the validation output text.
+    """
+    vr_lower = validation_results.lower() if validation_results else ""
+    suggestions: List[str] = []
+
+    # Missing/required
+    if ("mincount" in vr_lower) or ("missing" in vr_lower) or ("required" in vr_lower):
+        suggestions.append("• Some required fields are missing. Add the missing information where indicated.")
+
+    # Too many values
+    if ("maxcount" in vr_lower) or ("too many" in vr_lower) or ("more than allowed" in vr_lower):
+        suggestions.append("• Some fields have too many values. Keep only the main/one value as required.")
+
+    # Datatype/format issues
+    if ("datatype" in vr_lower) or ("type mismatch" in vr_lower) or ("expected" in vr_lower and "datatype" in vr_lower):
+        suggestions.append("• Some values are in the wrong format. Use the expected format (e.g., dates like YYYY-MM-DD).")
+
+    # URI/identifier issues
+    if ("iri" in vr_lower) or ("uri" in vr_lower) or ("identifier" in vr_lower and "invalid" in vr_lower):
+        suggestions.append("• Some identifiers look malformed. Use complete, valid web addresses or proper identifiers.")
+
+    # Namespace/prefix issues
+    if ("namespace" in vr_lower) or ("prefix" in vr_lower):
+        suggestions.append("• Define all XML namespace prefixes at the top and use them consistently.")
+
+    # XML syntax/structure
+    if ("xml" in vr_lower) or ("syntax" in vr_lower) or ("well-formed" in vr_lower):
+        suggestions.append("• Fix XML structure issues (unclosed tags, invalid characters, or nesting problems).")
+
+    # Fallback
     if not suggestions:
-        suggestions.append("• Review detailed validation results for specific issues")
-        suggestions.append("• Ensure your RDF follows the selected template requirements")
-    
+        suggestions.append("• Review the validation details and update the record where issues are highlighted.")
+        suggestions.append("• Follow the selected template; add missing fields and correct formats as needed.")
+
     suggestions_text = "\n".join(suggestions)
-    
+
     return f"""
-📋 **Manual Analysis:**
+📋 **What needs fixing:**
 
 {suggestions_text}
 
-💡 **General Tips:**
-• Check namespace declarations at the top of your RDF
-• Ensure all required properties are present
-• Verify data types match expected formats
-• Make sure XML structure is well-formed
-
-🔧 **Common Fixes:**
-• Add missing namespace prefixes
-• Include required properties like rdf:type
-• Fix malformed URIs or literals
-• Ensure proper XML syntax
+💡 **Quick tips:**
+• Include required fields when noted
+• Keep single-value fields to one value
+• Use the expected formats (e.g., for dates)
+• Declare and use XML namespace prefixes consistently
+• Ensure the XML is well‑formed
+
+Need help? Load an example and compare the structure.
 """
 
+def clean_technical_jargon(text: str) -> str:
+    """Replace technical RDF/SHACL terms with plain language for end users."""
+    if not text:
+        return text
+    replacements = {
+        # RDF/SHACL jargon
+        "URIRef": "identifier",
+        "URI": "identifier",
+        "IRI": "identifier",
+        "Literal": "text value",
+        "triple": "field entry",
+        "graph": "dataset",
+        "node": "record",
+        "subject": "record",
+        "predicate": "field type",
+        "object": "value",
+        "SHACL": "validation",
+        "constraint": "rule",
+        "conformance": "compliance",
+        "violation": "issue",
+        "sh:": "",
+        "rdf:": "",
+        "rdfs:": "",
+        "xsd:": "",
+        # Tone softening
+        "Error:": "Issue:",
+        "Invalid": "Incorrect",
+        "Failed": "Did not pass",
+        "Missing": "Not found",
+    }
+    cleaned = text
+    for k, v in replacements.items():
+        cleaned = cleaned.replace(k, v)
+    return cleaned
+
+def parse_shacl_results_for_ai(results_text: str) -> str:
+    """Simplify SHACL results into clearer sentences for AI processing.
+
+    Pattern-based only; does not depend on any SHACL rule definitions.
+    """
+    if not results_text:
+        return ""
+    import re
+    simplified: List[str] = []
+
+    # Generic patterns
+    patterns = [
+        (re.compile(r"minCount", re.IGNORECASE), "A required field is missing."),
+        (re.compile(r"maxCount", re.IGNORECASE), "A field has more values than allowed; only one may be permitted."),
+        (re.compile(r"datatype", re.IGNORECASE), "A field has a value in the wrong format."),
+        (re.compile(r"iri|uri", re.IGNORECASE), "An identifier looks malformed or incomplete."),
+        (re.compile(r"namespace|prefix", re.IGNORECASE), "A namespace prefix is undefined or inconsistent."),
+        (re.compile(r"xml|syntax|well-formed", re.IGNORECASE), "The XML structure has an error (e.g., unclosed tag)."),
+    ]
+
+    lines = [ln.strip() for ln in results_text.splitlines() if ln.strip()]
+    for ln in lines:
+        matched = False
+        for regex, message in patterns:
+            if regex.search(ln):
+                simplified.append(message)
+                matched = True
+                break
+        if not matched and ("Constraint Violation" in ln or "Violation" in ln):
+            simplified.append("A record rule was not met.")
+
+    # Deduplicate while preserving order
+    seen = set()
+    unique = []
+    for s in simplified:
+        if s not in seen:
+            unique.append(s)
+            seen.add(s)
+
+    return "\n".join(unique) if unique else results_text
+
 def generate_manual_correction_hints(validation_results: str, rdf_content: str) -> str:
     """Generate manual correction hints when AI is not available"""
     return f"""<!-- Manual correction hints based on validation results -->
@@ -861,35 +953,35 @@ MANUAL CORRECTION STEPS:
 5. Validate data types
 -->"""
 
-def validate_rdf_interface(rdf_content: str, template: str, use_ai: bool = True, include_warnings: bool = False):
-    """
-    Main validation function for Gradio interface and MCP server.
-    
-    This function provides comprehensive RDF/XML validation with AI-powered
-    suggestions and corrections. It serves as the primary interface for both
-    the Gradio web UI and MCP client tools.
-    
-    Args:
-        rdf_content (str): The RDF/XML content to validate
-        template (str): Validation template to use ('monograph' or 'custom')
-        use_ai (bool): Whether to enable AI-powered suggestions and corrections
-        include_warnings (bool): Whether to include warnings in AI corrections (violations only by default)
-    
-    Returns:
-        tuple: (status, results_text, suggestions, corrected_rdf) containing:
-            - status: Validation status message
-            - results_text: Detailed validation results
-            - suggestions: AI or manual fix suggestions
-            - corrected_rdf: AI-generated corrections or success message
+def extract_xml_from_text(text: str) -> str:
+    """Extract RDF/XML from model output that may include extra formatting.
+
+    Looks for the first <rdf:RDF ...> ... </rdf:RDF> block. If not found,
+    returns the original text unchanged.
     """
+    if not text:
+        return text
+    import re
+    # Try to capture XML block even if fenced in code blocks
+    # Use DOTALL to span multiple lines
+    pattern = re.compile(r"<rdf:RDF[\s\S]*?</rdf:RDF>", re.IGNORECASE)
+    m = pattern.search(text)
+    if m:
+        return m.group(0)
+    # Strip common markdown fences if present
+    fenced = re.sub(r"^```[a-zA-Z]*\n|```$", "", text.strip())
+    return fenced if fenced else text
+
+def validate_rdf_interface(rdf_content: str, template: str, use_ai: bool = True):
+    """Main validation function for Gradio interface"""
     if not rdf_content.strip():
-        return "❌ Error", "No RDF/XML data provided", "", ""
-    
+        return "❌ Error", "No RDF/XML data provided", "", "", "", ""
+
     # Validate RDF
     result = validate_rdf_tool(rdf_content, template)
     
     if "error" in result:
-        return f"❌ Error: {result['error']}", "", "", ""
+        return f"❌ Error: {result['error']}", "", "", "", "", ""
     
     status = result["status"]
     results_text = result["results"]
@@ -1102,6 +1194,20 @@ def create_interface():
                     show_copy_button=True,
                     placeholder="Corrected RDF will appear here after validation..."
                 )
+
+                with gr.Row():
+                    corrected_status_output = gr.Textbox(
+                        label="Re-validation Status (Corrected RDF)",
+                        interactive=False,
+                        lines=1,
+                        elem_classes=["status-box"]
+                    )
+                    corrected_results_output = gr.Textbox(
+                        label="Re-validation Details",
+                        interactive=False,
+                        lines=6,
+                        show_copy_button=True
+                    )
         
         # Event handlers
         validate_btn.click(
@@ -1129,8 +1235,8 @@ def create_interface():
         )
         
         clear_btn.click(
-            lambda: ("", "", "", "", ""),
-            outputs=[rdf_input, status_output, results_output, suggestions_output, corrected_output]
+            lambda: ("", "", "", "", "", "", ""),
+            outputs=[rdf_input, status_output, results_output, suggestions_output, corrected_output, corrected_status_output, corrected_results_output]
         )
         
         # Footer with instructions

copilot-instructions.md

@@ -0,0 +1,311 @@
+# GitHub Copilot Instructions for MCP4RDF Project
+
+## Project Context
+
+This is an RDF validation tool with AI features, deployed on Hugging Face Spaces. It validates RDF/XML against SHACL schemas and provides AI-powered suggestions for fixing validation errors.
+
+### Key Technologies
+- **Frontend**: Gradio 5.33.0
+- **RDF Processing**: rdflib, pyshacl
+- **AI Integration**: Hugging Face Inference API
+- **Protocol**: MCP (Model Context Protocol)
+- **Deployment**: Hugging Face Spaces
+
+### Project Structure
+```
+mcp4rdf-hf-space/
+├── app.py                    # Main Gradio application
+├── validator.py              # Core SHACL validation logic
+├── mcp_server_gradio.py      # MCP server implementation
+├── MonographDCTAP/           # TSV files with SHACL definitions
+├── electronic_MonographDCTAP/ # Electronic format SHACL definitions
+└── requirements.txt          # Python dependencies
+```
+
+## Code Style Guidelines
+
+### Python Standards
+- Use type hints for function parameters and return values
+- Follow PEP 8 naming conventions
+- Add docstrings for all public functions
+- Use logging instead of print statements
+
+### RDF/SHACL Patterns
+```python
+# Always bind common namespaces
+STANDARD_NAMESPACES = {
+    "bf": "http://id.loc.gov/ontologies/bibframe/",
+    "bflc": "http://id.loc.gov/ontologies/bflc/",
+    "rdfs": "http://www.w3.org/2000/01/rdf-schema#",
+    "sh": "http://www.w3.org/ns/shacl#"
+}
+
+# Use URIRef for RDF predicates
+from rdflib import URIRef, Literal, Graph
+sh_path = URIRef("http://www.w3.org/ns/shacl#path")
+```
+
+## Common Tasks and Templates
+
+### 1. Adding New SHACL Validation Rules
+```python
+# Template for adding a new property constraint
+def add_property_constraint(shape_node, property_id, constraints):
+    """
+    Add SHACL property constraints to a shape.
+    
+    Args:
+        shape_node: RDF node representing the shape
+        property_id: Property identifier (e.g., "bf:title")
+        constraints: Dict with keys like 'mandatory', 'repeatable', 'datatype'
+    """
+    # Copilot: implement SHACL property shape creation
+```
+
+### 2. Parsing TSV to SHACL
+```python
+# When converting TSV rows to SHACL shapes, use this pattern:
+def tsv_row_to_shacl(row, graph, prefixes):
+    """
+    Convert a TSV row to SHACL constraints.
+    Expected columns: shapeID, propertyID, mandatory, repeatable, valueShape
+    """
+    # Copilot: handle prefix expansion and constraint mapping
+```
+
+### 3. Error Message Formatting
+```python
+# Format validation errors for user display
+def format_validation_error(result):
+    """
+    Format pyshacl validation result for Gradio display.
+    Include: severity, focus node, property path, and message
+    """
+    # Copilot: create user-friendly error messages with context
+```
+
+### 4. AI Integration Patterns
+```python
+# Template for AI API calls
+async def get_ai_suggestion(error_context, rdf_snippet):
+    """
+    Get AI suggestions for fixing RDF validation errors.
+    Uses Hugging Face Inference API with proper error handling.
+    """
+    # Copilot: implement with retry logic and timeout handling
+```
+
+## Debugging Helpers
+
+### SHACL Validation Issues
+```python
+# Debug template for missing validations
+def debug_shacl_targeting():
+    """
+    Log all target classes and matching nodes in the data graph.
+    Helps diagnose why validations aren't triggering.
+    """
+    # Copilot: implement comprehensive logging of shapes and targets
+```
+
+### Namespace Resolution
+```python
+# Helper for namespace issues
+def resolve_prefixed_uri(prefixed_id, namespace_map):
+    """
+    Resolve prefixed identifiers like 'bf:Work' to full URIs.
+    Handle edge cases: no prefix, already full URI, unknown prefix
+    """
+    # Copilot: implement robust prefix resolution
+```
+
+## MCP Server Implementation
+
+### Tool Registration Pattern
+```python
+# MCP tool definition template
+@mcp_server.tool()
+async def new_mcp_tool(param1: str, param2: Optional[str] = None) -> dict:
+    """
+    MCP tool implementation.
+    Returns: {"success": bool, "result": Any, "error": Optional[str]}
+    """
+    # Copilot: implement with proper error handling and logging
+```
+
+### SSE Event Formatting
+```python
+# Server-Sent Events response pattern
+def format_sse_response(tool_name, result):
+    """
+    Format MCP tool response as SSE event.
+    Include proper event type and JSON encoding.
+    """
+    # Copilot: implement SSE formatting with error states
+```
+
+## Testing Patterns
+
+### Unit Test Templates
+```python
+# Test SHACL shape generation
+def test_shape_generation():
+    """
+    Test that TSV rows correctly generate SHACL shapes.
+    Include: basic properties, cardinality, value shapes
+    """
+    # Copilot: generate comprehensive test cases
+
+# Test RDF validation
+def test_rdf_validation():
+    """
+    Test validation with various RDF inputs.
+    Include: valid, invalid, edge cases
+    """
+    # Copilot: create test data and assertions
+```
+
+### Integration Test Patterns
+```python
+# Test MCP server endpoints
+async def test_mcp_endpoints():
+    """
+    Test all MCP tools with realistic inputs.
+    Verify: response format, error handling, performance
+    """
+    # Copilot: implement async test scenarios
+```
+
+## Performance Optimization
+
+### Caching Strategies
+```python
+# Cache compiled SHACL graphs
+@lru_cache(maxsize=10)
+def get_compiled_shacl_graph(template_name):
+    """
+    Cache parsed SHACL graphs to avoid repeated parsing.
+    """
+    # Copilot: implement with proper cache invalidation
+
+# Cache namespace resolutions
+@lru_cache(maxsize=1000)
+def cached_uri_resolution(prefixed_id, namespace_json):
+    """
+    Cache URI resolutions to improve performance.
+    """
+    # Copilot: implement with hashable inputs
+```
+
+### Batch Processing
+```python
+# Process multiple RDF documents efficiently
+async def batch_validate_rdf(rdf_documents: List[str]):
+    """
+    Validate multiple RDF documents in parallel.
+    Use asyncio for concurrent processing.
+    """
+    # Copilot: implement with progress tracking
+```
+
+## Common Pitfalls to Avoid
+
+1. **Namespace Conflicts**: Always use `override=True` when binding namespaces
+2. **Graph Parsing**: Specify format explicitly, don't rely on auto-detection
+3. **SPARQL Queries**: Escape special characters in URIs
+4. **Async/Await**: Don't mix synchronous and asynchronous code
+5. **Error Messages**: Always include context for debugging
+
+## Gradio UI Enhancements
+
+### Adding New UI Components
+```python
+# Template for new Gradio components
+def create_validation_interface():
+    """
+    Create Gradio interface with:
+    - File upload for RDF
+    - Template selection
+    - Real-time validation
+    - Export functionality
+    """
+    # Copilot: implement with proper event handlers
+```
+
+### Custom CSS/Theming
+```python
+# Apply custom styling to Gradio components
+custom_css = """
+    .validation-error { color: red; font-weight: bold; }
+    .validation-warning { color: orange; }
+    .validation-info { color: blue; }
+"""
+# Copilot: suggest CSS for better UX
+```
+
+## Deployment Considerations
+
+### Hugging Face Spaces Configuration
+```python
+# Environment variable handling
+HF_API_KEY = os.environ.get("HF_API_KEY")
+if not HF_API_KEY:
+    logger.warning("HF_API_KEY not set, AI features disabled")
+    
+# Gradio launch configuration for Spaces
+demo.launch(
+    server_name="0.0.0.0",
+    server_port=7860,
+    share=False  # Don't use share=True on Spaces
+)
+```
+
+### Error Recovery
+```python
+# Implement graceful degradation
+def safe_ai_call(func):
+    """
+    Decorator for AI calls that falls back gracefully.
+    """
+    # Copilot: implement with fallback behavior
+```
+
+## Quick Reference
+
+### Essential Imports
+```python
+import gradio as gr
+import rdflib
+from rdflib import Graph, URIRef, Literal, Namespace, RDF, RDFS
+from pyshacl import validate
+import pandas as pd
+import logging
+import asyncio
+from typing import Optional, Dict, List, Any
+```
+
+### Debugging Commands
+```python
+# Log graph contents
+logger.debug(f"Graph has {len(graph)} triples")
+logger.debug(graph.serialize(format='turtle'))
+
+# Log validation details
+conforms, results_graph, results_text = validate(
+    data_graph, 
+    shacl_graph=shapes, 
+    debug=True,
+    inference='rdfs'
+)
+```
+
+### Common SHACL Properties
+- `sh:targetClass` - Define which RDF types to validate
+- `sh:path` - Property to validate
+- `sh:minCount` - Minimum occurrences (1 for mandatory)
+- `sh:maxCount` - Maximum occurrences (1 for non-repeatable)
+- `sh:datatype` - Expected datatype
+- `sh:node` - Link to another shape (valueShape)
+- `sh:severity` - sh:Violation, sh:Warning, or sh:Info
+
+Remember: Always test with real BIBFRAME data and verify MCP endpoints are accessible!

system-architecture.mmd

@@ -0,0 +1,53 @@
+graph TB
+    subgraph "User Interfaces"
+        UI1[Web Browser<br/>Gradio UI]
+        UI2[Claude Desktop<br/>MCP Client]
+        UI3[External Apps<br/>via API]
+    end
+    
+    subgraph "MCP4RDF Service Layer"
+        APP[app.py<br/>Main Application]
+        API[API Endpoints]
+        MCP[MCP Server<br/>SSE Protocol]
+        
+        APP --> API
+        APP --> MCP
+    end
+    
+    subgraph "Core Functions"
+        F1[validate_rdf_tool]
+        F2[get_ai_suggestions]
+        F3[get_ai_correction]
+        F4[get_rdf_examples]
+    end
+    
+    subgraph "Validation Engine"
+        VAL[validator.py]
+        SHACL[SHACL Templates]
+        PY[pyshacl Library]
+        
+        VAL --> PY
+        PY --> SHACL
+    end
+    
+    subgraph "AI Enhancement"
+        OAI[OpenAI Client]
+        HF[HF Inference Endpoint]
+        LLM[Llama-3.3-70B]
+        
+        OAI --> HF
+        HF --> LLM
+    end
+    
+    UI1 --> APP
+    UI2 --> MCP
+    UI3 --> API
+    
+    API --> F1
+    API --> F2
+    API --> F3
+    API --> F4
+    
+    F1 --> VAL
+    F2 --> OAI
+    F3 --> OAI