Upload folder using huggingface_hub

README.md

@@ -1,12 +1,10 @@
 ---
 title: Havelock Demo Substring
-emoji: 🏃
-colorFrom: green
-colorTo: pink
+emoji: 🔬
+colorFrom: purple
+colorTo: gray
 sdk: gradio
-sdk_version: 6.9.0
+sdk_version: 6.3.0
 app_file: app.py
 pinned: false
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,783 @@
+# v1.7 - Substring-level marker highlighting
+"""
+Havelock.AI - Orality Analyzer (Substring Staging)
+
+Adds token-level span predictions from the trained MultiLabelTokenClassifier
+(HavelockAI/bert-token-classifier) alongside existing sentence-level models.
+
+Only Tier 1 markers (F1 >= 0.50) produce substring spans.
+"""
+
+import gradio as gr
+import torch
+import torch.nn as nn
+from transformers import (
+    AutoModel,
+    AutoTokenizer,
+    BertTokenizerFast,
+    BertModel,
+    BertForSequenceClassification,
+)
+from huggingface_hub import hf_hub_download
+import json
+import re
+import threading
+import time
+import random
+
+# Tracking endpoint
+TRACK_URL = "https://havelock.ai/api/track"
+
+# Language detection
+try:
+    from langdetect import detect, DetectorFactory
+    DetectorFactory.seed = 0
+    HAS_LANGDETECT = True
+except ImportError:
+    HAS_LANGDETECT = False
+    print("Warning: langdetect not installed. Language detection disabled.")
+
+# Model repositories
+MODEL_REPO = "thestalwart/havelock-orality"
+TOKEN_MODEL_REPO = "HavelockAI/bert-token-classifier"
+
+# Device
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# Ensemble weights
+DOC_MODEL_WEIGHT = 0.35
+SENTENCE_WEIGHT = 0.65
+
+# Sentence analysis cap
+MAX_SENTENCES = 100
+MAX_INPUT_CHARS = 50000
+MAX_WORDS_PER_SENTENCE = 150
+
+# Tier 1 markers: F1 >= 0.50 from manifest
+TIER1_MARKERS = {
+    "oral_vocative",
+    "literate_technical_abbreviation",
+    "oral_phatic_check",
+    "oral_imperative",
+    "oral_specific_place",
+    "literate_citation",
+    "literate_agentless_passive",
+    "oral_rhetorical_question",
+    "oral_inclusive_we",
+    "oral_second_person",
+    "oral_named_individual",
+    "literate_nominalization",
+    "literate_probability",
+}
+
+
+class BertOralityRegressor(nn.Module):
+    """BERT model with regression head for orality scoring."""
+    def __init__(self, bert_model_name='bert-base-uncased', dropout=0.1):
+        super().__init__()
+        self.bert = BertModel.from_pretrained(bert_model_name)
+        self.dropout = nn.Dropout(dropout)
+        self.regressor = nn.Linear(self.bert.config.hidden_size, 1)
+        self.sigmoid = nn.Sigmoid()
+
+    def forward(self, input_ids, attention_mask):
+        outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask)
+        pooled_output = outputs.pooler_output
+        pooled_output = self.dropout(pooled_output)
+        logits = self.regressor(pooled_output)
+        return self.sigmoid(logits).squeeze(-1)
+
+
+def load_models():
+    """Download and load all models from HuggingFace Hub."""
+    print("Loading sentence-level models...")
+
+    tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')
+
+    # Load document regressor
+    doc_model_path = hf_hub_download(repo_id=MODEL_REPO, filename="bert_orality_regressor.pt")
+    doc_model = BertOralityRegressor().to(DEVICE)
+    doc_model.load_state_dict(torch.load(doc_model_path, map_location=DEVICE))
+    doc_model.eval()
+
+    # Load category classifier
+    cat_model_path = hf_hub_download(repo_id=MODEL_REPO, filename="bert_marker_category.pt")
+    cat_labels_path = hf_hub_download(repo_id=MODEL_REPO, filename="bert_marker_category_labels.json")
+    with open(cat_labels_path) as f:
+        category_labels = json.load(f)
+    category_model = BertForSequenceClassification.from_pretrained(
+        'bert-base-uncased', num_labels=len(category_labels)
+    ).to(DEVICE)
+    category_model.load_state_dict(torch.load(cat_model_path, map_location=DEVICE))
+    category_model.eval()
+
+    # Load subtype classifier
+    sub_model_path = hf_hub_download(repo_id=MODEL_REPO, filename="bert_marker_subtype.pt")
+    sub_labels_path = hf_hub_download(repo_id=MODEL_REPO, filename="bert_marker_subtype_labels.json")
+    with open(sub_labels_path) as f:
+        subtype_labels = json.load(f)
+    subtype_model = BertForSequenceClassification.from_pretrained(
+        'bert-base-uncased', num_labels=len(subtype_labels),
+        attn_implementation="eager"
+    ).to(DEVICE)
+    subtype_model.load_state_dict(torch.load(sub_model_path, map_location=DEVICE))
+    subtype_model.eval()
+
+    print("Sentence-level models loaded!")
+    return tokenizer, doc_model, category_model, category_labels, subtype_model, subtype_labels
+
+
+def load_token_classifier():
+    """Load the token-level classifier from HuggingFace Hub."""
+    print("Loading token classifier...")
+
+    tok_tokenizer = AutoTokenizer.from_pretrained(TOKEN_MODEL_REPO)
+    tok_model = AutoModel.from_pretrained(TOKEN_MODEL_REPO, trust_remote_code=True)
+    tok_model.to(DEVICE)
+    tok_model.eval()
+
+    type_map_path = hf_hub_download(TOKEN_MODEL_REPO, "type_to_idx.json")
+    with open(type_map_path) as f:
+        type_to_idx = json.load(f)
+    idx_to_type = {v: k for k, v in type_to_idx.items()}
+
+    print(f"Token classifier loaded! ({len(type_to_idx)} marker types)")
+    return tok_tokenizer, tok_model, idx_to_type
+
+
+# Load all models at startup
+tokenizer, doc_model, category_model, category_labels, subtype_model, subtype_labels = load_models()
+tok_tokenizer, tok_model, idx_to_type = load_token_classifier()
+
+
+def split_sentences(text):
+    """Split text into sentences."""
+    sentences = re.split(r'(?<=[.!?])\s+', text)
+    return [s.strip() for s in sentences if s.strip()]
+
+
+def predict_doc_score(text):
+    """Predict document-level orality score with chunking for long texts."""
+    full_encoding = tokenizer(text, truncation=False, return_tensors='pt')
+    total_tokens = full_encoding['input_ids'].shape[1]
+
+    if total_tokens <= 512:
+        encoding = tokenizer(
+            text, truncation=True, max_length=512,
+            padding='max_length', return_tensors='pt'
+        )
+        with torch.no_grad():
+            score = doc_model(
+                encoding['input_ids'].to(DEVICE),
+                encoding['attention_mask'].to(DEVICE)
+            )
+        return score.item()
+
+    input_ids = full_encoding['input_ids'][0]
+    chunk_size = 512
+    stride = 448
+    scores = []
+
+    for start in range(0, total_tokens, stride):
+        end = min(start + chunk_size, total_tokens)
+        chunk_ids = input_ids[start:end].unsqueeze(0)
+
+        if chunk_ids.shape[1] < chunk_size:
+            pad_length = chunk_size - chunk_ids.shape[1]
+            chunk_ids = torch.nn.functional.pad(chunk_ids, (0, pad_length), value=tokenizer.pad_token_id)
+
+        attention_mask = (chunk_ids != tokenizer.pad_token_id).long()
+
+        with torch.no_grad():
+            score = doc_model(
+                chunk_ids.to(DEVICE),
+                attention_mask.to(DEVICE)
+            )
+        scores.append(score.item())
+
+        if end >= total_tokens:
+            break
+
+    return sum(scores) / len(scores)
+
+
+def predict_category(text):
+    """Predict oral vs literate for a sentence with confidence."""
+    encoding = tokenizer(
+        text, truncation=True, max_length=128,
+        padding='max_length', return_tensors='pt'
+    )
+    with torch.no_grad():
+        outputs = category_model(
+            encoding['input_ids'].to(DEVICE),
+            encoding['attention_mask'].to(DEVICE)
+        )
+        probs = torch.softmax(outputs.logits, dim=1)
+        pred = torch.argmax(probs, dim=1).item()
+        confidence = probs[0][pred].item()
+
+    id_to_label = {v: k for k, v in category_labels.items()}
+    return id_to_label[pred], confidence
+
+
+def predict_subtype(text, top_k=3, threshold=0.05):
+    """Predict marker subtype(s) for a sentence."""
+    encoding = tokenizer(
+        text, truncation=True, max_length=128,
+        padding='max_length', return_tensors='pt'
+    )
+    with torch.no_grad():
+        outputs = subtype_model(
+            encoding['input_ids'].to(DEVICE),
+            encoding['attention_mask'].to(DEVICE)
+        )
+        probs = torch.softmax(outputs.logits, dim=1)
+        top_probs, top_indices = torch.topk(probs, k=top_k, dim=1)
+
+    id_to_label = {v: k for k, v in subtype_labels.items()}
+
+    markers = []
+    for prob, idx in zip(top_probs[0], top_indices[0]):
+        conf = prob.item()
+        if conf >= threshold:
+            markers.append({
+                'marker': id_to_label[idx.item()],
+                'confidence': round(conf, 3)
+            })
+
+    if not markers:
+        markers = [{
+            'marker': id_to_label[top_indices[0][0].item()],
+            'confidence': round(top_probs[0][0].item(), 3)
+        }]
+
+    return markers
+
+
+def predict_spans(text):
+    """Run token classifier and return character-level spans for Tier 1 markers.
+
+    Returns a list of span dicts with character offsets into the original text.
+    Only markers in TIER1_MARKERS are included.
+    """
+    encoding = tok_tokenizer(
+        text,
+        return_tensors="pt",
+        truncation=True,
+        max_length=128,
+        return_offsets_mapping=True,
+    )
+
+    offset_mapping = encoding.pop("offset_mapping")[0]  # (seq_len, 2)
+    input_ids = encoding["input_ids"].to(DEVICE)
+    attention_mask = encoding["attention_mask"].to(DEVICE)
+
+    with torch.no_grad():
+        if hasattr(tok_model, "decode"):
+            preds = tok_model.decode(input_ids, attention_mask)
+        else:
+            logits = tok_model(input_ids, attention_mask)
+            preds = logits.argmax(dim=-1)
+
+    # preds shape: (1, seq_len, num_types) where values are 0=O, 1=B, 2=I
+    preds = preds[0]  # (seq_len, num_types)
+    seq_len = attention_mask.sum().item()
+
+    # Collect spans per marker type using BIO transitions
+    spans = []
+
+    for type_idx in range(preds.shape[1]):
+        marker_name = idx_to_type.get(type_idx)
+        if marker_name is None or marker_name not in TIER1_MARKERS:
+            continue
+
+        span_start_tok = None
+
+        for tok_pos in range(seq_len):
+            tag = preds[tok_pos, type_idx].item()
+            offsets = offset_mapping[tok_pos].tolist()
+
+            # Skip special tokens (offset 0,0)
+            if offsets[0] == 0 and offsets[1] == 0 and tok_pos > 0:
+                if span_start_tok is not None:
+                    _emit_token_span(spans, text, offset_mapping, span_start_tok, tok_pos, marker_name)
+                    span_start_tok = None
+                continue
+
+            if tag == 1:  # B
+                if span_start_tok is not None:
+                    _emit_token_span(spans, text, offset_mapping, span_start_tok, tok_pos, marker_name)
+                span_start_tok = tok_pos
+            elif tag == 2:  # I
+                if span_start_tok is None:
+                    span_start_tok = tok_pos  # orphan I, treat as B
+            else:  # O
+                if span_start_tok is not None:
+                    _emit_token_span(spans, text, offset_mapping, span_start_tok, tok_pos, marker_name)
+                    span_start_tok = None
+
+        if span_start_tok is not None:
+            _emit_token_span(spans, text, offset_mapping, span_start_tok, seq_len, marker_name)
+
+    # Sort by start position
+    spans.sort(key=lambda s: (s["start"], s["end"]))
+    return spans
+
+
+def _emit_token_span(spans, text, offset_mapping, start_tok, end_tok, marker_name):
+    """Convert token indices to a character-level span dict."""
+    char_start = int(offset_mapping[start_tok][0])
+    char_end = int(offset_mapping[end_tok - 1][1])
+    if char_end > char_start:
+        span_text = text[char_start:char_end]
+        category = "oral" if marker_name.startswith("oral_") else "literate"
+        spans.append({
+            "text": span_text,
+            "marker": marker_name,
+            "category": category,
+            "start": char_start,
+            "end": char_end,
+        })
+
+
+def get_attention_spans(text, threshold=0.15):
+    """Extract high-attention token spans from the subtype model."""
+    encoding = tokenizer(
+        text, truncation=True, max_length=128,
+        padding='max_length', return_tensors='pt',
+        return_offsets_mapping=True
+    )
+
+    offset_mapping = encoding.pop('offset_mapping')[0]
+    input_ids = encoding['input_ids'].to(DEVICE)
+    attention_mask = encoding['attention_mask'].to(DEVICE)
+
+    with torch.no_grad():
+        outputs = subtype_model.bert(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            output_attentions=True
+        )
+
+    last_layer = outputs.attentions[-1][0]
+    cls_attention = last_layer.mean(dim=0)[0]
+
+    max_attn = cls_attention.max()
+    if max_attn > 0:
+        cls_attention = cls_attention / max_attn
+
+    seq_len = attention_mask.sum().item()
+    tokens = tokenizer.convert_ids_to_tokens(input_ids[0][:seq_len])
+    offsets = offset_mapping[:seq_len].tolist()
+    attn_values = cls_attention[:seq_len].tolist()
+
+    PUNCT = {'.', ',', '!', '?', ';', ':', '-', '--', "'", '"',
+             '(', ')', '[', ']', '...', '\u2013', '\u2014'}
+
+    spans = []
+    in_span = False
+    span_start_idx = 0
+    span_attentions = []
+
+    for i, (token, attn, (cs, ce)) in enumerate(zip(tokens, attn_values, offsets)):
+        if cs == 0 and ce == 0:
+            if in_span and span_attentions:
+                _emit_attn_span(spans, text, offsets, span_start_idx, i, span_attentions)
+            in_span = False
+            span_attentions = []
+            continue
+
+        if token in PUNCT:
+            if in_span and span_attentions:
+                _emit_attn_span(spans, text, offsets, span_start_idx, i, span_attentions)
+            in_span = False
+            span_attentions = []
+            continue
+
+        if attn >= threshold:
+            if not in_span:
+                in_span = True
+                span_start_idx = i
+                span_attentions = [attn]
+            else:
+                span_attentions.append(attn)
+        else:
+            if in_span and span_attentions:
+                _emit_attn_span(spans, text, offsets, span_start_idx, i, span_attentions)
+            in_span = False
+            span_attentions = []
+
+    if in_span and span_attentions:
+        _emit_attn_span(spans, text, offsets, span_start_idx, seq_len, span_attentions)
+
+    return spans
+
+
+def _emit_attn_span(spans, text, offsets, start_idx, end_idx, attentions):
+    """Helper to build an attention span dict from token indices."""
+    char_start = int(offsets[start_idx][0])
+    char_end = int(offsets[end_idx - 1][1])
+    if char_end > char_start:
+        spans.append({
+            'char_start': char_start,
+            'char_end': char_end,
+            'text': text[char_start:char_end],
+            'attention': round(sum(attentions) / len(attentions), 3)
+        })
+
+
+def detect_language(text):
+    """Detect language of text."""
+    if not HAS_LANGDETECT:
+        return None, True, None
+    try:
+        if len(text.split()) < 10:
+            return None, True, None
+        lang = detect(text[:1000])
+        is_english = lang == 'en'
+        if not is_english:
+            warning = f"Non-English text detected ({lang}). Results may be unreliable as the model was trained primarily on English text."
+            return lang, False, warning
+        return lang, True, None
+    except Exception:
+        return None, True, None
+
+
+def log_usage(text, score, oral_count, literate_count, sentence_results):
+    """Log API usage to Cloudflare D1 via track endpoint (fire and forget)."""
+    def _send():
+        try:
+            import urllib.request
+            import urllib.error
+
+            words = text.split() if text else []
+            sentences = [s for s in re.split(r'[.!?]+', text) if s.strip()] if text else []
+            unique_words = len(set(w.lower() for w in words if w.isalpha()))
+
+            word_count = len(words)
+            sentence_count = len(sentences)
+            avg_sentence_length = round(word_count / sentence_count, 1) if sentence_count else 0
+            alpha_words = [w for w in words if w.isalpha()]
+            avg_word_length = round(sum(len(w) for w in alpha_words) / len(alpha_words), 1) if alpha_words else 0
+            lexical_diversity = round((unique_words / word_count) * 100) if word_count else 0
+
+            marker_counts = {}
+            for sent in sentence_results:
+                marker = sent.get('marker', '')
+                if marker:
+                    marker_counts[marker] = marker_counts.get(marker, 0) + 1
+
+            analysis_id = hex(int(time.time()))[2:] + hex(random.randint(0, 0xFFFFFFFF))[2:]
+
+            payload = {
+                "page": "huggingface_api",
+                "text": text[:10000] if text else "",
+                "score": score,
+                "word_count": word_count,
+                "sentence_count": sentence_count,
+                "avg_sentence_length": avg_sentence_length,
+                "avg_word_length": avg_word_length,
+                "lexical_diversity": lexical_diversity,
+                "oral_marker_count": oral_count,
+                "literate_marker_count": literate_count,
+                "markers_json": json.dumps(marker_counts) if marker_counts else None,
+                "analysis_id": analysis_id,
+            }
+
+            data = json.dumps(payload).encode('utf-8')
+            req = urllib.request.Request(
+                TRACK_URL,
+                data=data,
+                headers={'Content-Type': 'application/json', 'User-Agent': 'HavelockSpace/1.7'},
+                method='POST'
+            )
+            urllib.request.urlopen(req, timeout=5)
+        except Exception as e:
+            print(f"Tracking failed: {e}")
+
+    thread = threading.Thread(target=_send, daemon=True)
+    thread.start()
+
+
+def analyze_api(text):
+    """JSON API for website integration.
+
+    v1.7: Adds spans field with token-level marker predictions (Tier 1 only).
+    """
+    if not text or len(text.strip()) < 10:
+        return {"error": "Please enter at least 10 characters of text."}
+
+    if len(text) > MAX_INPUT_CHARS:
+        return {"error": f"Text too long. Maximum {MAX_INPUT_CHARS:,} characters allowed ({len(text):,} provided)."}
+
+    # Language detection
+    lang_code, is_english, lang_warning = detect_language(text)
+
+    # Document-level score
+    doc_score = predict_doc_score(text)
+
+    # Sentence-level analysis
+    sentences = split_sentences(text)
+    total_sentence_count = len(sentences)
+    truncated = total_sentence_count > MAX_SENTENCES
+    if truncated:
+        sentences = sentences[:MAX_SENTENCES]
+
+    oral_count = 0
+    literate_count = 0
+    oral_weighted = 0.0
+    literate_weighted = 0.0
+    sentence_results = []
+
+    for sent in sentences:
+        word_count = len(sent.split())
+        if word_count < 3 or word_count > MAX_WORDS_PER_SENTENCE:
+            continue
+
+        category, cat_confidence = predict_category(sent)
+        markers = predict_subtype(sent)
+        attention_spans = get_attention_spans(sent)
+        spans = predict_spans(sent)
+
+        if category == 'oral':
+            oral_count += 1
+            oral_weighted += cat_confidence
+        else:
+            literate_count += 1
+            literate_weighted += cat_confidence
+
+        sentence_results.append({
+            'text': sent,
+            'category': category,
+            'category_confidence': round(cat_confidence, 3),
+            'marker': markers[0]['marker'],
+            'confidence': markers[0]['confidence'],
+            'markers': markers,
+            'attention_spans': attention_spans,
+            'spans': spans,
+        })
+
+    # Ensemble scoring
+    total = oral_count + literate_count
+    total_weighted = oral_weighted + literate_weighted
+    sentence_ratio_binary = oral_count / total if total > 0 else 0.5
+    sentence_ratio_weighted = oral_weighted / total_weighted if total_weighted > 0 else 0.5
+    ensemble_score = (DOC_MODEL_WEIGHT * doc_score) + (SENTENCE_WEIGHT * sentence_ratio_weighted)
+
+    result = {
+        'score': round(ensemble_score * 100),
+        'doc_score': round(doc_score, 3),
+        'sentence_ratio': round(sentence_ratio_weighted, 3),
+        'sentence_ratio_binary': round(sentence_ratio_binary, 3),
+        'oral_count': oral_count,
+        'literate_count': literate_count,
+        'oral_weighted': round(oral_weighted, 3),
+        'literate_weighted': round(literate_weighted, 3),
+        'sentences': sentence_results
+    }
+
+    if lang_warning:
+        result['language_warning'] = lang_warning
+        result['detected_language'] = lang_code
+
+    if truncated:
+        result['truncation_warning'] = f"This text has {total_sentence_count} sentences. Only the first {MAX_SENTENCES} were analyzed."
+        result['total_sentences'] = total_sentence_count
+        result['analyzed_sentences'] = MAX_SENTENCES
+
+    log_usage(text, result['score'], oral_count, literate_count, sentence_results)
+
+    return result
+
+
+def analyze_text(text):
+    """Main analysis function for Gradio UI."""
+    if not text or len(text.strip()) < 10:
+        return "Please enter some text to analyze.", "", ""
+
+    if len(text) > MAX_INPUT_CHARS:
+        return f"Text too long. Maximum {MAX_INPUT_CHARS:,} characters allowed.", "", ""
+
+    lang_code, is_english, lang_warning = detect_language(text)
+    doc_score = predict_doc_score(text)
+
+    sentences = split_sentences(text)
+    total_sentence_count = len(sentences)
+    truncated = total_sentence_count > MAX_SENTENCES
+    if truncated:
+        sentences = sentences[:MAX_SENTENCES]
+
+    oral_count = 0
+    literate_count = 0
+    oral_weighted = 0.0
+    literate_weighted = 0.0
+    sentence_results = []
+
+    for sent in sentences:
+        word_count = len(sent.split())
+        if word_count < 3 or word_count > MAX_WORDS_PER_SENTENCE:
+            continue
+
+        category, cat_confidence = predict_category(sent)
+        markers = predict_subtype(sent)
+
+        if category == 'oral':
+            oral_count += 1
+            oral_weighted += cat_confidence
+        else:
+            literate_count += 1
+            literate_weighted += cat_confidence
+
+        sentence_results.append({
+            'sentence': sent[:200] + '...' if len(sent) > 200 else sent,
+            'category': category,
+            'cat_confidence': cat_confidence,
+            'subtype': markers[0]['marker'],
+            'confidence': markers[0]['confidence']
+        })
+
+    total = oral_count + literate_count
+    total_weighted = oral_weighted + literate_weighted
+    sentence_ratio = oral_weighted / total_weighted if total_weighted > 0 else 0.5
+    ensemble_score = (DOC_MODEL_WEIGHT * doc_score) + (SENTENCE_WEIGHT * sentence_ratio)
+
+    if ensemble_score >= 0.65:
+        mode = "ORAL"
+        mode_desc = "High oral characteristics - repetition, direct address, concrete imagery"
+        color = "#228B22"
+    elif ensemble_score >= 0.35:
+        mode = "MIXED"
+        mode_desc = "Mixed oral and literate characteristics"
+        color = "#B8860B"
+    else:
+        mode = "LITERATE"
+        mode_desc = "High literate characteristics - abstraction, subordination, hedging"
+        color = "#4169E1"
+
+    bar_len = int(ensemble_score * 30)
+    bar = "\u2588" * bar_len + "\u2591" * (30 - bar_len)
+
+    warnings_html = ""
+    if lang_warning:
+        warnings_html += f"""
+        <div style="background: #f8d7da; border: 1px solid #f5c6cb; color: #721c24; padding: 12px; border-radius: 8px; margin-bottom: 15px;">
+            <strong>Warning:</strong> {lang_warning}
+        </div>
+        """
+    if truncated:
+        warnings_html += f"""
+        <div style="background: #fff3cd; border: 1px solid #ffc107; color: #856404; padding: 12px; border-radius: 8px; margin-bottom: 15px;">
+            <strong>Note:</strong> This text has {total_sentence_count} sentences. Only the first {MAX_SENTENCES} were analyzed.
+        </div>
+        """
+
+    score_html = f"""
+    <div style="font-family: system-ui; padding: 20px; background: #f8f9fa; border-radius: 10px;">
+        {warnings_html}
+        <h2 style="color: {color}; margin-bottom: 10px;">Orality Score: {ensemble_score:.2f} ({mode})</h2>
+        <p style="color: #666; margin-bottom: 15px;">{mode_desc}</p>
+        <div style="font-family: monospace; font-size: 14px; margin-bottom: 15px;">
+            <span style="color: #4169E1;">Literate</span> [{bar}] <span style="color: #228B22;">Oral</span>
+        </div>
+        <div style="font-size: 13px; color: #555;">
+            <strong>Score Components:</strong><br>
+            - Sentence analysis: {oral_count}/{total} oral ({sentence_ratio:.0%}) [confidence-weighted]<br>
+            - Document model: {doc_score:.2f}<br>
+            - Ensemble weights: {int(SENTENCE_WEIGHT*100)}% sentence + {int(DOC_MODEL_WEIGHT*100)}% document
+        </div>
+    </div>
+    """
+
+    if sentence_results:
+        rows = ""
+        for r in sentence_results[:20]:
+            cat_color = "#228B22" if r['category'] == 'oral' else "#4169E1"
+            rows += f"""
+            <tr>
+                <td style="color: {cat_color}; font-weight: bold; padding: 8px;">{r['category'].upper()}</td>
+                <td style="padding: 8px;">{r['subtype']} ({r['confidence']:.0%})</td>
+                <td style="padding: 8px; font-style: italic;">"{r['sentence']}"</td>
+            </tr>
+            """
+
+        sentences_html = f"""
+        <div style="font-family: system-ui; padding: 20px; background: #f8f9fa; border-radius: 10px; margin-top: 20px;">
+            <h3 style="margin-bottom: 15px;">Sentence-Level Analysis</h3>
+            <table style="width: 100%; border-collapse: collapse; font-size: 13px;">
+                <tr style="background: #e9ecef;">
+                    <th style="padding: 8px; text-align: left; width: 80px;">Category</th>
+                    <th style="padding: 8px; text-align: left; width: 150px;">Marker Type</th>
+                    <th style="padding: 8px; text-align: left;">Sentence</th>
+                </tr>
+                {rows}
+            </table>
+            {f'<p style="color: #666; margin-top: 10px; font-size: 12px;">...and {len(sentence_results) - 20} more sentences analyzed</p>' if len(sentence_results) > 20 else ''}
+        </div>
+        """
+    else:
+        sentences_html = ""
+
+    reference_html = """
+    <div style="font-family: system-ui; padding: 20px; background: #fff3cd; border-radius: 10px; margin-top: 20px;">
+        <h4 style="margin-bottom: 10px;">Reference: Orality Scores by Genre</h4>
+        <table style="font-size: 12px; width: 100%;">
+            <tr><td><strong>0.9+</strong></td><td>Epic poetry, hip-hop, spoken word</td></tr>
+            <tr><td><strong>0.7-0.9</strong></td><td>Speeches, sermons, podcasts</td></tr>
+            <tr><td><strong>0.4-0.7</strong></td><td>Essays, blogs, casual writing</td></tr>
+            <tr><td><strong>0.1-0.4</strong></td><td>Journalism, technical writing</td></tr>
+            <tr><td><strong><0.1</strong></td><td>Academic papers, legal documents, philosophy</td></tr>
+        </table>
+    </div>
+    """
+
+    return score_html, sentences_html, reference_html
+
+
+# Example texts
+examples = [
+    ["Tell me, O Muse, of that ingenious hero who travelled far and wide after he had sacked the famous town of Troy. Many cities did he visit, and many were the nations with whose manners and customs he was acquainted."],
+    ["We will fight on the beaches, we will fight on the landing grounds, we will fight in the fields and in the streets, we will fight in the hills; we will never surrender."],
+    ["The analysis of variance revealed a statistically significant effect of treatment condition on participant response latency, F(2, 147) = 4.23, p < .05, suggesting that the experimental manipulation influenced cognitive processing speed."],
+    ["So like, I was just thinking about this the other day, right? And it's crazy because we never really talk about how much social media has changed everything. You know what I mean?"],
+]
+
+# Build interface
+with gr.Blocks(title="Havelock.AI - Orality Analyzer (Substring)", theme=gr.themes.Soft()) as demo:
+    gr.Markdown("""
+    # Havelock.AI - Orality Analyzer (Substring Staging)
+
+    Analyze text for **oral vs literate characteristics** based on Walter Ong's linguistic framework.
+
+    **v1.7**: Token-level span predictions for Tier 1 markers (F1 >= 0.50).
+    Use the API endpoint for span data — the Gradio UI shows sentence-level results only.
+    """)
+
+    with gr.Row():
+        with gr.Column(scale=1):
+            text_input = gr.Textbox(
+                label="Enter text to analyze",
+                placeholder="Paste your text here...",
+                lines=8
+            )
+            analyze_btn = gr.Button("Analyze", variant="primary")
+            gr.Examples(examples, inputs=text_input, label="Try these examples")
+
+        with gr.Column(scale=1):
+            score_output = gr.HTML(label="Orality Score")
+
+    sentences_output = gr.HTML(label="Sentence Analysis")
+    reference_output = gr.HTML(label="Reference")
+
+    analyze_btn.click(
+        fn=analyze_text,
+        inputs=text_input,
+        outputs=[score_output, sentences_output, reference_output]
+    )
+
+    # Hidden API interface for website integration
+    api_input = gr.Textbox(visible=False)
+    api_output = gr.JSON(visible=False)
+    api_btn = gr.Button(visible=False)
+    api_btn.click(fn=analyze_api, inputs=api_input, outputs=api_output, api_name="analyze")
+
+demo.launch()

requirements.txt

@@ -0,0 +1,5 @@
+torch
+transformers
+huggingface_hub
+langdetect
+safetensors